Merge remote-tracking branch 'origin/closedown-5.0.x'

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

common/workunit/pkgimpl.hpp

@@ -382,7 +382,8 @@ public:
 
     void load(const char *id)
     {
-        load(getPackageMapById(id, true));
+        Owned<IPropertyTree> xml = getPackageMapById(id, true);
+        load(xml);
     }
 
     virtual void gatherFileMappingForQuery(const char *queryname, IPropertyTree *fileInfo) const

ecl/hql/hqlgram.y

@@ -8573,21 +8573,21 @@ simpleDataSet
                             $$.setExpr(dataset);
                             $$.setPosition($1);
                         }
-    | DATASET '(' '[' beginList inlineDatasetValueList ']' ',' recordDef ')'
+    | DATASET '(' '[' beginList inlineDatasetValueList ']' ',' recordDef optDatasetFlags ')'
                         {
                             HqlExprArray values;
                             parser->endList(values);
-                            OwnedHqlExpr table = createDataset(no_temptable, createValue(no_recordlist, NULL, values), $8.getExpr());
+                            OwnedHqlExpr table = createDataset(no_temptable, createValue(no_recordlist, NULL, values), createComma($8.getExpr(), $9.getExpr()));
                             $$.setExpr(convertTempTableToInlineTable(*parser->errorHandler, $5.pos, table));
                             $$.setPosition($1);
                         }
-    | DATASET '(' '[' beginList transformList ']' ')'
+    | DATASET '(' '[' beginList transformList ']' optDatasetFlags ')'
                         {
                             HqlExprArray values;
                             parser->endList(values);
                             IHqlExpression * record = values.item(0).queryRecord();
                             parser->checkCompatibleTransforms(values, record, $5);
-                            $$.setExpr(createDataset(no_inlinetable, createValue(no_transformlist, makeNullType(), values), LINK(record)));
+                            $$.setExpr(createDataset(no_inlinetable, createValue(no_transformlist, makeNullType(), values), createComma(LINK(record), $7.getExpr())));
                             $$.setPosition($1);
                         }
     | DATASET '(' thorFilenameOrList ',' beginCounterScope recordDef endCounterScope ')'

ecl/hql/hqlutil.cpp

@@ -5922,6 +5922,7 @@ static IHqlExpression * convertTempTableToInline(IErrorReceiver & errorProcessor
     HqlExprArray children;
     children.append(*createValue(no_transformlist, makeNullType(), transforms));
     children.append(*LINK(record));
+    unwindChildren(children, expr, 2);
     OwnedHqlExpr ret = createDataset(no_inlinetable, children);
     return expr->cloneAllAnnotations(ret);
 }

ecl/regress/dataset3.ecl

@@ -32,7 +32,7 @@ namesTable := dataset([
         {'Page','John',62},
         {'Page','Chris',26},
         {'Smithe','Abigail',13},
-        {'X','Za'}], namesRecord);
+        {'X','Za'}], namesRecord, DISTRIBUTED);
 
 x := group(namesTable, surname);// : persist('~GroupedNames2');
 

esp/services/ws_packageprocess/ws_packageprocessService.cpp

@@ -944,7 +944,8 @@ bool CWsPackageProcessEx::onGetQueryFileMapping(IEspContext &context, IEspGetQue
     const char *pmid = req.getPMID();
     if (pmid && *pmid)
     {
-        ownedmap.setown(createPackageMapFromPtree(getPackageMapById(req.getGlobalScope() ? NULL : target, pmid, true), target, pmid));
+        Owned<IPropertyTree> pm = getPackageMapById(req.getGlobalScope() ? NULL : target, pmid, true);
+        ownedmap.setown(createPackageMapFromPtree(pm, target, pmid));
         if (!ownedmap)
             throw MakeStringException(PKG_LOAD_PACKAGEMAP_FAILED, "Error loading package map %s", req.getPMID());
         map = ownedmap;

esp/src/eclwatch/DFUQueryWidget.js

@@ -179,7 +179,7 @@ define([
             if (this.copyForm.validate()) {
                 var context = this;
                 arrayUtil.forEach(this.copyGrid.store.data, function (item, idx) {
-                    var logicalFile = ESPLogicalFile.Get(item.ClusterName, item.Name);
+                    var logicalFile = ESPLogicalFile.Get(item.NodeGroup, item.Name);
                     var request = domForm.toObject(context.id + "CopyForm");
                     request.RenameSourceName = item.Name;
                     request.destLogicalName = item.targetCopyName;
@@ -198,7 +198,7 @@ define([
             if (this.renameForm.validate()) {
                 var context = this;
                 arrayUtil.forEach(this.renameGrid.store.data, function (item, idx) {
-                    var logicalFile = ESPLogicalFile.Get(item.ClusterName, item.Name);
+                    var logicalFile = ESPLogicalFile.Get(item.NodeGroup, item.Name);
                     var request = domForm.toObject(context.id + "RenameForm");
                     request.RenameSourceName = item.Name;
                     request.dstname = item.targetRenameName;
@@ -258,14 +258,14 @@ define([
             if (item) {
                 this.menuFilterOwner.set("label", this.i18n.Owner + ":  " + item.Owner);
                 this.menuFilterOwner.set("hpcc_value", item.Owner);
-                this.menuFilterCluster.set("label", this.i18n.Cluster + ":  " + item.ClusterName);
-                this.menuFilterCluster.set("hpcc_value", item.ClusterName);
+                this.menuFilterCluster.set("label", this.i18n.Cluster + ":  " + item.NodeGroup);
+                this.menuFilterCluster.set("hpcc_value", item.NodeGroup);
             }
             if (item.Owner == "") {
                 this.menuFilterOwner.set("disabled", true);
                 this.menuFilterOwner.set("label", this.i18n.Owner + ":  " + this.i18n.NA);
             }
-            if (item.ClusterName == "") {
+            if (item.NodeGroup == "") {
                 this.menuFilterCluster.set("disabled", true);
                 this.menuFilterCluster.set("label", this.i18n.Cluster + ":  " + this.i18n.NA);
             }
@@ -462,7 +462,7 @@ define([
                     }),
                     Owner: { label: this.i18n.Owner, width: 72 },
                     Description: { label: this.i18n.Description, width: 153 },
-                    ClusterName: { label: this.i18n.Cluster, width: 108 },
+                    NodeGroup: { label: this.i18n.Cluster, width: 108 },
                     RecordCount: { label: this.i18n.Records, width: 72},
                     Totalsize: { label: this.i18n.Size, width: 72},
                     Parts: { label: this.i18n.Parts, width: 45},
@@ -646,7 +646,7 @@ define([
                         closable: true,
                         delayWidget: "LFDetailsWidget",
                         _hpccParams: {
-                            ClusterName: params.ClusterName,
+                            NodeGroup: params.NodeGroup,
                             Name: params.Name
                         }
                     });

esp/src/eclwatch/DFUWUDetailsWidget.js

@@ -264,15 +264,19 @@ define([
                 case "hasCompleted":
                     this.refreshActionState();
                     break;
-                case "SourceLogicalName":
-                    this.ensurePane("SourceLogicalName", this.i18n.Source, {
-                        Name: newValue
-                    });
-                    break;
-                case "DestLogicalName":
-                    this.ensurePane("DestLogicalName", this.i18n.Target, {
-                        Name: newValue
-                    });
+                case "changedCount":
+                    if (this.wu.SourceLogicalName) {
+                        this.ensurePane("SourceLogicalName", this.i18n.Source, {
+                            NodeGroup: this.wu.SourceGroupName,
+                            Name: this.wu.SourceLogicalName
+                        });
+                    }
+                    if (this.wu.DestLogicalName) {
+                        this.ensurePane("DestLogicalName", this.i18n.Target, {
+                            NodeGroup: this.wu.DestGroupName,
+                            Name: this.wu.DestLogicalName
+                        });
+                    }
                     break;
             }
         },

esp/src/eclwatch/ESPLogicalFile.js

@@ -55,14 +55,11 @@ define([
         },
         preRequest: function (request) {
             switch (request.Sortby) {
-                case "ClusterName":
-                    request.Sortby = "Cluster";
-                    break;
                 case "RecordCount":
                     request.Sortby = "Records";
                     break;
                 case "Totalsize":
-                    request.Sortby = "Size";
+                    request.Sortby = "FileSize";
                     break;
             }
         },
@@ -80,7 +77,7 @@ define([
         },
         preProcessRow: function (item, request, query, options) {
             lang.mixin(item, {
-                __hpcc_id: createID(item.ClusterName, item.Name),
+                __hpcc_id: createID(item.NodeGroup, item.Name),
                 __hpcc_isDir: false,
                 __hpcc_displayName: item.Name
             });

esp/src/eclwatch/LFDetailsWidget.js

@@ -169,7 +169,7 @@ define([
 
             var context = this;
             if (params.Name) {
-                this.logicalFile = ESPLogicalFile.Get(params.ClusterName, params.Name);
+                this.logicalFile = ESPLogicalFile.Get(params.NodeGroup, params.Name);
                 var data = this.logicalFile.getData();
                 for (var key in data) {
                     this.updateInput(key, null, data[key]);
@@ -197,7 +197,7 @@ define([
                 if (currSel.id == this.summaryWidget.id) {
                 } else if (currSel.id == this.contentWidget.id) {
                     this.contentWidget.init({
-                        ClusterName: this.logicalFile.ClusterName,
+                        ClusterName: this.logicalFile.NodeGroup,
                         LogicalName: this.logicalFile.Name
                     });
                 } else if (currSel.id == this.sourceWidget.id) {
@@ -222,6 +222,10 @@ define([
                     this.filePartsWidget.init({
                         fileParts: lang.exists("logicalFile.DFUFileParts.DFUPart", this) ? this.logicalFile.DFUFileParts.DFUPart : []
                     });
+                } else if (currSel.id == this.widget._Queries.id && !this.widget._Queries.__hpcc_initalized) {
+                    this.widget._Queries.init({
+                        LogicalName: this.logicalFile.Name
+                    });
                 } else if (this.workunitWidget && currSel.id == this.workunitWidget.id) {
                     this.workunitWidget.init({
                         Wuid: this.logicalFile.Wuid

esp/src/eclwatch/QuerySetQueryWidget.js

@@ -113,6 +113,8 @@ define([
             });
             if (params.Wuid) {
                 this.filter.setValue(this.id + "Wuid", params.Wuid);
+            } else if (params.LogicalName) {
+                this.filter.setValue(this.id + "FileName", params.LogicalName);
             }
             this.initQuerySetGrid();
 

esp/src/eclwatch/QueryTestWidget.js

@@ -78,6 +78,7 @@ define([
                 return;
 
             this.query = ESPQuery.Get(params.QuerySetId, params.Id);
+            this.initTab();
         },
 
         setContent: function (target, type, postfix) {

esp/src/eclwatch/WsDfu.js

@@ -95,7 +95,7 @@ define([
 
         DFUArrayAction: function (logicalFiles, actionType, callback) {
             arrayUtil.forEach(logicalFiles, function (item, idx) {
-                item.qualifiedName = item.Name + "@" + item.ClusterName;
+                item.qualifiedName = item.Name + "@" + item.NodeGroup;
             });
             var request = {
                 LogicalFiles: logicalFiles,

esp/src/eclwatch/nls/bs/hpcc.js

@@ -361,7 +361,7 @@
     SearchResults: "Rezultati Pretraživanja",
     SecondsRemaining: "Preostalo Sekundi",
     SelectPackageFile: "Izaberi Paket",
-    Seperators: "Seperatori",
+    Separators: "Seperatori",
     SetBanner: "Postavite Zastavicu",
     SetTextError: "Neuspješno prikazivanje teksta (da li je tekst predug?). Koristite 'pomagače' za preuzimanje datoteka.",
     SetToFailed: "Postavite Na Neuspješan",

esp/src/eclwatch/nls/es/hpcc.js

@@ -361,7 +361,7 @@ define(
     SearchResults: "Resultados de búsqueda",
     SecondsRemaining: "Segundos que faltan",
     SelectPackageFile: "Seleccione 'Package File'",
-    Seperators: "Seperadores",
+    Separators: "Seperadores",
     SetBanner: "Configurar la bandera",
     SetTextError: "No se pudo visualizar el texto (muy grande?). Use 'helpers' para descargar.",
     SetToFailed: "Marcar como fallado",

esp/src/eclwatch/nls/hpcc.js

@@ -327,6 +327,7 @@ define({root:
     QuerySet: "Query Set",
     Queue: "Queue",
     Quote: "Quote",
+    QuotedTerminator: "Quoted Terminator",
     RecordCount: "Record Count",
     RecordLength: "Record Length",
     Records: "Records",
@@ -370,7 +371,7 @@ define({root:
     SearchResults: "Search Results",
     SecondsRemaining: "Seconds Remaining",
     SelectPackageFile: "Select Package File",
-    Seperators: "Seperators",
+    Separators: "Separators",
     SetBanner: "Set Banner",
     SetTextError: "Failed to display text (too large?).  Use 'helpers' to download.",
     SetToFailed: "Set To Failed",

esp/src/eclwatch/nls/hr/hpcc.js

@@ -361,7 +361,7 @@
     SearchResults: "Rezultati Pretraživanja",
     SecondsRemaining: "Preostalo Sekundi",
     SelectPackageFile: "Izaberi Paket",
-    Seperators: "Seperatori",
+    Separators: "Seperatori",
     SetBanner: "Postavite Zastavicu",
     SetTextError: "Neuspješno prikazivanje teksta (da li je tekst predug?). Koristite 'pomagače' za preuzimanje datoteka.",
     SetToFailed: "Postavite Na Neuspješan",

esp/src/eclwatch/nls/hu/hpcc.js

@@ -362,7 +362,7 @@ define(
     SearchResults: "Keresések",
     SecondsRemaining: "másodperc maradt",
     SelectPackageFile: "Csomagfájl választás",
-    Seperators: "Elválasztó jelek",
+    Separators: "Elválasztó jelek",
     SetBanner: "Fejléc megadása",
     SetTextError: "Nem sikerült a szöveget megjeleníteni. Lehetséges, hogy túl hosszú. Használja a 'segédleteket' a letöltéshez.",
     SetToFailed: "Elront",

esp/src/eclwatch/nls/pt-br/hpcc.js

@@ -212,7 +212,7 @@
     Scope: "Escopo",
     SearchResults: "Resultado da Busca",
     SecondsRemaining: "Segundos Restantes",
-    Seperators: "Seperadores",
+    Separators: "Seperadores",
     SetBanner: "Fixar Banner",
     SetToFailed: "Indicar como Falhado",
     Severity: "Severidade",

esp/src/eclwatch/nls/sr/hpcc.js

@@ -361,7 +361,7 @@
     SearchResults: "Резултати Претраживања",
     SecondsRemaining: "Преостало Секунди",
     SelectPackageFile: "Изаберитe Пакет",
-    Seperators: "Сеператори",
+    Separators: "Сеператори",
     SetBanner: "Поставите Заставицу",
     SetTextError: "Неуспешно приказивање текста (да ли је текст предуг?). Користите 'помагаче' за преузимање датотека.",
     SetToFailed: "Поставите На Неуспешан",

esp/src/eclwatch/nls/zh/hpcc.js

@@ -360,7 +360,7 @@
     SearchResults: "查询结果",
     SecondsRemaining: "剩余时间(秒)",
     SelectPackageFile: "选择文件包文件",
-    Seperators: "分割符",
+    Separators: "分割符",
     SetBanner: "标语设置",
     SetTextError: "文本太大无法显示. 请用'帮助'下载",
     SetToFailed: "设为失误状态",

esp/src/eclwatch/templates/DFUWUDetailsWidget.html

@@ -126,20 +126,20 @@
                                     <div id="${id}RowTag"></div>
                                 </li>
                                 <li>
-                                    <label for="${id}SourceSeperators">${i18n.Seperators}:</label>
-                                    <div id="${id}SourceSeperators"></div>
+                                    <label for="${id}SourceCsvSeparate">${i18n.Separators}:</label>
+                                    <div id="${id}SourceCsvSeparate"></div>
                                 </li>
                                 <li>
-                                    <label for="${id}SourceEscape">${i18n.Escape}:</label>
-                                    <div id="${id}SourceEscape"></div>
+                                    <label for="${id}SourceCsvEscape">${i18n.Escape}:</label>
+                                    <div id="${id}SourceCsvEscape"></div>
                                 </li>
                                 <li>
-                                    <label for="${id}SourceTerminators">${i18n.Terminators}:</label>
-                                    <div id="${id}SourceTerminators"></div>
+                                    <label for="${id}SourceCsvTerminate">${i18n.Terminators}:</label>
+                                    <div id="${id}SourceCsvTerminate"></div>
                                 </li>
                                 <li>
-                                    <label for="${id}SourceQuote">${i18n.Quote}:</label>
-                                    <div id="${id}SourceQuote"></div>
+                                    <label for="${id}SourceCsvQuote">${i18n.Quote}:</label>
+                                    <div id="${id}SourceCsvQuote"></div>
                                 </li>
                             </ul>
                         </form>

esp/src/eclwatch/templates/LFDetailsWidget.html

@@ -78,7 +78,7 @@
                             </li>
                             <li>
                                 <label for="${id}ClusterName">${i18n.ClusterName}: </label>
-                                <div id="${id}ClusterName"></div>
+                                <div id="${id}NodeGroup"></div>
                             </li>
                             <li>
                                 <label for="${id}Description">${i18n.Description}: </label>
@@ -134,6 +134,8 @@
             </div>
             <div id="${id}_FileParts" title="${i18n.FileParts}" data-dojo-props="delayWidget: 'FilePartsWidget'" data-dojo-type="DelayLoadWidget">
             </div>
+            <div id="${id}_Queries" title="${i18n.Queries}" data-dojo-props="delayWidget: 'QuerySetQueryWidget', iconClass:'iconFind'" data-dojo-type="DelayLoadWidget">
+            </div>
             <div id="${id}_Workunit" title="${i18n.Workunit}" data-dojo-props="delayWidget: 'WUDetailsWidget'" data-dojo-type="DelayLoadWidget">
             </div>
             <div id="${id}_DFUWorkunit" title="${i18n.Workunit}" data-dojo-props="delayWidget: 'DFUWUDetailsWidget'" data-dojo-type="DelayLoadWidget">

esp/src/eclwatch/templates/LZBrowseWidget.html

@@ -47,7 +47,6 @@
                                         <input title="${i18n.Overwrite}:" name="overwrite" data-dojo-type="dijit.form.CheckBox" />
                                         <input title="${i18n.Compress}:" name="compress" data-dojo-type="dijit.form.CheckBox" />
                                         <input title="${i18n.FailIfNoSourceFile}:" name="failIfNoSourceFile" data-dojo-type="dijit.form.CheckBox" />
-                                        <input title="${i18n.RecordStructurePresent}:" name="recordStructurePresent" data-dojo-type="dijit.form.CheckBox" />
                                     </div>
                                 </div>
                                 <div class="dijitDialogPaneActionBar">
@@ -84,15 +83,16 @@
                                             <option value="9">UTF-32BE</option>
                                         </select>
                                         <input id="${id}SprayDelimitedMaxRecordLength" title="${i18n.MaxRecordLength}:" style="width: 95%;" name="sourceMaxRecordSize" value="8192" required="true" colspan="2" data-dojo-props="trim: true, placeHolder:'8192'" data-dojo-type="dijit.form.ValidationTextBox" />
-                                        <input id="${id}SprayDelimitedSeperators" title="${i18n.Seperators}:" style="width: 95%;" name="sourceCsvSeparate" value="," data-dojo-props="trim: true, placeHolder:'\,'" colspan="2" data-dojo-type="dijit.form.ValidationTextBox" />
+                                        <input id="${id}SprayDelimitedSeparators" title="${i18n.Separators}:" style="width: 95%;" name="sourceCsvSeparate" value="," data-dojo-props="trim: true, placeHolder:'\,'" colspan="2" data-dojo-type="dijit.form.ValidationTextBox" />
                                         <input title="${i18n.OmitSeparator}:" name="NoSourceCsvSeparator" colspan="2" data-dojo-type="dijit.form.CheckBox" />
                                         <input id="${id}SprayDelimitedEscape" title="${i18n.Escape}:" style="width: 95%;" name="sourceCsvEscape" data-dojo-props="trim: true" colspan="2" data-dojo-type="dijit.form.TextBox" />
                                         <input id="${id}SprayDelimitedTerminators" title="${i18n.LineTerminators}:" name="sourceCsvTerminate" style="width: 95%;" value="\n,\r\n" required="true" colspan="2" data-dojo-props="trim: true, placeHolder:'\\n,\\r\\n'" data-dojo-type="dijit.form.ValidationTextBox" />
-                                        <input id="${id}SprayDelimitedQuote" title="${i18n.Quote}:" style="width: 95%;" name="sourceCsvQuote" value="'" data-data-dojo-props="trim: true, placeHolder:'\''" colspan="2" data-dojo-type="dijit.form.TextBox" />
+                                        <input id="${id}SprayDelimitedQuote" title="${i18n.Quote}:" style="width: 95%;" name="sourceCsvQuote" value='"' data-data-dojo-props="trim: true, placeHolder:'\''" colspan="2" data-dojo-type="dijit.form.TextBox" />
                                         <input title="${i18n.Overwrite}:" name="overwrite" data-dojo-type="dijit.form.CheckBox" />
                                         <input title="${i18n.Compress}:" name="compress" data-dojo-type="dijit.form.CheckBox" />
                                         <input title="${i18n.FailIfNoSourceFile}:" name="failIfNoSourceFile" data-dojo-type="dijit.form.CheckBox" />
                                         <input title="${i18n.RecordStructurePresent}:" name="recordStructurePresent" data-dojo-type="dijit.form.CheckBox" />
+                                        <input title="${i18n.QuotedTerminator}:" name="quotedTerminator" data-dojo-type="dijit.form.CheckBox" />
                                     </div>
                                 </div>
                                 <div class="dijitDialogPaneActionBar">
@@ -132,7 +132,6 @@
                                         <input title="${i18n.Overwrite}:" name="overwrite" data-dojo-type="dijit.form.CheckBox" />
                                         <input title="${i18n.Compress}:" name="compress" data-dojo-type="dijit.form.CheckBox" />
                                         <input title="${i18n.FailIfNoSourceFile}:" name="failIfNoSourceFile" data-dojo-type="dijit.form.CheckBox" />
-                                        <input title="${i18n.RecordStructurePresent}:" name="recordStructurePresent" data-dojo-type="dijit.form.CheckBox" />
                                     </div>
                                 </div>
                                 <div class="dijitDialogPaneActionBar">
@@ -166,7 +165,6 @@
                                         <input title="${i18n.Overwrite}:" name="overwrite" data-dojo-type="dijit.form.CheckBox" />
                                         <input title="${i18n.Compress}:" name="compress" data-dojo-type="dijit.form.CheckBox" />
                                         <input title="${i18n.FailIfNoSourceFile}:" name="failIfNoSourceFile" data-dojo-type="dijit.form.CheckBox" />
-                                        <input title="${i18n.RecordStructurePresent}:" name="recordStructurePresent" data-dojo-type="dijit.form.CheckBox" />
                                     </div>
                                 </div>
                                 <div class="dijitDialogPaneActionBar">
@@ -195,7 +193,6 @@
                                         <input title="${i18n.Overwrite}:" name="overwrite" data-dojo-type="dijit.form.CheckBox" />
                                         <input title="${i18n.Compress}:" name="compress" data-dojo-type="dijit.form.CheckBox" />
                                         <input title="${i18n.FailIfNoSourceFile}:" name="failIfNoSourceFile" data-dojo-type="dijit.form.CheckBox" />
-                                        <input title="${i18n.RecordStructurePresent}:" name="recordStructurePresent" data-dojo-type="dijit.form.CheckBox" />
                                     </div>
                                 </div>
                                 <div class="dijitDialogPaneActionBar">

esp/src/eclwatch/templates/QuerySetQueryWidget.html

@@ -16,7 +16,8 @@
                     <span data-dojo-type="dijit.ToolbarSeparator"></span>
                     <div id="${id}Filter" data-dojo-type="FilterDropDownWidget">
                         <input id="${id}Wuid" title="${i18n.WUID}:" name="WUID" colspan="2" data-dojo-props="trim: true, placeHolder:'W20130222-171723'" data-dojo-type="dijit.form.TextBox" />
-                        <input id="${id}ClusterTargetSelect" title="${i18n.Cluster}:" name="QuerySetName" colspan="2"  data-dojo-type="TargetSelectWidget" />
+                        <input id="${id}ClusterTargetSelect" title="${i18n.Cluster}:" name="QuerySetName" colspan="2" data-dojo-type="TargetSelectWidget" />
+                        <input id="${id}FileName" title="${i18n.LogicalFile}:" name="FileName" colspan="2" data-dojo-props="trim: true, placeHolder: '${i18n.TargetNamePlaceholder}'" data-dojo-type="dijit.form.TextBox" />
                         <select id="${id}SuspendedStates" title="${i18n.Suspended}:" name="SuspendedByUser" colspan="2" data-dojo-type="dijit.form.Select">
                             <option value="" selected="selected">${i18n.All}</option>
                             <option value="1">${i18n.SuspendedByUser}</option>

roxie/ccd/ccdlistener.cpp

@@ -96,7 +96,7 @@ static void sendJsonException(SafeSocket &client, IException *E, const char *que
         if (!queryName)
             queryName = "Unknown"; // Exceptions when parsing query XML can leave queryName unset/unknowable....
 
-        StringBuffer response("{");
+        StringBuffer response;
         appendfJSONName(response, "%sResponse", queryName).append(" {");
         appendJSONName(response, "Results").append(" {");
         appendJSONName(response, "Exception").append(" [{");
@@ -104,7 +104,7 @@ static void sendJsonException(SafeSocket &client, IException *E, const char *que
         appendJSONValue(response, "Code", E->errorCode());
         StringBuffer s;
         appendJSONValue(response, "Message", E->errorMessage(s).str());
-        response.append("}]}}}");
+        response.append("}]}}");
         client.write(response.str(), response.length());
     }
     catch(IException *EE)
@@ -1794,7 +1794,7 @@ readAnother:
             if (client)
             {
                 if (isHTTP)
-                    sendSoapException(*client, E, queryName);
+                    sendHttpException(*client, httpHelper.queryContentFormat(), E, queryName);
                 else
                     client->sendException("Roxie", code, error.str(), isBlocked, logctx);
             }
@@ -1819,7 +1819,7 @@ readAnother:
             if (client)
             {
                 if (isHTTP)
-                    sendHttpException(*client, mlFmt, E, queryName);
+                    sendHttpException(*client, httpHelper.queryContentFormat(), E, queryName);
                 else
                     client->sendException("Roxie", code, error.str(), isBlocked, logctx);
             }
@@ -1839,7 +1839,7 @@ readAnother:
                 if (isHTTP)
                 {
                     Owned<IException> E = MakeStringException(ROXIE_INTERNAL_ERROR, "Unknown exception");
-                    sendSoapException(*client, E, queryName);
+                    sendHttpException(*client, httpHelper.queryContentFormat(), E, queryName);
                 }
                 else
                     client->sendException("Roxie", ROXIE_INTERNAL_ERROR, "Unknown exception", isBlocked, logctx);

roxie/ccd/ccdquery.cpp

@@ -1282,12 +1282,12 @@ public:
     }
     virtual int getDebugValueInt(const char * propname, int defVal) const
     {
-        assertex(dll->queryWorkUnit());
+        assertex(dll && dll->queryWorkUnit());
         return dll->queryWorkUnit()->getDebugValueInt(propname, defVal);
     }
     virtual bool getDebugValueBool(const char * propname, bool defVal) const
     {
-        assertex(dll->queryWorkUnit());
+        assertex(dll && dll->queryWorkUnit());
         return dll->queryWorkUnit()->getDebugValueBool(propname, defVal);
     }
     bool getEnableFieldTranslation() const

system/jlib/jptree.cpp

@@ -6464,7 +6464,9 @@ public:
     void readValueNotify(const char *name, bool skipAttributes)
     {
         StringBuffer value;
-        readValue(value);
+        if (readValue(value)==elementTypeNull)
+            return;
+
         if ('@'==*name)
         {
             if (!skipAttributes)

testing/ecl/textsearch_ao.ecl

@@ -1,387 +0,0 @@
-/*##############################################################################
-
-    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
-
-    Licensed under the Apache License, Version 2.0 (the "License");
-    you may not use this file except in compliance with the License.
-    You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-    Unless required by applicable law or agreed to in writing, software
-    distributed under the License is distributed on an "AS IS" BASIS,
-    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-    See the License for the specific language governing permissions and
-    limitations under the License.
-############################################################################## */
-
-//UseStandardFiles
-//UseTextSearch
-//tidyoutput
-//nothor
-//nothorlcr
-//nohthor
-//DoesntReallyUseIndexes    Thje test is primarily here to check everything works when the index optimizer is always on
-//xxvarskip type==roxie && setuptype==thor && !local
-
-#option ('checkAsserts',false)
-
-//SingleQuery := 'AND(NOTAT(AND("twinkle", "little"), 9999), NOTAT(AND("how", "wonder"),8888))';
-
-q1 := dataset([
-
-#if (#ISDEFINED(SingleQuery))
-            SingleQuery
-#else
-            'AND("black","sheep")',
-            'ANDNOT("black","sheep")',
-            'MOFN(2,"black","sheep","white")',
-            'MOFN(2,2,"black","sheep","white")',
-
-//Word tests
-            '("nonexistant")',
-            '("one")',
-            'CAPS("one")',
-            'NOCAPS("one")',
-            'ALLCAPS("one")',
-            '"ibm"',                                        // simple word, and an alias
-
-//Or tests
-            'OR("nonexistant1", "nonexistant2")',           // neither side matches
-            'OR("nonexistant1", "sheep")',                  // RHS matches
-            'OR("twinkle", "nonexistant2")',                // LHS matches
-            'OR("twinkle", "twinkle")',                     // should dedup
-            'OR("sheep", "black")',                         // matches in same document
-            'OR("sheep", "twinkle")',                       // matches in different documents
-            'OR("one", "sheep", "sheep", "black", "fish")', // matches in different documents
-            'OR(OR("one", "sheep"), OR("sheep", "black", "fish"))', // matches in different documents
-
-//And tests
-            'AND("nonexistant1", "nonexistant2")',          // neither side matches
-            'AND("nonexistant1", "sheep")',                 // RHS matches
-            'AND("twinkle", "nonexistant2")',               // LHS matches
-            'AND("twinkle", "twinkle")',                    // should dedup
-            'AND("sheep", "black")',                        // matches in same document
-            'AND("sheep", "twinkle")',                      // matches in different documents
-            'AND("in", "a")',                               // high frequencies
-            'AND("twinkle", "little", "how", "star")',      // Nary
-            'AND(AND("twinkle", "little"), AND("how", "star"))',        // Nested
-            'AND(NOTAT(AND("twinkle", "little"), 9999), NOTAT(AND("how", "wonder"),8888))',     // Nested
-            'AND(AND("twinkle", "little"), AND("how", "wonder"))',      // Nested
-
-//MORE: Should also test segment restriction....
-
-            'ANDNOT("nonexistant1", "nonexistant2")',       // neither side matches
-            'ANDNOT("nonexistant1", "sheep")',              // RHS matches
-            'ANDNOT("twinkle", "nonexistant2")',            // LHS matches
-            'ANDNOT("twinkle", "twinkle")',                 // should dedup
-            'ANDNOT("sheep", "black")',                     // matches in same document
-            'ANDNOT("sheep", "twinkle")',                   // matches in different documents
-            'ANDNOT("one", OR("sheep", "black", "fish"))',      // matches one, but none of the others
-
-//Phrases
-            'PHRASE("nonexistant1", "nonexistant2")',       // words don't occour
-            'PHRASE("in", "are")',                          // doesn't occur, but words do
-            'PHRASE("baa", "black")',                       // occurs, but
-            'PHRASE("x", "y", "x", "x", "y")',              // a partial match, first - doesn't actually make it more complicatied to implement
-            'PHRASE("james","arthur","stuart")',            // check that next occurence of stuart takes note of the change of document.
-            'PHRASE(OR("black","white"),"sheep")',          // proximity on a non-word input
-            'PHRASE("one", "for", OR(PHRASE("the","master"),PHRASE("the","dame"),PHRASE("the","little","boy")))',
-                                                            // more complex again
-
-//Testing range
-            'PHRASE1to5("humpty","dumpty")',
-            'PHRASE1to5("together","again")',
-
-//M of N
-            'MOFN(2, "humpty", "horses", "together", "beansprout")',    // m<matches
-            'MOFN(3, "humpty", "horses", "together", "beansprout")',    // m=matches
-            'MOFN(4, "humpty", "horses", "together", "beansprout")',    // m>matches
-            'MOFN(2,2, "humpty", "horses", "together", "beansprout")',  // too many matches
-            'MOFN(2, "nonexistant", "little", "bo")',                   // first input fails to match any
-            'MOFN(2, "little", "bo", "nonexistant")',                   // lose an input while finising candidates
-            'MOFN(2, "one", "two", "three", "four", "five")',
-            'MOFN(2, "nonexistant", "two", "three", "four", "five")',
-            'MOFN(2, "one", "nonexistant", "three", "four", "five")',
-            'MOFN(2, "nonexistant1", "nonexistant2", "three", "four", "five")',
-            'MOFN(2, "nonexistant1", "nonexistant2", "nonexistant3", "four", "five")',
-            'MOFN(2, "nonexistant1", "nonexistant2", "nonexistant3", "nonexistant4", "five")',
-            'MOFN(2, PHRASE("little","bo"),PHRASE("three","bags"),"sheep")',    // m of n on phrases
-            'MOFN(2, PHRASE("Little","Bo"),PHRASE("three","bags"),"sheep")',    // m of n on phrases - capital letters don't match
-            'MOFN(2, OR("little","big"), OR("king", "queen"), OR("star", "sheep", "twinkle"))',
-
-//Proximity
-            'PROXIMITY("nonexistant1", "nonexistant2", -1, 1)',
-            'PROXIMITY("black", "nonexistant2", -1, 1)',
-            'PROXIMITY("nonexistant1", "sheep", -1, 1)',
-
-//Adjacent checks
-            'PROXIMITY("ship", "sank", 0, 0)',                      // either order but adjacent
-            'NORM(PROXIMITY("ship", "sank", 0, 0))',
-            'PROXIMITY("ship", "sank", -1, 0)',                     // must follow
-            'PROXIMITY("ship", "sank", 0, -1)',                     // must preceed
-            'PROXIMITY("sank", "ship", 0, 0)',                      // either order but adjacent
-            'PROXIMITY("sank", "ship", -1, 0)',                     // must follow
-            'PROXIMITY("sank", "ship", 0, -1)',                     // must preceed
-
-//Within a distance of 1
-            'PROXIMITY("ship", "sank", 1, 1)',                      // either order but only 1 intervening word
-            'PROXIMITY("ship", "sank", -1, 1)',                     // must follow
-            'PROXIMITY("ship", "sank", 1, -1)',                     // must preceed
-            'PROXIMITY("sank", "ship", 1, 1)',                      // either order but only 1 intervening word
-            'PROXIMITY("sank", "ship", -1, 1)',                     // must follow
-            'PROXIMITY("sank", "ship", 1, -1)',                     // must preceed
-
-            'PROXIMITY("ship", "sank", 0, 2)',                      // asymetric range
-
-//Within a distance of 2
-            'PROXIMITY("ship", "ship", 2, 2)',                      // either order but only 2 intervening word, no dups
-                                                                    // *** currently fails because of lack of duplication in lowest merger
-            'PROXIMITY("zx", "zx", 5, 5)',                          // "zx (za) zx", "zx (za zx zb zc zd) zx" and "zx (zb zc zd zx)"
-            'PROXIMITY(PROXIMITY("zx", "zx", 5, 5), "zx", 1, 1)',   // "zx (za) zx (zb zc zd) zx" - obtained two different ways.
-            'NORM(PROXIMITY(PROXIMITY("zx", "zx", 5, 5), "zx", 1, 1))', // as above, but normalized
-            'PROXIMITY(PROXIMITY("zx", "zx", 5, 5), "zx", 0, 0)',   // "zx (za) zx (zb zc zd) zx" - can obly be obtained from first
-                                                                    // you could imagine -ve left and right to mean within - would need -1,0 in stepping, and appropriate hard condition.
-
-            'PROXIMITY("ibm", "business", 2, 2)',                   // alias doesn't allow matching within itself.
-            'PROXIMITY("ibm", "business", 3, 3)',                   // alias should match now with other word
-            'PROXIMITY("ibm", "ibm", 0, 0)',                        // aliases and non aliases cause fun.
-
-//More combinations of operators
-            'AND(OR("twinkle", "black"), OR("sheep", "wonder"))',
-            'OR(AND("twinkle", "sheep"), AND("star", "black"))',
-            'OR(AND("twinkle", "star"), AND("sheep", "black"))',
-            'AND(SET("twinkle", "black"), SET("sheep", "wonder"))',
-
-//Silly queries
-            'OR("star","star","star","star","star")',
-            'AND("star","star","star","star","star")',
-            'MOFN(4,"star","star","star","star","star")',
-
-
-//Other operators
-            'PRE("twinkle", "twinkle")',
-            'PRE(PHRASE("twinkle", "twinkle"), PHRASE("little","star"))',
-            'PRE(PHRASE("little","star"), PHRASE("twinkle", "twinkle"))',
-            'PRE(PROXIMITY("twinkle","twinkle", 3, 3), PROXIMITY("little", "star", 2, 2))',
-            'AFT("twinkle", "twinkle")',
-            'AFT(PHRASE("little","star"), PHRASE("twinkle", "twinkle"))',
-            'AFT(PHRASE("twinkle", "twinkle"), PHRASE("little","star"))',
-            'AFT(PROXIMITY("twinkle","twinkle", 3, 3), PROXIMITY("little", "star", 2, 2))',
-
-// Left outer joins for ranking.
-            'RANK("sheep", OR("peep", "baa"))',
-            'RANK("three", OR("bags", "full"))',
-            'RANK("three", OR("one", "bags"))',
-
-
-//Non standard variants - AND, generating a single record for the match.  Actually for each cross product as it is currently (and logically) implemented
-            'ANDJOIN("nonexistant1", "nonexistant2")',          // neither side matches
-            'ANDJOIN("nonexistant1", "sheep")',                 // RHS matches
-            'ANDJOIN("twinkle", "nonexistant2")',               // LHS matches
-            'ANDJOIN("twinkle", "twinkle")',                    // should dedup
-            'ANDJOIN("sheep", "black")',                        // matches in same document
-            'ANDJOIN("sheep", "twinkle")',                      // matches in different documents
-            'ANDJOIN("in", "a")',                               // high frequencies
-            'ANDJOIN("twinkle", "little", "how", "star")',      // Nary
-
-            'ANDNOTJOIN("nonexistant1", "nonexistant2")',       // neither side matches
-            'ANDNOTJOIN("nonexistant1", "sheep")',              // RHS matches
-            'ANDNOTJOIN("twinkle", "nonexistant2")',            // LHS matches
-            'ANDNOTJOIN("twinkle", "twinkle")',                 // should dedup
-            'ANDNOTJOIN("sheep", "black")',                     // matches in same document
-            'ANDNOTJOIN("sheep", "twinkle")',                   // matches in different documents
-            'ANDNOTJOIN("one", OR("sheep", "black", "fish"))',  // matches one, but none of the others
-
-            'MOFNJOIN(2, "humpty", "horses", "together", "beansprout")',    // m<matches
-            'MOFNJOIN(3, "humpty", "horses", "together", "beansprout")',    // m=matches
-            'MOFNJOIN(4, "humpty", "horses", "together", "beansprout")',    // m>matches
-            'MOFNJOIN(2,2, "humpty", "horses", "together", "beansprout")',  // too many matches
-            'MOFNJOIN(2, "nonexistant", "little", "bo")',                   // first input fails to match any
-            'MOFNJOIN(2, "little", "bo", "nonexistant")',                   // lose an input while finising candidates
-            'MOFNJOIN(2, "one", "two", "three", "four", "five")',
-            'MOFNJOIN(2, "nonexistant", "two", "three", "four", "five")',
-            'MOFNJOIN(2, "one", "nonexistant", "three", "four", "five")',
-            'MOFNJOIN(2, "nonexistant1", "nonexistant2", "three", "four", "five")',
-            'MOFNJOIN(2, "nonexistant1", "nonexistant2", "nonexistant3", "four", "five")',
-            'MOFNJOIN(2, "nonexistant1", "nonexistant2", "nonexistant3", "nonexistant4", "five")',
-            'MOFNJOIN(2, PHRASE("little","bo"),PHRASE("three","bags"),"sheep")',    // m of n on phrases
-            'MOFNJOIN(2, PHRASE("Little","Bo"),PHRASE("three","bags"),"sheep")',    // m of n on phrases - capital letters don't match
-            'MOFNJOIN(2, OR("little","big"), OR("king", "queen"), OR("star", "sheep", "twinkle"))',
-
-            'RANKJOIN("SHEEP", "BLACK")',
-            'RANKJOIN("sheep", OR("peep", "baa"))',
-            'RANKJOIN("three", OR("bags", "full"))',
-            'RANKJOIN("three", OR("one", "bags"))',
-
-
-//ROLLAND - does AND, followed by a rollup by doc.  Should also check that smart stepping still works through the grouped rollup
-            'ROLLAND("nonexistant1", "nonexistant2")',          // neither side matches
-            'ROLLAND("nonexistant1", "sheep")',                 // RHS matches
-            'ROLLAND("twinkle", "nonexistant2")',               // LHS matches
-            'ROLLAND("twinkle", "twinkle")',                    // should dedup
-            'ROLLAND("sheep", "black")',                        // matches in same document
-            'ROLLAND("sheep", "twinkle")',                      // matches in different documents
-            'ROLLAND("in", "a")',                               // high frequencies
-            'ROLLAND("twinkle", "little", "how", "star")',      // Nary
-            'AND(ROLLAND("twinkle", "little"), ROLLAND("how", "star"))',        // Nary
-
-//Same tests as proximity above, but not calling a transform - merging instead
-            'PROXMERGE("ship", "sank", 0, 0)',                      // either order but adjacent
-            'PROXMERGE("ship", "sank", -1, 0)',                     // must follow
-            'PROXMERGE("ship", "sank", 0, -1)',                     // must preceed
-            'PROXMERGE("sank", "ship", 0, 0)',                      // either order but adjacent
-            'PROXMERGE("sank", "ship", -1, 0)',                     // must follow
-            'PROXMERGE("sank", "ship", 0, -1)',                     // must preceed
-            'PROXMERGE("ship", "sank", 1, 1)',                      // either order but only 1 intervening word
-            'PROXMERGE("ship", "sank", -1, 1)',                     // must follow
-            'PROXMERGE("ship", "sank", 1, -1)',                     // must preceed
-            'PROXMERGE("sank", "ship", 1, 1)',                      // either order but only 1 intervening word
-            'PROXMERGE("sank", "ship", -1, 1)',                     // must follow
-            'PROXMERGE("sank", "ship", 1, -1)',                     // must preceed
-            'PROXMERGE("ship", "sank", 0, 2)',                      // asymetric range
-
-//SET should be equivalent to OR
-            'SET("nonexistant1", "nonexistant2")',          // neither side matches
-            'SET("nonexistant1", "sheep")',                 // RHS matches
-            'SET("twinkle", "nonexistant2")',               // LHS matches
-            'SET("twinkle", "twinkle")',                    // should dedup
-            'SET("sheep", "black")',                        // matches in same document
-            'SET("sheep", "twinkle")',                      // matches in different documents
-            'SET("one", "sheep", "sheep", "black", "fish")',    // matches in different documents
-            'OR(SET("one", "sheep"), SET("sheep", "black", "fish"))',   // matches in different documents
-
-//Testing range
-            'PHRASE1to5(PHRASE1to5("what","you"),"are")',
-            'PHRASE1to5("what", PHRASE1to5("you","are"))',
-            'PHRASE1to5(PHRASE1to5("open","source"),"software")',
-            'PHRASE1to5("open", PHRASE1to5("source","software"))',
-
-//Atleast
-            'ATLEAST(2, "twinkle")',                                // would something like UNIQUEAND("twinkle", "twinkle") be more efficient???
-            'ATLEAST(4, "twinkle")',
-            'ATLEAST(5, "twinkle")',
-            'ATLEAST(5, AND("twinkle","star"))',
-            'AND(ATLEAST(4, "twinkle"),"star")',                    // make sure this still smart steps!
-            'AND(ATLEAST(5, "twinkle"),"star")',
-            'ATLEAST(1, PHRASE("humpty","dumpty"))',
-            'ATLEAST(2, PHRASE("humpty","dumpty"))',
-            'ATLEAST(3, PHRASE("humpty","dumpty"))',
-
-            '"little"',
-            'IN(name, "little")',
-            'NOTIN(name, "little")',
-            'IN(suitcase, AND("sock", "shirt"))',
-            'IN(suitcase, AND("sock", "dress"))',
-            'IN(suitcase, AND("shirt", "dress"))',                  //no, different suitcases..
-            'IN(suitcase, OR("cat", "dog"))',                       //no - wrong container
-            'IN(box, OR("cat", "dog"))',                            //yes
-            'IN(box, IN(suitcase, "shirt"))',
-            'IN(suitcase, IN(box, "shirt"))',                       // no other elements in the suitcase, so not valid
-            'IN(box, AND(IN(suitcase, "shirt"), "car"))',
-            'IN(box, AND(IN(suitcase, "shirt"), "lego"))',          // no, lego isn't in the box...
-            'IN(box, MOFN(2, "car", "train", "glue"))',             // really nasty - need to modify the m of n to add position equality!
-            'IN(box, MOFN(2, "car", "glue", "train"))',             // and check works in all positions.
-            'IN(box, MOFN(2, "glue", "car", "train"))',             //   " ditto "
-            'IN(box, MOFN(3, "car", "train", "glue"))',
-            'NOTIN(box, AND("computer", "lego"))',
-            'NOTIN(box, AND("train", "lego"))',
-            'IN(suitcase, PROXIMITY("trouser", "sock", 1, 2))',     // ok.
-            'IN(suitcase, PROXIMITY("trouser", "train", 1, 2))',    // no, close enough, but not both in the suitcase
-            'IN(suitcase, PROXIMITY("trouser", "dress", 6, 6))',    // no, close enough, but not both in the same suitcase
-            'PROXIMITY(IN(suitcase, "trouser"), IN(suitcase, "dress"), 6, 6)',  // yes - testing the proximity of the suitcases, not the contents.
-
-            'IN(S, AND("fish", "alive"))',                          // <s> is the sentence container
-            'S(AND("fish", "alive"))',                              // pseudonym for within same sentence -
-            'S(AND("fish", "finger"))',                             //
-            'S(AND("sheep", "wagging"))',
-            'P(AND("sheep", "wagging"))',                           // same paragraph...
-            'AND(IN(socks, "fox"),IN(socks, "knox"))',
-            'AND(IN(box, "fox"),IN(box, "knox"))',
-            'AND(IN(box, IN(socks, "fox")),IN(box, "knox"))',
-            'AND(IN(socks, IN(box, "fox")),IN(box, "knox"))',           // yes - because no extra elements in the box.
-            'S(PHRASE("black", "sheep"))',
-            'IN(name, PHRASE("little", "bo", "peep"))',
-            'IN(name, PHRASE("little", "bo", "peep", "has"))',
-
-            'IN(range1, IN(range2, "seven"))',                      // only match 5.3
-            'SAME(IN(range1, "seven"), IN(range2, "seven"))',       // only match 5.3
-            'OVERLAP(IN(range1, "five"), IN(range2, "ten"))',       // overlapping, match 5.4
-            'PROXIMITY(IN(range1, "five"), IN(range2, "ten"), 0, 0)',   // adjacent match 5.4, 5.5
-            'PROXIMITY(IN(range1, "five"), IN(range2, "ten"), 1, 1)',   // proximity match 5.4, 5.5
-            'PROXIMITY(IN(range1, "five"), IN(range2, "ten"), 2, 2)',   // adjacent match 5.4, 5.5, 5.6
-
-            'ATLEAST(2, IN(suitcase, "sock"))',                     // at least two suitcases containing a sock.
-            'ATLEAST(3, IN(suitcase, "sock"))',                     // no
-
-            'IN(box, "train")',                                     // should be 4 matches (since inside nested boxes)
-
-            'IN(suitcase, ATLEAST(1, "sock"))',                     // suitcases containing at least one sock.   (should really optimize away)
-            'IN(suitcase, ATLEAST(2, "sock"))',                     // at least two suit cases containing a sock.
-            'IN(suitcase, ATLEAST(3, "sock"))',
-            'IN(suitcase, ATLEAST(3, OR("sock", "dress")))',        //no
-            'IN(suitcase, ATLEAST(3, SET("sock", "dress")))',       //no
-            'IN(suitcase, ATLEAST(3, OR("sock", "jacket")))',       //yes...
-            'IN(suitcase, ATLEAST(3, SET("sock", "jacket")))',      //yes...
-            'IN(box, IN(suitcase, ATLEAST(2, "sock")))',            //yes - box, with one match
-            'IN(box, IN(suitcase, ATLEAST(3, "sock")))',            //no -
-            'IN(box, ATLEAST(2, IN(suitcase, "sock")))',            //yes -
-            'IN(box, ATLEAST(3, IN(suitcase, "sock")))',            //no -
-            'IN(box, ATLEAST(2, IN(suitcase, ATLEAST(2, "sock"))))',            //no...
-            'IN(box, AND(ATLEAST(2, "train"), ATLEAST(2, "sock")))',    // yes
-            'IN(box, AND(ATLEAST(3, "train"), ATLEAST(2, "sock")))',    // no
-            'IN(suitcase, AND(ATLEAST(2, "sock"), ATLEAST(2, OR("tights", "dress"))))', // no - different suitcases.
-            'IN(suitcase, ATLEAST(2, "sock"))', // yes
-            'IN(suitcase, ATLEAST(2, OR("tights", "dress")))',  // yes
-
-//The following example fails - not quite sure how to fix it.
-//          'IN(suitcase, OR(ATLEAST(2, "sock"), ATLEAST(2, OR("tights", "dress"))))',  // yes
-            'IN(suitcase, ATLEAST(4, AND(ATLEAST(2, "sock"), OR("shirt", "trouser"))))',    // yes - nested atleasts...
-            'IN(suitcase, ATLEAST(5, AND(ATLEAST(2, "sock"), OR("shirt", "trouser"))))',    // no
-
-            '_ATLEASTIN_(1, IN(suitcase, "sock"), 1)',                      // suitcases containing at least one sock.   (should really optimize away)
-            '_ATLEASTIN_(2, IN(suitcase, "sock"), 1)',                      // at least two suit cases containing a sock.
-            '_ATLEASTIN_(3, IN(suitcase, "sock"), 1)',
-
-            'S(ANDNOT("fish", "alive"))',                               // pseudonym for within same sentence -
-            'S(ANDNOT("fish", "finger"))',                              //
-
-            'AT("the", 2)',                                             // occurences of 'the' at position 2
-            'AT("the", 18)',
-            'AT("is", 17)',
-            'AND(AT("the", 18),AT("is",17))',
-
-            'AND("gch01", "gch02", "gch04")',
-            'AND("gch01", "gch02", "gch10")',
-
-            'AND(SET("and","a"), SET("the", "one"), PHRASE("for","the","dame"))',
-            'AND(CAPS("sheep"), "spotted")',
-            'AND(CAPS("sheep"), NOCAPS("spotted"))',
-            'AND(SET(CAPS("sheep","little")), SET(CAPS("Up","go")))',
-            'AND(SET(CAPS("sheep","little")), SET(NOCAPS("Up","go")))',
-            'AND(OR(CAPS("sheep"),CAPS("Little")), OR(CAPS("Up"),NOCAPS("go")))',
-
-            'ANDNOT(AND("black","sheep"), "family")',
-            'ANDNOT(AND("little","and"), "jack")',
-
-            'BUTNOT("little", PHRASE("little", "star"))',
-            'BUTNOTJOIN("little", PHRASE("little", "star"))',
-            'BUTNOT("black", PHRASE("black", OR("spotted", "sheep")))',
-            'BUTNOTJOIN("black", PHRASE("black", OR("spotted", "sheep")))',
-
-//MORE:
-// STEPPED flag on merge to give an error if input doesn't support stepping.
-// What about the duplicates that can come out of the proximity operators?
-// where the next on the rhs is at a compatible position, but in a different document
-// What about inverse of proximity x not w/n y
-// Can inverse proximity be used for sentance/paragraph.  Can we combine them so short circuited before temporaries created.
-//MORE: What other boundary conditions can we think of.
-
-                ' '
-
-#end
-
-            ], queryInputRecord);
-
-p := project(nofold(q1), doBatchExecute(TS_wordIndex, LEFT, 0x80000000));
-output(p);

testing/ecl/textsearch_no.ecl

@@ -1,387 +0,0 @@
-/*##############################################################################
-
-    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
-
-    Licensed under the Apache License, Version 2.0 (the "License");
-    you may not use this file except in compliance with the License.
-    You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-    Unless required by applicable law or agreed to in writing, software
-    distributed under the License is distributed on an "AS IS" BASIS,
-    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-    See the License for the specific language governing permissions and
-    limitations under the License.
-############################################################################## */
-
-//UseStandardFiles
-//UseTextSearch
-//tidyoutput
-//nothor
-//nothorlcr
-//nohthor
-//DoesntReallyUseIndexes    Thje test is primarily here to check everything works when the join index optimizer is always on
-//xxvarskip type==roxie && setuptype==thor && !local
-
-#option ('checkAsserts',false)
-
-//SingleQuery := 'AND(NOTAT(AND("twinkle", "little"), 9999), NOTAT(AND("how", "wonder"),8888))';
-
-q1 := dataset([
-
-#if (#ISDEFINED(SingleQuery))
-            SingleQuery
-#else
-            'AND("black","sheep")',
-            'ANDNOT("black","sheep")',
-            'MOFN(2,"black","sheep","white")',
-            'MOFN(2,2,"black","sheep","white")',
-
-//Word tests
-            '("nonexistant")',
-            '("one")',
-            'CAPS("one")',
-            'NOCAPS("one")',
-            'ALLCAPS("one")',
-            '"ibm"',                                        // simple word, and an alias
-
-//Or tests
-            'OR("nonexistant1", "nonexistant2")',           // neither side matches
-            'OR("nonexistant1", "sheep")',                  // RHS matches
-            'OR("twinkle", "nonexistant2")',                // LHS matches
-            'OR("twinkle", "twinkle")',                     // should dedup
-            'OR("sheep", "black")',                         // matches in same document
-            'OR("sheep", "twinkle")',                       // matches in different documents
-            'OR("one", "sheep", "sheep", "black", "fish")', // matches in different documents
-            'OR(OR("one", "sheep"), OR("sheep", "black", "fish"))', // matches in different documents
-
-//And tests
-            'AND("nonexistant1", "nonexistant2")',          // neither side matches
-            'AND("nonexistant1", "sheep")',                 // RHS matches
-            'AND("twinkle", "nonexistant2")',               // LHS matches
-            'AND("twinkle", "twinkle")',                    // should dedup
-            'AND("sheep", "black")',                        // matches in same document
-            'AND("sheep", "twinkle")',                      // matches in different documents
-            'AND("in", "a")',                               // high frequencies
-            'AND("twinkle", "little", "how", "star")',      // Nary
-            'AND(AND("twinkle", "little"), AND("how", "star"))',        // Nested
-            'AND(NOTAT(AND("twinkle", "little"), 9999), NOTAT(AND("how", "wonder"),8888))',     // Nested
-            'AND(AND("twinkle", "little"), AND("how", "wonder"))',      // Nested
-
-//MORE: Should also test segment restriction....
-
-            'ANDNOT("nonexistant1", "nonexistant2")',       // neither side matches
-            'ANDNOT("nonexistant1", "sheep")',              // RHS matches
-            'ANDNOT("twinkle", "nonexistant2")',            // LHS matches
-            'ANDNOT("twinkle", "twinkle")',                 // should dedup
-            'ANDNOT("sheep", "black")',                     // matches in same document
-            'ANDNOT("sheep", "twinkle")',                   // matches in different documents
-            'ANDNOT("one", OR("sheep", "black", "fish"))',      // matches one, but none of the others
-
-//Phrases
-            'PHRASE("nonexistant1", "nonexistant2")',       // words don't occour
-            'PHRASE("in", "are")',                          // doesn't occur, but words do
-            'PHRASE("baa", "black")',                       // occurs, but
-            'PHRASE("x", "y", "x", "x", "y")',              // a partial match, first - doesn't actually make it more complicatied to implement
-            'PHRASE("james","arthur","stuart")',            // check that next occurence of stuart takes note of the change of document.
-            'PHRASE(OR("black","white"),"sheep")',          // proximity on a non-word input
-            'PHRASE("one", "for", OR(PHRASE("the","master"),PHRASE("the","dame"),PHRASE("the","little","boy")))',
-                                                            // more complex again
-
-//Testing range
-            'PHRASE1to5("humpty","dumpty")',
-            'PHRASE1to5("together","again")',
-
-//M of N
-            'MOFN(2, "humpty", "horses", "together", "beansprout")',    // m<matches
-            'MOFN(3, "humpty", "horses", "together", "beansprout")',    // m=matches
-            'MOFN(4, "humpty", "horses", "together", "beansprout")',    // m>matches
-            'MOFN(2,2, "humpty", "horses", "together", "beansprout")',  // too many matches
-            'MOFN(2, "nonexistant", "little", "bo")',                   // first input fails to match any
-            'MOFN(2, "little", "bo", "nonexistant")',                   // lose an input while finising candidates
-            'MOFN(2, "one", "two", "three", "four", "five")',
-            'MOFN(2, "nonexistant", "two", "three", "four", "five")',
-            'MOFN(2, "one", "nonexistant", "three", "four", "five")',
-            'MOFN(2, "nonexistant1", "nonexistant2", "three", "four", "five")',
-            'MOFN(2, "nonexistant1", "nonexistant2", "nonexistant3", "four", "five")',
-            'MOFN(2, "nonexistant1", "nonexistant2", "nonexistant3", "nonexistant4", "five")',
-            'MOFN(2, PHRASE("little","bo"),PHRASE("three","bags"),"sheep")',    // m of n on phrases
-            'MOFN(2, PHRASE("Little","Bo"),PHRASE("three","bags"),"sheep")',    // m of n on phrases - capital letters don't match
-            'MOFN(2, OR("little","big"), OR("king", "queen"), OR("star", "sheep", "twinkle"))',
-
-//Proximity
-            'PROXIMITY("nonexistant1", "nonexistant2", -1, 1)',
-            'PROXIMITY("black", "nonexistant2", -1, 1)',
-            'PROXIMITY("nonexistant1", "sheep", -1, 1)',
-
-//Adjacent checks
-            'PROXIMITY("ship", "sank", 0, 0)',                      // either order but adjacent
-            'NORM(PROXIMITY("ship", "sank", 0, 0))',
-            'PROXIMITY("ship", "sank", -1, 0)',                     // must follow
-            'PROXIMITY("ship", "sank", 0, -1)',                     // must preceed
-            'PROXIMITY("sank", "ship", 0, 0)',                      // either order but adjacent
-            'PROXIMITY("sank", "ship", -1, 0)',                     // must follow
-            'PROXIMITY("sank", "ship", 0, -1)',                     // must preceed
-
-//Within a distance of 1
-            'PROXIMITY("ship", "sank", 1, 1)',                      // either order but only 1 intervening word
-            'PROXIMITY("ship", "sank", -1, 1)',                     // must follow
-            'PROXIMITY("ship", "sank", 1, -1)',                     // must preceed
-            'PROXIMITY("sank", "ship", 1, 1)',                      // either order but only 1 intervening word
-            'PROXIMITY("sank", "ship", -1, 1)',                     // must follow
-            'PROXIMITY("sank", "ship", 1, -1)',                     // must preceed
-
-            'PROXIMITY("ship", "sank", 0, 2)',                      // asymetric range
-
-//Within a distance of 2
-            'PROXIMITY("ship", "ship", 2, 2)',                      // either order but only 2 intervening word, no dups
-                                                                    // *** currently fails because of lack of duplication in lowest merger
-            'PROXIMITY("zx", "zx", 5, 5)',                          // "zx (za) zx", "zx (za zx zb zc zd) zx" and "zx (zb zc zd zx)"
-            'PROXIMITY(PROXIMITY("zx", "zx", 5, 5), "zx", 1, 1)',   // "zx (za) zx (zb zc zd) zx" - obtained two different ways.
-            'NORM(PROXIMITY(PROXIMITY("zx", "zx", 5, 5), "zx", 1, 1))', // as above, but normalized
-            'PROXIMITY(PROXIMITY("zx", "zx", 5, 5), "zx", 0, 0)',   // "zx (za) zx (zb zc zd) zx" - can obly be obtained from first
-                                                                    // you could imagine -ve left and right to mean within - would need -1,0 in stepping, and appropriate hard condition.
-
-            'PROXIMITY("ibm", "business", 2, 2)',                   // alias doesn't allow matching within itself.
-            'PROXIMITY("ibm", "business", 3, 3)',                   // alias should match now with other word
-            'PROXIMITY("ibm", "ibm", 0, 0)',                        // aliases and non aliases cause fun.
-
-//More combinations of operators
-            'AND(OR("twinkle", "black"), OR("sheep", "wonder"))',
-            'OR(AND("twinkle", "sheep"), AND("star", "black"))',
-            'OR(AND("twinkle", "star"), AND("sheep", "black"))',
-            'AND(SET("twinkle", "black"), SET("sheep", "wonder"))',
-
-//Silly queries
-            'OR("star","star","star","star","star")',
-            'AND("star","star","star","star","star")',
-            'MOFN(4,"star","star","star","star","star")',
-
-
-//Other operators
-            'PRE("twinkle", "twinkle")',
-            'PRE(PHRASE("twinkle", "twinkle"), PHRASE("little","star"))',
-            'PRE(PHRASE("little","star"), PHRASE("twinkle", "twinkle"))',
-            'PRE(PROXIMITY("twinkle","twinkle", 3, 3), PROXIMITY("little", "star", 2, 2))',
-            'AFT("twinkle", "twinkle")',
-            'AFT(PHRASE("little","star"), PHRASE("twinkle", "twinkle"))',
-            'AFT(PHRASE("twinkle", "twinkle"), PHRASE("little","star"))',
-            'AFT(PROXIMITY("twinkle","twinkle", 3, 3), PROXIMITY("little", "star", 2, 2))',
-
-// Left outer joins for ranking.
-            'RANK("sheep", OR("peep", "baa"))',
-            'RANK("three", OR("bags", "full"))',
-            'RANK("three", OR("one", "bags"))',
-
-
-//Non standard variants - AND, generating a single record for the match.  Actually for each cross product as it is currently (and logically) implemented
-            'ANDJOIN("nonexistant1", "nonexistant2")',          // neither side matches
-            'ANDJOIN("nonexistant1", "sheep")',                 // RHS matches
-            'ANDJOIN("twinkle", "nonexistant2")',               // LHS matches
-            'ANDJOIN("twinkle", "twinkle")',                    // should dedup
-            'ANDJOIN("sheep", "black")',                        // matches in same document
-            'ANDJOIN("sheep", "twinkle")',                      // matches in different documents
-            'ANDJOIN("in", "a")',                               // high frequencies
-            'ANDJOIN("twinkle", "little", "how", "star")',      // Nary
-
-            'ANDNOTJOIN("nonexistant1", "nonexistant2")',       // neither side matches
-            'ANDNOTJOIN("nonexistant1", "sheep")',              // RHS matches
-            'ANDNOTJOIN("twinkle", "nonexistant2")',            // LHS matches
-            'ANDNOTJOIN("twinkle", "twinkle")',                 // should dedup
-            'ANDNOTJOIN("sheep", "black")',                     // matches in same document
-            'ANDNOTJOIN("sheep", "twinkle")',                   // matches in different documents
-            'ANDNOTJOIN("one", OR("sheep", "black", "fish"))',  // matches one, but none of the others
-
-            'MOFNJOIN(2, "humpty", "horses", "together", "beansprout")',    // m<matches
-            'MOFNJOIN(3, "humpty", "horses", "together", "beansprout")',    // m=matches
-            'MOFNJOIN(4, "humpty", "horses", "together", "beansprout")',    // m>matches
-            'MOFNJOIN(2,2, "humpty", "horses", "together", "beansprout")',  // too many matches
-            'MOFNJOIN(2, "nonexistant", "little", "bo")',                   // first input fails to match any
-            'MOFNJOIN(2, "little", "bo", "nonexistant")',                   // lose an input while finising candidates
-            'MOFNJOIN(2, "one", "two", "three", "four", "five")',
-            'MOFNJOIN(2, "nonexistant", "two", "three", "four", "five")',
-            'MOFNJOIN(2, "one", "nonexistant", "three", "four", "five")',
-            'MOFNJOIN(2, "nonexistant1", "nonexistant2", "three", "four", "five")',
-            'MOFNJOIN(2, "nonexistant1", "nonexistant2", "nonexistant3", "four", "five")',
-            'MOFNJOIN(2, "nonexistant1", "nonexistant2", "nonexistant3", "nonexistant4", "five")',
-            'MOFNJOIN(2, PHRASE("little","bo"),PHRASE("three","bags"),"sheep")',    // m of n on phrases
-            'MOFNJOIN(2, PHRASE("Little","Bo"),PHRASE("three","bags"),"sheep")',    // m of n on phrases - capital letters don't match
-            'MOFNJOIN(2, OR("little","big"), OR("king", "queen"), OR("star", "sheep", "twinkle"))',
-
-            'RANKJOIN("SHEEP", "BLACK")',
-            'RANKJOIN("sheep", OR("peep", "baa"))',
-            'RANKJOIN("three", OR("bags", "full"))',
-            'RANKJOIN("three", OR("one", "bags"))',
-
-
-//ROLLAND - does AND, followed by a rollup by doc.  Should also check that smart stepping still works through the grouped rollup
-            'ROLLAND("nonexistant1", "nonexistant2")',          // neither side matches
-            'ROLLAND("nonexistant1", "sheep")',                 // RHS matches
-            'ROLLAND("twinkle", "nonexistant2")',               // LHS matches
-            'ROLLAND("twinkle", "twinkle")',                    // should dedup
-            'ROLLAND("sheep", "black")',                        // matches in same document
-            'ROLLAND("sheep", "twinkle")',                      // matches in different documents
-            'ROLLAND("in", "a")',                               // high frequencies
-            'ROLLAND("twinkle", "little", "how", "star")',      // Nary
-            'AND(ROLLAND("twinkle", "little"), ROLLAND("how", "star"))',        // Nary
-
-//Same tests as proximity above, but not calling a transform - merging instead
-            'PROXMERGE("ship", "sank", 0, 0)',                      // either order but adjacent
-            'PROXMERGE("ship", "sank", -1, 0)',                     // must follow
-            'PROXMERGE("ship", "sank", 0, -1)',                     // must preceed
-            'PROXMERGE("sank", "ship", 0, 0)',                      // either order but adjacent
-            'PROXMERGE("sank", "ship", -1, 0)',                     // must follow
-            'PROXMERGE("sank", "ship", 0, -1)',                     // must preceed
-            'PROXMERGE("ship", "sank", 1, 1)',                      // either order but only 1 intervening word
-            'PROXMERGE("ship", "sank", -1, 1)',                     // must follow
-            'PROXMERGE("ship", "sank", 1, -1)',                     // must preceed
-            'PROXMERGE("sank", "ship", 1, 1)',                      // either order but only 1 intervening word
-            'PROXMERGE("sank", "ship", -1, 1)',                     // must follow
-            'PROXMERGE("sank", "ship", 1, -1)',                     // must preceed
-            'PROXMERGE("ship", "sank", 0, 2)',                      // asymetric range
-
-//SET should be equivalent to OR
-            'SET("nonexistant1", "nonexistant2")',          // neither side matches
-            'SET("nonexistant1", "sheep")',                 // RHS matches
-            'SET("twinkle", "nonexistant2")',               // LHS matches
-            'SET("twinkle", "twinkle")',                    // should dedup
-            'SET("sheep", "black")',                        // matches in same document
-            'SET("sheep", "twinkle")',                      // matches in different documents
-            'SET("one", "sheep", "sheep", "black", "fish")',    // matches in different documents
-            'OR(SET("one", "sheep"), SET("sheep", "black", "fish"))',   // matches in different documents
-
-//Testing range
-            'PHRASE1to5(PHRASE1to5("what","you"),"are")',
-            'PHRASE1to5("what", PHRASE1to5("you","are"))',
-            'PHRASE1to5(PHRASE1to5("open","source"),"software")',
-            'PHRASE1to5("open", PHRASE1to5("source","software"))',
-
-//Atleast
-            'ATLEAST(2, "twinkle")',                                // would something like UNIQUEAND("twinkle", "twinkle") be more efficient???
-            'ATLEAST(4, "twinkle")',
-            'ATLEAST(5, "twinkle")',
-            'ATLEAST(5, AND("twinkle","star"))',
-            'AND(ATLEAST(4, "twinkle"),"star")',                    // make sure this still smart steps!
-            'AND(ATLEAST(5, "twinkle"),"star")',
-            'ATLEAST(1, PHRASE("humpty","dumpty"))',
-            'ATLEAST(2, PHRASE("humpty","dumpty"))',
-            'ATLEAST(3, PHRASE("humpty","dumpty"))',
-
-            '"little"',
-            'IN(name, "little")',
-            'NOTIN(name, "little")',
-            'IN(suitcase, AND("sock", "shirt"))',
-            'IN(suitcase, AND("sock", "dress"))',
-            'IN(suitcase, AND("shirt", "dress"))',                  //no, different suitcases..
-            'IN(suitcase, OR("cat", "dog"))',                       //no - wrong container
-            'IN(box, OR("cat", "dog"))',                            //yes
-            'IN(box, IN(suitcase, "shirt"))',
-            'IN(suitcase, IN(box, "shirt"))',                       // no other elements in the suitcase, so not valid
-            'IN(box, AND(IN(suitcase, "shirt"), "car"))',
-            'IN(box, AND(IN(suitcase, "shirt"), "lego"))',          // no, lego isn't in the box...
-            'IN(box, MOFN(2, "car", "train", "glue"))',             // really nasty - need to modify the m of n to add position equality!
-            'IN(box, MOFN(2, "car", "glue", "train"))',             // and check works in all positions.
-            'IN(box, MOFN(2, "glue", "car", "train"))',             //   " ditto "
-            'IN(box, MOFN(3, "car", "train", "glue"))',
-            'NOTIN(box, AND("computer", "lego"))',
-            'NOTIN(box, AND("train", "lego"))',
-            'IN(suitcase, PROXIMITY("trouser", "sock", 1, 2))',     // ok.
-            'IN(suitcase, PROXIMITY("trouser", "train", 1, 2))',    // no, close enough, but not both in the suitcase
-            'IN(suitcase, PROXIMITY("trouser", "dress", 6, 6))',    // no, close enough, but not both in the same suitcase
-            'PROXIMITY(IN(suitcase, "trouser"), IN(suitcase, "dress"), 6, 6)',  // yes - testing the proximity of the suitcases, not the contents.
-
-            'IN(S, AND("fish", "alive"))',                          // <s> is the sentence container
-            'S(AND("fish", "alive"))',                              // pseudonym for within same sentence -
-            'S(AND("fish", "finger"))',                             //
-            'S(AND("sheep", "wagging"))',
-            'P(AND("sheep", "wagging"))',                           // same paragraph...
-            'AND(IN(socks, "fox"),IN(socks, "knox"))',
-            'AND(IN(box, "fox"),IN(box, "knox"))',
-            'AND(IN(box, IN(socks, "fox")),IN(box, "knox"))',
-            'AND(IN(socks, IN(box, "fox")),IN(box, "knox"))',           // yes - because no extra elements in the box.
-            'S(PHRASE("black", "sheep"))',
-            'IN(name, PHRASE("little", "bo", "peep"))',
-            'IN(name, PHRASE("little", "bo", "peep", "has"))',
-
-            'IN(range1, IN(range2, "seven"))',                      // only match 5.3
-            'SAME(IN(range1, "seven"), IN(range2, "seven"))',       // only match 5.3
-            'OVERLAP(IN(range1, "five"), IN(range2, "ten"))',       // overlapping, match 5.4
-            'PROXIMITY(IN(range1, "five"), IN(range2, "ten"), 0, 0)',   // adjacent match 5.4, 5.5
-            'PROXIMITY(IN(range1, "five"), IN(range2, "ten"), 1, 1)',   // proximity match 5.4, 5.5
-            'PROXIMITY(IN(range1, "five"), IN(range2, "ten"), 2, 2)',   // adjacent match 5.4, 5.5, 5.6
-
-            'ATLEAST(2, IN(suitcase, "sock"))',                     // at least two suitcases containing a sock.
-            'ATLEAST(3, IN(suitcase, "sock"))',                     // no
-
-            'IN(box, "train")',                                     // should be 4 matches (since inside nested boxes)
-
-            'IN(suitcase, ATLEAST(1, "sock"))',                     // suitcases containing at least one sock.   (should really optimize away)
-            'IN(suitcase, ATLEAST(2, "sock"))',                     // at least two suit cases containing a sock.
-            'IN(suitcase, ATLEAST(3, "sock"))',
-            'IN(suitcase, ATLEAST(3, OR("sock", "dress")))',        //no
-            'IN(suitcase, ATLEAST(3, SET("sock", "dress")))',       //no
-            'IN(suitcase, ATLEAST(3, OR("sock", "jacket")))',       //yes...
-            'IN(suitcase, ATLEAST(3, SET("sock", "jacket")))',      //yes...
-            'IN(box, IN(suitcase, ATLEAST(2, "sock")))',            //yes - box, with one match
-            'IN(box, IN(suitcase, ATLEAST(3, "sock")))',            //no -
-            'IN(box, ATLEAST(2, IN(suitcase, "sock")))',            //yes -
-            'IN(box, ATLEAST(3, IN(suitcase, "sock")))',            //no -
-            'IN(box, ATLEAST(2, IN(suitcase, ATLEAST(2, "sock"))))',            //no...
-            'IN(box, AND(ATLEAST(2, "train"), ATLEAST(2, "sock")))',    // yes
-            'IN(box, AND(ATLEAST(3, "train"), ATLEAST(2, "sock")))',    // no
-            'IN(suitcase, AND(ATLEAST(2, "sock"), ATLEAST(2, OR("tights", "dress"))))', // no - different suitcases.
-            'IN(suitcase, ATLEAST(2, "sock"))', // yes
-            'IN(suitcase, ATLEAST(2, OR("tights", "dress")))',  // yes
-
-//The following example fails - not quite sure how to fix it.
-//          'IN(suitcase, OR(ATLEAST(2, "sock"), ATLEAST(2, OR("tights", "dress"))))',  // yes
-            'IN(suitcase, ATLEAST(4, AND(ATLEAST(2, "sock"), OR("shirt", "trouser"))))',    // yes - nested atleasts...
-            'IN(suitcase, ATLEAST(5, AND(ATLEAST(2, "sock"), OR("shirt", "trouser"))))',    // no
-
-            '_ATLEASTIN_(1, IN(suitcase, "sock"), 1)',                      // suitcases containing at least one sock.   (should really optimize away)
-            '_ATLEASTIN_(2, IN(suitcase, "sock"), 1)',                      // at least two suit cases containing a sock.
-            '_ATLEASTIN_(3, IN(suitcase, "sock"), 1)',
-
-            'S(ANDNOT("fish", "alive"))',                               // pseudonym for within same sentence -
-            'S(ANDNOT("fish", "finger"))',                              //
-
-            'AT("the", 2)',                                             // occurences of 'the' at position 2
-            'AT("the", 18)',
-            'AT("is", 17)',
-            'AND(AT("the", 18),AT("is",17))',
-
-            'AND("gch01", "gch02", "gch04")',
-            'AND("gch01", "gch02", "gch10")',
-
-            'AND(SET("and","a"), SET("the", "one"), PHRASE("for","the","dame"))',
-            'AND(CAPS("sheep"), "spotted")',
-            'AND(CAPS("sheep"), NOCAPS("spotted"))',
-            'AND(SET(CAPS("sheep","little")), SET(CAPS("Up","go")))',
-            'AND(SET(CAPS("sheep","little")), SET(NOCAPS("Up","go")))',
-            'AND(OR(CAPS("sheep"),CAPS("Little")), OR(CAPS("Up"),NOCAPS("go")))',
-
-            'ANDNOT(AND("black","sheep"), "family")',
-            'ANDNOT(AND("little","and"), "jack")',
-
-            'BUTNOT("little", PHRASE("little", "star"))',
-            'BUTNOTJOIN("little", PHRASE("little", "star"))',
-            'BUTNOT("black", PHRASE("black", OR("spotted", "sheep")))',
-            'BUTNOTJOIN("black", PHRASE("black", OR("spotted", "sheep")))',
-
-//MORE:
-// STEPPED flag on merge to give an error if input doesn't support stepping.
-// What about the duplicates that can come out of the proximity operators?
-// where the next on the rhs is at a compatible position, but in a different document
-// What about inverse of proximity x not w/n y
-// Can inverse proximity be used for sentance/paragraph.  Can we combine them so short circuited before temporaries created.
-//MORE: What other boundary conditions can we think of.
-
-                ' '
-
-#end
-
-            ], queryInputRecord);
-
-p := project(nofold(q1), doBatchExecute(TS_wordIndex, LEFT, 0x40000000));
-output(p);

testing/regress/ecl-test

@@ -31,7 +31,7 @@ from hpcc.util.ecl.file import ECLFile
 from hpcc.util.util import checkPqParam,  getVersionNumbers,  checkXParam
 from hpcc.common.error import Error
 
-prog_version = "0.0.19"
+prog_version = "0.0.20"
 
 # For coverage
 if ('coverage' in os.environ) and (os.environ['coverage'] == '1'):

testing/regress/ecl/aggds1.ecl

@@ -1,40 +0,0 @@
-/*##############################################################################
-
-    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
-
-    Licensed under the Apache License, Version 2.0 (the "License");
-    you may not use this file except in compliance with the License.
-    You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-    Unless required by applicable law or agreed to in writing, software
-    distributed under the License is distributed on an "AS IS" BASIS,
-    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-    See the License for the specific language governing permissions and
-    limitations under the License.
-############################################################################## */
-
-import $.setup.sq;
-
-//Check fixed size disk count correctly checks canMatchAny()
-inlineDs := dataset([1,2],{integer value});
-
-//Simple disk aggregate
-output(table(sq.SimplePersonBookDs, { sum(group, aage),exists(group),exists(group,aage>0),exists(group,aage>100),count(group,aage>20) })) : independent;
-
-//Filtered disk aggregate, which also requires a beenProcessed flag
-output(table(sq.SimplePersonBookDs(surname != 'Halliday'), { max(group, aage) })): independent;
-
-//Special case count.
-output(table(sq.SimplePersonBookDs(forename = 'Gavin'), { count(group) })): independent;
-
-output(count(sq.SimplePersonBookDs)): independent;
-
-//Special case count.
-output(table(sq.SimplePersonBookDs, { count(group, (forename = 'Gavin')) })): independent;
-
-//existance checks
-output(exists(sq.SimplePersonBookDs)): independent;
-output(exists(sq.SimplePersonBookDs(forename = 'Gavin'))): independent;
-output(exists(sq.SimplePersonBookDs(forename = 'Joshua'))): independent;

testing/regress/ecl/aggds1_hthor.ecl

@@ -0,0 +1,19 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+import $.common;
+common.aggds1('hthor');

testing/regress/ecl/aggds1_roxie.ecl

@@ -0,0 +1,19 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+import $.common;
+common.aggds1('roxie');

testing/regress/ecl/aggds1_thor.ecl

@@ -0,0 +1,19 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+import $.common;
+common.aggds1('thor');

testing/regress/ecl/aggds1seq.ecl

@@ -15,10 +15,8 @@
     limitations under the License.
 ############################################################################## */
 
-import $.setup.sq;
-
-//Check fixed size disk count correctly checks canMatchAny()
-inlineDs := dataset([1,2],{integer value});
+import $.setup;
+sq := setup.sq('hthor');
 
 sequential(
 //Simple disk aggregate

testing/regress/ecl/aggds2.ecl

@@ -15,15 +15,5 @@
     limitations under the License.
 ############################################################################## */
 
-import $.setup.sq;
-
-pr:= table(sq.SimplePersonBookDs, { fullname := trim(surname) + ', ' + trim(forename), aage });
-
-//Aggregate on a projected table that can't be merged.  seq is actually set to aage
-pr2:= table(sq.SimplePersonBookDs, { surname, forename, aage, unsigned8 seq := (random() % 100) / 2000 + aage; });
-
-//Filtered Aggregate on a projected table.
-output(table(pr(aage > 20), { max(group, fullname) }));
-
-//Filtered Aggregate on a projected table.
-output(table(pr2(seq > 30), { ave(group, aage) }));
+import $.common;
+common.aggds2('hthor');

testing/regress/ecl/aggds2_thor.ecl

@@ -0,0 +1,19 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+import $.common;
+common.aggds2('thor');

testing/regress/ecl/aggds2seq.ecl

@@ -15,7 +15,8 @@
     limitations under the License.
 ############################################################################## */
 
-import $.setup.sq;
+import $.setup;
+sq := setup.sq('hthor');
 
 pr:= table(sq.SimplePersonBookDs, { fullname := trim(surname) + ', ' + trim(forename), aage });
 

testing/regress/ecl/aggds3.ecl

@@ -15,29 +15,5 @@
     limitations under the License.
 ############################################################################## */
 
-import $.setup.sq;
-
-//A not-so-simple out of line subquery
-secondBookName := (string20)sort(sq.SimplePersonBookDs.books, name)[2].name;
-
-//Simple disk aggregate
-output(sort(table(sq.SimplePersonBookDs, { surname, sumage := sum(group, aage) }, surname, few),surname));
-
-//Filtered disk aggregate, which also requires a beenProcessed flag
-output(sort(table(sq.SimplePersonBookDs(surname != 'Halliday'), { max(group, aage), surname }, surname, few),surname));
-
-//check literals are assigned
-output(sort(table(sq.SimplePersonBookDs(forename = 'Gavin'), { 'Count: ', count(group), 'Name: ', surname }, surname, few),surname));
-
-//Sub query needs serializing or repeating....
-
-// A simple inline subquery
-output(sort(table(sq.SimplePersonBookDs, { cnt := count(books), sumage := sum(group, aage) }, count(books), few),cnt));
-
-output(sort(table(sq.SimplePersonBookDs, { sbn := secondBookName, sumage := sum(group, aage) }, secondBookName, few),sbn));
-
-// An out of line subquery (caused problems accessing parent inside sort criteria
-output(sort(table(sq.SimplePersonBookDs, { cnt := count(books(id != 0)), sumage := sum(group, aage) }, count(books(id != 0)), few),cnt));
-
-//Bizarre - add a dataset that needs serialization/deserialisation to enusre cloned correctly
-output(sort(table(nofold(sq.SimplePersonBookDs)(surname != 'Halliday'), { max(group, aage), surname, dataset books := books}, surname, few),surname));
+import $.common;
+common.aggds3('hthor');

testing/regress/ecl/aggds3_thor.ecl

@@ -0,0 +1,19 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+import $.common;
+common.aggds3('thor');

testing/regress/ecl/aggds3seq.ecl

@@ -15,7 +15,8 @@
     limitations under the License.
 ############################################################################## */
 
-import $.setup.sq;
+import $.setup;
+sq := setup.sq('hthor');
 
 //A not-so-simple out of line subquery
 secondBookName := (string20)sort(sq.SimplePersonBookDs.books, name)[2].name;

testing/regress/ecl/aggds4.ecl

@@ -15,15 +15,5 @@
     limitations under the License.
 ############################################################################## */
 
-import $.setup.sq;
-
-pr:= table(sq.SimplePersonBookDs, { fullname := trim(surname) + ', ' + trim(forename), aage });
-
-//Aggregate on a projected table that can't be merged
-pr2:= table(sq.SimplePersonBookDs, { surname, forename, aage, unsigned8 seq := (random() % 100) / 2000 + aage; });
-
-//Filtered Aggregate on a projected table.
-output(sort(table(pr(aage > 20), { aage, max(group, fullname) }, aage, few), record));
-
-//Filtered Aggregate on a projected table.
-output(sort(table(pr2(seq > 10), { surname, ave(group, aage) }, surname, few), record));
+import $.common;
+common.aggds4('hthor');

testing/regress/ecl/aggds4_thor.ecl

@@ -0,0 +1,19 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+import $.common;
+common.aggds4('thor');

testing/regress/ecl/aggds4seq.ecl

@@ -15,7 +15,8 @@
     limitations under the License.
 ############################################################################## */
 
-import $.setup.sq;
+import $.setup;
+sq := setup.sq('hthor');
 
 pr:= table(sq.SimplePersonBookDs, { fullname := trim(surname) + ', ' + trim(forename), aage });
 

testing/regress/ecl/aggidx1.ecl

@@ -15,28 +15,5 @@
     limitations under the License.
 ############################################################################## */
 
-import $.setup.sq;
-
-//Check correctly checks canMatchAny()
-inlineDs := dataset([1,2],{integer value});
-
-//Simple disk aggregate
-output(table(sq.SimplePersonBookIndex, { sum(group, aage),exists(group),exists(group,aage>0),exists(group,aage>100),count(group,aage>20) }));
-
-//Filtered disk aggregate, which also requires a beenProcessed flag
-output(table(sq.SimplePersonBookIndex(surname != 'Halliday'), { max(group, aage) }));
-
-//Special case count.
-output(table(sq.SimplePersonBookIndex(forename = 'Gavin'), { count(group) }));
-
-output(count(sq.SimplePersonBookIndex));
-
-//Special case count.
-output(table(sq.SimplePersonBookIndex, { count(group, (forename = 'Gavin')) }));
-
-output(table(inlineDs, { count(sq.SimplePersonBookIndex(inlineDs.value = 1)); }));
-
-//existance checks
-output(exists(sq.SimplePersonBookIndex));
-output(exists(sq.SimplePersonBookIndex(forename = 'Gavin')));
-output(exists(sq.SimplePersonBookIndex(forename = 'Joshua')));
+import $.common;
+common.aggidx1('hthor');

testing/regress/ecl/aggidx1_thor.ecl

@@ -0,0 +1,19 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+import $.common;
+common.aggidx1('thor');

testing/regress/ecl/aggidx1seq.ecl

@@ -15,7 +15,8 @@
     limitations under the License.
 ############################################################################## */
 
-import $.setup.sq;
+import $.setup;
+sq := setup.sq('hthor');
 
 //Check correctly checks canMatchAny()
 inlineDs := dataset([1,2],{integer value});

testing/regress/ecl/aggidx2.ecl

@@ -15,15 +15,5 @@
     limitations under the License.
 ############################################################################## */
 
-import $.setup.sq;
-
-pr:= table(sq.SimplePersonBookIndex, { fullname := trim(surname) + ', ' + trim(forename), aage });
-
-//Aggregate on a projected table that can't be merged
-pr2:= table(sq.SimplePersonBookIndex, { surname, forename, aage, unsigned8 seq := (random() % 100) / 2000 + aage; });
-
-//Filtered Aggregate on a projected table.
-output(table(pr(aage > 20), { max(group, fullname) }));
-
-//Filtered Aggregate on a projected table.
-output(table(pr2(seq > 30), { ave(group, aage) }));
+import $.common;
+common.aggidx2('hthor');

testing/regress/ecl/aggidx2_thor.ecl

@@ -0,0 +1,19 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+import $.common;
+common.aggidx2('thor');

testing/regress/ecl/aggidx2seq.ecl

@@ -15,7 +15,8 @@
     limitations under the License.
 ############################################################################## */
 
-import $.setup.sq;
+import $.setup;
+sq := setup.sq('hthor');
 
 pr:= table(sq.SimplePersonBookIndex, { fullname := trim(surname) + ', ' + trim(forename), aage });
 

testing/regress/ecl/aggidx3.ecl

@@ -15,30 +15,5 @@
     limitations under the License.
 ############################################################################## */
 
-import $.setup.sq;
-
-//A not-so-simple out of line subquery
-secondBookName := (string20)sort(sq.SimplePersonBookIndex.books, name)[2].name;
-
-//Simple disk aggregate
-output(sort(table(sq.SimplePersonBookIndex, { string surname := trim(sq.SimplePersonBookIndex.surname), sum(group, aage) }, trim(surname), few), surname));
-
-
-//Filtered disk aggregate, which also requires a beenProcessed flag
-output(sort(table(sq.SimplePersonBookIndex(surname != 'Halliday'), { max(group, aage), surname }, surname, few), surname));
-
-//Special case count.
-output(sort(table(sq.SimplePersonBookIndex(forename = 'Gavin'), { count(group), surname }, surname, few), surname));
-
-//Sub query needs serializing or repeating....
-
-// A simple inline subquery
-output(sort(table(sq.SimplePersonBookIndex, { cntBooks:= count(books), sumage := sum(group, aage) }, count(books), few), cntBooks, sumage));
-
-output(sort(table(sq.SimplePersonBookIndex, { secondBookName, sumage := sum(group, aage) }, secondBookName, few), secondbookname));
-
-//Access to file position - ensure it is serialized correctly!
-output(sort(table(sq.SimplePersonBookIndex, { pos := filepos DIV 8192, sumage := sum(group, aage) }, filepos DIV 8192, few), pos, sumage));
-
-// An out of line subquery (caused problems accessing parent inside sort criteria
-output(sort(table(sq.SimplePersonBookIndex, { numBooks := count(books(id != 0)), sumage := sum(group, aage) }, count(books(id != 0)), few), numbooks, sumage));
+import $.common;
+common.aggidx3('hthor');

testing/regress/ecl/aggidx3_thor.ecl

@@ -0,0 +1,19 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+import $.common;
+common.aggidx3('thor');

testing/regress/ecl/aggidx3seq.ecl

@@ -15,7 +15,8 @@
     limitations under the License.
 ############################################################################## */
 
-import $.setup.sq;
+import $.setup;
+sq := setup.sq('hthor');
 
 //A not-so-simple out of line subquery
 secondBookName := (string20)sort(sq.SimplePersonBookIndex.books, name)[2].name;

testing/regress/ecl/aggidx4.ecl

@@ -15,15 +15,5 @@
     limitations under the License.
 ############################################################################## */
 
-import $.setup.sq;
-
-pr:= table(sq.SimplePersonBookIndex, { fullname := trim(surname) + ', ' + trim(forename), aage });
-
-//Aggregate on a projected table that can't be merged
-pr2:= table(sq.SimplePersonBookIndex, { surname, forename, aage, unsigned8 seq := (random() % 100) / 2000 + aage; });
-
-//Filtered Aggregate on a projected table.
-output(sort(table(pr(aage > 20), { aage, max(group, fullname) }, aage, few), aage));
-
-//Filtered Aggregate on a projected table.
-output(sort(table(pr2(seq > 10), { surname, ave(group, aage) }, surname, few), surname));
+import $.common;
+common.aggidx4('hthor');

testing/regress/ecl/aggidx4_thor.ecl

@@ -0,0 +1,19 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+import $.common;
+common.aggidx4('thor');

testing/regress/ecl/aggidx4seq.ecl

@@ -15,7 +15,8 @@
     limitations under the License.
 ############################################################################## */
 
-import $.setup.sq;
+import $.setup;
+sq := setup.sq('hthor');
 
 pr:= table(sq.SimplePersonBookIndex, { fullname := trim(surname) + ', ' + trim(forename), aage });
 

testing/regress/ecl/aggsqx1.ecl

@@ -15,7 +15,8 @@
     limitations under the License.
 ############################################################################## */
 
-import $.setup.sq;
+import $.setup;
+sq := setup.sq('hthor');
 
 forceSubQuery(a) := macro
     { dedup(a,true)[1] }

testing/regress/ecl/aggsqx1b.ecl

@@ -15,7 +15,8 @@
     limitations under the License.
 ############################################################################## */
 
-import $.setup.sq;
+import $.setup;
+sq := setup.sq('hthor');
 
 forceSubQuery(a) := macro
     { dedup(a,true)[1] }

testing/regress/ecl/aggsqx2.ecl

@@ -15,7 +15,8 @@
     limitations under the License.
 ############################################################################## */
 
-import $.setup.sq;
+import $.setup;
+sq := setup.sq('hthor');
 
 forceSubQuery(a) := macro
     { dedup(a,true)[1] }

testing/regress/ecl/aggsqx3.ecl

@@ -15,7 +15,8 @@
     limitations under the License.
 ############################################################################## */
 
-import $.setup.sq;
+import $.setup;
+sq := setup.sq('hthor');
 
 //Simple disk aggregate
 output(preload(sq.HousePersonBookDs), { dataset people := sort(table(persons, { surname, sum(group, aage) }, surname, few), surname)});

testing/regress/ecl/aggsqx3err.ecl

@@ -15,7 +15,8 @@
     limitations under the License.
 ############################################################################## */
 
-import $.setup.sq;
+import $.setup;
+sq := setup.sq('hthor');
 
 //A mistyped query, but interesting never the less - needs to access a stored variable from the child.
 secondBookNameX := (string20)sort(sq.SimplePersonBookDs.books, name)[2].name;

testing/regress/ecl/aggsqx4.ecl

@@ -15,7 +15,8 @@
     limitations under the License.
 ############################################################################## */
 
-import $.setup.sq;
+import $.setup;
+sq := setup.sq('hthor');
 
 pr:= table(sq.HousePersonBookDs.persons, { fullname := trim(surname) + ', ' + trim(forename), aage });
 

testing/ecl/setup/textsearch.txt

@@ -15,38 +15,41 @@
     limitations under the License.
 ############################################################################## */
 
-//Code that is common to text searching
+import $.^.Setup;
+import Setup.Options;
+import Setup.TS;
 
+EXPORT TextSearch := FUNCTION
 
 INCLUDE_DEBUG_INFO := false;
 import lib_stringlib;
 
-MaxTerms            := TS_MaxTerms;
-MaxStages           := TS_MaxStages;
-MaxProximity        := TS_MaxProximity;
-MaxWildcard     := TS_MaxWildcard;
-MaxMatchPerDocument := TS_MaxMatchPerDocument;
-MaxFilenameLength := TS_MaxFilenameLength;
-MaxActions       := TS_MaxActions;
-
-kindType        := TS_kindType;
-sourceType      := TS_sourceType;
-wordCountType   := TS_wordCountType;
-segmentType     := TS_segmentType;
-wordPosType     := TS_wordPosType;
-docPosType      := TS_docPosType;
-documentId      := TS_documentId;
-termType        := TS_termType;
-distanceType    := TS_distanceType;
-stageType       := TS_stageType;
-dateType        := TS_dateType;
+MaxTerms            := TS.MaxTerms;
+MaxStages           := TS.MaxStages;
+MaxProximity        := TS.MaxProximity;
+MaxWildcard     := TS.MaxWildcard;
+MaxMatchPerDocument := TS.MaxMatchPerDocument;
+MaxFilenameLength := TS.MaxFilenameLength;
+MaxActions       := TS.MaxActions;
+
+kindType        := TS.kindType;
+sourceType      := TS.sourceType;
+wordCountType   := TS.wordCountType;
+segmentType     := TS.segmentType;
+wordPosType     := TS.wordPosType;
+docPosType      := TS.docPosType;
+documentId      := TS.documentId;
+termType        := TS.termType;
+distanceType    := TS.distanceType;
+stageType       := TS.stageType;
+dateType        := TS.dateType;
 matchCountType  := unsigned2;
-wordType        := TS_wordType;
-wordFlags       := TS_wordFlags;
-wordIdType      := TS_wordIdType;
+wordType        := TS.wordType;
+wordFlags       := TS.wordFlags;
+wordIdType      := TS.wordIdType;
 
 //May want the following, probably not actually implemented as an index - would save having dpos in the index, but more importantly storing it in the candidate match results because the mapping could be looked up later.
-wordIndexRecord := TS_wordIndexRecord;
+wordIndexRecord := TS.wordIndexRecord;
 
 MaxWipWordOrAlias  := 4;
 MaxWipTagContents  := 65535;
@@ -61,9 +64,9 @@ actionEnum := ENUM(
 //Minimal operations required to implement the searching.
     ReadWord,           // termNum, source, segment, word, wordFlagMask, wordFlagCompare,
     ReadWordSet,        // termNum, source, segment, words, wordFlagMask, wordFlagCompare,
-    AndTerms,           // 
-    OrTerms,            // 
-    AndNotTerms,        // 
+    AndTerms,           //
+    OrTerms,            //
+    AndNotTerms,        //
     PhraseAnd,          //
     ProximityAnd,       // distanceBefore, distanceAfter
     MofNTerms,          // minMatches, maxMatches
@@ -75,16 +78,16 @@ actionEnum := ENUM(
     ContainedAtLeast,
     TagContainsSearch,  // Used for the outermost IN() expression - check it overlaps and rolls up
     TagContainsTerm,    // Used for an inner tag contains - checks, but doesn't roll up
-    TagNotContainsTerm, // 
+    TagNotContainsTerm, //
     SameContainer,
-    NotSameContainer,   // 
-    MofNContainer,      // 
+    NotSameContainer,   //
+    MofNContainer,      //
     RankContainer,      // NOTIMPLEMENTED
     OverlapProximityAnd,
 
 //The following aren't very sensible as far as text searching goes, but are here to test the underlying functionality
     AndJoinTerms,       // join on non-proximity
-    AndNotJoinTerms,    // 
+    AndNotJoinTerms,    //
     MofNJoinTerms,      // minMatches, maxMatches
     RankJoinTerms,      // left outer join
     ProximityMergeAnd,  // merge join on proximity
@@ -102,13 +105,13 @@ actionEnum := ENUM(
 
     //The following are only used in the production
     FlagModifier,       // wordFlagMask, wordFlagCompare
-    QuoteModifier,      // 
+    QuoteModifier,      //
     Max
 );
 
 //  FAIL(stageType, 'Missing entry: ' + (string)action));
 
-boolean definesTerm(actionEnum action) := 
+boolean definesTerm(actionEnum action) :=
     (action in [actionEnum.ReadWord, actionEnum.ReadWordSet]);
 
 booleanRecord := { boolean value };
@@ -124,7 +127,7 @@ createStage(stageType stage) := transform(stageRecord, self.stage := stage);
 createTerm(termType term) := transform(termRecord, self.term := term);
 
 //should have an option to optimize the order
-searchRecord := 
+searchRecord :=
             RECORD  //,PACK
 stageType       stage;
 termType        term;
@@ -183,8 +186,8 @@ StageSetToDataset(stageSet x) := dataset(x, stageRecord);
 StageDatasetToSet(dataset(stageRecord) x) := set(x, stage);
 
 hasSingleRowPerMatch(actionEnum kind) :=
-    (kind IN [  actionEnum.ReadWord, 
-                actionEnum.ReadWordSet, 
+    (kind IN [  actionEnum.ReadWord,
+                actionEnum.ReadWordSet,
                 actionEnum.PhraseAnd,
                 actionEnum.ProximityAnd,
                 actionEnum.ContainedAtLeast,
@@ -193,7 +196,7 @@ hasSingleRowPerMatch(actionEnum kind) :=
                 actionEnum.OverlapProximityAnd]);
 
 inheritsSingleRowPerMatch(actionEnum kind) :=
-    (kind IN [  actionEnum.OrTerms, 
+    (kind IN [  actionEnum.OrTerms,
 //              actionEnum.AndNotTerms,                 // move container inside an andnot
                 actionEnum.TagNotContainsTerm,
                 actionEnum.NotSameContainer]);
@@ -236,8 +239,8 @@ combineChildren(dataset(childMatchRecord) l, dataset(childMatchRecord) r) := fun
     return dedup(mergedDs, wpos, wip, term);
 end;
 
-SearchExecutor(dataset(TS_wordIndexRecord) wordIndex, unsigned4 internalFlags) := MODULE
-    
+SearchExecutor(dataset(TS.wordIndexRecord) wordIndex, unsigned4 internalFlags) := MODULE
+
     ///////////////////////////////////////////////////////////////////////////////////////////////////////////
     // Matching helper functions
 
@@ -254,7 +257,7 @@ SearchExecutor(dataset(TS_wordIndexRecord) wordIndex, unsigned4 internalFlags) :
         matchSearchFlags(wIndex, search);
 
     EXPORT matchSearchSource(wordIndex wIndex, searchRecord search) :=
-        keyed(search.source = 0 OR TS_docMatchesSource(wIndex.doc, search.source), opt);
+        keyed(search.source = 0 OR TS.docMatchesSource(wIndex.doc, search.source), opt);
 
     ///////////////////////////////////////////////////////////////////////////////////////////////////////////
     // ReadWord
@@ -271,7 +274,7 @@ SearchExecutor(dataset(TS_wordIndexRecord) wordIndex, unsigned4 internalFlags) :
             self := [];
         end;
 
-        steppedMatches :=   IF(search.priority <> 0, 
+        steppedMatches :=   IF(search.priority <> 0,
                                 stepped(matches, doc, segment, wpos, priority(search.priority)),
                                 stepped(matches, doc, segment, wpos)
                             );
@@ -298,7 +301,7 @@ SearchExecutor(dataset(TS_wordIndexRecord) wordIndex, unsigned4 internalFlags) :
             self := [];
         end;
 
-        steppedMatches :=   IF(search.priority <> 0, 
+        steppedMatches :=   IF(search.priority <> 0,
                                 stepped(matches, doc, segment, wpos, priority(search.priority)),
                                 stepped(matches, doc, segment, wpos)
                             );
@@ -312,7 +315,7 @@ SearchExecutor(dataset(TS_wordIndexRecord) wordIndex, unsigned4 internalFlags) :
     // OrTerms
 
     EXPORT doOrTerms(searchRecord search, SetOfInputs inputs) := FUNCTION
-        return merge(inputs, doc, segment, wpos, dedup);        // MORE  option to specify priority?
+        return merge(inputs, doc, segment, wpos, dedup, SORTED(doc, segment, wpos));        // MORE  option to specify priority?
     END;
 
 
@@ -336,7 +339,7 @@ SearchExecutor(dataset(TS_wordIndexRecord) wordIndex, unsigned4 internalFlags) :
     // ButNotTerms
 
     EXPORT doButNotTerms(searchRecord search, SetOfInputs inputs) := FUNCTION
-        return mergejoin(inputs, STEPPED(left.doc = right.doc and left.segment=right.segment) and 
+        return mergejoin(inputs, STEPPED(left.doc = right.doc and left.segment=right.segment) and
                                  (LEFT.wpos BETWEEN RIGHT.wpos AND RIGHT.wpos+RIGHT.wip), doc, segment, wpos, left only, internal(internalFlags));
     END;
 
@@ -345,7 +348,7 @@ SearchExecutor(dataset(TS_wordIndexRecord) wordIndex, unsigned4 internalFlags) :
     // ButNotJoinTerms
 
     EXPORT doButNotJoinTerms(searchRecord search, SetOfInputs inputs) := FUNCTION
-        return join(inputs, STEPPED(left.doc = right.doc and left.segment=right.segment) and 
+        return join(inputs, STEPPED(left.doc = right.doc and left.segment=right.segment) and
                                  (LEFT.wpos BETWEEN RIGHT.wpos AND RIGHT.wpos+RIGHT.wip), transform(left), sorted(doc, segment, wpos), left only, internal(internalFlags));
     END;
 
@@ -418,7 +421,7 @@ SearchExecutor(dataset(TS_wordIndexRecord) wordIndex, unsigned4 internalFlags) :
     ///////////////////////////////////////////////////////////////////////////////////////////////////////////
     // PhraseAnd
 
-    steppedPhraseCondition(matchRecord l, matchRecord r, distanceType maxWip) := 
+    steppedPhraseCondition(matchRecord l, matchRecord r, distanceType maxWip) :=
             (l.doc = r.doc) and (l.segment = r.segment) and
             (r.wpos between l.wpos+1 and l.wpos+maxWip);
 
@@ -426,7 +429,7 @@ SearchExecutor(dataset(TS_wordIndexRecord) wordIndex, unsigned4 internalFlags) :
 
         steppedCondition(matchRecord l, matchRecord r) := steppedPhraseCondition(l, r, search.maxWipLeft);
 
-        condition(matchRecord l, matchRecord r) :=  
+        condition(matchRecord l, matchRecord r) :=
             (r.wpos = l.wpos + l.wip);
 
         matchRecord createMatch(matchRecord l, dataset(matchRecord) allRows) := transform
@@ -444,7 +447,7 @@ SearchExecutor(dataset(TS_wordIndexRecord) wordIndex, unsigned4 internalFlags) :
     ///////////////////////////////////////////////////////////////////////////////////////////////////////////
     // PhraseAnd
 
-    steppedPhrase1To5Condition(matchRecord l, matchRecord r, distanceType maxWip) := 
+    steppedPhrase1To5Condition(matchRecord l, matchRecord r, distanceType maxWip) :=
             (l.doc = r.doc) and (l.segment = r.segment) and
             (r.wpos between l.wpos+1 and l.wpos+5);
 
@@ -452,7 +455,7 @@ SearchExecutor(dataset(TS_wordIndexRecord) wordIndex, unsigned4 internalFlags) :
 
         steppedCondition(matchRecord l, matchRecord r) := steppedPhrase1To5Condition(l, r, search.maxWipLeft);
 
-        condition(matchRecord l, matchRecord r) :=  
+        condition(matchRecord l, matchRecord r) :=
             (r.wpos = l.wpos + l.wip);
 
         matchRecord createMatch(matchRecord l, dataset(matchRecord) allRows) := transform
@@ -654,7 +657,7 @@ SearchExecutor(dataset(TS_wordIndexRecord) wordIndex, unsigned4 internalFlags) :
             maxRightBeforeLeft := IF(maxDistanceRightBeforeLeft >= 0, maxDistanceRightBeforeLeft + maxWipRight, maxDistanceRightBeforeLeft);
             maxRightAfterLeft := IF(maxDistanceRightAfterLeft >= 0, maxDistanceRightAfterLeft + maxWipLeft, maxDistanceRightAfterLeft);
 
-            return 
+            return
                 (l.doc = r.doc) and (l.segment = r.segment) and
                 (r.wpos + maxRightBeforeLeft >= l.wpos) and             // (right.wpos + right.wip + maxRightBeforeLeft >= left.wpos)
                 (r.wpos <= l.wpos + (maxRightAfterLeft));               // (right.wpos <= left.wpos + left.wip + maxRightAfterLeft)
@@ -665,14 +668,14 @@ SearchExecutor(dataset(TS_wordIndexRecord) wordIndex, unsigned4 internalFlags) :
 
         steppedCondition(matchRecord l, matchRecord r) := steppedProximityCondition(l, r, search.maxWipLeft, search.maxWipRight, search.maxDistanceRightBeforeLeft, search.maxDistanceRightAfterLeft);
 
-        condition(matchRecord l, matchRecord r) :=  
+        condition(matchRecord l, matchRecord r) :=
             (r.wpos + r.wip + search.maxDistanceRightBeforeLeft >= l.wpos) and
             (r.wpos <= l.wpos + l.wip + search.maxDistanceRightAfterLeft);
 
         overlaps(wordPosType wpos, childMatchRecord r) := (wpos between r.wpos and r.wpos + (r.wip - 1));
 
         createMatch(matchRecord l, matchRecord r) := function
-        
+
             wpos := min(l.wpos, r.wpos);
             wend := max(l.wpos + l.wip, r.wpos + r.wip);
 
@@ -704,14 +707,14 @@ SearchExecutor(dataset(TS_wordIndexRecord) wordIndex, unsigned4 internalFlags) :
 
         steppedCondition(matchRecord l, matchRecord r) := steppedProximityCondition(l, r, search.maxWipLeft, search.maxWipRight, search.maxDistanceRightBeforeLeft, search.maxDistanceRightAfterLeft);
 
-        condition(matchRecord l, matchRecord r) :=  
+        condition(matchRecord l, matchRecord r) :=
             (r.wpos + r.wip + search.maxDistanceRightBeforeLeft >= l.wpos) and
             (r.wpos <= l.wpos + l.wip + search.maxDistanceRightAfterLeft);
 
         overlaps(wordPosType wpos, childMatchRecord r) := (wpos between r.wpos and r.wpos + (r.wip - 1));
 
         anyOverlaps (matchRecord l, matchRecord r) := function
-        
+
             wpos := min(l.wpos, r.wpos);
             wend := max(l.wpos + l.wip, r.wpos + r.wip);
 
@@ -735,17 +738,17 @@ SearchExecutor(dataset(TS_wordIndexRecord) wordIndex, unsigned4 internalFlags) :
 
     EXPORT doOverlapProximityAnd(searchRecord search, SetOfInputs inputs) := FUNCTION
 
-        steppedCondition(matchRecord l, matchRecord r) := 
+        steppedCondition(matchRecord l, matchRecord r) :=
                 (l.doc = r.doc) and (l.segment = r.segment) and
                 (r.wpos + search.maxWipRight >= l.wpos) and
                 (r.wpos <= l.wpos + search.maxWipLeft);
 
-        condition(matchRecord l, matchRecord r) :=  
+        condition(matchRecord l, matchRecord r) :=
             (r.wpos + r.wip >= l.wpos) and (r.wpos <= l.wpos + l.wip);
 
 
         createMatch(matchRecord l, matchRecord r) := function
-        
+
             wpos := min(l.wpos, r.wpos);
             wend := max(l.wpos + l.wip, r.wpos + r.wip);
 
@@ -855,26 +858,33 @@ SearchExecutor(dataset(TS_wordIndexRecord) wordIndex, unsigned4 internalFlags) :
     EXPORT convertToUserOutput(dataset(matchRecord) results) := function
 
         simpleUserOutputRecord createUserOutput(matchRecord l) := transform
-                self.source := TS_docid2source(l.doc);
-                self.subDoc := TS_docid2doc(l.doc);
+                self.source := TS.docid2source(l.doc);
+                self.subDoc := TS.docid2doc(l.doc);
                 self.words := l.children;
-                SELF.line := l.dpos DIV TS_MaxColumnsPerLine;
-                SELF.column := l.dpos % TS_MaxColumnsPerLine;
+                SELF.line := l.dpos DIV TS.MaxColumnsPerLine;
+                SELF.column := l.dpos % TS.MaxColumnsPerLine;
                 SELF := l;
             END;
 
         return project(results, createUserOutput(left));
     end;
 
-    EXPORT doExecuteQuery(dataset(searchRecord) queryDefinition, dataset(matchRecord) initialResults = dataset([], matchRecord)) := function
+    EXPORT doExecuteQuery(dataset(searchRecord) queryDefinition, boolean useLocal) := function
+
+        initialResults := dataset([], matchRecord);
+        localResults := FUNCTION
+            executionPlan := thisnode(global(queryDefinition, opt, few));           // Store globally for efficient access
+            results := ALLNODES(LOCAL(graph(initialResults, count(executionPlan), processStage(executionPlan[NOBOUNDCHECK COUNTER], rowset(left)), parallel)));
+            RETURN results;
+        END;
+
+        globalResults := FUNCTION
+            executionPlan := global(queryDefinition, opt, few);         // Store globally for efficient access
+            results := graph(initialResults, count(executionPlan), processStage(executionPlan[NOBOUNDCHECK COUNTER], rowset(left)), parallel);
+            RETURN results;
+        END;
 
-    #if (useLocal=true)
-        executionPlan := thisnode(global(queryDefinition, opt, few));           // Store globally for efficient access
-        results := ALLNODES(LOCAL(graph(initialResults, count(executionPlan), processStage(executionPlan[NOBOUNDCHECK COUNTER], rowset(left)), parallel)));
-    #else
-        executionPlan := global(queryDefinition, opt, few);         // Store globally for efficient access
-        results := graph(initialResults, count(executionPlan), processStage(executionPlan[NOBOUNDCHECK COUNTER], rowset(left)), parallel);
-    #end
+        results := IF(useLocal, localResults, globalResults);
         userOutput := convertToUserOutput(results);
 
         return userOutput;
@@ -887,12 +897,12 @@ end;
 // A simplified query language
 parseQuery(string queryText) := function
 
-searchParseRecord := 
+searchParseRecord :=
             RECORD(searchRecord)
 unsigned        numInputs;
             END;
 
-productionRecord  := 
+productionRecord  :=
             record
 unsigned        termCount;
 dataset(searchParseRecord) actions{maxcount(MaxActions)};
@@ -924,7 +934,7 @@ END;
 
 PRULE forwardExpr := use(productionRecord, 'ExpressionRule');
 
-ARULE term0 
+ARULE term0
     := quotedword                               transform(searchParseRecord,
                                                     SELF.action := actionEnum.ReadWord;
                                                     SELF.word := StringLib.StringToLowerCase($1[2..length($1)-1]);
@@ -1020,11 +1030,11 @@ PRULE term1
                                                     )
                                                 )
     | 'RANK' '(' forwardExpr ',' forwardExpr ')'
-                                                transform(productionRecord, 
+                                                transform(productionRecord,
                                                     self.termCount := 1;
                                                     self.actions := $3.actions + $5.actions + row(
-                                                        transform(searchParseRecord, 
-                                                            self.action := actionEnum.RankMergeTerms; 
+                                                        transform(searchParseRecord,
+                                                            self.action := actionEnum.RankMergeTerms;
                                                             self.numInputs := 2;
                                                             self := []
                                                         )
@@ -1085,11 +1095,11 @@ PRULE term1
                                                     )
                                                 )
     | 'PROXIMITY' '(' forwardExpr ',' forwardExpr ',' number ',' number ')'
-                                                transform(productionRecord, 
+                                                transform(productionRecord,
                                                     self.termCount := 1;
                                                     self.actions := $3.actions + $5.actions + row(
-                                                        transform(searchParseRecord, 
-                                                            self.action := actionEnum.ProximityAnd; 
+                                                        transform(searchParseRecord,
+                                                            self.action := actionEnum.ProximityAnd;
                                                             self.numInputs := 2;
                                                             self.maxDistanceRightBeforeLeft := (integer)$7;
                                                             self.maxDistanceRightAfterLeft := (integer)$9;
@@ -1098,22 +1108,22 @@ PRULE term1
                                                     )
                                                 )
     | 'OVERLAP' '(' forwardExpr ',' forwardExpr ')'
-                                                transform(productionRecord, 
+                                                transform(productionRecord,
                                                     self.termCount := 1;
                                                     self.actions := $3.actions + $5.actions + row(
-                                                        transform(searchParseRecord, 
-                                                            self.action := actionEnum.OverlapProximityAnd; 
+                                                        transform(searchParseRecord,
+                                                            self.action := actionEnum.OverlapProximityAnd;
                                                             self.numInputs := 2;
                                                             self := []
                                                         )
                                                     )
                                                 )
     | 'PRE' '(' forwardExpr ',' forwardExpr ')'
-                                                transform(productionRecord, 
+                                                transform(productionRecord,
                                                     self.termCount := 1;
                                                     self.actions := $3.actions + $5.actions + row(
-                                                        transform(searchParseRecord, 
-                                                            self.action := actionEnum.ProximityAnd; 
+                                                        transform(searchParseRecord,
+                                                            self.action := actionEnum.ProximityAnd;
                                                             self.numInputs := 2;
                                                             self.maxDistanceRightBeforeLeft := -1;
                                                             self.maxDistanceRightAfterLeft := MaxWordsInDocument;
@@ -1122,11 +1132,11 @@ PRULE term1
                                                     )
                                                 )
     | 'AFT' '(' forwardExpr ',' forwardExpr ')'
-                                                transform(productionRecord, 
+                                                transform(productionRecord,
                                                     self.termCount := 1;
                                                     self.actions := $3.actions + $5.actions + row(
-                                                        transform(searchParseRecord, 
-                                                            self.action := actionEnum.ProximityAnd; 
+                                                        transform(searchParseRecord,
+                                                            self.action := actionEnum.ProximityAnd;
                                                             self.numInputs := 2;
                                                             self.maxDistanceRightBeforeLeft := MaxWordsInDocument;
                                                             self.maxDistanceRightAfterLeft := -1;
@@ -1135,10 +1145,10 @@ PRULE term1
                                                     )
                                                 )
     | 'PROXMERGE' '(' forwardExpr ',' forwardExpr ',' number ',' number ')'
-                                                transform(productionRecord, 
+                                                transform(productionRecord,
                                                     self.termCount := 1;
                                                     self.actions := $3.actions + $5.actions + row(
-                                                        transform(searchParseRecord, 
+                                                        transform(searchParseRecord,
                                                             self.action := actionEnum.ProximityMergeAnd;
                                                             self.numInputs := 2;
                                                             self.maxDistanceRightBeforeLeft := (integer)$7;
@@ -1193,11 +1203,11 @@ PRULE term1
                                                     )
                                                 )
     | 'RANKJOIN' '(' forwardExpr ',' forwardExpr ')'
-                                                transform(productionRecord, 
+                                                transform(productionRecord,
                                                     self.termCount := 1;
                                                     self.actions := $3.actions + $5.actions + row(
-                                                        transform(searchParseRecord, 
-                                                            self.action := actionEnum.RankJoinTerms; 
+                                                        transform(searchParseRecord,
+                                                            self.action := actionEnum.RankJoinTerms;
                                                             self.numInputs := 2;
                                                             self := []
                                                         )
@@ -1263,11 +1273,11 @@ PRULE term1
                                                     )
                                                 )
     | 'SAME' '(' forwardExpr ',' forwardExpr ')'
-                                                transform(productionRecord, 
+                                                transform(productionRecord,
                                                     self.termCount := 1;
                                                     self.actions := $3.actions + $5.actions + row(
-                                                        transform(searchParseRecord, 
-                                                            self.action := actionEnum.SameContainer; 
+                                                        transform(searchParseRecord,
+                                                            self.action := actionEnum.SameContainer;
                                                             self.numInputs := 2;
                                                             self := []
                                                         )
@@ -1295,7 +1305,7 @@ PRULE term1
                                                         )
                                                     )
                                                 )
-    | 'AT' '(' forwardExpr ',' number ')'   
+    | 'AT' '(' forwardExpr ',' number ')'
                                                 transform(productionRecord,
                                                     self.termCount := 1;
                                                     self.actions := $3.actions + row(
@@ -1307,7 +1317,7 @@ PRULE term1
                                                         )
                                                     )
                                                 )
-    | 'NOTAT' '(' forwardExpr ',' number ')'    
+    | 'NOTAT' '(' forwardExpr ',' number ')'
                                                 transform(productionRecord,
                                                     self.termCount := 1;
                                                     self.actions := $3.actions + row(
@@ -1373,11 +1383,11 @@ dataset(searchParseRecord) actions{maxcount(MaxActions)};
         end;
 
 
-resultsRecord extractResults(dataset(searchParseRecord) actions) := 
+resultsRecord extractResults(dataset(searchParseRecord) actions) :=
         TRANSFORM
             SELF.actions := actions;
         END;
-            
+
 p1 := PARSE(infile,line,expr,extractResults($1.actions),first,whole,skip(ws),nocase,parse);
 
 pnorm := normalize(p1, left.actions, transform(right));
@@ -1547,14 +1557,14 @@ transformAtLeast(dataset(searchRecord) parsed) := function
         shared inputTerms := r.mapping(stage in set(l.inputs, stage)).outputTerms;
         shared inputAtleast := r.mapping(stage in set(l.inputs, stage)).activeAtleast;
         shared outputTerms := IF(hasSingleRowPerMatch(l.action), dataset([createTerm(l.term)]), inputTerms);
-        shared outputAtleast := 
+        shared outputAtleast :=
             MAP(l.action = actionEnum.GlobalAtLeast=>inputAtleast + row(createAtleast(l, inputTerms)),
                 l.action <> actionEnum.TagContainsSearch=>inputAtleast);
 
         export processRecord nextRow := transform
             SELF.moved := IF(l.action = actionEnum.TagContainsSearch, inputAtLeast);
 #if (INCLUDE_DEBUG_INFO)
-            SELF.debug := trim(l.debug) + '[' + 
+            SELF.debug := trim(l.debug) + '[' +
                     (string)COUNT(inputTerms) + ':' +
                     (string)COUNT(inputAtleast) + ':' +
                     (string)COUNT(outputTerms) + ':' +
@@ -1663,8 +1673,8 @@ transformNotIn(dataset(searchRecord) input) := function
     doStage2(processRecord l, stage2Record r) := module
         shared newContainer := r.map(stage = l.stage)[1].container;
         shared newTerm := r.map(stage = l.stage)[1].term;
-        shared numStages := 
-                    MAP(l.singleRowPerMatch and newContainer <> ''=>2, 
+        shared numStages :=
+                    MAP(l.singleRowPerMatch and newContainer <> ''=>2,
                         l.action = actionEnum.TagNotContainsTerm and not l.singleRowPerMatch=>0,
                         1);
         export processRecord nextRow := transform
@@ -1792,7 +1802,7 @@ transformIn(dataset(searchRecord) input) := function
                  actionEnum.RankMergeTerms  => IF(l.newContainer <> '', actionEnum.RankContainer, l.action),
                  actionEnum.TagContainsSearch => IF(l.inputsSingleRowPerMatch, actionEnum.TagContainsSearch, actionEnum.rollupContainer),
                  l.action);
-                 
+
 
         SELF.stage := l.stage + (c - 1);
         SELF.inputs := IF(c=2, dataset([createStage(l.stage)]), l.inputs);
@@ -1817,37 +1827,48 @@ applySearchTransformations(dataset(searchRecord) input) := FUNCTION
     RETURN processed4;
 END;
 
-queryProcessor(dataset(TS_wordIndexRecord) wordIndex, string query, unsigned4 internalFlags) := module//,library('TextSearch',1,0)
+queryProcessor(dataset(TS.wordIndexRecord) wordIndex, string query, boolean useLocal, unsigned4 internalFlags) := module//,library('TextSearch',1,0)
 export string queryText := query;
 export request := parseQuery(query);
 export processed := applySearchTransformations(request);
-export result := SearchExecutor(wordIndex, internalFlags).doExecuteQuery(processed);
+export result := SearchExecutor(wordIndex, internalFlags).doExecuteQuery(processed, useLocal);
     end;
 
 
-queryInputRecord := { string query{maxlength(2048)}; };
-
 MaxResults := 10000;
 
-processedRecord := record(queryInputRecord)
-dataset(searchRecord) request{maxcount(MaxActions)};
-dataset(simpleUserOutputRecord) result{maxcount(MaxResults)};
-        end;
+publicExports := MODULE
 
+    EXPORT queryInputRecord := { string query{maxlength(2048)}; };
 
-processedRecord doBatchExecute(dataset(TS_wordIndexRecord) wordIndex, queryInputRecord l, unsigned4 internalFlags=0) := transform
-    processed := queryProcessor(wordIndex, l.query, internalFlags);
-    self.request := processed.processed;
-    self.result := choosen(processed.result, MaxResults);
-    self := l;
-end;
+    EXPORT processedRecord := record(queryInputRecord)
+        dataset(searchRecord) request{maxcount(MaxActions)};
+        dataset(simpleUserOutputRecord) result{maxcount(MaxResults)};
+    END;
 
+    EXPORT processedRecord doBatchExecute(dataset(TS.wordIndexRecord) wordIndex, queryInputRecord l, boolean useLocal, unsigned4 internalFlags=0) := transform
+        processed := queryProcessor(wordIndex, l.query, useLocal, internalFlags);
+        self.request := processed.processed;
+        self.result := choosen(processed.result, MaxResults);
+        self := l;
+    end;
 
-doSingleExecute(dataset(TS_wordIndexRecord) wordIndex, string queryText, unsigned4 internalFlags=0) := function
-    request := parseQuery(queryText);
-    result := SearchExecutor(wordIndex, internalFlags).doExecuteQuery(request);
-    return result;
-end;
 
+    EXPORT doSingleExecute(dataset(TS.wordIndexRecord) wordIndex, string queryText, boolean useLocal, unsigned4 internalFlags=0) := function
+        request := parseQuery(queryText);
+        result := SearchExecutor(wordIndex, internalFlags).doExecuteQuery(request, useLocal);
+        return result;
+    end;
+
+    EXPORT executeBatchAgainstWordIndex(DATASET(queryInputRecord) queries, boolean useLocal, string source, unsigned4 internalFlags=0) := FUNCTION
+        Files := Setup.Files(source);
+        wordIndex := index(TS.textSearchIndex, Files.NameWordIndex(useLocal));
+        p := project(nofold(queries), doBatchExecute(wordIndex, LEFT, useLocal, internalFlags));
+        RETURN p;
+    END;
+
+END;
 
-#line(0)
+   RETURN publicExports;
+
+END;

testing/ecl/textsearch.ecl

@@ -15,24 +15,12 @@
     limitations under the License.
 ############################################################################## */
 
-//UseStandardFiles
-//UseTextSearch
-//tidyoutput
-//nothor
-//nothorlcr
-//UseIndexes
-//xxvarskip type==roxie && setuptype==thor && !local
+import $.TextSearch;
 
-#option ('checkAsserts',false)
+EXPORT TextSearchQueries := module
 
-//SingleQuery := 'AND("the":1, "software":2, "source":3)';
-//SingleQuery := 'AND("the", "software", "source")';
+EXPORT WordTests := dataset([
 
-q1 := dataset([
-
-#if (#ISDEFINED(SingleQuery))
-            SingleQuery
-#else
             'AND("black","sheep")',
             'ANDNOT("black","sheep")',
             'MOFN(2,"black","sheep","white")',
@@ -383,9 +371,9 @@ q1 := dataset([
 
                 ' '
 
-#end
+            ], TextSearch.queryInputRecord);
+
 
-            ], queryInputRecord);
+EXPORT SingleBatchQuery(string search) := dataset([search], TextSearch.queryInputRecord);
 
-p := project(nofold(q1), doBatchExecute(TS_wordIndex, LEFT, 0x00000200));           // 0x200 forces paranoid order checking on
-output(p);
+END;

testing/regress/ecl/common/aggds1.ecl

@@ -0,0 +1,45 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+import $.^.setup;
+
+EXPORT aggds1(string source) := function
+
+sq := setup.sq(source);
+
+    RETURN SEQUENTIAL(
+    //Simple disk aggregate
+    output(table(sq.SimplePersonBookDs, { sum(group, aage),exists(group),exists(group,aage>0),exists(group,aage>100),count(group,aage>20) }));
+
+    //Filtered disk aggregate, which also requires a beenProcessed flag
+    output(table(sq.SimplePersonBookDs(surname != 'Halliday'), { max(group, aage) }));
+
+    //Special case count.
+    output(table(sq.SimplePersonBookDs(forename = 'Gavin'), { count(group) }));
+
+    output(count(sq.SimplePersonBookDs));
+
+    //Special case count.
+    output(table(sq.SimplePersonBookDs, { count(group, (forename = 'Gavin')) }));
+
+    //existence checks
+    output(exists(sq.SimplePersonBookDs));
+    output(exists(sq.SimplePersonBookDs(forename = 'Gavin')));
+    output(exists(sq.SimplePersonBookDs(forename = 'Joshua')));
+    );
+
+END;

testing/regress/ecl/common/aggds2.ecl

@@ -0,0 +1,36 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+import $.^.setup;
+
+EXPORT aggds2(string source) := FUNCTION
+
+sq := setup.sq(source);
+
+pr:= table(sq.SimplePersonBookDs, { fullname := trim(surname) + ', ' + trim(forename), aage });
+
+//Aggregate on a projected table that can't be merged.  seq is actually set to aage
+pr2:= table(sq.SimplePersonBookDs, { surname, forename, aage, unsigned8 seq := (random() % 100) / 2000 + aage; });
+
+    RETURN ORDERED(
+        //Filtered Aggregate on a projected table.
+        output(table(pr(aage > 20), { max(group, fullname) }));
+
+        //Filtered Aggregate on a projected table.
+        output(table(pr2(seq > 30), { ave(group, aage) }));
+    );
+END;

testing/regress/ecl/common/aggds3.ecl

@@ -0,0 +1,50 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+import $.^.setup;
+
+EXPORT aggds3(string source) := function
+
+    sq := setup.sq(source);
+
+    //A not-so-simple out of line subquery
+    secondBookName := (string20)sort(sq.SimplePersonBookDs.books, name)[2].name;
+
+    RETURN ORDERED(
+        //Simple disk aggregate
+        output(sort(table(sq.SimplePersonBookDs, { surname, sumage := sum(group, aage) }, surname, few),surname));
+
+        //Filtered disk aggregate, which also requires a beenProcessed flag
+        output(sort(table(sq.SimplePersonBookDs(surname != 'Halliday'), { max(group, aage), surname }, surname, few),surname));
+
+        //check literals are assigned
+        output(sort(table(sq.SimplePersonBookDs(forename = 'Gavin'), { 'Count: ', count(group), 'Name: ', surname }, surname, few),surname));
+
+        //Sub query needs serializing or repeating....
+
+        // A simple inline subquery
+        output(sort(table(sq.SimplePersonBookDs, { cnt := count(books), sumage := sum(group, aage) }, count(books), few),cnt));
+
+        output(sort(table(sq.SimplePersonBookDs, { sbn := secondBookName, sumage := sum(group, aage) }, secondBookName, few),sbn));
+
+        // An out of line subquery (caused problems accessing parent inside sort criteria
+        output(sort(table(sq.SimplePersonBookDs, { cnt := count(books(id != 0)), sumage := sum(group, aage) }, count(books(id != 0)), few),cnt));
+
+        //Bizarre - add a dataset that needs serialization/deserialisation to enusre cloned correctly
+        output(sort(table(nofold(sq.SimplePersonBookDs)(surname != 'Halliday'), { max(group, aage), surname, dataset books := books}, surname, few),surname));
+    );
+END;

testing/regress/ecl/common/aggds4.ecl

@@ -0,0 +1,36 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+import $.^.setup;
+
+EXPORT aggds4(string source) := function
+
+    sq := setup.sq(source);
+
+    pr:= table(sq.SimplePersonBookDs, { fullname := trim(surname) + ', ' + trim(forename), aage });
+
+    //Aggregate on a projected table that can't be merged
+    pr2:= table(sq.SimplePersonBookDs, { surname, forename, aage, unsigned8 seq := (random() % 100) / 2000 + aage; });
+
+    RETURN ORDERED(
+        //Filtered Aggregate on a projected table.
+        output(sort(table(pr(aage > 20), { aage, max(group, fullname) }, aage, few), record));
+
+        //Filtered Aggregate on a projected table.
+        output(sort(table(pr2(seq > 10), { surname, ave(group, aage) }, surname, few), record));
+    );
+END;

testing/regress/ecl/common/aggidx1.ecl

@@ -0,0 +1,50 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+import $.^.setup;
+
+EXPORT aggidx1(string source) := function
+
+    sq := setup.sq(source);
+
+    //Check correctly checks canMatchAny()
+    inlineDs := dataset([1,2],{integer value});
+
+    RETURN ORDERED(
+        #onwarning (4515, ignore);
+        //Simple disk aggregate
+        output(table(sq.SimplePersonBookIndex, { sum(group, aage),exists(group),exists(group,aage>0),exists(group,aage>100),count(group,aage>20) }));
+
+        //Filtered disk aggregate, which also requires a beenProcessed flag
+        output(table(sq.SimplePersonBookIndex(surname != 'Halliday'), { max(group, aage) }));
+
+        //Special case count.
+        output(table(sq.SimplePersonBookIndex(forename = 'Gavin'), { count(group) }));
+
+        output(count(sq.SimplePersonBookIndex));
+
+        //Special case count.
+        output(table(sq.SimplePersonBookIndex, { count(group, (forename = 'Gavin')) }));
+
+        output(table(inlineDs, { count(sq.SimplePersonBookIndex(inlineDs.value = 1)); }));
+
+        //existence checks
+        output(exists(sq.SimplePersonBookIndex));
+        output(exists(sq.SimplePersonBookIndex(forename = 'Gavin')));
+        output(exists(sq.SimplePersonBookIndex(forename = 'Joshua')));
+    );
+END;

testing/regress/ecl/common/aggidx2.ecl

@@ -0,0 +1,37 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+import $.^.setup;
+
+EXPORT aggidx2(string source) := function
+
+    sq := setup.sq(source);
+
+    pr:= table(sq.SimplePersonBookIndex, { fullname := trim(surname) + ', ' + trim(forename), aage });
+
+    //Aggregate on a projected table that can't be merged
+    pr2:= table(sq.SimplePersonBookIndex, { surname, forename, aage, unsigned8 seq := (random() % 100) / 2000 + aage; });
+
+    RETURN ORDERED(
+        //Filtered Aggregate on a projected table.
+        output(table(pr(aage > 20), { max(group, fullname) }));
+
+        //Filtered Aggregate on a projected table.
+        output(table(pr2(seq > 30), { ave(group, aage) }));
+    );
+
+END;

testing/regress/ecl/common/aggidx3.ecl

@@ -0,0 +1,51 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+import $.^.setup;
+
+EXPORT aggidx3(string source) := function
+
+    sq := setup.sq(source);
+
+    //A not-so-simple out of line subquery
+    secondBookName := (string20)sort(sq.SimplePersonBookIndex.books, name)[2].name;
+
+    RETURN ORDERED(
+        //Simple disk aggregate
+        output(sort(table(sq.SimplePersonBookIndex, { string surname := trim(sq.SimplePersonBookIndex.surname), sum(group, aage) }, trim(surname), few), surname));
+
+
+        //Filtered disk aggregate, which also requires a beenProcessed flag
+        output(sort(table(sq.SimplePersonBookIndex(surname != 'Halliday'), { max(group, aage), surname }, surname, few), surname));
+
+        //Special case count.
+        output(sort(table(sq.SimplePersonBookIndex(forename = 'Gavin'), { count(group), surname }, surname, few), surname));
+
+        //Sub query needs serializing or repeating....
+
+        // A simple inline subquery
+        output(sort(table(sq.SimplePersonBookIndex, { cntBooks:= count(books), sumage := sum(group, aage) }, count(books), few), cntBooks, sumage));
+
+        output(sort(table(sq.SimplePersonBookIndex, { secondBookName, sumage := sum(group, aage) }, secondBookName, few), secondbookname));
+
+        //Access to file position - ensure it is serialized correctly!
+        output(sort(table(sq.SimplePersonBookIndex, { pos := filepos DIV 8192, sumage := sum(group, aage) }, filepos DIV 8192, few), pos, sumage));
+
+        // An out of line subquery (caused problems accessing parent inside sort criteria
+        output(sort(table(sq.SimplePersonBookIndex, { numBooks := count(books(id != 0)), sumage := sum(group, aage) }, count(books(id != 0)), few), numbooks, sumage));
+    );
+END;

testing/regress/ecl/common/aggidx4.ecl

@@ -0,0 +1,36 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+import $.^.setup;
+
+EXPORT aggidx4(string source) := function
+
+    sq := setup.sq(source);
+
+    pr:= table(sq.SimplePersonBookIndex, { fullname := trim(surname) + ', ' + trim(forename), aage });
+
+    //Aggregate on a projected table that can't be merged
+    pr2:= table(sq.SimplePersonBookIndex, { surname, forename, aage, unsigned8 seq := (random() % 100) / 2000 + aage; });
+
+    RETURN ORDERED(
+        //Filtered Aggregate on a projected table.
+        output(sort(table(pr(aage > 20), { aage, max(group, fullname) }, aage, few), aage));
+
+        //Filtered Aggregate on a projected table.
+        output(sort(table(pr2(seq > 10), { surname, ave(group, aage) }, surname, few), surname));
+    );
+END;

testing/regress/ecl/common/sqagg.ecl

@@ -0,0 +1,119 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+boolean missingLevelOk := false;
+
+export sqagg(string source) := function
+
+import $.^.setup;
+sq := setup.sq(source);
+
+// Test the different child operators.  Try and test inline and out of line, also part of a compound
+// source activity and not part.
+
+udecimal8 todaysDate := 20040602D;
+unsigned4 age(udecimal8 dob) := ((todaysDate - dob) / 10000D);
+
+//MORE: books[1] ave(books)
+
+// Different child operators, all inline.
+persons := sq.HousePersonBookDs.persons;
+books := persons.books;
+personByAgeDesc := sort(sq.HousePersonBookDs.persons, dob);
+
+// Grand children, again all inline.
+
+booksByRatingDesc := sort(sq.HousePersonBookDs.persons.books, -rating100);
+
+//sort order is deliberately different from anything that will be used later
+xpersons := sort(sq.HousePersonBookDs.persons, surname + (string)dob + forename)[1..200];
+xpersonByAgeDesc := sort(xpersons, dob);
+xbooks := sort(sq.HousePersonBookDs.persons.books, name + (string)rating100 + author)[1..200];
+xbooksByRatingDesc := sort(xbooks, -rating100);
+xxbooks := sort(xpersons.books, name + (string)rating100 + author)[1..200];
+xxbooksByRatingDesc := sort(xxbooks, -rating100);
+
+RETURN ORDERED(
+output(sq.HousePersonBookDs, { addr, numPeople := count(persons), aveAge:= ave(persons, age(dob)), maxDob := max(persons, dob)});
+output(sq.HousePersonBookDs, { addr, oldest := personByAgeDesc[1].forename + ' ' + personByAgeDesc[1].surname });
+output(sq.HousePersonBookDs, { addr, firstPerson := persons[1].forename + ' ' + persons[1].surname });
+
+//More: Need to think about walking 3rd level children e.g., in ave, and [1]:
+output(sq.HousePersonBookDs, { addr, numBooks := sum(persons, count(books)), maxRating := max(persons, max(books, rating100))});
+output(sq.HousePersonBookDs, { addr, firstBook := persons[1].books[1].name + ': ' + persons[1].books[1].author });
+#if (true)
+output(sq.HousePersonBookDs, { addr, numBooks := count(persons.books), ave(persons.books, rating100), max(persons.books, rating100)});
+output(sq.HousePersonBookDs, { addr, ave(persons.books(persons.booklimit > 0), rating100)});
+output(sq.HousePersonBookDs, { addr, bestBook := booksByRatingDesc[1].name + ': ' + booksByRatingDesc[1].author});
+output(sq.HousePersonBookDs, { addr, firstBook := persons.books[1].name + ': ' + persons.books[1].author });     //NB: Different from above.
+#end
+
+//Now do the same, but unsure the children and main activity are not inline or compound
+// Different child operators, out of line inline.
+output('**** The following results should be identical - calculated using a subquery ****');
+
+output(sq.HousePersonBookDs, { addr, numPeople := count(xpersons), aveAge := ave(xpersons, age(dob)), maxDob := max(xpersons, dob)});
+output(sq.HousePersonBookDs, { addr, oldest := xpersonByAgeDesc[1].forename + ' ' + xpersonByAgeDesc[1].surname });
+output(sq.HousePersonBookDs, { addr, firstPerson := xpersons[1].forename + ' ' + xpersons[1].surname });
+
+// Grand children out of line, children are inline
+
+output(sq.HousePersonBookDs, { addr, numBooks := sum(persons, count(xbooks)), maxRating := max(persons, max(xbooks, rating100))});
+output(sq.HousePersonBookDs, { addr, firstBook := evaluate(persons[1], xbooks[1].name) + ': ' + evaluate(persons[1], xbooks[1].author) });
+#if (true)
+output(sq.HousePersonBookDs, { addr, numBooks := count(xbooks), ave(xbooks, rating100), max(xbooks, rating100)});
+output(sq.HousePersonBookDs, { addr, bestBook := xbooksByRatingDesc[1].name + ': ' + xbooksByRatingDesc[1].author});
+output(sq.HousePersonBookDs, { addr, firstBook := xbooks[1].name + ': ' + xbooks[1].author });       //NB: Different from above.
+#end
+
+// Grand children out of line, children also out of line
+
+output('**** The following results should be similar persons are reordered ****');
+output(sq.HousePersonBookDs, { addr, numBooks := sum(xpersons, count(xxbooks)), max(xpersons, max(xxbooks, rating100))});
+output(sq.HousePersonBookDs, { addr, firstBook := evaluate(xpersons[1], xxbooks[1].name) + ': ' + evaluate(xpersons[1], xxbooks[1].author) });
+#if (true)
+output(sq.HousePersonBookDs, { addr, numBooks := count(xxbooks), ave(xxbooks, rating100), max(xxbooks, rating100)});
+output(sq.HousePersonBookDs, { addr, bestBook := xxbooksByRatingDesc[1].name + ': ' + xxbooksByRatingDesc[1].author});
+output(sq.HousePersonBookDs, { addr, firstBook := xxbooks[1].name + ': ' + xxbooks[1].author });     //NB: Different from above.
+#end
+
+//--------- Now perform the aggregate operations with person as outer iteration ----------
+// note: sq.HousePersonDs fields are still accessible!
+output(sq.HousePersonBookDs.persons, { surname, numBooks := count(books), ave(books, rating100), max(books, rating100)});
+output(sq.HousePersonBookDs.persons, { surname, bestBook := booksByRatingDesc[1].name + ': ' + booksByRatingDesc[1].author});
+output(sq.HousePersonBookDs.persons, { surname, firstBook := books[1].name + ': ' + books[1].author });
+
+output(xpersons, { surname, numBooks := count(xxbooks), ave(xxbooks, rating100), max(xxbooks, rating100)});
+output(xpersons, { surname, bestBook := xxbooksByRatingDesc[1].name + ': ' + xxbooksByRatingDesc[1].author});
+output(xpersons, { surname, firstBook := xxbooks[1].name + ': ' + xxbooks[1].author });
+
+// note: sq.HousePersonDs fields are still accessible!
+
+#if (false)
+output(sq.HousePersonBookDs.persons, { sq.HousePersonBookDs.addr, surname, numBooks := count(books), ave(books, rating100), max(books, rating100)});
+output(sq.HousePersonBookDs.persons, { sq.HousePersonBookDs.addr, surname, bestBook := booksByRatingDesc[1].name + ': ' + booksByRatingDesc[1].author});
+output(sq.HousePersonBookDs.persons, { sq.HousePersonBookDs.addr, surname, firstBook := books[1].name + ': ' + books[1].author });
+#end
+
+#if (missingLevelOk)
+output(xpersons, { sq.HousePersonBookDs.addr, surname, numBooks := count(xxbooks), ave(xxbooks, rating100), max(xxbooks, rating100)});
+output(xpersons, { sq.HousePersonBookDs.addr, surname, bestBook := xxbooksByRatingDesc[1].name + ': ' + xxbooksByRatingDesc[1].author});
+output(xpersons, { sq.HousePersonBookDs.addr, surname, firstBook := xxbooks[1].name + ': ' + xxbooks[1].author });
+#end
+);
+
+END;

testing/regress/ecl/diskAggregate.ecl

@@ -15,7 +15,8 @@
     limitations under the License.
 ############################################################################## */
 
-import $.setup.sq;
+import $.setup;
+sq := setup.sq('hthor');
 
 // Test a case that needs serialize due to child dataset...
 table(sq.SimplePersonBookds, { dataset books := sq.SimplePersonBookDs.books, sq.SimplePersonBookDs.surname, count(group) });

testing/regress/ecl/diskGroupAggregate.ecl

@@ -15,7 +15,8 @@
     limitations under the License.
 ############################################################################## */
 
-import $.setup.sq;
+import $.setup;
+sq := setup.sq('hthor');
 
 // Test a case that needs serialize due to child dataset...
 sort(table(sq.SimplePersonBookds, { dataset books := sq.SimplePersonBookDs.books, sq.SimplePersonBookDs.surname, sq.SimplePersonBookDs.forename, count(group) }, sq.SimplePersonBookDs.surname, few), surname, forename);

testing/regress/ecl/fetch2.ecl

@@ -15,7 +15,8 @@
     limitations under the License.
 ############################################################################## */
 
-import $.setup.sq;
+import $.setup;
+sq := setup.sq('hthor');
 
 IMPORT Std;
 

testing/regress/ecl/indexAggregate.ecl

@@ -15,7 +15,8 @@
     limitations under the License.
 ############################################################################## */
 
-import $.setup.sq;
+import $.setup;
+sq := setup.sq('hthor');
 
 // Test a case that needs serialize due to child dataset...
 table(sq.SimplePersonBookIndex, { dataset books := sq.SimplePersonBookIndex.books, sq.SimplePersonBookIndex.surname, count(group) });

testing/regress/ecl/indexGroupAggregate.ecl

@@ -15,7 +15,8 @@
     limitations under the License.
 ############################################################################## */
 
-import $.setup.sq;
+import $.setup;
+sq := setup.sq('hthor');
 
 // Test a case that needs serialize due to child dataset...
 sort(table(sq.SimplePersonBookIndex, { dataset books := sq.SimplePersonBookIndex.books, sq.SimplePersonBookIndex.surname, sq.SimplePersonBookIndex.forename, count(group) }, sq.SimplePersonBookIndex.surname, few), surname, forename);

testing/regress/ecl/indexread5.ecl

@@ -15,7 +15,8 @@
     limitations under the License.
 ############################################################################## */
 
-import $.setup.sq;
+import $.setup;
+sq := setup.sq('hthor');
 
 somePeople := sq.PersonBookDs(id % 2 = 1);
 

testing/regress/ecl/key/aggds1_roxie.xml

@@ -0,0 +1,24 @@
+<Dataset name='Result 1'>
+ <Row><_unnamed_sum_aage1>600</_unnamed_sum_aage1><_unnamed_exists_2>true</_unnamed_exists_2><_unnamed_exists_3>true</_unnamed_exists_3><_unnamed_exists_4>false</_unnamed_exists_4><_unnamed_cnt_5>9</_unnamed_cnt_5></Row>
+</Dataset>
+<Dataset name='Result 2'>
+ <Row><_unnamed_max_aage1>99</_unnamed_max_aage1></Row>
+</Dataset>
+<Dataset name='Result 3'>
+ <Row><_unnamed_cnt_1>1</_unnamed_cnt_1></Row>
+</Dataset>
+<Dataset name='Result 4'>
+ <Row><Result_4>10</Result_4></Row>
+</Dataset>
+<Dataset name='Result 5'>
+ <Row><_unnamed_cnt_1>1</_unnamed_cnt_1></Row>
+</Dataset>
+<Dataset name='Result 6'>
+ <Row><Result_6>true</Result_6></Row>
+</Dataset>
+<Dataset name='Result 7'>
+ <Row><Result_7>true</Result_7></Row>
+</Dataset>
+<Dataset name='Result 8'>
+ <Row><Result_8>false</Result_8></Row>
+</Dataset>

testing/regress/ecl/key/aggds1_thor.xml

@@ -0,0 +1,24 @@
+<Dataset name='Result 1'>
+ <Row><_unnamed_sum_aage1>600</_unnamed_sum_aage1><_unnamed_exists_2>true</_unnamed_exists_2><_unnamed_exists_3>true</_unnamed_exists_3><_unnamed_exists_4>false</_unnamed_exists_4><_unnamed_cnt_5>9</_unnamed_cnt_5></Row>
+</Dataset>
+<Dataset name='Result 2'>
+ <Row><_unnamed_max_aage1>99</_unnamed_max_aage1></Row>
+</Dataset>
+<Dataset name='Result 3'>
+ <Row><_unnamed_cnt_1>1</_unnamed_cnt_1></Row>
+</Dataset>
+<Dataset name='Result 4'>
+ <Row><Result_4>10</Result_4></Row>
+</Dataset>
+<Dataset name='Result 5'>
+ <Row><_unnamed_cnt_1>1</_unnamed_cnt_1></Row>
+</Dataset>
+<Dataset name='Result 6'>
+ <Row><Result_6>true</Result_6></Row>
+</Dataset>
+<Dataset name='Result 7'>
+ <Row><Result_7>true</Result_7></Row>
+</Dataset>
+<Dataset name='Result 8'>
+ <Row><Result_8>false</Result_8></Row>
+</Dataset>

testing/regress/ecl/key/aggds2_thor.xml

@@ -0,0 +1,6 @@
+<Dataset name='Result 1'>
+ <Row><_unnamed_max_fullname1>Windsor, Philip</_unnamed_max_fullname1></Row>
+</Dataset>
+<Dataset name='Result 2'>
+ <Row><_unnamed_avg_aage1>66.11111111111111</_unnamed_avg_aage1></Row>
+</Dataset>

testing/regress/ecl/key/aggds4_thor.xml

@@ -0,0 +1,17 @@
+<Dataset name='Result 1'>
+ <Row><aage>34</aage><_unnamed_max_fullname2>Halliday, Liz</_unnamed_max_fullname2></Row>
+ <Row><aage>35</aage><_unnamed_max_fullname2>Halliday, Gavin</_unnamed_max_fullname2></Row>
+ <Row><aage>49</aage><_unnamed_max_fullname2>Jones, Julia</_unnamed_max_fullname2></Row>
+ <Row><aage>50</aage><_unnamed_max_fullname2>Jones, Brian</_unnamed_max_fullname2></Row>
+ <Row><aage>68</aage><_unnamed_max_fullname2>Grabdale, John</_unnamed_max_fullname2></Row>
+ <Row><aage>78</aage><_unnamed_max_fullname2>Windsor, Elizabeth</_unnamed_max_fullname2></Row>
+ <Row><aage>83</aage><_unnamed_max_fullname2>Windsor, Philip</_unnamed_max_fullname2></Row>
+ <Row><aage>99</aage><_unnamed_max_fullname2>Flintstone, Wilma</_unnamed_max_fullname2></Row>
+</Dataset>
+<Dataset name='Result 2'>
+ <Row><surname>Flintstone          </surname><_unnamed_avg_aage2>99.0</_unnamed_avg_aage2></Row>
+ <Row><surname>Grabdale            </surname><_unnamed_avg_aage2>68.0</_unnamed_avg_aage2></Row>
+ <Row><surname>Halliday            </surname><_unnamed_avg_aage2>34.5</_unnamed_avg_aage2></Row>
+ <Row><surname>Jones               </surname><_unnamed_avg_aage2>49.5</_unnamed_avg_aage2></Row>
+ <Row><surname>Windsor             </surname><_unnamed_avg_aage2>80.5</_unnamed_avg_aage2></Row>
+</Dataset>

testing/regress/ecl/key/aggidx1_thor.xml

@@ -0,0 +1,28 @@
+<Dataset name='Result 1'>
+ <Row><_unnamed_sum_aage1>600</_unnamed_sum_aage1><_unnamed_exists_2>true</_unnamed_exists_2><_unnamed_exists_3>true</_unnamed_exists_3><_unnamed_exists_4>false</_unnamed_exists_4><_unnamed_cnt_5>9</_unnamed_cnt_5></Row>
+</Dataset>
+<Dataset name='Result 2'>
+ <Row><_unnamed_max_aage1>99</_unnamed_max_aage1></Row>
+</Dataset>
+<Dataset name='Result 3'>
+ <Row><_unnamed_cnt_1>1</_unnamed_cnt_1></Row>
+</Dataset>
+<Dataset name='Result 4'>
+ <Row><Result_4>10</Result_4></Row>
+</Dataset>
+<Dataset name='Result 5'>
+ <Row><_unnamed_cnt_1>1</_unnamed_cnt_1></Row>
+</Dataset>
+<Dataset name='Result 6'>
+ <Row><_unnamed_1>10</_unnamed_1></Row>
+ <Row><_unnamed_1>0</_unnamed_1></Row>
+</Dataset>
+<Dataset name='Result 7'>
+ <Row><Result_7>true</Result_7></Row>
+</Dataset>
+<Dataset name='Result 8'>
+ <Row><Result_8>true</Result_8></Row>
+</Dataset>
+<Dataset name='Result 9'>
+ <Row><Result_9>false</Result_9></Row>
+</Dataset>

testing/regress/ecl/key/aggidx2_thor.xml

@@ -0,0 +1,6 @@
+<Dataset name='Result 1'>
+ <Row><_unnamed_max_fullname1>Windsor, Philip</_unnamed_max_fullname1></Row>
+</Dataset>
+<Dataset name='Result 2'>
+ <Row><_unnamed_avg_aage1>66.11111111111111</_unnamed_avg_aage1></Row>
+</Dataset>

testing/regress/ecl/key/aggidx3_thor.xml

@@ -0,0 +1,46 @@
+<Dataset name='Result 1'>
+ <Row><surname>Flintstone</surname><_unnamed_sum_aage2>198</_unnamed_sum_aage2></Row>
+ <Row><surname>Grabdale</surname><_unnamed_sum_aage2>68</_unnamed_sum_aage2></Row>
+ <Row><surname>Halliday</surname><_unnamed_sum_aage2>74</_unnamed_sum_aage2></Row>
+ <Row><surname>Jones</surname><_unnamed_sum_aage2>99</_unnamed_sum_aage2></Row>
+ <Row><surname>Windsor</surname><_unnamed_sum_aage2>161</_unnamed_sum_aage2></Row>
+</Dataset>
+<Dataset name='Result 2'>
+ <Row><_unnamed_max_aage1>99</_unnamed_max_aage1><surname>Flintstone          </surname></Row>
+ <Row><_unnamed_max_aage1>68</_unnamed_max_aage1><surname>Grabdale            </surname></Row>
+ <Row><_unnamed_max_aage1>50</_unnamed_max_aage1><surname>Jones               </surname></Row>
+ <Row><_unnamed_max_aage1>83</_unnamed_max_aage1><surname>Windsor             </surname></Row>
+</Dataset>
+<Dataset name='Result 3'>
+ <Row><_unnamed_cnt_1>1</_unnamed_cnt_1><surname>Halliday            </surname></Row>
+</Dataset>
+<Dataset name='Result 4'>
+ <Row><cntbooks>0</cntbooks><sumage>99</sumage></Row>
+ <Row><cntbooks>1</cntbooks><sumage>99</sumage></Row>
+ <Row><cntbooks>2</cntbooks><sumage>118</sumage></Row>
+ <Row><cntbooks>3</cntbooks><sumage>34</sumage></Row>
+ <Row><cntbooks>4</cntbooks><sumage>137</sumage></Row>
+ <Row><cntbooks>5</cntbooks><sumage>113</sumage></Row>
+</Dataset>
+<Dataset name='Result 5'>
+ <Row><secondbookname>                    </secondbookname><sumage>198</sumage></Row>
+ <Row><secondbookname>Compilers           </secondbookname><sumage>35</sumage></Row>
+ <Row><secondbookname>Fox in Socks        </secondbookname><sumage>50</sumage></Row>
+ <Row><secondbookname>Girl guide annual   </secondbookname><sumage>78</sumage></Row>
+ <Row><secondbookname>Pride and prejudice,</secondbookname><sumage>68</sumage></Row>
+ <Row><secondbookname>Social etiquette    </secondbookname><sumage>83</sumage></Row>
+ <Row><secondbookname>The Life of Pi      </secondbookname><sumage>34</sumage></Row>
+ <Row><secondbookname>The cat in the hat  </secondbookname><sumage>49</sumage></Row>
+ <Row><secondbookname>The story of Jonah  </secondbookname><sumage>5</sumage></Row>
+</Dataset>
+<Dataset name='Result 6'>
+ <Row><pos>0</pos><sumage>600</sumage></Row>
+</Dataset>
+<Dataset name='Result 7'>
+ <Row><numbooks>0</numbooks><sumage>99</sumage></Row>
+ <Row><numbooks>1</numbooks><sumage>99</sumage></Row>
+ <Row><numbooks>2</numbooks><sumage>118</sumage></Row>
+ <Row><numbooks>3</numbooks><sumage>34</sumage></Row>
+ <Row><numbooks>4</numbooks><sumage>137</sumage></Row>
+ <Row><numbooks>5</numbooks><sumage>113</sumage></Row>
+</Dataset>

testing/regress/ecl/key/aggidx4_thor.xml

@@ -0,0 +1,17 @@
+<Dataset name='Result 1'>
+ <Row><aage>34</aage><_unnamed_max_fullname2>Halliday, Liz</_unnamed_max_fullname2></Row>
+ <Row><aage>35</aage><_unnamed_max_fullname2>Halliday, Gavin</_unnamed_max_fullname2></Row>
+ <Row><aage>49</aage><_unnamed_max_fullname2>Jones, Julia</_unnamed_max_fullname2></Row>
+ <Row><aage>50</aage><_unnamed_max_fullname2>Jones, Brian</_unnamed_max_fullname2></Row>
+ <Row><aage>68</aage><_unnamed_max_fullname2>Grabdale, John</_unnamed_max_fullname2></Row>
+ <Row><aage>78</aage><_unnamed_max_fullname2>Windsor, Elizabeth</_unnamed_max_fullname2></Row>
+ <Row><aage>83</aage><_unnamed_max_fullname2>Windsor, Philip</_unnamed_max_fullname2></Row>
+ <Row><aage>99</aage><_unnamed_max_fullname2>Flintstone, Wilma</_unnamed_max_fullname2></Row>
+</Dataset>
+<Dataset name='Result 2'>
+ <Row><surname>Flintstone          </surname><_unnamed_avg_aage2>99.0</_unnamed_avg_aage2></Row>
+ <Row><surname>Grabdale            </surname><_unnamed_avg_aage2>68.0</_unnamed_avg_aage2></Row>
+ <Row><surname>Halliday            </surname><_unnamed_avg_aage2>34.5</_unnamed_avg_aage2></Row>
+ <Row><surname>Jones               </surname><_unnamed_avg_aage2>49.5</_unnamed_avg_aage2></Row>
+ <Row><surname>Windsor             </surname><_unnamed_avg_aage2>80.5</_unnamed_avg_aage2></Row>
+</Dataset>

testing/regress/ecl/key/setuptext.xml

@@ -0,0 +1,6 @@
+<Dataset name='Result 1'>
+</Dataset>
+<Dataset name='Result 2'>
+</Dataset>
+<Dataset name='Result 3'>
+</Dataset>

testing/regress/ecl/key/sqagg_thor.xml

@@ -0,0 +1,232 @@
+<Dataset name='Result 1'>
+ <Row><addr>99 Maltings Road</addr><numpeople>3</numpeople><aveage>23.66666666666667</aveage><maxdob>20000101</maxdob></Row>
+ <Row><addr>Buckingham Palace</addr><numpeople>2</numpeople><aveage>80.0</aveage><maxdob>19260421</maxdob></Row>
+ <Row><addr>Bedrock</addr><numpeople>2</numpeople><aveage>2003.0</aveage><maxdob>20202</maxdob></Row>
+ <Row><addr>Wimpole Hall</addr><numpeople>1</numpeople><aveage>67.0</aveage><maxdob>19361008</maxdob></Row>
+ <Row><addr>56 New Road</addr><numpeople>2</numpeople><aveage>49.5</aveage><maxdob>19550312</maxdob></Row>
+</Dataset>
+<Dataset name='Result 2'>
+ <Row><addr>99 Maltings Road</addr><oldest>Gavin Halliday</oldest></Row>
+ <Row><addr>Buckingham Palace</addr><oldest>Philip Windsor</oldest></Row>
+ <Row><addr>Bedrock</addr><oldest>Fred Flintstone</oldest></Row>
+ <Row><addr>Wimpole Hall</addr><oldest>John Grabdale</oldest></Row>
+ <Row><addr>56 New Road</addr><oldest>Brian Jones</oldest></Row>
+</Dataset>
+<Dataset name='Result 3'>
+ <Row><addr>99 Maltings Road</addr><firstperson>Gavin Halliday</firstperson></Row>
+ <Row><addr>Buckingham Palace</addr><firstperson>Elizabeth Windsor</firstperson></Row>
+ <Row><addr>Bedrock</addr><firstperson>Fred Flintstone</firstperson></Row>
+ <Row><addr>Wimpole Hall</addr><firstperson>John Grabdale</firstperson></Row>
+ <Row><addr>56 New Road</addr><firstperson>Brian Jones</firstperson></Row>
+</Dataset>
+<Dataset name='Result 4'>
+ <Row><addr>99 Maltings Road</addr><numbooks>12</numbooks><maxrating>98</maxrating></Row>
+ <Row><addr>Buckingham Palace</addr><numbooks>9</numbooks><maxrating>93</maxrating></Row>
+ <Row><addr>Bedrock</addr><numbooks>1</numbooks><maxrating>55</maxrating></Row>
+ <Row><addr>Wimpole Hall</addr><numbooks>2</numbooks><maxrating>95</maxrating></Row>
+ <Row><addr>56 New Road</addr><numbooks>6</numbooks><maxrating>99</maxrating></Row>
+</Dataset>
+<Dataset name='Result 5'>
+ <Row><addr>99 Maltings Road</addr><firstbook>To kill a mocking bird: Harper Lee</firstbook></Row>
+ <Row><addr>Buckingham Palace</addr><firstbook>The bible: Various</firstbook></Row>
+ <Row><addr>Bedrock</addr><firstbook>: </firstbook></Row>
+ <Row><addr>Wimpole Hall</addr><firstbook>Pride and prejudice, 1st edition: Jane Austen</firstbook></Row>
+ <Row><addr>56 New Road</addr><firstbook>All quiet on the western front: Erich Maria Remarque</firstbook></Row>
+</Dataset>
+<Dataset name='Result 6'>
+ <Row><addr>99 Maltings Road</addr><numbooks>12</numbooks><_unnamed_3>73.25</_unnamed_3><_unnamed_4>98</_unnamed_4></Row>
+ <Row><addr>Buckingham Palace</addr><numbooks>9</numbooks><_unnamed_3>52.0</_unnamed_3><_unnamed_4>93</_unnamed_4></Row>
+ <Row><addr>Bedrock</addr><numbooks>1</numbooks><_unnamed_3>55.0</_unnamed_3><_unnamed_4>55</_unnamed_4></Row>
+ <Row><addr>Wimpole Hall</addr><numbooks>2</numbooks><_unnamed_3>87.5</_unnamed_3><_unnamed_4>95</_unnamed_4></Row>
+ <Row><addr>56 New Road</addr><numbooks>6</numbooks><_unnamed_3>72.33333333333333</_unnamed_3><_unnamed_4>99</_unnamed_4></Row>
+</Dataset>
+<Dataset name='Result 7'>
+ <Row><addr>99 Maltings Road</addr><_unnamed_2>71.55555555555556</_unnamed_2></Row>
+ <Row><addr>Buckingham Palace</addr><_unnamed_2>0.0</_unnamed_2></Row>
+ <Row><addr>Bedrock</addr><_unnamed_2>0.0</_unnamed_2></Row>
+ <Row><addr>Wimpole Hall</addr><_unnamed_2>0.0</_unnamed_2></Row>
+ <Row><addr>56 New Road</addr><_unnamed_2>0.0</_unnamed_2></Row>
+</Dataset>
+<Dataset name='Result 8'>
+ <Row><addr>99 Maltings Road</addr><bestbook>The bible: Various</bestbook></Row>
+ <Row><addr>Buckingham Palace</addr><bestbook>The bible: Various</bestbook></Row>
+ <Row><addr>Bedrock</addr><bestbook>Dinosaur stews: Trog</bestbook></Row>
+ <Row><addr>Wimpole Hall</addr><bestbook>Pride and prejudice, 1st edition: Jane Austen</bestbook></Row>
+ <Row><addr>56 New Road</addr><bestbook>Fox in Socks: Dr. Seuss</bestbook></Row>
+</Dataset>
+<Dataset name='Result 9'>
+ <Row><addr>99 Maltings Road</addr><firstbook>To kill a mocking bird: Harper Lee</firstbook></Row>
+ <Row><addr>Buckingham Palace</addr><firstbook>The bible: Various</firstbook></Row>
+ <Row><addr>Bedrock</addr><firstbook>Dinosaur stews: Trog</firstbook></Row>
+ <Row><addr>Wimpole Hall</addr><firstbook>Pride and prejudice, 1st edition: Jane Austen</firstbook></Row>
+ <Row><addr>56 New Road</addr><firstbook>All quiet on the western front: Erich Maria Remarque</firstbook></Row>
+</Dataset>
+<Dataset name='Result 10'>
+ <Row><Result_10>**** The following results should be identical - calculated using a subquery ****</Result_10></Row>
+</Dataset>
+<Dataset name='Result 11'>
+ <Row><addr>99 Maltings Road</addr><numpeople>3</numpeople><aveage>23.66666666666667</aveage><maxdob>20000101</maxdob></Row>
+ <Row><addr>Buckingham Palace</addr><numpeople>2</numpeople><aveage>80.0</aveage><maxdob>19260421</maxdob></Row>
+ <Row><addr>Bedrock</addr><numpeople>2</numpeople><aveage>2003.0</aveage><maxdob>20202</maxdob></Row>
+ <Row><addr>Wimpole Hall</addr><numpeople>1</numpeople><aveage>67.0</aveage><maxdob>19361008</maxdob></Row>
+ <Row><addr>56 New Road</addr><numpeople>2</numpeople><aveage>49.5</aveage><maxdob>19550312</maxdob></Row>
+</Dataset>
+<Dataset name='Result 12'>
+ <Row><addr>99 Maltings Road</addr><oldest>Gavin Halliday</oldest></Row>
+ <Row><addr>Buckingham Palace</addr><oldest>Philip Windsor</oldest></Row>
+ <Row><addr>Bedrock</addr><oldest>Fred Flintstone</oldest></Row>
+ <Row><addr>Wimpole Hall</addr><oldest>John Grabdale</oldest></Row>
+ <Row><addr>56 New Road</addr><oldest>Brian Jones</oldest></Row>
+</Dataset>
+<Dataset name='Result 13'>
+ <Row><addr>99 Maltings Road</addr><firstperson>Gavin Halliday</firstperson></Row>
+ <Row><addr>Buckingham Palace</addr><firstperson>Philip Windsor</firstperson></Row>
+ <Row><addr>Bedrock</addr><firstperson>Fred Flintstone</firstperson></Row>
+ <Row><addr>Wimpole Hall</addr><firstperson>John Grabdale</firstperson></Row>
+ <Row><addr>56 New Road</addr><firstperson>Brian Jones</firstperson></Row>
+</Dataset>
+<Dataset name='Result 14'>
+ <Row><addr>99 Maltings Road</addr><numbooks>12</numbooks><maxrating>98</maxrating></Row>
+ <Row><addr>Buckingham Palace</addr><numbooks>9</numbooks><maxrating>93</maxrating></Row>
+ <Row><addr>Bedrock</addr><numbooks>1</numbooks><maxrating>55</maxrating></Row>
+ <Row><addr>Wimpole Hall</addr><numbooks>2</numbooks><maxrating>95</maxrating></Row>
+ <Row><addr>56 New Road</addr><numbooks>6</numbooks><maxrating>99</maxrating></Row>
+</Dataset>
+<Dataset name='Result 15'>
+ <Row><addr>99 Maltings Road</addr><firstbook>Clarion Language Manual: Richard Taylor</firstbook></Row>
+ <Row><addr>Buckingham Palace</addr><firstbook>Atlas of the world: Times</firstbook></Row>
+ <Row><addr>Bedrock</addr><firstbook>: </firstbook></Row>
+ <Row><addr>Wimpole Hall</addr><firstbook>Mein Kampf, 1st edition: Adolph Hitler</firstbook></Row>
+ <Row><addr>56 New Road</addr><firstbook>All quiet on the western front: Erich Maria Remarque</firstbook></Row>
+</Dataset>
+<Dataset name='Result 16'>
+ <Row><addr>99 Maltings Road</addr><numbooks>12</numbooks><_unnamed_3>73.25</_unnamed_3><_unnamed_4>98</_unnamed_4></Row>
+ <Row><addr>Buckingham Palace</addr><numbooks>9</numbooks><_unnamed_3>52.0</_unnamed_3><_unnamed_4>93</_unnamed_4></Row>
+ <Row><addr>Bedrock</addr><numbooks>1</numbooks><_unnamed_3>55.0</_unnamed_3><_unnamed_4>55</_unnamed_4></Row>
+ <Row><addr>Wimpole Hall</addr><numbooks>2</numbooks><_unnamed_3>87.5</_unnamed_3><_unnamed_4>95</_unnamed_4></Row>
+ <Row><addr>56 New Road</addr><numbooks>6</numbooks><_unnamed_3>72.33333333333333</_unnamed_3><_unnamed_4>99</_unnamed_4></Row>
+</Dataset>
+<Dataset name='Result 17'>
+ <Row><addr>99 Maltings Road</addr><bestbook>The bible: Various</bestbook></Row>
+ <Row><addr>Buckingham Palace</addr><bestbook>The bible: Various</bestbook></Row>
+ <Row><addr>Bedrock</addr><bestbook>Dinosaur stews: Trog</bestbook></Row>
+ <Row><addr>Wimpole Hall</addr><bestbook>Pride and prejudice, 1st edition: Jane Austen</bestbook></Row>
+ <Row><addr>56 New Road</addr><bestbook>Fox in Socks: Dr. Seuss</bestbook></Row>
+</Dataset>
+<Dataset name='Result 18'>
+ <Row><addr>99 Maltings Road</addr><firstbook>BNF: Various</firstbook></Row>
+ <Row><addr>Buckingham Palace</addr><firstbook>Atlas of the world: Times</firstbook></Row>
+ <Row><addr>Bedrock</addr><firstbook>Dinosaur stews: Trog</firstbook></Row>
+ <Row><addr>Wimpole Hall</addr><firstbook>Mein Kampf, 1st edition: Adolph Hitler</firstbook></Row>
+ <Row><addr>56 New Road</addr><firstbook>All quiet on the western front: Erich Maria Remarque</firstbook></Row>
+</Dataset>
+<Dataset name='Result 19'>
+ <Row><Result_19>**** The following results should be similar persons are reordered ****</Result_19></Row>
+</Dataset>
+<Dataset name='Result 20'>
+ <Row><addr>99 Maltings Road</addr><numbooks>12</numbooks><_unnamed_3>98</_unnamed_3></Row>
+ <Row><addr>Buckingham Palace</addr><numbooks>9</numbooks><_unnamed_3>93</_unnamed_3></Row>
+ <Row><addr>Bedrock</addr><numbooks>1</numbooks><_unnamed_3>55</_unnamed_3></Row>
+ <Row><addr>Wimpole Hall</addr><numbooks>2</numbooks><_unnamed_3>95</_unnamed_3></Row>
+ <Row><addr>56 New Road</addr><numbooks>6</numbooks><_unnamed_3>99</_unnamed_3></Row>
+</Dataset>
+<Dataset name='Result 21'>
+ <Row><addr>99 Maltings Road</addr><firstbook>Clarion Language Manual: Richard Taylor</firstbook></Row>
+ <Row><addr>Buckingham Palace</addr><firstbook>Greek tragedies: Various</firstbook></Row>
+ <Row><addr>Bedrock</addr><firstbook>: </firstbook></Row>
+ <Row><addr>Wimpole Hall</addr><firstbook>Mein Kampf, 1st edition: Adolph Hitler</firstbook></Row>
+ <Row><addr>56 New Road</addr><firstbook>All quiet on the western front: Erich Maria Remarque</firstbook></Row>
+</Dataset>
+<Dataset name='Result 22'>
+ <Row><addr>99 Maltings Road</addr><numbooks>12</numbooks><_unnamed_3>73.25</_unnamed_3><_unnamed_4>98</_unnamed_4></Row>
+ <Row><addr>Buckingham Palace</addr><numbooks>9</numbooks><_unnamed_3>52.0</_unnamed_3><_unnamed_4>93</_unnamed_4></Row>
+ <Row><addr>Bedrock</addr><numbooks>1</numbooks><_unnamed_3>55.0</_unnamed_3><_unnamed_4>55</_unnamed_4></Row>
+ <Row><addr>Wimpole Hall</addr><numbooks>2</numbooks><_unnamed_3>87.5</_unnamed_3><_unnamed_4>95</_unnamed_4></Row>
+ <Row><addr>56 New Road</addr><numbooks>6</numbooks><_unnamed_3>72.33333333333333</_unnamed_3><_unnamed_4>99</_unnamed_4></Row>
+</Dataset>
+<Dataset name='Result 23'>
+ <Row><addr>99 Maltings Road</addr><bestbook>The bible: Various</bestbook></Row>
+ <Row><addr>Buckingham Palace</addr><bestbook>The bible: Various</bestbook></Row>
+ <Row><addr>Bedrock</addr><bestbook>Dinosaur stews: Trog</bestbook></Row>
+ <Row><addr>Wimpole Hall</addr><bestbook>Pride and prejudice, 1st edition: Jane Austen</bestbook></Row>
+ <Row><addr>56 New Road</addr><bestbook>Fox in Socks: Dr. Seuss</bestbook></Row>
+</Dataset>
+<Dataset name='Result 24'>
+ <Row><addr>99 Maltings Road</addr><firstbook>BNF: Various</firstbook></Row>
+ <Row><addr>Buckingham Palace</addr><firstbook>Atlas of the world: Times</firstbook></Row>
+ <Row><addr>Bedrock</addr><firstbook>Dinosaur stews: Trog</firstbook></Row>
+ <Row><addr>Wimpole Hall</addr><firstbook>Mein Kampf, 1st edition: Adolph Hitler</firstbook></Row>
+ <Row><addr>56 New Road</addr><firstbook>All quiet on the western front: Erich Maria Remarque</firstbook></Row>
+</Dataset>
+<Dataset name='Result 25'>
+ <Row><surname>Halliday</surname><numbooks>5</numbooks><_unnamed_3>73.8</_unnamed_3><_unnamed_4>98</_unnamed_4></Row>
+ <Row><surname>Halliday</surname><numbooks>4</numbooks><_unnamed_3>68.75</_unnamed_3><_unnamed_4>90</_unnamed_4></Row>
+ <Row><surname>Halliday</surname><numbooks>3</numbooks><_unnamed_3>78.33333333333333</_unnamed_3><_unnamed_4>90</_unnamed_4></Row>
+ <Row><surname>Windsor</surname><numbooks>5</numbooks><_unnamed_3>56.6</_unnamed_3><_unnamed_4>93</_unnamed_4></Row>
+ <Row><surname>Windsor</surname><numbooks>4</numbooks><_unnamed_3>46.25</_unnamed_3><_unnamed_4>85</_unnamed_4></Row>
+ <Row><surname>Flintstone</surname><numbooks>0</numbooks><_unnamed_3>0.0</_unnamed_3><_unnamed_4>0</_unnamed_4></Row>
+ <Row><surname>Flintstone</surname><numbooks>1</numbooks><_unnamed_3>55.0</_unnamed_3><_unnamed_4>55</_unnamed_4></Row>
+ <Row><surname>Grabdale</surname><numbooks>2</numbooks><_unnamed_3>87.5</_unnamed_3><_unnamed_4>95</_unnamed_4></Row>
+ <Row><surname>Jones</surname><numbooks>2</numbooks><_unnamed_3>92.0</_unnamed_3><_unnamed_4>99</_unnamed_4></Row>
+ <Row><surname>Jones</surname><numbooks>4</numbooks><_unnamed_3>62.5</_unnamed_3><_unnamed_4>90</_unnamed_4></Row>
+</Dataset>
+<Dataset name='Result 26'>
+ <Row><surname>Halliday</surname><bestbook>The bible: Various</bestbook></Row>
+ <Row><surname>Halliday</surname><bestbook>The thinks you can think: Dr. Seuss</bestbook></Row>
+ <Row><surname>Halliday</surname><bestbook>The Life of Pi: Yan Martel</bestbook></Row>
+ <Row><surname>Windsor</surname><bestbook>The bible: Various</bestbook></Row>
+ <Row><surname>Windsor</surname><bestbook>The cat in the hat: Dr. Seuss</bestbook></Row>
+ <Row><surname>Flintstone</surname><bestbook>: </bestbook></Row>
+ <Row><surname>Flintstone</surname><bestbook>Dinosaur stews: Trog</bestbook></Row>
+ <Row><surname>Grabdale</surname><bestbook>Pride and prejudice, 1st edition: Jane Austen</bestbook></Row>
+ <Row><surname>Jones</surname><bestbook>Fox in Socks: Dr. Seuss</bestbook></Row>
+ <Row><surname>Jones</surname><bestbook>Zen and the art of motorcyle maintenance: Robert Pirsig</bestbook></Row>
+</Dataset>
+<Dataset name='Result 27'>
+ <Row><surname>Halliday</surname><firstbook>To kill a mocking bird: Harper Lee</firstbook></Row>
+ <Row><surname>Halliday</surname><firstbook>The thinks you can think: Dr. Seuss</firstbook></Row>
+ <Row><surname>Halliday</surname><firstbook>The Life of Pi: Yan Martel</firstbook></Row>
+ <Row><surname>Windsor</surname><firstbook>The bible: Various</firstbook></Row>
+ <Row><surname>Windsor</surname><firstbook>Greek tragedies: Various</firstbook></Row>
+ <Row><surname>Flintstone</surname><firstbook>: </firstbook></Row>
+ <Row><surname>Flintstone</surname><firstbook>Dinosaur stews: Trog</firstbook></Row>
+ <Row><surname>Grabdale</surname><firstbook>Pride and prejudice, 1st edition: Jane Austen</firstbook></Row>
+ <Row><surname>Jones</surname><firstbook>All quiet on the western front: Erich Maria Remarque</firstbook></Row>
+ <Row><surname>Jones</surname><firstbook>Zen and the art of motorcyle maintenance: Robert Pirsig</firstbook></Row>
+</Dataset>
+<Dataset name='Result 28'>
+ <Row><surname>Flintstone</surname><numbooks>0</numbooks><_unnamed_3>0.0</_unnamed_3><_unnamed_4>0</_unnamed_4></Row>
+ <Row><surname>Flintstone</surname><numbooks>1</numbooks><_unnamed_3>55.0</_unnamed_3><_unnamed_4>55</_unnamed_4></Row>
+ <Row><surname>Grabdale</surname><numbooks>2</numbooks><_unnamed_3>87.5</_unnamed_3><_unnamed_4>95</_unnamed_4></Row>
+ <Row><surname>Halliday</surname><numbooks>5</numbooks><_unnamed_3>73.8</_unnamed_3><_unnamed_4>98</_unnamed_4></Row>
+ <Row><surname>Halliday</surname><numbooks>3</numbooks><_unnamed_3>78.33333333333333</_unnamed_3><_unnamed_4>90</_unnamed_4></Row>
+ <Row><surname>Halliday</surname><numbooks>4</numbooks><_unnamed_3>68.75</_unnamed_3><_unnamed_4>90</_unnamed_4></Row>
+ <Row><surname>Jones</surname><numbooks>2</numbooks><_unnamed_3>92.0</_unnamed_3><_unnamed_4>99</_unnamed_4></Row>
+ <Row><surname>Jones</surname><numbooks>4</numbooks><_unnamed_3>62.5</_unnamed_3><_unnamed_4>90</_unnamed_4></Row>
+ <Row><surname>Windsor</surname><numbooks>4</numbooks><_unnamed_3>46.25</_unnamed_3><_unnamed_4>85</_unnamed_4></Row>
+ <Row><surname>Windsor</surname><numbooks>5</numbooks><_unnamed_3>56.6</_unnamed_3><_unnamed_4>93</_unnamed_4></Row>
+</Dataset>
+<Dataset name='Result 29'>
+ <Row><surname>Flintstone</surname><bestbook>: </bestbook></Row>
+ <Row><surname>Flintstone</surname><bestbook>Dinosaur stews: Trog</bestbook></Row>
+ <Row><surname>Grabdale</surname><bestbook>Pride and prejudice, 1st edition: Jane Austen</bestbook></Row>
+ <Row><surname>Halliday</surname><bestbook>The bible: Various</bestbook></Row>
+ <Row><surname>Halliday</surname><bestbook>The Life of Pi: Yan Martel</bestbook></Row>
+ <Row><surname>Halliday</surname><bestbook>The thinks you can think: Dr. Seuss</bestbook></Row>
+ <Row><surname>Jones</surname><bestbook>Fox in Socks: Dr. Seuss</bestbook></Row>
+ <Row><surname>Jones</surname><bestbook>Zen and the art of motorcyle maintenance: Robert Pirsig</bestbook></Row>
+ <Row><surname>Windsor</surname><bestbook>The cat in the hat: Dr. Seuss</bestbook></Row>
+ <Row><surname>Windsor</surname><bestbook>The bible: Various</bestbook></Row>
+</Dataset>
+<Dataset name='Result 30'>
+ <Row><surname>Flintstone</surname><firstbook>: </firstbook></Row>
+ <Row><surname>Flintstone</surname><firstbook>Dinosaur stews: Trog</firstbook></Row>
+ <Row><surname>Grabdale</surname><firstbook>Mein Kampf, 1st edition: Adolph Hitler</firstbook></Row>
+ <Row><surname>Halliday</surname><firstbook>Clarion Language Manual: Richard Taylor</firstbook></Row>
+ <Row><surname>Halliday</surname><firstbook>BNF: Various</firstbook></Row>
+ <Row><surname>Halliday</surname><firstbook>David and Goliath: </firstbook></Row>
+ <Row><surname>Jones</surname><firstbook>All quiet on the western front: Erich Maria Remarque</firstbook></Row>
+ <Row><surname>Jones</surname><firstbook>The bible: Various</firstbook></Row>
+ <Row><surname>Windsor</surname><firstbook>Greek tragedies: Various</firstbook></Row>
+ <Row><surname>Windsor</surname><firstbook>Atlas of the world: Times</firstbook></Row>
+</Dataset>