Merge remote-tracking branch 'origin/closedown-5.0.x'

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

common/fileview2/fvdisksource.cpp

@@ -360,7 +360,7 @@ void CsvRecordSize::init(IDistributedFile * df)
     addUtfActionList(matcher, separate ? separate : "\\,", SEPARATOR, NULL, utfType);
 
     const char * quote = props->queryProp("@csvQuote");
-    addUtfActionList(matcher, quote ? quote : "'", QUOTE, NULL, utfType);
+    addUtfActionList(matcher, quote ? quote : "\"", QUOTE, NULL, utfType);
 
     const char * escape = props->queryProp("@csvEscape");
     addUtfActionList(matcher, escape, ESCAPE, NULL, utfType);

common/fileview2/fvresultset.cpp

@@ -3294,13 +3294,25 @@ extern FILEVIEW_API void writeFullWorkUnitResults(const char *username, const ch
     Owned<IConstWUExceptionIterator> exceptions = &cw->getExceptions();
     ForEach(*exceptions)
     {
-        WUExceptionSeverity severity = exceptions->query().getSeverity();
+        IConstWUException & exception = exceptions->query();
+        WUExceptionSeverity severity = exception.getSeverity();
         if (severity>=minSeverity)
         {
-            SCMStringBuffer src, msg;
-            exceptions->query().getExceptionSource(src);
-            exceptions->query().getExceptionMessage(msg);
+            SCMStringBuffer src, msg, filename;
+            exception.getExceptionSource(src);
+            exception.getExceptionMessage(msg);
+            exception.getExceptionFileName(filename);
+            unsigned lineno = exception.getExceptionLineNo();
+            unsigned code = exception.getExceptionCode();
+
             writer.outputBeginNested(getSeverityTagname(severity, flags), false);
+            if (code)
+                writer.outputUInt(code, "Code");
+            if (filename.length())
+                writer.outputCString(filename.str(), "Filename");
+            if (lineno)
+                writer.outputUInt(lineno, "Line");
+
             writer.outputCString(src.str(), "Source");
             writer.outputCString(msg.str(), "Message");
             writer.outputEndNested(getSeverityTagname(severity, flags));
@@ -3317,7 +3329,7 @@ extern FILEVIEW_API void writeFullWorkUnitResults(const char *username, const ch
             ForEach(*results)
             {
                 IConstWUResult &ds = results->query();
-                if (ds.getResultSequence()>=0)
+                if (ds.getResultSequence()>=0 && (ds.getResultStatus() != ResultStatusUndefined))
                 {
                     SCMStringBuffer name;
                     ds.getResultName(name);

common/workunit/pkgimpl.hpp

@@ -291,11 +291,12 @@ public:
     StringAttr packageId;
     StringAttr querySet;
     bool active;
+    bool compulsory;
     StringArray wildMatches, wildIds;
 public:
     IMPLEMENT_IINTERFACE;
     CPackageMapOf(const char *_packageId, const char *_querySet, bool _active)
-        : packageId(_packageId), querySet(_querySet), active(_active), packages(true)
+        : packageId(_packageId), querySet(_querySet), active(_active), packages(true), compulsory(false)
     {
     }
 
@@ -355,6 +356,7 @@ public:
     {
         if (!xml)
             return;
+        compulsory = xml->getPropBool("@compulsory");
         Owned<IPropertyTreeIterator> allpackages = xml->getElements("Package");
         ForEach(*allpackages)
         {

common/workunit/referencedfilelist.cpp

@@ -123,7 +123,7 @@ class ReferencedFile : public CInterface, implements IReferencedFile
 {
 public:
     IMPLEMENT_IINTERFACE;
-    ReferencedFile(const char *lfn, const char *sourceIP, const char *srcCluster, const char *prefix, bool isSubFile, unsigned _flags, const char *_pkgid) : flags(_flags), pkgid(_pkgid)
+    ReferencedFile(const char *lfn, const char *sourceIP, const char *srcCluster, const char *prefix, bool isSubFile, unsigned _flags, const char *_pkgid, bool noDfs) : flags(_flags), pkgid(_pkgid), noDfsResolution(noDfs)
     {
         logicalName.set(skipForeign(lfn, &daliip)).toLowerCase();
         if (daliip.length())
@@ -172,6 +172,7 @@ public:
     StringBuffer filePrefix;
     StringAttr fileSrcCluster;
     unsigned flags;
+    bool noDfsResolution;
 };
 
 class ReferencedFileList : public CInterface, implements IReferencedFileList
@@ -193,7 +194,7 @@ public:
             user.set(userDesc);
     }
 
-    void ensureFile(const char *ln, unsigned flags, const char *pkgid, const char *daliip=NULL, const char *srcCluster=NULL, const char *remotePrefix=NULL);
+    void ensureFile(const char *ln, unsigned flags, const char *pkgid, bool noDfsResolution, const char *daliip=NULL, const char *srcCluster=NULL, const char *remotePrefix=NULL);
 
     virtual void addFile(const char *ln, const char *daliip=NULL, const char *srcCluster=NULL, const char *remotePrefix=NULL);
     virtual void addFiles(StringArray &files);
@@ -287,8 +288,11 @@ void ReferencedFile::processRemoteFileTree(IPropertyTree *tree, const char *srcC
 
 void ReferencedFile::resolveLocal(const char *dstCluster, const char *srcCluster, IUserDescriptor *user, StringArray *subfiles)
 {
-    if (flags & RefFileInPackage)
+    if (noDfsResolution || (flags & RefFileInPackage))
+    {
+        flags |= RefFileNotFound;
         return;
+    }
     reset();
     Owned<IDistributedFile> df = queryDistributedFileDirectory().lookup(logicalName.str(), user);
     if(df)
@@ -330,8 +334,11 @@ void ReferencedFile::resolveRemote(IUserDescriptor *user, INode *remote, const c
 {
     if ((flags & RefFileForeign) && !resolveForeign)
         return;
-    if (flags & RefFileInPackage)
+    if (noDfsResolution || (flags & RefFileInPackage))
+    {
+        flags |= RefFileNotFound;
         return;
+    }
     reset();
     if (checkLocalFirst) //usually means we don't want to overwrite existing file info
     {
@@ -489,12 +496,12 @@ public:
     Owned<HashIterator> iter;
 };
 
-void ReferencedFileList::ensureFile(const char *ln, unsigned flags, const char *pkgid, const char *daliip, const char *srcCluster, const char *prefix)
+void ReferencedFileList::ensureFile(const char *ln, unsigned flags, const char *pkgid, bool noDfsResolution, const char *daliip, const char *srcCluster, const char *prefix)
 {
     if (!allowForeign && checkForeign(ln))
         throw MakeStringException(-1, "Foreign file not allowed%s: %s", (flags & RefFileInPackage) ? " (declared in package)" : "", ln);
 
-    Owned<ReferencedFile> file = new ReferencedFile(ln, daliip, srcCluster, prefix, false, flags, pkgid);
+    Owned<ReferencedFile> file = new ReferencedFile(ln, daliip, srcCluster, prefix, false, flags, pkgid, noDfsResolution);
     if (!file->logicalName.length())
         return;
     ReferencedFile *existing = map.getValue(file->getLogicalName());
@@ -600,14 +607,14 @@ void ReferencedFileList::addFilesFromQuery(IConstWorkUnit *cw, const IHpccPackag
                         {
                             StringBuffer subfile;
                             ssfe->getSubFileName(count, subfile);
-                            ensureFile(subfile, RefSubFile | RefFileInPackage, pkgid);
+                            ensureFile(subfile, RefSubFile | RefFileInPackage, pkgid, pkg->isCompulsory());
                         }
                     }
                 }
-                ensureFile(logicalName, flags, pkgid);
+                ensureFile(logicalName, flags, pkgid, pkg->isCompulsory());
             }
             else
-                ensureFile(logicalName, flags, NULL);
+                ensureFile(logicalName, flags, NULL, false);
         }
     }
 }
@@ -634,7 +641,7 @@ void ReferencedFileList::resolveSubFiles(StringArray &subfiles, bool checkLocalF
         if (!allowForeign && checkForeign(lfn))
             throw MakeStringException(-1, "Foreign sub file not allowed: %s", lfn);
 
-        Owned<ReferencedFile> file = new ReferencedFile(lfn, NULL, NULL, NULL, true, 0, NULL);
+        Owned<ReferencedFile> file = new ReferencedFile(lfn, NULL, NULL, NULL, true, 0, NULL, false);
         if (file->logicalName.length() && !map.getValue(file->getLogicalName()))
         {
             file->resolve(process.get(), srcCluster, user, remote, remotePrefix, checkLocalFirst, &childSubFiles, resolveForeign);

common/workunit/workunit.cpp

@@ -5381,7 +5381,7 @@ IConstWUQuery* CLocalWorkUnit::getQuery() const
     {
         IPropertyTree *s = p->getPropTree("Query");
         if (s)
-            query.setown(new CLocalWUQuery(s)); 
+            query.setown(new CLocalWUQuery(s)); // NB takes ownership of 's'
     }
     return query.getLink();
 }
@@ -5981,7 +5981,7 @@ IConstWUWebServicesInfo* CLocalWorkUnit::getWebServicesInfo() const
         assertex(!webServicesInfo);
         IPropertyTree *s = p->getPropTree("WebServicesInfo");
         if (s)
-            webServicesInfo.setown(new CLocalWUWebServicesInfo(s)); 
+            webServicesInfo.setown(new CLocalWUWebServicesInfo(s)); // NB takes ownership of 's'
         webServicesInfoCached = true;
     }
     return webServicesInfo.getLink();
@@ -6017,7 +6017,7 @@ IConstWURoxieQueryInfo* CLocalWorkUnit::getRoxieQueryInfo() const
         assertex(!roxieQueryInfo);
         IPropertyTree *s = p->getPropTree("RoxieQueryInfo");
         if (s)
-            roxieQueryInfo.setown(new CLocalWURoxieQueryInfo(s)); 
+            roxieQueryInfo.setown(new CLocalWURoxieQueryInfo(s)); // NB takes ownership of 's'
         roxieQueryInfoCached = true;
     }
     return roxieQueryInfo.getLink();

dali/dfu/dfuwu.cpp

@@ -1331,7 +1331,7 @@ public:
         const char *ter=t->queryProp("@csvTerminate");
         terminate.append(ter?ter:"\\n,\\r\\n");
         const char *quo=t->queryProp("@csvQuote");
-        quote.append(quo?quo:"'");
+        quote.append(quo?quo:"\"");
         const char *esc=t->queryProp("@csvEscape");
         if (esc && *esc)
             escape.set(esc);

dali/dfuplus/main.cpp

@@ -73,7 +73,7 @@ void handleSyntax()
     out.append("            maxrecordsize=<max-record-size> -- optional, default is 8192\n");
     out.append("            separator=<separator> -- optional, default is \\,\n");
     out.append("            terminator=<terminator> -- optional, default is \\r,\\r\\n\n");
-    out.append("            quote=<quote> -- optional, default is '\n");
+    out.append("            quote=<quote> -- optional, default is \"\n");
     out.append("            escape=<escape> -- optional, no default value \n");
     out.append("            recordstructurepresent=0|1 -- optional, default is 0 (no field names in first row) \n");
     out.append("            quotedTerminator=1|0 -- optional, default is 1 (quoted terminators in rows) \n");

dali/ft/daftformat.cpp

@@ -632,7 +632,7 @@ CCsvPartitioner::CCsvPartitioner(const FileFormat & _format) : CInputBasePartiti
     maxElementLength = 1;
     format.set(_format);
     addActionList(matcher, format.separate.get() ? format.separate.get() : "\\,", SEPARATOR, &maxElementLength);
-    addActionList(matcher, format.quote.get() ? format.quote.get() : "'", QUOTE, &maxElementLength);
+    addActionList(matcher, format.quote.get() ? format.quote.get() : "\"", QUOTE, &maxElementLength);
     addActionList(matcher, format.terminate.get() ? format.terminate.get() : "\\n,\\r\\n", TERMINATOR, &maxElementLength);
     const char * escape = format.escape.get();
     if (escape && *escape)
@@ -959,7 +959,7 @@ CUtfPartitioner::CUtfPartitioner(const FileFormat & _format) : CInputBasePartiti
     format.set(_format);
     utfFormat = getUtfFormatType(format.type);
     addUtfActionList(matcher, format.separate ? format.separate.get() : "\\,", SEPARATOR, &maxElementLength, utfFormat);
-    addUtfActionList(matcher, format.quote ? format.quote.get() : "'", QUOTE, &maxElementLength, utfFormat);
+    addUtfActionList(matcher, format.quote ? format.quote.get() : "\"", QUOTE, &maxElementLength, utfFormat);
     addUtfActionList(matcher, format.terminate ? format.terminate.get() : "\\n,\\r\\n", TERMINATOR, &maxElementLength, utfFormat);
     addUtfActionList(matcher, " ", WHITESPACE, NULL, utfFormat);
     addUtfActionList(matcher, "\t", WHITESPACE, NULL, utfFormat);

docs/HPCCClientTools/CT_Mods/CT_ECL_CLI.xml

@@ -40,7 +40,7 @@
 
             <tbody>
               <row>
-                <entry><emphasis>--version=</emphasis></entry>
+                <entry><emphasis>--version</emphasis></entry>
 
                 <entry>displays version info.</entry>
               </row>
@@ -96,9 +96,48 @@
               </row>
 
               <row>
+                <entry>roxie</entry>
+
+                <entry>execute commands for Roxie</entry>
+              </row>
+
+              <row>
                 <entry>packagemap</entry>
 
-                <entry>execute package commands (for Roxie)</entry>
+                <entry>execute packagemap commands (for Roxie)</entry>
+              </row>
+
+              <row>
+                <entry>bundle</entry>
+
+                <entry>execute a bundle command</entry>
+              </row>
+
+              <row>
+                <entry>abort</entry>
+
+                <entry>aborts one or more workunits from the given WUID or job
+                name</entry>
+              </row>
+
+              <row>
+                <entry>status</entry>
+
+                <entry>returns the status of a given workunit or job name. If
+                more than one is found, a list returns.</entry>
+              </row>
+
+              <row>
+                <entry>getname</entry>
+
+                <entry>returns the workunit name from the given WUID.</entry>
+              </row>
+
+              <row>
+                <entry>getwuid</entry>
+
+                <entry>returns the WUID(s) of the given workunit job
+                name.</entry>
               </row>
             </tbody>
           </tgroup>
@@ -292,14 +331,14 @@ ecl deploy --target=roxie --name=FindPersonService libW20120224-125557.so
                 <row>
                   <entry>-s, --server</entry>
 
-                  <entry>The IP Address or hostname of ESP server running
-                  eclwatch services</entry>
+                  <entry>The IP Address or hostname of ESP server running ECL
+                  Watch services</entry>
                 </row>
 
                 <row>
                   <entry>--port</entry>
 
-                  <entry>The eclwatch services port (Default is 8010)</entry>
+                  <entry>The ECL Watch services port (Default is 8010)</entry>
                 </row>
 
                 <row>
@@ -542,14 +581,14 @@ ecl publish --target=roxie --name=FindPersonService --no-activate findperson.ecl
                 <row>
                   <entry>-s, --server</entry>
 
-                  <entry>The IP Address or hostname of ESP server running
-                  eclwatch services</entry>
+                  <entry>The IP Address or hostname of ESP server running ECL
+                  Watch services</entry>
                 </row>
 
                 <row>
                   <entry>--port</entry>
 
-                  <entry>The eclwatch services port (Default is 8010)</entry>
+                  <entry>The ECL Watch services port (Default is 8010)</entry>
                 </row>
 
                 <row>
@@ -674,14 +713,14 @@ ecl unpublish roxie "FindpersonService*"
                 <row>
                   <entry>-s, --server</entry>
 
-                  <entry>The IP Address or hostname of ESP server running
-                  eclwatch services</entry>
+                  <entry>The IP Address or hostname of ESP server running ECL
+                  Watch services</entry>
                 </row>
 
                 <row>
                   <entry>--port</entry>
 
-                  <entry>The eclwatch services port (Default is 8010)</entry>
+                  <entry>The ECL Watch services port (Default is 8010)</entry>
                 </row>
 
                 <row>
@@ -828,14 +867,14 @@ ecl run --target=thor --input="&lt;request&gt;&lt;LName&gt;JONES&lt;/LName&gt;&l
                 <row>
                   <entry>-s, --server</entry>
 
-                  <entry>The IP Address or hostname of ESP server running
-                  eclwatch services</entry>
+                  <entry>The IP Address or hostname of ESP server running ECL
+                  Watch services</entry>
                 </row>
 
                 <row>
                   <entry>--port</entry>
 
-                  <entry>The eclwatch services port (Default is 8010)</entry>
+                  <entry>The ECL Watch services port (Default is 8010)</entry>
                 </row>
 
                 <row>
@@ -972,14 +1011,14 @@ ecl run --target=thor --input="&lt;request&gt;&lt;LName&gt;JONES&lt;/LName&gt;&l
                 <row>
                   <entry>-s, --server</entry>
 
-                  <entry>The IP Address or hostname of ESP server running
-                  eclwatch services</entry>
+                  <entry>The IP Address or hostname of ESP server running ECL
+                  Watch services</entry>
                 </row>
 
                 <row>
                   <entry>--port</entry>
 
-                  <entry>The eclwatch services port (Default is 8010)</entry>
+                  <entry>The ECL Watch services port (Default is 8010)</entry>
                 </row>
 
                 <row>
@@ -1062,14 +1101,14 @@ ecl run --target=thor --input="&lt;request&gt;&lt;LName&gt;JONES&lt;/LName&gt;&l
                 <row>
                   <entry>-s, --server</entry>
 
-                  <entry>The IP Address or hostname of ESP server running
-                  eclwatch services</entry>
+                  <entry>The IP Address or hostname of ESP server running ECL
+                  Watch services</entry>
                 </row>
 
                 <row>
                   <entry>--port</entry>
 
-                  <entry>The eclwatch services port (Default is 8010)</entry>
+                  <entry>The ECL Watch services port (Default is 8010)</entry>
                 </row>
 
                 <row>
@@ -1201,14 +1240,14 @@ ecl queries list roxie --target=roxie --show=A </programlisting>
                 <row>
                   <entry>-s, --server</entry>
 
-                  <entry>The IP Address or hostname of ESP server running
-                  eclwatch services</entry>
+                  <entry>The IP Address or hostname of ESP server running ECL
+                  Watch services</entry>
                 </row>
 
                 <row>
                   <entry>--port</entry>
 
-                  <entry>The eclwatch services port (Default is 8010)</entry>
+                  <entry>The ECL Watch services port (Default is 8010)</entry>
                 </row>
 
                 <row>
@@ -1261,7 +1300,7 @@ ecl queries copy //192.168.1.10:8010/thor/findperson thor
                   <entry>Copies a query from one queryset to another. A query
                   can be copied from one HPCC environment to another by using
                   a path which begins with '//' followed by the IP or hostname
-                  and Port of the source EclWatch and then followed by the
+                  and Port of the source ECL Watch and then followed by the
                   source queryset and query.</entry>
                 </row>
 
@@ -1373,14 +1412,14 @@ ecl queries copy //192.168.1.10:8010/thor/findperson thor
                 <row>
                   <entry>-s, --server</entry>
 
-                  <entry>The IP Address or hostname of ESP server running
-                  eclwatch services</entry>
+                  <entry>The IP Address or hostname of ESP server running ECL
+                  Watch services</entry>
                 </row>
 
                 <row>
                   <entry>--port</entry>
 
-                  <entry>The eclwatch services port (Default is 8010)</entry>
+                  <entry>The ECL Watch services port (Default is 8010)</entry>
                 </row>
 
                 <row>
@@ -1503,14 +1542,14 @@ ecl queries copy //192.168.1.10:8010/thor/findperson thor
                 <row>
                   <entry>-s, --server</entry>
 
-                  <entry>The IP Address or hostname of ESP server running
-                  eclwatch services</entry>
+                  <entry>The IP Address or hostname of ESP server running ECL
+                  Watch services</entry>
                 </row>
 
                 <row>
                   <entry>--port</entry>
 
-                  <entry>The eclwatch services port (Default is 8010)</entry>
+                  <entry>The ECL Watch services port (Default is 8010)</entry>
                 </row>
 
                 <row>
@@ -1639,14 +1678,14 @@ ecl packagemap add roxie mypackagemap.pkg --daliip=192.168.11.11
                 <row>
                   <entry>-s, --server</entry>
 
-                  <entry>The IP Address or hostname of ESP server running
-                  eclwatch services</entry>
+                  <entry>The IP Address or hostname of ESP server running ECL
+                  Watch services</entry>
                 </row>
 
                 <row>
                   <entry>--port</entry>
 
-                  <entry>The eclwatch services port (Default is 8010)</entry>
+                  <entry>The ECL Watch services port (Default is 8010)</entry>
                 </row>
 
                 <row>
@@ -1721,14 +1760,14 @@ ecl packagemap add roxie mypackagemap.pkg --daliip=192.168.11.11
                 <row>
                   <entry>-s, --server</entry>
 
-                  <entry>The IP Address or hostname of ESP server running
-                  eclwatch services</entry>
+                  <entry>The IP Address or hostname of ESP server running ECL
+                  Watch services</entry>
                 </row>
 
                 <row>
                   <entry>--port</entry>
 
-                  <entry>The eclwatch services port (Default is 8010)</entry>
+                  <entry>The ECL Watch services port (Default is 8010)</entry>
                 </row>
 
                 <row>
@@ -1811,14 +1850,14 @@ ecl packagemap add roxie mypackagemap.pkg --daliip=192.168.11.11
                 <row>
                   <entry>-s, --server</entry>
 
-                  <entry>The IP Address or hostname of ESP server running
-                  eclwatch services</entry>
+                  <entry>The IP Address or hostname of ESP server running ECL
+                  Watch services</entry>
                 </row>
 
                 <row>
                   <entry>--port</entry>
 
-                  <entry>The eclwatch services port (Default is 8010)</entry>
+                  <entry>The ECL Watch services port (Default is 8010)</entry>
                 </row>
 
                 <row>
@@ -1900,14 +1939,14 @@ ecl packagemap add roxie mypackagemap.pkg --daliip=192.168.11.11
                 <row>
                   <entry>-s, --server</entry>
 
-                  <entry>The IP Address or hostname of ESP server running
-                  eclwatch services</entry>
+                  <entry>The IP Address or hostname of ESP server running ECL
+                  Watch services</entry>
                 </row>
 
                 <row>
                   <entry>--port</entry>
 
-                  <entry>The eclwatch services port (Default is 8010)</entry>
+                  <entry>The ECL Watch services port (Default is 8010)</entry>
                 </row>
 
                 <row>
@@ -1992,14 +2031,14 @@ ecl packagemap add roxie mypackagemap.pkg --daliip=192.168.11.11
                 <row>
                   <entry>-s, --server</entry>
 
-                  <entry>The IP Address or hostname of ESP server running
-                  eclwatch services</entry>
+                  <entry>The IP Address or hostname of ESP server running ECL
+                  Watch services</entry>
                 </row>
 
                 <row>
                   <entry>--port</entry>
 
-                  <entry>The eclwatch services port (Default is 8010)</entry>
+                  <entry>The ECL Watch services port (Default is 8010)</entry>
                 </row>
 
                 <row>
@@ -2085,14 +2124,14 @@ ecl packagemap add roxie mypackagemap.pkg --daliip=192.168.11.11
                 <row>
                   <entry>-s, --server</entry>
 
-                  <entry>The IP Address or hostname of ESP server running
-                  eclwatch services</entry>
+                  <entry>The IP Address or hostname of ESP server running ECL
+                  Watch services</entry>
                 </row>
 
                 <row>
                   <entry>--port</entry>
 
-                  <entry>The eclwatch services port (Default is 8010)</entry>
+                  <entry>The ECL Watch services port (Default is 8010)</entry>
                 </row>
 
                 <row>
@@ -2211,8 +2250,8 @@ ecl packagemap validate roxie --active</programlisting>
                 <row>
                   <entry>-s, --server</entry>
 
-                  <entry>The IP Address or hostname of ESP server running
-                  eclwatch services</entry>
+                  <entry>The IP Address or hostname of ESP server running ECL
+                  Watch services</entry>
                 </row>
 
                 <row>
@@ -2238,7 +2277,7 @@ ecl packagemap validate roxie --active</programlisting>
                 <row>
                   <entry>--port</entry>
 
-                  <entry>The eclwatch services port (Default is 8010)</entry>
+                  <entry>The ECL Watch services port (Default is 8010)</entry>
                 </row>
 
                 <row>
@@ -2303,14 +2342,14 @@ ecl packagemap validate roxie --active</programlisting>
                 <row>
                   <entry>-s, --server</entry>
 
-                  <entry>The IP Address or hostname of ESP server running
-                  eclwatch services</entry>
+                  <entry>The IP Address or hostname of ESP server running ECL
+                  Watch services</entry>
                 </row>
 
                 <row>
                   <entry>--port</entry>
 
-                  <entry>The eclwatch services port (Default is 8010)</entry>
+                  <entry>The ECL Watch services port (Default is 8010)</entry>
                 </row>
 
                 <row>
@@ -2375,14 +2414,14 @@ ecl packagemap validate roxie --active</programlisting>
                 <row>
                   <entry>-s, --server</entry>
 
-                  <entry>The IP Address or hostname of ESP server running
-                  eclwatch services</entry>
+                  <entry>The IP Address or hostname of ESP server running ECL
+                  Watch services</entry>
                 </row>
 
                 <row>
                   <entry>--port</entry>
 
-                  <entry>The eclwatch services port (Default is 8010)</entry>
+                  <entry>The ECL Watch services port (Default is 8010)</entry>
                 </row>
 
                 <row>
@@ -2447,14 +2486,14 @@ ecl packagemap validate roxie --active</programlisting>
                 <row>
                   <entry>-s, --server</entry>
 
-                  <entry>The IP Address or hostname of ESP server running
-                  eclwatch services</entry>
+                  <entry>The IP Address or hostname of ESP server running ECL
+                  Watch services</entry>
                 </row>
 
                 <row>
                   <entry>--port</entry>
 
-                  <entry>The eclwatch services port (Default is 8010)</entry>
+                  <entry>The ECL Watch services port (Default is 8010)</entry>
                 </row>
 
                 <row>
@@ -2519,14 +2558,14 @@ ecl packagemap validate roxie --active</programlisting>
                 <row>
                   <entry>-s, --server</entry>
 
-                  <entry>The IP Address or hostname of ESP server running
-                  eclwatch services</entry>
+                  <entry>The IP Address or hostname of ESP server running ECL
+                  Watch services</entry>
                 </row>
 
                 <row>
                   <entry>--port</entry>
 
-                  <entry>The eclwatch services port (Default is 8010)</entry>
+                  <entry>The ECL Watch services port (Default is 8010)</entry>
                 </row>
 
                 <row>
@@ -2966,8 +3005,8 @@ ecl bundle use myBundle 2
                 <row>
                   <entry>-s, --server</entry>
 
-                  <entry>The IP Address or hostname of ESP server running
-                  eclwatch services</entry>
+                  <entry>The IP Address or hostname of ESP server running ECL
+                  Watch services</entry>
                 </row>
 
                 <row>
@@ -2980,7 +3019,339 @@ ecl bundle use myBundle 2
                 <row>
                   <entry>--port</entry>
 
-                  <entry>The eclwatch services port (Default is 8010)</entry>
+                  <entry>The ECL Watch services port (Default is 8010)</entry>
+                </row>
+
+                <row>
+                  <entry>-u, --username</entry>
+
+                  <entry>The username (if necessary)</entry>
+                </row>
+
+                <row>
+                  <entry>-pw, --password</entry>
+
+                  <entry>The password (if necessary)</entry>
+                </row>
+              </tbody>
+            </tgroup>
+          </informaltable></para>
+      </sect2>
+
+      <sect2 role="brk">
+        <title>ecl abort</title>
+
+        <para><emphasis role="bold">ecl abort -wu &lt;WUID&gt; | -n
+        &lt;jobName&gt;</emphasis></para>
+
+        <para>Examples:</para>
+
+        <programlisting>ecl abort -wu W20150516-111213
+ecl abort -n MyJob</programlisting>
+
+        <para></para>
+
+        <para><informaltable colsep="0" frame="none" rowsep="0">
+            <tgroup cols="2">
+              <colspec align="left" colwidth="125.55pt" />
+
+              <colspec colwidth="384.85pt" />
+
+              <tbody>
+                <row>
+                  <entry>ecl abort</entry>
+
+                  <entry>aborts one or more Workunits from the given WUID or
+                  job name</entry>
+                </row>
+
+                <row>
+                  <entry><emphasis role="bold">Options</emphasis></entry>
+                </row>
+
+                <row>
+                  <entry>-wu</entry>
+
+                  <entry>The WUID (Workunit ID)</entry>
+                </row>
+
+                <row>
+                  <entry>-n</entry>
+
+                  <entry>The job name</entry>
+                </row>
+
+                <row>
+                  <entry>-v, --verbose</entry>
+
+                  <entry>Output additional tracing information</entry>
+                </row>
+
+                <row>
+                  <entry>-s, --server</entry>
+
+                  <entry>The IP Address or hostname of ESP server running ECL
+                  Watch services</entry>
+                </row>
+
+                <row>
+                  <entry>-ssl,--ssl</entry>
+
+                  <entry>Use SSL to secure the connection to the
+                  server.</entry>
+                </row>
+
+                <row>
+                  <entry>--port</entry>
+
+                  <entry>The ECL Watch services port (Default is 8010)</entry>
+                </row>
+
+                <row>
+                  <entry>-u, --username</entry>
+
+                  <entry>The username (if necessary)</entry>
+                </row>
+
+                <row>
+                  <entry>-pw, --password</entry>
+
+                  <entry>The password (if necessary)</entry>
+                </row>
+              </tbody>
+            </tgroup>
+          </informaltable></para>
+      </sect2>
+
+      <sect2 role="brk">
+        <title>ecl status</title>
+
+        <para><emphasis role="bold">ecl status -wu &lt;WUID&gt; | -n
+        &lt;jobName&gt;</emphasis></para>
+
+        <para>Examples:</para>
+
+        <programlisting>ecl status -wu W20150516-111213
+ecl status -n MyJob</programlisting>
+
+        <para></para>
+
+        <para><informaltable colsep="0" frame="none" rowsep="0">
+            <tgroup cols="2">
+              <colspec align="left" colwidth="125.55pt" />
+
+              <colspec colwidth="384.85pt" />
+
+              <tbody>
+                <row>
+                  <entry>ecl status</entry>
+
+                  <entry>returns the status of a given workunit or job name.
+                  If more than one is found, a CSV list returns.</entry>
+                </row>
+
+                <row>
+                  <entry><emphasis role="bold">Options</emphasis></entry>
+                </row>
+
+                <row>
+                  <entry>-wu</entry>
+
+                  <entry>The WUID (Workunit ID)</entry>
+                </row>
+
+                <row>
+                  <entry>-n</entry>
+
+                  <entry>The job name</entry>
+                </row>
+
+                <row>
+                  <entry>-v, --verbose</entry>
+
+                  <entry>Output additional tracing information</entry>
+                </row>
+
+                <row>
+                  <entry>-s, --server</entry>
+
+                  <entry>The IP Address or hostname of ESP server running ECL
+                  Watch services</entry>
+                </row>
+
+                <row>
+                  <entry>-ssl,--ssl</entry>
+
+                  <entry>Use SSL to secure the connection to the
+                  server.</entry>
+                </row>
+
+                <row>
+                  <entry>--port</entry>
+
+                  <entry>The ECL Watch services port (Default is 8010)</entry>
+                </row>
+
+                <row>
+                  <entry>-u, --username</entry>
+
+                  <entry>The username (if necessary)</entry>
+                </row>
+
+                <row>
+                  <entry>-pw, --password</entry>
+
+                  <entry>The password (if necessary)</entry>
+                </row>
+              </tbody>
+            </tgroup>
+          </informaltable></para>
+      </sect2>
+
+      <sect2 role="brk">
+        <title>ecl getwuid</title>
+
+        <para><emphasis role="bold">ecl getwuid -n &lt;jobName&gt;
+        [--limit=&lt;limitCount&gt;]</emphasis></para>
+
+        <para>Examples:</para>
+
+        <programlisting>ecl getwuid -n MyJobName
+ecl getwuid -n MyCommonJobName --limit=100</programlisting>
+
+        <para></para>
+
+        <para><informaltable colsep="0" frame="none" rowsep="0">
+            <tgroup cols="2">
+              <colspec align="left" colwidth="125.55pt" />
+
+              <colspec colwidth="384.85pt" />
+
+              <tbody>
+                <row>
+                  <entry>ecl getwuid</entry>
+
+                  <entry>returns the WUID(s) for a given job name. If more
+                  than one is found, a list returns.</entry>
+                </row>
+
+                <row>
+                  <entry><emphasis role="bold">Options</emphasis></entry>
+                </row>
+
+                <row>
+                  <entry>-n</entry>
+
+                  <entry>The job name</entry>
+                </row>
+
+                <row>
+                  <entry>--limit=<emphasis>nn</emphasis></entry>
+
+                  <entry>Integer to set result limit, default is 100</entry>
+                </row>
+
+                <row>
+                  <entry>-v, --verbose</entry>
+
+                  <entry>Output additional tracing information</entry>
+                </row>
+
+                <row>
+                  <entry>-s, --server</entry>
+
+                  <entry>The IP Address or hostname of ESP server running ECL
+                  Watch services</entry>
+                </row>
+
+                <row>
+                  <entry>-ssl,--ssl</entry>
+
+                  <entry>Use SSL to secure the connection to the
+                  server.</entry>
+                </row>
+
+                <row>
+                  <entry>--port</entry>
+
+                  <entry>The ECL Watch services port (Default is 8010)</entry>
+                </row>
+
+                <row>
+                  <entry>-u, --username</entry>
+
+                  <entry>The username (if necessary)</entry>
+                </row>
+
+                <row>
+                  <entry>-pw, --password</entry>
+
+                  <entry>The password (if necessary)</entry>
+                </row>
+              </tbody>
+            </tgroup>
+          </informaltable></para>
+      </sect2>
+
+      <sect2 role="brk">
+        <title>ecl getname</title>
+
+        <para><emphasis role="bold">ecl getname -wu &lt;WUID&gt;
+        </emphasis></para>
+
+        <para>Examples:</para>
+
+        <programlisting>ecl getname -wu W20140516-111213</programlisting>
+
+        <para></para>
+
+        <para><informaltable colsep="0" frame="none" rowsep="0">
+            <tgroup cols="2">
+              <colspec align="left" colwidth="125.55pt" />
+
+              <colspec colwidth="384.85pt" />
+
+              <tbody>
+                <row>
+                  <entry>ecl getname</entry>
+
+                  <entry>returns the job name for a given workunit.</entry>
+                </row>
+
+                <row>
+                  <entry><emphasis role="bold">Options</emphasis></entry>
+                </row>
+
+                <row>
+                  <entry>-wu</entry>
+
+                  <entry>The WUID (Workunit ID)</entry>
+                </row>
+
+                <row>
+                  <entry>-v, --verbose</entry>
+
+                  <entry>Output additional tracing information</entry>
+                </row>
+
+                <row>
+                  <entry>-s, --server</entry>
+
+                  <entry>The IP Address or hostname of ESP server running ECL
+                  Watch services</entry>
+                </row>
+
+                <row>
+                  <entry>-ssl,--ssl</entry>
+
+                  <entry>Use SSL to secure the connection to the
+                  server.</entry>
+                </row>
+
+                <row>
+                  <entry>--port</entry>
+
+                  <entry>The ECL Watch services port (Default is 8010)</entry>
                 </row>
 
                 <row>

ecl/hql/hqlgram.y

@@ -5644,8 +5644,7 @@ primexpr1
                         {
                             parser->normalizeExpression($5);
                             parser->normalizeExpression($7);
-                            ITypeInfo * type = parser->checkPromoteIfType($5, $7);
-                            $$.setExpr(createValue(no_if, type, $3.getExpr(), $5.getExpr(), $7.getExpr()), $1);
+                            $$.setExpr(parser->processIfProduction($3, $5, &$7), $1);
                         }
     | IFF '(' booleanExpr ',' expression ',' expression ')'
                         {
@@ -7787,14 +7786,15 @@ simpleDataSet
                             $$.setExpr(createDataset(no_distribute, $3.getExpr(), value.getClear()));
                             $$.setPosition($1);
                         }
-    | JOIN '(' startLeftDelaySeqFilter ',' startRightFilter ',' expression opt_join_transform_flags ')' endSelectorSequence
+    | JOIN '(' startLeftDelaySeqFilter ',' startRightFilter ',' expression beginCounterScope opt_join_transform_flags endCounterScope ')' endSelectorSequence
                         {
                             parser->normalizeExpression($7, type_boolean, false);
 
                             IHqlExpression * left = $3.getExpr();
                             IHqlExpression * right = $5.getExpr();
                             IHqlExpression * cond = $7.getExpr();
-                            OwnedHqlExpr flags = $8.getExpr();
+                            OwnedHqlExpr flags = $9.getExpr();
+                            OwnedHqlExpr counter = $10.getExpr();
                             OwnedHqlExpr transform;
                             if (flags)
                             {
@@ -7813,11 +7813,14 @@ simpleDataSet
 
                             if (!transform)
                             {
-                                IHqlExpression * seq = $10.queryExpr();
+                                IHqlExpression * seq = $12.queryExpr();
                                 transform.setown(parser->createDefJoinTransform(left,right,$1,seq,flags));
                             }
 
-                            IHqlExpression *join = createDataset(no_join, left, createComma(right, cond, createComma(transform.getClear(), flags.getClear(), $10.getExpr())));
+                            if (counter)
+                                flags.setown(createComma(flags.getClear(), createAttribute(_countProject_Atom, counter.getClear())));
+
+                            IHqlExpression *join = createDataset(no_join, left, createComma(right, cond, createComma(transform.getClear(), flags.getClear(), $12.getExpr())));
 
                             bool isLocal = join->hasAttribute(localAtom);
                             parser->checkDistribution($3, left, isLocal, true);

ecl/hql/hqlgram2.cpp

@@ -8759,20 +8759,19 @@ IHqlExpression * HqlGram::processIfProduction(attribute & condAttr, attribute &
             right.setown(createNullExpr(left));
     }
 
-    //MORE: Not sure about this!
-    if (parsingTemplateAttribute && cond->isConstant())
-    {
-        OwnedHqlExpr folded = quickFoldExpression(cond);
-        if (folded->queryValue())
-            return folded->queryValue()->getBoolValue() ? left.getClear() : right.getClear();
-    }
-
     if (left->queryRecord() && falseAttr)
         right.setown(checkEnsureRecordsMatch(left, right, *falseAttr, false));
 
     if (isGrouped(left) != isGrouped(right))
         reportError(ERR_GROUPING_MISMATCH, trueAttr, "Branches of the condition have different grouping");
 
+    if (cond->isConstant())
+    {
+        OwnedHqlExpr folded = quickFoldExpression(cond);
+        if (folded->queryValue())
+            return folded->queryValue()->getBoolValue() ? left.getClear() : right.getClear();
+    }
+
     return ::createIf(cond.getClear(), left.getClear(), right.getClear());
 }
 

ecl/hql/hqlopt.cpp

@@ -2996,7 +2996,7 @@ IHqlExpression * CTreeOptimizer::doCreateTransformed(IHqlExpression * transforme
 #endif
                 break;
             case no_selfjoin:
-                if (isPureActivity(child) && !hasUnknownTransform(child) && !isLimitedJoin(child) && !child->hasAttribute(fullouterAtom) && !child->hasAttribute(fullonlyAtom))
+                if (isPureActivity(child) && !hasUnknownTransform(child) && !isLimitedJoin(child) && !child->hasAttribute(fullouterAtom) && !child->hasAttribute(fullonlyAtom) && !child->hasAttribute(_countProject_Atom))
                 {
                     //Strictly speaking, we could hoist conditions that can be hoisted for left only (or even full) joins etc. if the fields that are filtered
                     //are based on equalities in the join condition.  However, that can wait....  (same for join below...)
@@ -3012,7 +3012,7 @@ IHqlExpression * CTreeOptimizer::doCreateTransformed(IHqlExpression * transforme
                 }
                 break;
             case no_join:
-                if (isPureActivity(child) && !hasUnknownTransform(child) && !isLimitedJoin(child) && !child->hasAttribute(fullouterAtom) && !child->hasAttribute(fullonlyAtom))
+                if (isPureActivity(child) && !hasUnknownTransform(child) && !isLimitedJoin(child) && !child->hasAttribute(fullouterAtom) && !child->hasAttribute(fullonlyAtom) && !child->hasAttribute(_countProject_Atom))
                 {
                     bool canHoistLeft = !child->hasAttribute(rightouterAtom) && !child->hasAttribute(rightonlyAtom);
                     bool canMergeLeft = isInnerJoin(child);

ecl/hql/hqlutil.cpp

@@ -1479,8 +1479,13 @@ IHqlExpression * createIf(IHqlExpression * cond, IHqlExpression * left, IHqlExpr
     if (left->isDatarow() || right->isDatarow())
         return createRow(no_if, cond, createComma(left, right));
 
-    ITypeInfo * type = ::getPromotedECLType(left->queryType(), right->queryType());
-    return createValue(no_if, type, cond, left, right);
+    ITypeInfo * leftType = left->queryType();
+    ITypeInfo * rightType = right->queryType();
+    Owned<ITypeInfo> type = ::getPromotedECLType(leftType, rightType);
+    if (isStringType(type) && (leftType->getStringLen() != rightType->getStringLen()))
+        type.setown(getStretchedType(UNKNOWN_LENGTH, type));
+
+    return createValue(no_if, type.getClear(), cond, left, right);
 }
 
 extern HQL_API unsigned numRealChildren(IHqlExpression * expr)

ecl/hqlcpp/hqlckey.cpp

@@ -578,10 +578,6 @@ void KeyedJoinInfo::buildTransform(BuildCtx & ctx)
     switch (expr->getOperator())
     {
     case no_join:
-        {
-            funcctx.addQuotedCompound("virtual size32_t transform(ARowBuilder & crSelf, const void * _left, const void * _right, unsigned __int64 _filepos)");
-            break;
-        }
     case no_denormalize:
         {
             funcctx.addQuotedCompound("virtual size32_t transform(ARowBuilder & crSelf, const void * _left, const void * _right, unsigned __int64 _filepos, unsigned counter)");

ecl/hqlcpp/hqlhtcpp.cpp

@@ -9516,7 +9516,7 @@ void HqlCppTranslator::buildCsvParameters(BuildCtx & subctx, IHqlExpression * cs
 
     doBuildSizetFunction(classctx, "queryMaxSize", getCsvMaxLength(csvAttr));
 
-    buildCsvListFunc(classctx, "getQuote", queryAttribute(quoteAtom, attrs), isReading ? "'" : NULL);
+    buildCsvListFunc(classctx, "getQuote", queryAttribute(quoteAtom, attrs), isReading ? "\"" : NULL);
     buildCsvListFunc(classctx, "getSeparator", separator, ",");
     buildCsvListFunc(classctx, "getTerminator", terminator, isReading ? "\r\n|\n" : "\n");
     buildCsvListFunc(classctx, "getEscape", escape, NULL);
@@ -12082,6 +12082,8 @@ ABoundActivity * HqlCppTranslator::doBuildActivityJoinOrDenormalize(BuildCtx & c
     BuildCtx transformctx(instance->startctx);
     switch (op)
     {
+    case no_join:
+    case no_selfjoin:
     case no_denormalize:
         {
             transformctx.addQuotedCompound("virtual size32_t transform(ARowBuilder & crSelf, const void * _left, const void * _right, unsigned counter)");
@@ -12104,14 +12106,6 @@ ABoundActivity * HqlCppTranslator::doBuildActivityJoinOrDenormalize(BuildCtx & c
             doBuildTransformBody(transformctx, transform, selfCursor);
             break;
         }
-    case no_join:
-    case no_selfjoin:
-        {
-            transformctx.addQuotedCompound("virtual size32_t transform(ARowBuilder & crSelf, const void * _left, const void * _right)");
-            ensureRowAllocated(transformctx, "crSelf");
-            buildTransformBody(transformctx, transform, dataset1, dataset2, instance->dataset, selSeq);
-            break;
-        }
     }
 
     IHqlExpression * onFail = expr->queryAttribute(onFailAtom);

ecl/hthor/hthor.cpp

@@ -4275,6 +4275,7 @@ void CHThorJoinActivity::ready()
     }
 
     rightIndex = 0;
+    joinCounter = 0;
     failingLimit.clear();
     state = JSfill;
     if ((helper.getJoinFlags() & JFlimitedprefixjoin) && helper.getJoinLimit()) 
@@ -4340,6 +4341,7 @@ void CHThorJoinActivity::fillLeft()
     if (limitedhelper && 0==rightIndex)
     {
         rightIndex = 0;
+        joinCounter = 0;
         right.clear();
         matchedRight.kill();
         if (left)
@@ -4367,6 +4369,7 @@ void CHThorJoinActivity::fillRight()
     else
         right.clear();
     rightIndex = 0;
+    joinCounter = 0;
     unsigned groupCount = 0;
     while(true)
     {
@@ -4446,12 +4449,12 @@ void CHThorJoinActivity::fillRight()
         matchedRight.append(false);
 }
 
-const void * CHThorJoinActivity::joinRecords(const void * curLeft, const void * curRight)
+const void * CHThorJoinActivity::joinRecords(const void * curLeft, const void * curRight, unsigned counter)
 {
     try
     {
         outBuilder.ensureRow();
-        size32_t thisSize = helper.transform(outBuilder, curLeft, curRight);
+        size32_t thisSize = helper.transform(outBuilder, curLeft, curRight, counter);
         if(thisSize)
             return outBuilder.finalizeRowClear(thisSize);
         else
@@ -4602,14 +4605,15 @@ const void *CHThorJoinActivity::nextInGroup()
                             if (!matchedRight.item(rightIndex))
                             {
                                 const void * rhs = right.item(rightIndex++);
-                                const void * ret = joinRecords(defaultLeft, rhs);
+                                const void * ret = joinRecords(defaultLeft, rhs, 0);
                                 if (ret)
                                 {
                                     processed++;
                                     return ret;
                                 }
                             }
-                            rightIndex++;
+                            else
+                                rightIndex++;
                         }
                         break;
                     }
@@ -4687,7 +4691,7 @@ const void *CHThorJoinActivity::nextInGroup()
                 switch (kind)
                 {
                 case TAKjoin:
-                    ret = joinRecords(left, defaultRight);
+                    ret = joinRecords(left, defaultRight, 0);
                     break;
                 case TAKdenormalize:
                     ret = left.getClear();
@@ -4726,7 +4730,7 @@ const void *CHThorJoinActivity::nextInGroup()
                                 matchedLeft = true;
                                 if (!exclude)
                                 {
-                                    const void *ret = joinRecords(left, rhs);
+                                    const void *ret = joinRecords(left, rhs, ++joinCounter);
                                     if (ret)
                                     {
                                         processed++;
@@ -4817,6 +4821,7 @@ const void *CHThorJoinActivity::nextInGroup()
             }
             state = JSleftonly;
             rightIndex = 0;
+            joinCounter = 0;
             break;
         }
     }
@@ -4896,6 +4901,7 @@ void CHThorSelfJoinActivity::ready()
         limitedhelper.setown(createRHLimitedCompareHelper());
         limitedhelper->init( helper.getJoinLimit(), dualcache->queryOut2(), collate, helper.queryPrefixCompare() );
     }
+    joinCounter = 0;
 }
 
 void CHThorSelfJoinActivity::done()
@@ -4976,6 +4982,7 @@ bool CHThorSelfJoinActivity::fillGroup()
     }
     leftIndex = 0;
     rightIndex = 0;
+    joinCounter = 0;
     rightOuterIndex = 0;
     joinLimit = keepLimit;
     ForEachItemIn(idx, group)
@@ -4994,6 +5001,7 @@ const void * CHThorSelfJoinActivity::nextInGroup()
                 if (lhs)
                 {
                     rightIndex = 0;
+                    joinCounter = 0;
                     group.clear();
                     limitedhelper->getGroup(group,lhs);
                 }
@@ -5006,7 +5014,7 @@ const void * CHThorSelfJoinActivity::nextInGroup()
                 const void * rhs = group.item(rightIndex++);
                 if(helper.match(lhs, rhs))
                 {
-                    const void * ret = joinRecords(lhs, rhs);
+                    const void * ret = joinRecords(lhs, rhs, ++joinCounter, NULL);
                     return ret;
                 }
             }
@@ -5024,7 +5032,7 @@ const void * CHThorSelfJoinActivity::nextInGroup()
         if(failingOuterAtmost)
             while(group.isItem(leftIndex))
             {
-                const void * ret = joinRecords(group.item(leftIndex++), defaultRight);
+                const void * ret = joinRecords(group.item(leftIndex++), defaultRight, 0, NULL);
                 if(ret)
                 {
                     processed++;
@@ -5035,7 +5043,7 @@ const void * CHThorSelfJoinActivity::nextInGroup()
         {
             if(leftOuterJoin && !matchedLeft && !failingLimit)
             {
-                const void * ret = joinRecords(group.item(leftIndex), defaultRight);
+                const void * ret = joinRecords(group.item(leftIndex), defaultRight, 0, NULL);
                 if(ret)
                 {
                     matchedLeft = true;
@@ -5046,6 +5054,7 @@ const void * CHThorSelfJoinActivity::nextInGroup()
             leftIndex++;
             matchedLeft = false;
             rightIndex = 0;
+            joinCounter = 0;
             joinLimit = keepLimit;
         }
         if(!group.isItem(leftIndex))
@@ -5055,7 +5064,7 @@ const void * CHThorSelfJoinActivity::nextInGroup()
                 OwnedConstRoxieRow lhs(input->nextInGroup());
                 while(lhs)
                 {
-                    const void * ret = joinRecords(lhs, defaultRight, failingLimit);
+                    const void * ret = joinRecords(lhs, defaultRight, 0, failingLimit);
                     if(ret)
                     {
                         processed++;
@@ -5069,7 +5078,7 @@ const void * CHThorSelfJoinActivity::nextInGroup()
                 while(group.isItem(rightOuterIndex))
                     if(!matchedRight.item(rightOuterIndex++))
                     {
-                        const void * ret = joinRecords(defaultLeft, group.item(rightOuterIndex-1));
+                        const void * ret = joinRecords(defaultLeft, group.item(rightOuterIndex-1), 0, NULL);
                         if(ret)
                         {
                             processed++;
@@ -5084,7 +5093,7 @@ const void * CHThorSelfJoinActivity::nextInGroup()
         if(failingLimit)
         {
             leftIndex++;
-            const void * ret = joinRecords(lhs, defaultRight, failingLimit);
+            const void * ret = joinRecords(lhs, defaultRight, 0, failingLimit);
             if(ret)
             {
                 processed++;
@@ -5100,7 +5109,7 @@ const void * CHThorSelfJoinActivity::nextInGroup()
                 matchedRight.replace(true, rightIndex-1);
                 if(!exclude)
                 {
-                    const void * ret = joinRecords(lhs, rhs);
+                    const void * ret = joinRecords(lhs, rhs, ++joinCounter, NULL);
                     if(ret)
                     {
                         processed++;
@@ -5114,12 +5123,12 @@ const void * CHThorSelfJoinActivity::nextInGroup()
     return NULL;
 }
 
-const void * CHThorSelfJoinActivity::joinRecords(const void * curLeft, const void * curRight, IException * except)
+const void * CHThorSelfJoinActivity::joinRecords(const void * curLeft, const void * curRight, unsigned counter, IException * except)
 {
     outBuilder.ensureRow();
     try
     {
-            size32_t thisSize = (except ? helper.onFailTransform(outBuilder, curLeft, curRight, except) : helper.transform(outBuilder, curLeft, curRight));
+            size32_t thisSize = (except ? helper.onFailTransform(outBuilder, curLeft, curRight, except) : helper.transform(outBuilder, curLeft, curRight, counter));
             if(thisSize){
                 return outBuilder.finalizeRowClear(thisSize);   
             }
@@ -5258,6 +5267,7 @@ void CHThorLookupJoinActivity::ready()
         createDefaultRight();   
     eog = false;
     matchedGroup = false;
+    joinCounter = 0;
 }
 
 void CHThorLookupJoinActivity::done()
@@ -5313,12 +5323,12 @@ void CHThorLookupJoinActivity::setInput(unsigned index, IHThorInput * _input)
 }
 
 //following are all copied from CHThorJoinActivity - should common up.
-const void * CHThorLookupJoinActivity::joinRecords(const void * left, const void * right)
+const void * CHThorLookupJoinActivity::joinRecords(const void * left, const void * right, unsigned counter)
 {
     try
     {
         outBuilder.ensureRow();
-        size32_t thisSize = helper.transform(outBuilder, left, right);
+        size32_t thisSize = helper.transform(outBuilder, left, right, counter);
         if(thisSize)
             return outBuilder.finalizeRowClear(thisSize);
         else
@@ -5430,7 +5440,7 @@ const void * CHThorLookupJoinActivity::nextInGroupJoin()
                     gotMatch = true;
                     if(exclude)
                         break;
-                    ret = joinRecords(left, right);
+                    ret = joinRecords(left, right, ++joinCounter);
                     if(ret)
                         break;
                 }
@@ -5439,7 +5449,7 @@ const void * CHThorLookupJoinActivity::nextInGroupJoin()
             }
             if(leftOuterJoin && !gotMatch)
             {
-                ret = joinRecords(left, defaultRight);
+                ret = joinRecords(left, defaultRight, 0);
                 gotMatch = true;
             }
         }
@@ -5450,11 +5460,13 @@ const void * CHThorLookupJoinActivity::nextInGroupJoin()
             if(!many || (--keepCount == 0) || failingLimit)
             {
                 left.clear();
+                joinCounter = 0;
                 failingLimit.clear();
             }
             return ret;
         }
         left.clear();
+        joinCounter = 0;
     }
 }
 
@@ -5681,15 +5693,16 @@ void CHThorAllJoinActivity::loadRight()
         matchedRight.append(false);
     }
     rightIndex = 0;
+    joinCounter = 0;
     rightOrdinality = rightset.ordinality();
 }
 
-const void * CHThorAllJoinActivity::joinRecords(const void * left, const void * right)
+const void * CHThorAllJoinActivity::joinRecords(const void * left, const void * right, unsigned counter)
 {
     try
     {
         outBuilder.ensureRow();
-        memsize_t thisSize = helper.transform(outBuilder, left, right);
+        memsize_t thisSize = helper.transform(outBuilder, left, right, counter);
         if(thisSize)
             return outBuilder.finalizeRowClear(thisSize);
         else
@@ -5763,7 +5776,7 @@ const void * CHThorAllJoinActivity::nextInGroup()
                 switch(kind)
                 {
                 case TAKalljoin:
-                    ret = joinRecords(left, defaultRight);
+                    ret = joinRecords(left, defaultRight, 0);
                     break;
                 case TAKalldenormalize:
                     ret = left.getClear();
@@ -5777,6 +5790,7 @@ const void * CHThorAllJoinActivity::nextInGroup()
                 }
             }
             rightIndex = 0;
+            joinCounter = 0;
             left.clear();
             if(ret)
             {
@@ -5822,7 +5836,7 @@ const void * CHThorAllJoinActivity::nextInGroup()
                     matchedLeft = true;
                     matchedRight.replace(true, rightIndex);
                     if(!exclude)
-                        ret = joinRecords(left, right);
+                        ret = joinRecords(left, right, ++joinCounter);
                 }
                 rightIndex++;
                 if(ret)

ecl/hthor/hthor.ipp

@@ -1227,6 +1227,7 @@ class CHThorJoinActivity : public CHThorActivityBase
     OwnedConstRoxieRow left;
     OwnedConstRoxieRow pendingRight;
     unsigned rightIndex;
+    unsigned joinCounter;
     BoolArray matchedRight;
     bool matchedLeft;
     Owned<IException> failingLimit;
@@ -1247,7 +1248,7 @@ private:
     void fillRight();
     //bool getMatchingRecords();
     //bool queryAdvanceCursors();
-    const void * joinRecords(const void * curLeft, const void * curRight);
+    const void * joinRecords(const void * curLeft, const void * curRight, unsigned counter);
     const void * groupDenormalizeRecords(const void * curLeft, ConstPointerArray & rows);
     const void * joinException(const void * curLeft, IException * except);
     void failLimit();
@@ -1297,6 +1298,7 @@ class CHThorSelfJoinActivity : public CHThorActivityBase
     unsigned leftIndex;
     unsigned rightIndex;
     unsigned rightOuterIndex;
+    unsigned joinCounter;
     bool eof;
     bool doneFirstFill;
 
@@ -1311,7 +1313,7 @@ class CHThorSelfJoinActivity : public CHThorActivityBase
     Owned<CRHDualCache> dualcache;
 private:
     bool fillGroup();
-    const void * joinRecords(const void * curLeft, const void * curRight, IException * except = NULL);
+    const void * joinRecords(const void * curLeft, const void * curRight, unsigned counter, IException * except);
     void failLimit(const void * next);
 
 public:
@@ -1368,6 +1370,7 @@ private:
     unsigned keepLimit;
     unsigned atmostLimit;
     unsigned limitLimit;
+    unsigned joinCounter;
     bool limitFail;
     bool limitOnFail;
     bool hasGroupLimit;
@@ -1390,7 +1393,7 @@ private:
 private:
     void loadRight();
     const void * groupDenormalizeRecords(const void * curLeft, ConstPointerArray & rows);
-    const void * joinRecords(const void * left, const void * right);
+    const void * joinRecords(const void * left, const void * right, unsigned counter);
     const void * joinException(const void * left, IException * except);
     const void * getRightFirst() { if(hasGroupLimit) return fillRightGroup(); else return table->find(left); }
     const void * getRightNext() { if(hasGroupLimit) return readRightGroup(); else return table->findNext(left); }
@@ -1445,6 +1448,7 @@ private:
     unsigned countForLeft;
     unsigned rightIndex;
     unsigned rightOrdinality;
+    unsigned joinCounter;
     OwnedRowArray rightset;
     ConstPointerArray filteredRight;
     BoolArray matchedRight;
@@ -1452,7 +1456,7 @@ private:
 
 private:
     void loadRight();
-    const void * joinRecords(const void * left, const void * right);
+    const void * joinRecords(const void * left, const void * right, unsigned counter);
     const void * groupDenormalizeRecords(const void * left, ConstPointerArray & rows);
     void createDefaultRight();  
 public:

ecl/hthor/hthorkey.cpp

@@ -3710,7 +3710,7 @@ public:
                         {
                             RtlDynamicRowBuilder rowBuilder(rowAllocator);
                             if (kind == TAKkeyedjoin)
-                                transformedSize = helper.transform(rowBuilder, left, defaultRight, 0);
+                                transformedSize = helper.transform(rowBuilder, left, defaultRight, (__uint64)0, (unsigned)0);
                             else if (kind == TAKkeyeddenormalizegroup)
                                 transformedSize = helper.transform(rowBuilder, left, defaultRight, 0, (const void * *)NULL);
                             if (transformedSize)
@@ -3750,6 +3750,7 @@ public:
                 {
                     if(jg->matches.start())
                     {
+                        unsigned counter = 0;
                         do
                         {
                             try
@@ -3759,7 +3760,7 @@ public:
                                 if(!row) continue;
                                 offset_t fpos = jg->matches.queryOffset();
                                 size32_t transformedSize;
-                                transformedSize = helper.transform(rowBuilder, left, row, fpos);
+                                transformedSize = helper.transform(rowBuilder, left, row, fpos, ++counter);
                                 if (transformedSize)
                                 {
                                     const void * shrunk = rowBuilder.finalizeRowClear(transformedSize);

esp/services/WsDeploy/WsDeployService.cpp

@@ -3952,7 +3952,7 @@ bool CWsDeployFileInfo::handleAttributeAdd(IEspContext &context, IEspHandleAttri
   if (attribName.length() == 0)
     throw MakeStringException(-1,"Attribute name can't be empty!");
 
-  IPropertyTree* pComp =  pEnvRoot->getPropTree(xpath.str());
+  IPropertyTree* pComp =  pEnvRoot->queryPropTree(xpath.str());
 
   if (pComp != NULL)
     pComp->addProp(attribName.str(), "");

esp/services/ws_dfu/ws_dfuService.cpp

@@ -5106,14 +5106,14 @@ void CWsDfuEx::mergeDataRow(StringBuffer& newRow, StringBuffer dataRow1, StringB
     newRow.append("<Row>");
 
     StringArray columnLabels;
-    IPropertyTreeIterator* it = data1->getElements("*");
+    Owned<IPropertyTreeIterator> it = data1->getElements("*");
     if (it)
     {
         StringArray columnLabels0;
         mergeDataRow(newRow, 0, it, columnLabels0, columnLabels);   
     }
 
-    IPropertyTreeIterator* it2 = data2->getElements("*");
+    Owned<IPropertyTreeIterator> it2 = data2->getElements("*");
     if (it2)
     {   
         mergeDataRow(newRow, 1, it2, columnsHide, columnLabels);

esp/services/ws_ecl/ws_ecl_service.cpp

@@ -1393,7 +1393,7 @@ void appendEclInputXsds(StringBuffer &content, IPropertyTree *xsd, BoolHash &add
                 {
 #if 0
                     content.appendf("<xsd:complexType name=\"%s\"><xsd:sequence>", schema_name);
-                    IPropertyTreeIterator *children = item.getElements("xs:complexType/xs:sequence/*");
+                    Owned<IPropertyTreeIterator> children = item.getElements("xs:complexType/xs:sequence/*");
                     ForEach(*children)
                     {
                         IPropertyTree &child = children->query();

esp/services/ws_fs/ws_fsService.cpp

@@ -2096,7 +2096,7 @@ bool CFileSprayEx::onSprayVariable(IEspContext &context, IEspSprayVariable &req,
                 ct = "\\n,\\r\\n";
             const char* cq = req.getSourceCsvQuote();
             if(cq== NULL)
-                cq = "'";
+                cq = "\"";
             source->setCsvOptions(cs, ct, cq, req.getSourceCsvEscape(), req.getQuotedTerminator());
 
             options->setQuotedTerminator(req.getQuotedTerminator());

esp/services/ws_packageprocess/ws_packageprocessService.cpp

@@ -422,7 +422,7 @@ bool deletePkgInfo(const char *name, const char *target, const char *process, bo
         IPropertyTree *pkgMaps = pkgMapsConn->queryRoot();
         if (!pkgMaps)
             throw MakeStringException(PKG_DALI_LOOKUP_ERROR, "Unable to retrieve PackageMaps information from dali [/PackageMaps]");
-        IPropertyTree *mapTree = pkgMaps->getPropTree(xpath.clear().appendf("PackageMap[@id='%s']", pmid.get()).str());
+        IPropertyTree *mapTree = pkgMaps->queryPropTree(xpath.clear().appendf("PackageMap[@id='%s']", pmid.get()).str());
         if (mapTree)
             pkgMaps->removeTree(mapTree);
     }

esp/services/ws_workunits/ws_workunitsQuerySets.cpp

@@ -365,6 +365,8 @@ void QueryFilesInUse::loadTarget(IPropertyTree *t, const char *target, unsigned
                 fileTree->setProp("@lfn", lfn);
                 if (rf.getFlags() & RefFileSuper)
                     fileTree->setPropBool("@super", true);
+                if (rf.getFlags() & RefFileNotFound)
+                    fileTree->setPropBool("@notFound", true);
                 const char *fpkgid = rf.queryPackageId();
                 if (fpkgid && *fpkgid)
                     fileTree->setProp("@pkgid", fpkgid);
@@ -404,7 +406,7 @@ IPropertyTreeIterator *QueryFilesInUse::findQueriesUsingFile(const char *target,
         return NULL;
     if (!target || !*target)
         return findAllQueriesUsingFile(lfn);
-    IPropertyTree *targetTree = tree->getPropTree(target);
+    IPropertyTree *targetTree = tree->queryPropTree(target);
     if (!targetTree)
         return NULL;
 

esp/src/eclwatch/templates/LFDetailsWidget.html

@@ -16,6 +16,7 @@
                                     <input id="${id}CopyTargetSelect" title="${i18n.Group}:" name="destGroup" colspan="2" style="width:100%;" data-dojo-type="TargetSelectWidget" style="display: inline-block; vertical-align: middle" />
                                     <input id="${id}CopyTargetName"  title="${i18n.TargetName}:" name="destLogicalName" colspan="2" style="width:100%;" required="true" data-dojo-props="trim: true" data-dojo-type="dijit.form.ValidationTextBox" />
                                     <input id="${id}CopyTargetOverwrite" title="${i18n.Overwrite}:" name="overwrite" data-dojo-type="dijit.form.CheckBox" />
+                                    <input id="${id}CopyTargetReplicate" title="${i18n.Replicate}:" name="replicate" data-dojo-type="dijit.form.CheckBox" />
                                     <input id="${id}CopyTargetNoSplit" title="${i18n.NoSplit}:" name="nosplit" data-dojo-type="dijit.form.CheckBox" />
                                     <input id="${id}CopyTargetCompress" title="${i18n.Compress}:" name="compress" data-dojo-type="dijit.form.CheckBox" />
                                     <input id="${id}CopyTargetWrap" title="${i18n.Wrap}:" name="Wrap" data-dojo-type="dijit.form.CheckBox" />

esp/src/eclwatch/templates/LZBrowseWidget.html

@@ -45,6 +45,8 @@
                                     <legend>Options</legend>
                                     <div data-dojo-props="cols:2" data-dojo-type="hpcc.TableContainer">
                                         <input title="${i18n.Overwrite}:" name="overwrite" data-dojo-type="dijit.form.CheckBox" />
+                                        <input title="${i18n.Replicate}:" name="replicate" data-dojo-type="dijit.form.CheckBox" />
+                                        <input title="${i18n.NoSplit}:" name="nosplit" data-dojo-type="dijit.form.CheckBox" />
                                         <input title="${i18n.Compress}:" name="compress" data-dojo-type="dijit.form.CheckBox" />
                                         <input title="${i18n.FailIfNoSourceFile}:" name="failIfNoSourceFile" data-dojo-type="dijit.form.CheckBox" />
                                     </div>
@@ -89,6 +91,8 @@
                                         <input id="${id}SprayDelimitedTerminators" title="${i18n.LineTerminators}:" name="sourceCsvTerminate" style="width: 95%;" value="\n,\r\n" required="true" colspan="2" data-dojo-props="trim: true, placeHolder:'\\n,\\r\\n'" data-dojo-type="dijit.form.ValidationTextBox" />
                                         <input id="${id}SprayDelimitedQuote" title="${i18n.Quote}:" style="width: 95%;" name="sourceCsvQuote" value='"' data-data-dojo-props="trim: true, placeHolder:'\''" colspan="2" data-dojo-type="dijit.form.TextBox" />
                                         <input title="${i18n.Overwrite}:" name="overwrite" data-dojo-type="dijit.form.CheckBox" />
+                                        <input title="${i18n.Replicate}:" name="replicate" data-dojo-type="dijit.form.CheckBox" />
+                                        <input title="${i18n.NoSplit}:" name="nosplit" data-dojo-type="dijit.form.CheckBox" />
                                         <input title="${i18n.Compress}:" name="compress" data-dojo-type="dijit.form.CheckBox" />
                                         <input title="${i18n.FailIfNoSourceFile}:" name="failIfNoSourceFile" data-dojo-type="dijit.form.CheckBox" />
                                         <input title="${i18n.RecordStructurePresent}:" name="recordStructurePresent" data-dojo-type="dijit.form.CheckBox" />
@@ -130,6 +134,8 @@
                                         </select>
                                         <input id="${id}SprayXmlMaxRecordLength" title="${i18n.MaxRecordLength}:" value="8192" style="width: 95%;" name="sourceMaxRecordSize" required="true" colspan="2" data-dojo-props="trim: true, placeHolder:'8192'" data-dojo-type="dijit.form.ValidationTextBox" />
                                         <input title="${i18n.Overwrite}:" name="overwrite" data-dojo-type="dijit.form.CheckBox" />
+                                        <input title="${i18n.Replicate}:" name="replicate" data-dojo-type="dijit.form.CheckBox" />
+                                        <input title="${i18n.NoSplit}:" name="nosplit" data-dojo-type="dijit.form.CheckBox" />
                                         <input title="${i18n.Compress}:" name="compress" data-dojo-type="dijit.form.CheckBox" />
                                         <input title="${i18n.FailIfNoSourceFile}:" name="failIfNoSourceFile" data-dojo-type="dijit.form.CheckBox" />
                                     </div>
@@ -163,6 +169,8 @@
                                             <option value="variablebigendian">${i18n.VariableBigendian}</option>
                                         </select>
                                         <input title="${i18n.Overwrite}:" name="overwrite" data-dojo-type="dijit.form.CheckBox" />
+                                        <input title="${i18n.Replicate}:" name="replicate" data-dojo-type="dijit.form.CheckBox" />
+                                        <input title="${i18n.NoSplit}:" name="nosplit" data-dojo-type="dijit.form.CheckBox" />
                                         <input title="${i18n.Compress}:" name="compress" data-dojo-type="dijit.form.CheckBox" />
                                         <input title="${i18n.FailIfNoSourceFile}:" name="failIfNoSourceFile" data-dojo-type="dijit.form.CheckBox" />
                                     </div>
@@ -191,6 +199,8 @@
                                     <div data-dojo-props="cols:2" data-dojo-type="hpcc.TableContainer">
                                         <input title="${i18n.BlobPrefix}:" style="width: 95%;" name="prefix" data-dojo-props="trim: true, placeHolder:'${i18n.PrefixPlaceholder}'" colspan="2" data-dojo-type="dijit.form.TextBox" />
                                         <input title="${i18n.Overwrite}:" name="overwrite" data-dojo-type="dijit.form.CheckBox" />
+                                        <input title="${i18n.Replicate}:" name="replicate" data-dojo-type="dijit.form.CheckBox" />
+                                        <input title="${i18n.NoSplit}:" name="nosplit" data-dojo-type="dijit.form.CheckBox" />
                                         <input title="${i18n.Compress}:" name="compress" data-dojo-type="dijit.form.CheckBox" />
                                         <input title="${i18n.FailIfNoSourceFile}:" name="failIfNoSourceFile" data-dojo-type="dijit.form.CheckBox" />
                                     </div>

esp/src/eclwatch/viz/DojoD3Choropleth.js

@@ -39,6 +39,11 @@ define([
 
         resize: function (args) {
             //  No resize (yet - its too slow)
+            this.calcGeom();
+            d3.select(this.target.domDivID).select("svg")
+                .attr("width", this.target.width)
+                .attr("height", this.target.height)
+            ;
         },
 
         renderTo: function (_target) {

initfiles/componentfiles/configxml/RoxieTopology.xsl

@@ -146,6 +146,12 @@
                     <xsl:copy-of select="@regex"/>
                 </xsl:copy>
             </xsl:for-each>
+            <xsl:for-each select="PreferredCluster">
+                <xsl:copy>
+                    <xsl:copy-of select="@name"/>
+                    <xsl:copy-of select="@priority"/>
+                </xsl:copy>
+            </xsl:for-each>
             <xsl:for-each select="RoxieFarmProcess">
                 <xsl:element name="RoxieFarmProcess">
                     <xsl:copy-of select="@*[name()!='name' and name()!='level']"/>

initfiles/componentfiles/configxml/roxie.xsd.in

@@ -212,6 +212,31 @@
             </xs:attribute>
           </xs:complexType>
         </xs:element>
+        <xs:element name="PreferredCluster" maxOccurs="unbounded">
+          <xs:annotation>
+            <xs:appinfo>
+              <title>Preferred Clusters</title>
+            </xs:appinfo>
+          </xs:annotation>
+          <xs:complexType>
+            <xs:attribute name="name" type="xs:string" use="required" >
+              <xs:annotation>
+                <xs:appinfo>
+                  <tooltip>Name of the cluster</tooltip>
+                                    <colIndex>1</colIndex>
+                </xs:appinfo>
+              </xs:annotation>
+            </xs:attribute>
+            <xs:attribute name="priority" type="xs:integer" use="required">
+              <xs:annotation>
+                <xs:appinfo>
+                  <tooltip>Priority (negative to disable)</tooltip>
+                                    <colIndex>2</colIndex>
+                </xs:appinfo>
+              </xs:annotation>
+            </xs:attribute>
+          </xs:complexType>
+        </xs:element>
         <xs:element name="UserMetric" maxOccurs="unbounded">
           <xs:annotation>
             <xs:appinfo>
@@ -690,6 +715,13 @@
         </xs:appinfo>
       </xs:annotation>
     </xs:attribute>
+    <xs:attribute name="slaveQueryReleaseDelaySeconds" type="xs:nonNegativeInteger" use="optional" default="60">
+      <xs:annotation>
+        <xs:appinfo>
+          <tooltip>Delay before unregistering slave queries to allow in-flight to complete</tooltip>
+        </xs:appinfo>
+      </xs:annotation>
+    </xs:attribute>
 
    <xs:attribute name="slaveThreads" type="xs:nonNegativeInteger" use="optional" default="30">
       <xs:annotation>

initfiles/etc/DIR_NAME/environment.xml.in

@@ -885,6 +885,7 @@
                 siteCertificate=""
                 slaTimeout="2000"
                 slaveConfig="simple"
+                slaveQueryReleaseDelaySeconds="60"
                 slaveThreads="30"
                 soapTraceLevel="1"
                 socketCheckInterval="5000"

roxie/ccd/ccd.hpp

@@ -324,8 +324,9 @@ extern unsigned slaTimeout;
 extern unsigned headRegionSize;
 extern unsigned ccdMulticastPort;
 extern CriticalSection ccdChannelsCrit;
-extern IPropertyTree* ccdChannels;
-extern IPropertyTree* topology;
+extern IPropertyTree *ccdChannels;
+extern IPropertyTree *topology;
+extern MapStringTo<int> *preferredClusters;
 extern StringArray allQuerySetNames;
 
 extern bool allFilesDynamic;

roxie/ccd/ccddali.cpp

@@ -173,6 +173,7 @@ private:
                     try
                     {
                         owner->disconnectSem.wait();
+                        Sleep(5000);   // Don't retry immediately, give Dali a chance to recover.
                     }
                     catch (IException *E)
                     {
@@ -434,12 +435,6 @@ public:
         return ret.getClear();
     }
 
-    static const char *getPackageMapPath(StringBuffer &buf, const char *id)
-    {
-        buf.appendf("PackageMaps/PackageMap[@id='%s']", id);
-        return buf.str();
-    }
-
     static const char *getSuperFilePath(StringBuffer &buf, const char *lfn)
     {
         CDfsLogicalFileName lfnParser;
@@ -676,10 +671,10 @@ public:
         return getSubscription("PackageSets", "PackageSets", notifier);
     }
 
-    virtual IDaliPackageWatcher *getPackageMapSubscription(const char *id, ISDSSubscription *notifier)
+    virtual IDaliPackageWatcher *getPackageMapsSubscription(ISDSSubscription *notifier)
     {
         StringBuffer xpath;
-        return getSubscription(id, getPackageMapPath(xpath, id), notifier);
+        return getSubscription("PackageMaps", "PackageMaps", notifier);
     }
 
     virtual IDaliPackageWatcher *getSuperFileSubscription(const char *lfn, ISDSSubscription *notifier)
@@ -721,14 +716,16 @@ public:
                     serverStatus->queryProperties()->setProp("@cluster", roxieName.str());
                     serverStatus->commitProperties();
                     initCache();
-                    isConnected = true;
                     ForEachItemIn(idx, watchers)
                     {
                         watchers.item(idx).onReconnect();
                     }
+                    isConnected = true;
                 }
                 catch(IException *e)
                 {
+                    delete serverStatus;
+                    serverStatus = NULL;
                     ::closedownClientProcess(); // undo any partial initialization
                     StringBuffer text;
                     e->errorMessage(text);

roxie/ccd/ccddali.hpp

@@ -47,8 +47,8 @@ interface IRoxieDaliHelper : extends IInterface
     virtual IDaliPackageWatcher *getQuerySetSubscription(const char *id, ISDSSubscription *notifier) = 0;
     virtual IPropertyTree *getPackageSets() = 0;
     virtual IDaliPackageWatcher *getPackageSetsSubscription(ISDSSubscription *notifier) = 0;
+    virtual IDaliPackageWatcher *getPackageMapsSubscription(ISDSSubscription *notifier) = 0;
     virtual IPropertyTree *getPackageMap(const char *id) = 0;
-    virtual IDaliPackageWatcher *getPackageMapSubscription(const char *id, ISDSSubscription *notifier) = 0;
     virtual IDaliPackageWatcher *getSuperFileSubscription(const char *lfn, ISDSSubscription *notifier) = 0;
     virtual void releaseSubscription(IDaliPackageWatcher *subscription) = 0;
     virtual bool connect(unsigned timeout) = 0;

roxie/ccd/ccdfile.cpp

@@ -485,32 +485,32 @@ static IPartDescriptor *queryMatchingRemotePart(IPartDescriptor *pdesc, IFileDes
     return NULL;
 }
 
-static bool checkClusterCount(UnsignedArray &counts, unsigned clusterNo, unsigned max)
+static int getClusterPriority(const char *clusterName)
 {
-    while (!counts.isItem(clusterNo))
-        counts.append(0);
-    unsigned count = counts.item(clusterNo);
-    if (count>=max)
-        return false;
-    counts.replace(++count, clusterNo);
-    return true;
-}
-
-static bool isCopyFromCluster(IPartDescriptor *pdesc, unsigned clusterNo, const char *name)
-{
-    StringBuffer s;
-    return strieq(name, pdesc->queryOwner().getClusterGroupName(clusterNo, s));
+    assertex(preferredClusters);
+    int *priority = preferredClusters->getValue(clusterName);
+    return priority ? *priority : 100;
 }
 
 static void appendRemoteLocations(IPartDescriptor *pdesc, StringArray &locations, const char *localFileName, const char *fromCluster, bool includeFromCluster)
 {
-    UnsignedArray clusterCounts;
+    IFileDescriptor &fdesc = pdesc->queryOwner();
     unsigned numCopies = pdesc->numCopies();
+    unsigned lastClusterNo = (unsigned) -1;
+    unsigned numThisCluster = 0;
+    int priority = 0;
+    IntArray priorities;
     for (unsigned copy = 0; copy < numCopies; copy++)
     {
         unsigned clusterNo = pdesc->copyClusterNum(copy);
-        if (fromCluster && *fromCluster && isCopyFromCluster(pdesc, clusterNo, fromCluster)!=includeFromCluster)
-            continue;
+        StringBuffer clusterName;
+        fdesc.getClusterGroupName(clusterNo, clusterName);
+        if (fromCluster && *fromCluster)
+        {
+            bool matches = strieq(clusterName.str(), fromCluster);
+            if (matches!=includeFromCluster)
+                continue;
+        }
         RemoteFilename r;
         pdesc->getFilename(copy,r);
         StringBuffer path;
@@ -522,26 +522,36 @@ static void appendRemoteLocations(IPartDescriptor *pdesc, StringArray &locations
             if (streq(l, localFileName))
                 continue; // don't add ourself
         }
-        if (!checkClusterCount(clusterCounts, clusterNo, 2))  // Don't add more than 2 from one cluster
-            continue;
-        locations.append(path.str());
-    }
-}
-
-static void appendPeerLocations(IPartDescriptor *pdesc, StringArray &locations, const char *localFileName)
-{
-    const char *peerCluster = pdesc->queryOwner().queryProperties().queryProp("@cloneFromPeerCluster");
-    if (peerCluster)
-    {
-        if (*peerCluster=='-') //a remote cluster was specified explicitly
-            return;
-        if (streq(peerCluster, roxieName))
-            peerCluster=NULL;
+        if (clusterNo == lastClusterNo)
+        {
+            numThisCluster++;
+            if (numThisCluster > 2)  // Don't add more than 2 from one cluster
+                continue;
+        }
+        else
+        {
+            numThisCluster = 1;
+            lastClusterNo = clusterNo;
+            if (preferredClusters)
+            {
+                priority = getClusterPriority(clusterName);
+            }
+            else
+                priority = copy;
+        }
+        if (priority >= 0)
+        {
+            ForEachItemIn(idx, priorities)
+            {
+                if (priorities.item(idx) < priority)
+                    break;
+            }
+            priorities.add(priority, idx);
+            locations.add(path.str(), idx);
+        }
     }
-    appendRemoteLocations(pdesc, locations, localFileName, peerCluster, true);
 }
 
-
 //----------------------------------------------------------------------------------------------
 
 typedef StringArray *StringArrayPtr;

roxie/ccd/ccdlistener.cpp

@@ -627,7 +627,7 @@ public:
         afor.For(activeChildren.ordinality()+(isMaster ? 0 : 1), 10);
         activeChildren.kill();
         if (mergedReply)
-            toXML(mergedReply, reply, 0, (mergeType == CascadeMergeQueries) ? XML_Format | XML_NewlinesOnly : XML_Format);
+            toXML(mergedReply, reply, 0, (mergeType == CascadeMergeQueries) ? XML_Embed|XML_LineBreak|XML_SortTags : XML_Format);
         if (logctx.queryTraceLevel() > 5)
             logctx.CTXLOG("doControlQuery (%d) finished: %.80s", isMaster, queryText);
     }

roxie/ccd/ccdmain.cpp

@@ -86,7 +86,8 @@ bool runOnce = false;
 unsigned udpMulticastBufferSize = 262142;
 bool roxieMulticastEnabled = true;
 
-IPropertyTree* topology;
+IPropertyTree *topology;
+MapStringTo<int> *preferredClusters;
 StringBuffer topologyFile;
 CriticalSection ccdChannelsCrit;
 IPropertyTree* ccdChannels;
@@ -512,7 +513,19 @@ int STARTQUERY_API start_query(int argc, const char *argv[])
             topology->setProp("@traceLevel", globals->queryProp("--traceLevel"));
             topology->setProp("@memTraceLevel", globals->queryProp("--memTraceLevel"));
         }
-
+        if (topology->hasProp("PreferredCluster"))
+        {
+            preferredClusters = new MapStringTo<int>(true);
+            Owned<IPropertyTreeIterator> clusters = topology->getElements("PreferredCluster");
+            ForEach(*clusters)
+            {
+                IPropertyTree &child = clusters->query();
+                const char *name = child.queryProp("@name");
+                int priority = child.getPropInt("@priority", 100);
+                if (name && *name)
+                    preferredClusters->setValue(name, priority);
+            }
+        }
         topology->getProp("@name", roxieName);
         Owned<const IQueryDll> standAloneDll;
         if (globals->hasProp("--loadWorkunit"))

roxie/ccd/ccdquery.cpp

@@ -1186,7 +1186,7 @@ public:
                 }
             }
         }
-        toXML(xref, reply, 1, XML_NewlinesOnly|XML_Format|XML_SortTags);
+        toXML(xref, reply, 1, XML_Embed|XML_LineBreak|XML_SortTags);
     }
     virtual void resetQueryTimings()
     {

roxie/ccd/ccdqueue.cpp

@@ -443,7 +443,7 @@ extern IRoxieQueryPacket *createRoxiePacket(void *_data, unsigned _len)
 extern IRoxieQueryPacket *createRoxiePacket(MemoryBuffer &m)
 {
     unsigned length = m.length(); // don't make assumptions about evaluation order of parameters...
-    return createRoxiePacket(m.detach(), length);
+    return createRoxiePacket(m.detachOwn(), length);
 }
 
 //=================================================================================

roxie/ccd/ccdserver.cpp

@@ -11644,6 +11644,7 @@ class CRoxieServerJoinActivity : public CRoxieServerTwoInputActivity
     const void * left;
     const void * pendingRight;
     unsigned rightIndex;
+    unsigned joinCounter;
     BoolArray matchedRight;
     bool matchedLeft;
     Owned<IException> failingLimit;
@@ -11706,6 +11707,7 @@ public:
             collate = collateupper = helper.queryCompareLeftRight();
         }
         rightIndex = 0;
+        joinCounter = 0;
         state = JSfill;
         matchedLeft = false;
         joinLimit = 0;
@@ -11725,6 +11727,7 @@ public:
     {
         left = NULL;
         rightIndex = 0;
+        joinCounter = 0;
         state = JSfill;
         matchedLeft = false;
 
@@ -11805,6 +11808,7 @@ public:
         if (limitedhelper && 0==rightIndex)
         {
             rightIndex = 0;
+            joinCounter = 0;
             right.clear();
             matchedRight.kill();
             if (left)
@@ -11832,6 +11836,7 @@ public:
         else
             right.clear();
         rightIndex = 0;
+        joinCounter = 0;
         unsigned groupCount = 0;
         const void * next;
         while(true)
@@ -11916,7 +11921,7 @@ public:
             matchedRight.append(false);
     }
 
-    const void * joinRecords(const void * curLeft, const void * curRight)
+    const void * joinRecords(const void * curLeft, const void * curRight, unsigned counter)
     {
         if (cloneLeft)
         {
@@ -11926,7 +11931,7 @@ public:
         try
         {
             RtlDynamicRowBuilder rowBuilder(rowAllocator);
-            size32_t thisSize = helper.transform(rowBuilder, curLeft, curRight);
+            size32_t thisSize = helper.transform(rowBuilder, curLeft, curRight, counter);
             if (thisSize)
                 return rowBuilder.finalizeRowClear(thisSize);
             else
@@ -12065,14 +12070,15 @@ public:
                                 if (!matchedRight.item(rightIndex))
                                 {
                                     const void * rhs = right.item(rightIndex++);
-                                    const void *ret = joinRecords(defaultLeft, rhs);
+                                    const void *ret = joinRecords(defaultLeft, rhs, 0);
                                     if (ret)
                                     {
                                         processed++;
                                         return ret;
                                     }
                                 }
-                                rightIndex++;
+                                else
+                                    rightIndex++;
                             }
                             break;
                         }
@@ -12149,7 +12155,7 @@ public:
                     switch (activityKind)
                     {
                     case TAKjoin:
-                        ret = joinRecords(left, defaultRight);
+                        ret = joinRecords(left, defaultRight, 0);
                         break;
                     case TAKdenormalize:
                         ret = left;
@@ -12188,7 +12194,7 @@ public:
                                     matchedLeft = true;
                                     if (!exclude)
                                     {
-                                        const void *ret = joinRecords(left, rhs);
+                                        const void *ret = joinRecords(left, rhs, ++joinCounter);
                                         if (ret)
                                         {
                                             processed++;
@@ -12277,6 +12283,7 @@ public:
                 }
                 state = JSleftonly;
                 rightIndex = 0;
+                joinCounter = 0;
                 break;
             }
         }
@@ -16863,6 +16870,7 @@ class CRoxieServerSelfJoinActivity : public CRoxieServerActivity
     unsigned atmostsTriggered;
     unsigned abortLimit;
     unsigned keepLimit;
+    unsigned joinCounter;
     bool leftOuterJoin;
     bool rightOuterJoin;
     bool exclude;
@@ -16954,6 +16962,7 @@ class CRoxieServerSelfJoinActivity : public CRoxieServerActivity
         leftIndex = 0;
         rightIndex = 0;
         rightOuterIndex = 0;
+        joinCounter = 0;
         joinLimit = keepLimit;
         ForEachItemIn(idx, group)
             matchedRight.append(false);
@@ -16973,7 +16982,7 @@ class CRoxieServerSelfJoinActivity : public CRoxieServerActivity
 
     virtual bool needsAllocator() const { return true; }
 
-    const void *joinRecords(const void * curLeft, const void * curRight, IException * except = NULL)
+    const void *joinRecords(const void * curLeft, const void * curRight, unsigned counter, IException * except)
     {
         try
         {
@@ -16983,7 +16992,7 @@ class CRoxieServerSelfJoinActivity : public CRoxieServerActivity
                 return curLeft;
             }
             RtlDynamicRowBuilder rowBuilder(rowAllocator);
-            size32_t outsize = except ? helper.onFailTransform(rowBuilder, curLeft, curRight, except) : helper.transform(rowBuilder, curLeft, curRight);
+            size32_t outsize = except ? helper.onFailTransform(rowBuilder, curLeft, curRight, except) : helper.transform(rowBuilder, curLeft, curRight, counter);
             if (outsize)
                 return rowBuilder.finalizeRowClear(outsize);
             else
@@ -17047,6 +17056,7 @@ public:
         leftIndex = 0;
         rightIndex = 0;
         rightOuterIndex = 0;
+        joinCounter = 0;
         dualCacheInput = NULL;
     }
 
@@ -17081,6 +17091,7 @@ public:
             matchedLeft = false;
             leftIndex = 0;
             rightOuterIndex = 0;
+            joinCounter = 0;
 
             limitedhelper.setown(createRHLimitedCompareHelper());
             limitedhelper->init( helper.getJoinLimit(), dualcache->queryOut2(), collate, helper.queryPrefixCompare() );
@@ -17110,6 +17121,7 @@ public:
                     if (lhs)
                     {
                         rightIndex = 0;
+                        joinCounter = 0;
                         group.clear();
                         limitedhelper->getGroup(group,lhs);
                     }
@@ -17124,7 +17136,7 @@ public:
                     const void * rhs = group.item(rightIndex++);
                     if(helper.match(lhs, rhs))
                     {
-                        const void * ret = joinRecords(lhs, rhs);
+                        const void * ret = joinRecords(lhs, rhs, ++joinCounter, NULL);
                         return ret;
                     }
                 }
@@ -17143,7 +17155,7 @@ public:
                 if(failingOuterAtmost)
                     while(group.isItem(leftIndex))
                     {
-                        const void * ret = joinRecords(group.item(leftIndex++), defaultRight);
+                        const void * ret = joinRecords(group.item(leftIndex++), defaultRight, 0, NULL);
                         if(ret)
                         {
                             processed++;
@@ -17154,7 +17166,7 @@ public:
                 {
                     if(leftOuterJoin && !matchedLeft && !failingLimit)
                     {
-                        const void * ret = joinRecords(group.item(leftIndex), defaultRight);
+                        const void * ret = joinRecords(group.item(leftIndex), defaultRight, 0, NULL);
                         if(ret)
                         {
                             matchedLeft = true;
@@ -17165,6 +17177,7 @@ public:
                     leftIndex++;
                     matchedLeft = false;
                     rightIndex = 0;
+                    joinCounter = 0;
                     joinLimit = keepLimit;
                 }
                 if(!group.isItem(leftIndex))
@@ -17174,7 +17187,7 @@ public:
                         const void * lhs;
                         while((lhs = input->nextInGroup()) != NULL)  // dualCache never active here
                         {
-                            const void * ret = joinRecords(lhs, defaultRight, failingLimit);
+                            const void * ret = joinRecords(lhs, defaultRight, 0, failingLimit);
                             ReleaseRoxieRow(lhs);
                             if(ret)
                             {
@@ -17188,7 +17201,7 @@ public:
                         while(group.isItem(rightOuterIndex))
                             if(!matchedRight.item(rightOuterIndex++))
                             {
-                                const void * ret = joinRecords(defaultLeft, group.item(rightOuterIndex-1));
+                                const void * ret = joinRecords(defaultLeft, group.item(rightOuterIndex-1), 0, NULL);
                                 if(ret)
                                 {
                                     processed++;
@@ -17203,7 +17216,7 @@ public:
                 if(failingLimit)
                 {
                     leftIndex++;
-                    const void * ret = joinRecords(lhs, defaultRight, failingLimit);
+                    const void * ret = joinRecords(lhs, defaultRight, 0, failingLimit);
                     if(ret)
                     {
                         processed++;
@@ -17219,7 +17232,7 @@ public:
                         matchedRight.replace(true, rightIndex-1);
                         if(!exclude)
                         {
-                            const void * ret = joinRecords(lhs, rhs);
+                            const void * ret = joinRecords(lhs, rhs, ++joinCounter, NULL);
                             if(ret)
                             {
                                 processed++;
@@ -17535,6 +17548,7 @@ private:
     unsigned atmostLimit;
     unsigned atmostsTriggered;
     unsigned limitLimit;
+    unsigned joinCounter;
     bool limitFail;
     bool limitOnFail;
     bool hasGroupLimit;
@@ -17580,6 +17594,7 @@ public:
         gotMatch = false;
         keepLimit = 0;
         keepCount = 0;
+        joinCounter = 0;
         atmostLimit = 0;
         atmostsTriggered = 0;
         limitLimit = 0;
@@ -17626,7 +17641,7 @@ public:
                             void **_rows = const_cast<void * *>(rightset.getArray());
                             memcpy(temp, _rows, rightord*sizeof(void **));
                             qsortvecstable(temp, rightord, *helper.queryCompareRight(), (void ***)_rows);
-                            for (int i = 0; i < rightord; i++)
+                            for (unsigned i = 0; i < rightord; i++)
                             {
                                 *_rows = **((void ***)_rows);
                                 _rows++;
@@ -17737,6 +17752,7 @@ private:
             {
                 left = input->nextInGroup();
                 keepCount = keepLimit;
+                joinCounter = 0;
                 if(!left)
                 {
                     if (isSmartJoin)
@@ -17774,7 +17790,7 @@ private:
                         gotMatch = true;
                         if(exclude)
                             break;
-                        ret = joinRecords(left, right);
+                        ret = joinRecords(left, right, ++joinCounter);
                         if(ret)
                             break;
                     }
@@ -17783,7 +17799,7 @@ private:
                 }
                 if(leftOuterJoin && !gotMatch)
                 {
-                    ret = joinRecords(left, defaultRight);
+                    ret = joinRecords(left, defaultRight, 0);
                     gotMatch = true;
                 }
             }
@@ -17900,7 +17916,7 @@ private:
         }
     }
 
-    const void * joinRecords(const void * left, const void * right)
+    const void * joinRecords(const void * left, const void * right, unsigned counter)
     {
         if (cloneLeft)
         {
@@ -17910,7 +17926,7 @@ private:
         try
         {
             RtlDynamicRowBuilder rowBuilder(rowAllocator);
-            unsigned outSize = helper.transform(rowBuilder, left, right);
+            unsigned outSize = helper.transform(rowBuilder, left, right, counter);
             if (outSize)
                 return rowBuilder.finalizeRowClear(outSize);
             else
@@ -18071,6 +18087,7 @@ private:
     bool leftIsGrouped;
     bool cloneLeft;
     unsigned rightIndex;
+    unsigned joinCounter;
     unsigned rightOrdinality;
     ThorActivityKind activityKind;
     ConstPointerArray filteredRight;
@@ -18122,6 +18139,7 @@ public:
         rightOrdinality = 0;
         leftIsGrouped = false;
         countForLeft = 0;
+        joinCounter = 0;
     }
 
     virtual void reset()
@@ -18149,6 +18167,7 @@ public:
         if(keepLimit==0)
             keepLimit = (unsigned) -1;
         countForLeft = keepLimit;
+        joinCounter = 0;
         leftIsGrouped = input->queryOutputMeta()->isGrouped();
         if((activityKind==TAKalljoin || activityKind==TAKalldenormalizegroup) && leftOuterJoin)
             createDefaultRight();
@@ -18170,6 +18189,7 @@ public:
             matchedRight.append(false);
         }
         rightIndex = 0;
+        joinCounter = 0;
         rightOrdinality = rightset.ordinality();
     }
 
@@ -18189,7 +18209,7 @@ public:
         }
     }
 
-    const void * joinRecords(const void * left, const void * right)
+    const void * joinRecords(const void * left, const void * right, unsigned counter)
     {
         // MORE - could share some code with lookup join
         if (cloneLeft)
@@ -18200,7 +18220,7 @@ public:
         try
         {
             RtlDynamicRowBuilder rowBuilder(rowAllocator);
-            unsigned outSize = helper.transform(rowBuilder, left, right);
+            unsigned outSize = helper.transform(rowBuilder, left, right, counter);
             if (outSize)
                 return rowBuilder.finalizeRowClear(outSize);
             else
@@ -18240,6 +18260,7 @@ public:
             left = input->nextInGroup();
             matchedLeft = false;
             countForLeft = keepLimit;
+            joinCounter = 0;
             if(left == NULL)
             {
                 eos = true;
@@ -18264,7 +18285,7 @@ public:
                     switch(activityKind)
                     {
                     case TAKalljoin:
-                        ret = joinRecords(left, defaultRight);
+                        ret = joinRecords(left, defaultRight, 0);
                         break;
                     case TAKalldenormalize:
                         ret = left;
@@ -18279,6 +18300,7 @@ public:
                     }
                 }
                 rightIndex = 0;
+                joinCounter = 0;
                 ReleaseRoxieRow(left);
                 left = NULL;
                 if(ret)
@@ -18294,6 +18316,7 @@ public:
                 left = input->nextInGroup();
                 matchedLeft = false;
                 countForLeft = keepLimit;
+                joinCounter = 0;
             }
             if(!left)
             {
@@ -18325,7 +18348,7 @@ public:
                         matchedLeft = true;
                         matchedRight.replace(true, rightIndex);
                         if(!exclude)
-                            ret = joinRecords(left, right);
+                            ret = joinRecords(left, right, ++joinCounter);
                     }
                     rightIndex++;
                     if(ret)
@@ -24422,7 +24445,7 @@ public:
 
     virtual bool needsAllocator() const { return true; }
 
-    unsigned doTransform(const void *left, const void *right, offset_t fpos_or_count, IException *except, const void **group)
+    unsigned doTransform(const void *left, const void *right, offset_t fpos_or_count, IException *except, const void **group, unsigned counter)
     {
         if (cloneLeft && !except)
         {
@@ -24437,7 +24460,7 @@ public:
         {   
             outSize = except ? helper.onFailTransform(rowBuilder, left, right, fpos_or_count, except) : 
                       (activityKind == TAKkeyeddenormalizegroup) ? helper.transform(rowBuilder, left, right, (unsigned) fpos_or_count, group) : 
-                      helper.transform(rowBuilder, left, right, fpos_or_count);
+                      helper.transform(rowBuilder, left, right, fpos_or_count, counter);
         }
         catch (IException *E)
         {
@@ -24476,7 +24499,7 @@ public:
                 {
                     except.setown(e);
                 }
-                added = doTransform(left, defaultRight, 0, except, NULL);
+                added = doTransform(left, defaultRight, 0, except, NULL, 0);
             }
         }
         else if (!matched || jg->candidateCount() > atMost)
@@ -24491,7 +24514,7 @@ public:
                 {
                 case TAKkeyedjoin:
                 case TAKkeyeddenormalizegroup:
-                    added = doTransform(left, defaultRight, 0, NULL, NULL);
+                    added = doTransform(left, defaultRight, 0, NULL, NULL, 0);
                     break;
                 case TAKkeyeddenormalize:
                     LinkRoxieRow(left);
@@ -24511,7 +24534,7 @@ public:
                 while (idx < matched)
                 {
                     const KeyedJoinHeader *rhs = jg->queryRow(idx);
-                    added += doTransform(left, &rhs->rhsdata, rhs->fpos, NULL, NULL);
+                    added += doTransform(left, &rhs->rhsdata, rhs->fpos, NULL, NULL, idx+1);
                     if (added==keepLimit)
                         break;
                     idx++;
@@ -24561,7 +24584,7 @@ public:
                         extractedRows.append((void *) &rhs->rhsdata);
                         idx++;
                     }
-                    added += doTransform(left, extractedRows.item(0), extractedRows.ordinality(), NULL, (const void * *)extractedRows.getArray());
+                    added += doTransform(left, extractedRows.item(0), extractedRows.ordinality(), NULL, (const void * *)extractedRows.getArray(), 0);
                 }
                 break;
             }

roxie/ccd/ccdstate.cpp

@@ -1201,13 +1201,20 @@ public:
 * look up queries that are received - this corresponds to the currently active package.
 *-----------------------------------------------------------------------------------------------*/
 
+static hash64_t hashXML(const IPropertyTree *tree)
+{
+    StringBuffer xml;
+    toXML(tree, xml, 0, XML_SortTags);
+    return rtlHash64Data(xml.length(), xml.str(), 877029);
+}
+
 class CRoxieQueryPackageManager : public CInterface
 {
 public:
     IMPLEMENT_IINTERFACE;
 
-    CRoxieQueryPackageManager(unsigned _numChannels, const char *_querySet, const IRoxiePackageMap *_packages)
-        : numChannels(_numChannels), packages(_packages), querySet(_querySet)
+    CRoxieQueryPackageManager(unsigned _numChannels, const char *_querySet, const IRoxiePackageMap *_packages, hash64_t _xmlHash)
+        : numChannels(_numChannels), packages(_packages), querySet(_querySet), xmlHash(_xmlHash)
     {
         queryHash = 0;
     }
@@ -1228,6 +1235,11 @@ public:
 
     virtual void load(bool forceReload) = 0;
 
+    bool matches(hash64_t _xmlHash, bool _active) const
+    {
+        return _xmlHash == xmlHash && _active==packages->isActive();
+    }
+
     virtual hash64_t getHash()
     {
         CriticalBlock b2(updateCrit);
@@ -1315,6 +1327,7 @@ public:
         serverManager->getAllQueryInfo(reply, full, slaveManagers, logctx);
     }
 protected:
+
     void reloadQueryManagers(CRoxieSlaveQuerySetManagerSet *newSlaveManagers, IRoxieQuerySetManager *newServerManager, hash64_t newHash)
     {
         Owned<CRoxieSlaveQuerySetManagerSet> oldSlaveManagers;
@@ -1339,6 +1352,7 @@ protected:
     Owned<const IRoxiePackageMap> packages;
     unsigned numChannels;
     hash64_t queryHash;
+    hash64_t xmlHash;
     StringAttr querySet;
 };
 
@@ -1369,8 +1383,8 @@ class CRoxieDaliQueryPackageManager : public CRoxieQueryPackageManager, implemen
 
 public:
     IMPLEMENT_IINTERFACE;
-    CRoxieDaliQueryPackageManager(unsigned _numChannels, const IRoxiePackageMap *_packages, const char *_querySet)
-        : CRoxieQueryPackageManager(_numChannels, _querySet, _packages)
+    CRoxieDaliQueryPackageManager(unsigned _numChannels, const IRoxiePackageMap *_packages, const char *_querySet, hash64_t _xmlHash)
+        : CRoxieQueryPackageManager(_numChannels, _querySet, _packages, _xmlHash)
     {
         daliHelper.setown(connectToDali());
     }
@@ -1415,7 +1429,7 @@ public:
     IMPLEMENT_IINTERFACE;
 
     CStandaloneQueryPackageManager(unsigned _numChannels, const char *_querySet, const IRoxiePackageMap *_packages, IPropertyTree *_standaloneDll)
-        : CRoxieQueryPackageManager(_numChannels, _querySet, _packages), standaloneDll(_standaloneDll)
+        : CRoxieQueryPackageManager(_numChannels, _querySet, _packages, 0), standaloneDll(_standaloneDll)
     {
         assertex(standaloneDll);
     }
@@ -1447,21 +1461,21 @@ extern IRoxieDebugSessionManager &queryRoxieDebugSessionManager()
     return *debugSessionManager;
 }
 
-class CRoxiePackageSetWatcher : public CInterface, implements ISDSSubscription
+class CRoxiePackageSetWatcher : public CInterface
 {
 public:
     IMPLEMENT_IINTERFACE;
-    CRoxiePackageSetWatcher(IRoxieDaliHelper *_daliHelper, ISDSSubscription *_owner, unsigned numChannels, bool forceReload)
-    : stateHash(0), daliHelper(_daliHelper), owner(_owner)
+    CRoxiePackageSetWatcher(IRoxieDaliHelper *_daliHelper, unsigned numChannels, CRoxiePackageSetWatcher *oldPackages, bool forceReload)
+    : stateHash(0), daliHelper(_daliHelper)
     {
         ForEachItemIn(idx, allQuerySetNames)
         {
-            createQueryPackageManagers(numChannels, allQuerySetNames.item(idx), forceReload);
+            createQueryPackageManagers(numChannels, allQuerySetNames.item(idx), oldPackages, forceReload);
         }
     }
 
-    CRoxiePackageSetWatcher(IRoxieDaliHelper *_daliHelper, ISDSSubscription *_owner, const IQueryDll *standAloneDll, unsigned numChannels, const char *querySet, bool forceReload)
-    : stateHash(0), daliHelper(_daliHelper), owner(_owner)
+    CRoxiePackageSetWatcher(IRoxieDaliHelper *_daliHelper, const IQueryDll *standAloneDll, unsigned numChannels, const char *querySet, bool forceReload)
+    : stateHash(0), daliHelper(_daliHelper)
     {
         Owned<IPropertyTree> standAloneDllTree;
         standAloneDllTree.setown(createPTree("Query"));
@@ -1473,16 +1487,6 @@ public:
         allQueryPackages.append(*qpm.getClear());
     }
 
-    ~CRoxiePackageSetWatcher()
-    {
-        unsubscribe();
-    }
-
-    virtual void notify(SubscriptionId id, const char *xpath, SDSNotifyFlags flags, unsigned valueLen, const void *valueData)
-    {
-        owner->notify(id, xpath, flags, valueLen, valueData);
-    }
-
     IQueryFactory *lookupLibrary(const char *libraryName, unsigned expectedInterfaceHash, const IRoxieContextLogger &logctx) const
     {
         ForEachItemIn(idx, allQueryPackages)
@@ -1554,8 +1558,11 @@ public:
         ForEachItemIn(idx, allQueryPackages)
         {
             Owned<IRoxieQuerySetManager> serverManager = allQueryPackages.item(idx).getRoxieServerManager();
-            Owned<IRoxieQuerySetManagerSet> slaveManagers = allQueryPackages.item(idx).getRoxieSlaveManagers();
-            serverManager->getAllQueryInfo(reply, full, slaveManagers, logctx);
+            if (serverManager->isActive())
+            {
+                Owned<IRoxieQuerySetManagerSet> slaveManagers = allQueryPackages.item(idx).getRoxieSlaveManagers();
+                serverManager->getAllQueryInfo(reply, full, slaveManagers, logctx);
+            }
         }
     }
 
@@ -1595,21 +1602,30 @@ public:
     }
 
 private:
-    ISDSSubscription *owner;
     CIArrayOf<CRoxieQueryPackageManager> allQueryPackages;
-    IArrayOf<IDaliPackageWatcher> notifiers;
     Linked<IRoxieDaliHelper> daliHelper;
     hash64_t stateHash;
 
-    void createQueryPackageManager(unsigned numChannels, const IRoxiePackageMap *packageMap, const char *querySet, bool forceReload)
+    CRoxieQueryPackageManager *getPackageManager(const char *id)
+    {
+        ForEachItemIn(idx, allQueryPackages)
+        {
+            CRoxieQueryPackageManager &pm = allQueryPackages.item(idx);
+            if (strcmp(pm.queryPackageId(), id)==0)
+                return LINK(&pm);
+        }
+        return NULL;
+    }
+
+    void createQueryPackageManager(unsigned numChannels, const IRoxiePackageMap *packageMap, const char *querySet, hash64_t xmlHash, bool forceReload)
     {
-        Owned<CRoxieQueryPackageManager> qpm = new CRoxieDaliQueryPackageManager(numChannels, packageMap, querySet);
+        Owned<CRoxieQueryPackageManager> qpm = new CRoxieDaliQueryPackageManager(numChannels, packageMap, querySet, xmlHash);
         qpm->load(forceReload);
         stateHash = rtlHash64Data(sizeof(stateHash), &stateHash, qpm->getHash());
         allQueryPackages.append(*qpm.getClear());
     }
 
-    void createQueryPackageManagers(unsigned numChannels, const char *querySet, bool forceReload)
+    void createQueryPackageManagers(unsigned numChannels, const char *querySet, CRoxiePackageSetWatcher *oldPackages, bool forceReload)
     {
         int loadedPackages = 0;
         int activePackages = 0;
@@ -1634,19 +1650,33 @@ private:
                     const char *packageMapFilter = pm.queryProp("@querySet");
                     if (packageMapId && *packageMapId && (!packageMapFilter || WildMatch(querySet, packageMapFilter, false)))
                     {
-                        bool isActive = pm.getPropBool("@active", true);
-                        if (traceLevel)
-                            DBGLOG("Loading package map %s, active %s", packageMapId, isActive ? "true" : "false");
                         try
                         {
-                            Owned<CRoxiePackageMap> packageMap = new CRoxiePackageMap(packageMapId, packageMapFilter, isActive);
+                            bool isActive = pm.getPropBool("@active", true);
                             Owned<IPropertyTree> xml = daliHelper->getPackageMap(packageMapId);
-                            packageMap->load(xml);
-                            createQueryPackageManager(numChannels, packageMap.getLink(), querySet, forceReload);
+                            hash64_t xmlHash = hashXML(xml);
+                            Owned<CRoxieQueryPackageManager> oldPackageManager;
+                            if (oldPackages)
+                            {
+                                oldPackageManager.setown(oldPackages->getPackageManager(packageMapId));
+                            }
+                            if (oldPackageManager && oldPackageManager->matches(xmlHash, isActive))
+                            {
+                                if (traceLevel)
+                                    DBGLOG("Package map %s, active %s already loaded", packageMapId, isActive ? "true" : "false");
+                                allQueryPackages.append(*oldPackageManager.getClear());
+                            }
+                            else
+                            {
+                                if (traceLevel)
+                                    DBGLOG("Loading package map %s, active %s", packageMapId, isActive ? "true" : "false");
+                                Owned<CRoxiePackageMap> packageMap = new CRoxiePackageMap(packageMapId, packageMapFilter, isActive);
+                                packageMap->load(xml);
+                                createQueryPackageManager(numChannels, packageMap.getLink(), querySet, xmlHash, forceReload);
+                            }
                             loadedPackages++;
                             if (isActive)
                                 activePackages++;
-                            notifiers.append(*daliHelper->getPackageMapSubscription(packageMapId, this));
                         }
                         catch (IException *E)
                         {
@@ -1663,25 +1693,18 @@ private:
         {
             if (traceLevel)
                 DBGLOG("Loading empty package for QuerySet %s", querySet);
-            createQueryPackageManager(numChannels, LINK(&queryEmptyRoxiePackageMap()), querySet, forceReload);
+            createQueryPackageManager(numChannels, LINK(&queryEmptyRoxiePackageMap()), querySet, 0, forceReload);
         }
         else if (traceLevel)
             DBGLOG("Loaded %d packages (%d active)", loadedPackages, activePackages);
     }
 
-    void unsubscribe()
-    {
-        ForEachItemIn(idx, notifiers)
-        {
-            daliHelper->releaseSubscription(&notifiers.item(idx));
-        }
-        notifiers.kill();
-    }
 };
 
 class CRoxiePackageSetManager : public CInterface, implements IRoxieQueryPackageManagerSet, implements ISDSSubscription
 {
-    Owned<IDaliPackageWatcher> notifier;
+    Owned<IDaliPackageWatcher> pSetsNotifier;
+    Owned<IDaliPackageWatcher> pMapsNotifier;
 public:
     IMPLEMENT_IINTERFACE;
     CRoxiePackageSetManager(const IQueryDll *_standAloneDll) :
@@ -1690,7 +1713,8 @@ public:
         daliHelper.setown(connectToDali(ROXIE_DALI_CONNECT_TIMEOUT));
         atomic_set(&autoPending, 0);
         autoReloadThread.start();
-        notifier.setown(daliHelper->getPackageSetsSubscription(this));
+        pSetsNotifier.setown(daliHelper->getPackageSetsSubscription(this));
+        pMapsNotifier.setown(daliHelper->getPackageMapsSubscription(this));
     }
 
     ~CRoxiePackageSetManager()
@@ -1796,6 +1820,9 @@ private:
             while (!closing)
             {
                 owner.autoReloadTrigger.wait();
+                if (closing)
+                    break;
+                Sleep(500); // Typically notifications come in clumps - this avoids reloading too often
                 if (atomic_read(&owner.autoPending))
                 {
                     atomic_set(&owner.autoPending, 0);
@@ -1834,9 +1861,9 @@ private:
         // So that the query/dll caching will work for anything that is not affected by the changes
         Owned<CRoxiePackageSetWatcher> newPackages;
         if (standAloneDll)
-            newPackages.setown(new CRoxiePackageSetWatcher(daliHelper, this, standAloneDll, numChannels, "roxie", forceRetry));
+            newPackages.setown(new CRoxiePackageSetWatcher(daliHelper, standAloneDll, numChannels, "roxie", forceRetry));
         else
-            newPackages.setown(new CRoxiePackageSetWatcher(daliHelper, this, numChannels, forceRetry));
+            newPackages.setown(new CRoxiePackageSetWatcher(daliHelper, numChannels, allQueryPackages, forceRetry));
         // Hold the lock for as little time as we can
         // Note that we must NOT hold the lock during the delete of the old object - or we deadlock.
         // Hence the slightly convoluted code below

roxie/ccd/ccdstate.hpp

@@ -149,4 +149,7 @@ extern const char *queryNodeIndexName(const IPropertyTree &graphNode, ThorActivi
 extern void createDelayedReleaser();
 extern void stopDelayedReleaser();
 
+extern void createDelayedReleaser();
+extern void stopDelayedReleaser();
+
 #endif

rtl/include/eclhelper.hpp

@@ -39,8 +39,8 @@ if the supplied pointer was not from the roxiemem heap. Usually an OwnedRoxieStr
 
 //Should be incremented whenever the virtuals in the context or a helper are changed, so
 //that a work unit can't be rerun.  Try as hard as possible to retain compatibility.
-#define ACTIVITY_INTERFACE_VERSION      154
-#define MIN_ACTIVITY_INTERFACE_VERSION  154             //minimum value that is compatible with current interface - without using selectInterface
+#define ACTIVITY_INTERFACE_VERSION      155
+#define MIN_ACTIVITY_INTERFACE_VERSION  155             //minimum value that is compatible with current interface - without using selectInterface
 
 typedef unsigned char byte;
 
@@ -1666,7 +1666,6 @@ struct IHThorAnyJoinBaseArg : public IHThorArg
     virtual unsigned getKeepLimit() = 0;
 
 //Join:
-    virtual size32_t transform(ARowBuilder & rowBuilder, const void * _left, const void * _right) { return 0; }
 //Denormalize
     virtual size32_t transform(ARowBuilder & rowBuilder, const void * _left, const void * _right, unsigned _count) { return 0; }
 //Denormalize group
@@ -1747,7 +1746,6 @@ struct IHThorKeyedJoinBaseArg : public IHThorArg
 
     virtual size32_t onFailTransform(ARowBuilder & rowBuilder, const void * _dummyRight, const void * _origRow, unsigned __int64 keyedFpos, IException * e) { return 0; }
 //Join:
-    virtual size32_t transform(ARowBuilder & rowBuilder, const void * _joinFields, const void * _origRow, unsigned __int64 keyedFpos) { return 0; }
 //Denormalize:
     virtual size32_t transform(ARowBuilder & rowBuilder, const void * _joinFields, const void * _origRow, unsigned __int64 keyedFpos, unsigned counter) { return 0; }
 //Denormalize group:

system/jlib/jbuff.cpp

@@ -376,6 +376,14 @@ void *MemoryBuffer::detach()
     return ret;
 }
 
+void *MemoryBuffer::detachOwn()
+{
+    assertex(ownBuffer);
+    void *ret = buffer;
+    init();
+    return ret;
+}
+
 void MemoryBuffer::setLength(unsigned len)
 {
     if (len > curLen)

system/jlib/jbuff.hpp

@@ -173,6 +173,7 @@ public:
     void            setLength(unsigned len);
     void            setWritePos(unsigned len);      // only use for back patching data
     void *          detach();
+    void *          detachOwn();  // Never reallocates
     //Non-standard functions:
     void *          reserve(unsigned size);
     void            truncate();                     // truncates (i.e. minimizes allocation) to current size

system/jlib/jptree.cpp

@@ -5224,13 +5224,13 @@ IPropertyTree *createPTreeFromXMLString(unsigned len, const char *xml, byte flag
 //////////////////////////
 /////////////////////////
 
-static void _toXML(const IPropertyTree *tree, IIOStream &out, unsigned indent, byte flags)
+static void _toXML(const IPropertyTree *tree, IIOStream &out, unsigned indent, unsigned flags)
 {
     const char *name = tree->queryName();
     if (!name) name = "__unnamed__";
     bool isBinary = tree->isBinary(NULL);
     bool inlinebody = true;
-    if (flags & XML_Format) writeCharsNToStream(out, ' ', indent);
+    if (flags & XML_Embed) writeCharsNToStream(out, ' ', indent);
     writeCharToStream(out, '<');
     writeStringToStream(out, name);
     Owned<IAttributeIterator> it = tree->getAttributes(true);
@@ -5247,11 +5247,11 @@ static void _toXML(const IPropertyTree *tree, IIOStream &out, unsigned indent, b
             {
                 if (first)
                 {
-                    if (flags & (XML_Format|XML_NewlinesOnly)) inlinebody = false;
+                    if (flags & XML_LineBreak) inlinebody = false;
                     first = false;
                     writeCharToStream(out, ' ');
                 }
-                else if ((flags & XML_Format) && it->count() > 3)
+                else if ((flags & XML_LineBreakAttributes) && it->count() > 3)
                 {
                     writeStringToStream(out, "\n");
                     writeCharsNToStream(out, ' ', attributeindent);
@@ -5295,7 +5295,7 @@ static void _toXML(const IPropertyTree *tree, IIOStream &out, unsigned indent, b
     bool empty;
     if (isBinary)
     {
-        if (flags & (XML_Format|XML_NewlinesOnly)) inlinebody = false;
+        if (flags & XML_LineBreak) inlinebody = false;
         writeStringToStream(out, " xsi:type=\"SOAP-ENC:base64\"");
         empty = (!tree->getPropBin(NULL, thislevelbin))||(thislevelbin.length()==0);
     }
@@ -5312,11 +5312,11 @@ static void _toXML(const IPropertyTree *tree, IIOStream &out, unsigned indent, b
     }
     if (sub->first())
     {
-        if (flags & (XML_Format|XML_NewlinesOnly)) inlinebody = false;
+        if (flags & XML_LineBreak) inlinebody = false;
     }
     else if (empty && !(flags & XML_Sanitize))
     {
-        if (flags & (XML_Format|XML_NewlinesOnly))
+        if (flags & XML_LineBreak)
             writeStringToStream(out, "/>\n");
         else
             writeStringToStream(out, "/>");
@@ -5403,13 +5403,13 @@ static void _toXML(const IPropertyTree *tree, IIOStream &out, unsigned indent, b
 
     writeStringToStream(out, "</");
     writeStringToStream(out, name);
-    if (flags & (XML_Format|XML_NewlinesOnly))
+    if (flags & XML_LineBreak)
         writeStringToStream(out, ">\n");
     else
         writeCharToStream(out, '>');
 }
 
-jlib_decl StringBuffer &toXML(const IPropertyTree *tree, StringBuffer &ret, unsigned indent, byte flags)
+jlib_decl StringBuffer &toXML(const IPropertyTree *tree, StringBuffer &ret, unsigned indent, unsigned flags)
 {
     class CAdapter : public CInterface, implements IIOStream
     {
@@ -5425,18 +5425,18 @@ jlib_decl StringBuffer &toXML(const IPropertyTree *tree, StringBuffer &ret, unsi
     return ret;
 }
 
-void toXML(const IPropertyTree *tree, IIOStream &out, unsigned indent, byte flags)
+void toXML(const IPropertyTree *tree, IIOStream &out, unsigned indent, unsigned flags)
 {
     _toXML(tree, out, indent, flags);
 }
 
-void saveXML(const char *filename, const IPropertyTree *tree, unsigned indent, byte flags)
+void saveXML(const char *filename, const IPropertyTree *tree, unsigned indent, unsigned flags)
 {
     OwnedIFile ifile = createIFile(filename);
     saveXML(*ifile, tree, indent, flags);
 }
 
-void saveXML(IFile &ifile, const IPropertyTree *tree, unsigned indent, byte flags)
+void saveXML(IFile &ifile, const IPropertyTree *tree, unsigned indent, unsigned flags)
 {
     OwnedIFileIO ifileio = ifile.open(IFOcreate);
     if (!ifileio)
@@ -5444,14 +5444,14 @@ void saveXML(IFile &ifile, const IPropertyTree *tree, unsigned indent, byte flag
     saveXML(*ifileio, tree, indent, flags);
 }
 
-void saveXML(IFileIO &ifileio, const IPropertyTree *tree, unsigned indent, byte flags)
+void saveXML(IFileIO &ifileio, const IPropertyTree *tree, unsigned indent, unsigned flags)
 {
     Owned<IIOStream> stream = createIOStream(&ifileio);
     stream.setown(createBufferedIOStream(stream));
     saveXML(*stream, tree, indent, flags);
 }
 
-void saveXML(IIOStream &stream, const IPropertyTree *tree, unsigned indent, byte flags)
+void saveXML(IIOStream &stream, const IPropertyTree *tree, unsigned indent, unsigned flags)
 {
     toXML(tree, stream, indent, flags);
 }

system/jlib/jptree.hpp

@@ -207,20 +207,22 @@ jlib_decl IPropertyTree *createPTreeFromJSONString(const char *json, byte flags=
 jlib_decl IPropertyTree *createPTreeFromJSONString(unsigned len, const char *json, byte flags=ipt_none, PTreeReaderOptions readFlags=ptr_ignoreWhiteSpace, IPTreeMaker *iMaker=NULL);
 
 #define XML_SortTags 0x01
-#define XML_Format   0x02
+#define XML_Embed    0x02
 #define XML_NoEncode 0x04
 #define XML_Sanitize 0x08
 #define XML_SanitizeAttributeValues 0x10
 #define XML_SingleQuoteAttributeValues 0x20
 #define XML_NoBinaryEncode64 0x40
-#define XML_NewlinesOnly 0x80
-
-jlib_decl StringBuffer &toXML(const IPropertyTree *tree, StringBuffer &ret, unsigned indent = 0, byte flags=XML_Format);
-jlib_decl void toXML(const IPropertyTree *tree, IIOStream &out, unsigned indent = 0, byte flags=XML_Format);
-jlib_decl void saveXML(const char *filename, const IPropertyTree *tree, unsigned indent = 0, byte flags=XML_Format);
-jlib_decl void saveXML(IFile &ifile, const IPropertyTree *tree, unsigned indent = 0, byte flags=XML_Format);
-jlib_decl void saveXML(IFileIO &ifileio, const IPropertyTree *tree, unsigned indent = 0, byte flags=XML_Format);
-jlib_decl void saveXML(IIOStream &stream, const IPropertyTree *tree, unsigned indent = 0, byte flags=XML_Format);
+#define XML_LineBreak 0x100
+#define XML_LineBreakAttributes 0x80
+#define XML_Format (XML_Embed|XML_LineBreakAttributes|XML_LineBreak)
+
+jlib_decl StringBuffer &toXML(const IPropertyTree *tree, StringBuffer &ret, unsigned indent = 0, unsigned flags=XML_Format);
+jlib_decl void toXML(const IPropertyTree *tree, IIOStream &out, unsigned indent = 0, unsigned flags=XML_Format);
+jlib_decl void saveXML(const char *filename, const IPropertyTree *tree, unsigned indent = 0, unsigned flags=XML_Format);
+jlib_decl void saveXML(IFile &ifile, const IPropertyTree *tree, unsigned indent = 0, unsigned=XML_Format);
+jlib_decl void saveXML(IFileIO &ifileio, const IPropertyTree *tree, unsigned indent = 0, unsigned flags=XML_Format);
+jlib_decl void saveXML(IIOStream &stream, const IPropertyTree *tree, unsigned indent = 0, unsigned flags=XML_Format);
 
 #define JSON_SortTags 0x01
 #define JSON_Format   0x02

testing/regress/ecl/joinattr2.ecl

@@ -25,6 +25,7 @@ END;
 RECORD
         string1 idL;
         string1 idR;
+        unsigned cnt;
 END;
 
  
@@ -40,69 +41,72 @@ ds8 := DATASET([{1,'A'}, {1,'B'}, {1,'C'}], rec);
 ds9 := DATASET([{2,'A'}], rec);
 ds10 := DATASET([{0,'A'}], rec);
 
-recout trans(rec L, rec R) :=
+recout trans(rec L, rec R, unsigned cnt) :=
 TRANSFORM
 
             SELF.idl := L.id;
             SELF.idr := R.id;
+            SELF.cnt := cnt;
 END;
 
-recout transkip(rec L, rec R) :=
+recout transkip(rec L, rec R, unsigned cnt) :=
 TRANSFORM
 
             SELF.idl := if(L.id='A',skip,L.id);
             SELF.idr := if(R.id='A',skip,R.id);
+            SELF.cnt := cnt;
 END;
 
-j1 := JOIN(ds1, ds2, LEFT.i = RIGHT.i, trans(LEFT, RIGHT), KEEP(1), LEFT OUTER);
-j2 := JOIN(ds1, ds2, LEFT.i = RIGHT.i, trans(LEFT, RIGHT), ATMOST(1), LEFT OUTER);
-j3 := JOIN(ds2, ds1, LEFT.i = RIGHT.i, trans(LEFT, RIGHT), RIGHT OUTER);
-j4 := JOIN(ds1, ds7, (LEFT.i = RIGHT.i), transkip(LEFT, RIGHT));
-j5 := JOIN(ds1, ds2, LEFT.i = RIGHT.i, trans(LEFT, RIGHT), LEFT OUTER);
-j6 := JOIN(ds2, ds1, LEFT.i = RIGHT.i, trans(LEFT, RIGHT), RIGHT OUTER);
-j7 := JOIN(ds4, ds3, (LEFT.i = RIGHT.i)and(RIGHT.id>'E'), trans(LEFT, RIGHT), KEEP(2));
-j8 := JOIN(ds1, ds3, (LEFT.i = RIGHT.i), trans(LEFT, RIGHT), KEEP(2),LEFT OUTER);
-j9 := JOIN(ds6, ds7, (LEFT.i = RIGHT.i), trans(LEFT, RIGHT), KEEP(5));
-j10 := JOIN(ds6, ds7, (LEFT.i = RIGHT.i), trans(LEFT, RIGHT), RIGHT OUTER);
-j11 := JOIN(ds1, ds7, (LEFT.i = RIGHT.i), transkip(LEFT, RIGHT));
-j12 := JOIN(ds1, ds8, LEFT.i = RIGHT.i, transkip(LEFT, RIGHT), RIGHT OUTER);
-j13 := JOIN(ds1, ds3, LEFT.i = RIGHT.i, trans(LEFT, RIGHT), RIGHT OUTER);
-j14 := JOIN(ds1, ds8, (LEFT.i = RIGHT.i)and(RIGHT.id>'E'), transkip(LEFT, RIGHT), KEEP(2));
-j15 := JOIN(ds1, ds1, (LEFT.i = RIGHT.i), trans(LEFT, RIGHT), KEEP(1));
-j16 := JOIN(ds1, ds8, (LEFT.i = RIGHT.i), trans(LEFT, RIGHT), KEEP(1));
-j17 := JOIN(ds1, ds8, (LEFT.i = RIGHT.i), trans(LEFT, RIGHT), LOOKUP);
-j18 := JOIN(ds1, ds8, (LEFT.i = RIGHT.i), trans(LEFT, RIGHT), ATMOST(1));
-j19 := JOIN(ds7, ds6, (LEFT.i = RIGHT.i)and(LEFT.id<=RIGHT.id), trans(LEFT, RIGHT), LEFT OUTER, LOOKUP);
-j20 := JOIN(ds1, ds9, LEFT.i = RIGHT.i, trans(LEFT, RIGHT), KEEP(1), LEFT OUTER);
-j21 := JOIN(ds1, ds9, LEFT.i = RIGHT.i, trans(LEFT, RIGHT), LOOKUP, LEFT OUTER);
-j22 := JOIN(ds1, ds9, LEFT.i = RIGHT.i, trans(LEFT, RIGHT), ATMOST(1), LEFT OUTER);
-j23 := JOIN(ds1, ds9, LEFT.i = RIGHT.i, trans(LEFT, RIGHT), RIGHT OUTER);
-j24 := JOIN(ds1, ds9, (LEFT.i = RIGHT.i)and(RIGHT.id>'E'), trans(LEFT, RIGHT), KEEP(2));
-j25 := JOIN(ds9, ds1, LEFT.i = RIGHT.i, trans(LEFT, RIGHT), KEEP(1), LEFT OUTER);
-j26 := JOIN(ds9, ds1, LEFT.i = RIGHT.i, trans(LEFT, RIGHT), LOOKUP, LEFT OUTER);
-j27 := JOIN(ds9, ds1, LEFT.i = RIGHT.i, trans(LEFT, RIGHT), ATMOST(1), LEFT OUTER);
-j28 := JOIN(ds9, ds1, LEFT.i = RIGHT.i, trans(LEFT, RIGHT), RIGHT OUTER);
-j29 := JOIN(ds9, ds1, (LEFT.i = RIGHT.i)and(RIGHT.id>'E'), trans(LEFT, RIGHT), KEEP(2));
-j30 := JOIN(ds1, ds8, LEFT.i = RIGHT.i, transkip(LEFT, RIGHT), KEEP(1), LEFT OUTER);
-j31 := JOIN(ds1, ds8, LEFT.i = RIGHT.i, trans(LEFT, RIGHT), LOOKUP);
-j32 := JOIN(ds1, ds1, LEFT.i = RIGHT.i, transkip(LEFT, RIGHT), LOOKUP);
-j33 := JOIN(ds1, ds8, LEFT.i = RIGHT.i, trans(LEFT, RIGHT), LOOKUP, LEFT OUTER);
-j34 := JOIN(ds1, ds8, LEFT.i = RIGHT.i, transkip(LEFT, RIGHT), LOOKUP, LEFT OUTER);
-j35 := JOIN(ds1, ds8, (LEFT.i = RIGHT.i)and(RIGHT.id>'E'), transkip(LEFT, RIGHT), LOOKUP);
-j36 := JOIN(ds1, ds8, (LEFT.i = RIGHT.i)and(RIGHT.id>'E'), transkip(LEFT, RIGHT), LOOKUP, LEFT OUTER);
-j37 := JOIN(ds1, ds8, LEFT.i = RIGHT.i, transkip(LEFT, RIGHT), LOOKUP, LEFT OUTER);
-j38 := JOIN(ds1, ds8, LEFT.i = RIGHT.i, transkip(LEFT, RIGHT), KEEP(1), LEFT OUTER);
-j39 := JOIN(ds1, ds8, LEFT.i = RIGHT.i, transkip(LEFT, RIGHT), LOOKUP, LEFT OUTER);
-j40 := JOIN(ds1, ds8, LEFT.i = RIGHT.i, transkip(LEFT, RIGHT), ATMOST(1), LEFT OUTER);
-
-j41 := JOIN(ds1, ds8, (LEFT.i = RIGHT.i), trans(LEFT, RIGHT), MANY LOOKUP);
-j42 := JOIN(ds1, ds8, (LEFT.i = RIGHT.i), trans(LEFT, RIGHT), MANY LOOKUP, KEEP(1));
-j43 := JOIN(ds1, ds8, (LEFT.i = RIGHT.i), trans(LEFT, RIGHT), MANY LOOKUP, ATMOST(1));
-j44 := JOIN(ds1, ds8, (LEFT.i = RIGHT.i), trans(LEFT, RIGHT), MANY LOOKUP, LEFT OUTER);
-j45 := JOIN(ds1, ds8, RIGHT.id<'B', trans(LEFT, RIGHT), ALL);
-j46 := JOIN(ds1, ds8, RIGHT.id<'B', trans(LEFT, RIGHT), ALL, KEEP(1));
-j47 := JOIN(ds1, ds8, RIGHT.id>'E', trans(LEFT, RIGHT), ALL, LEFT OUTER);
-j48 := JOIN(ds1, ds8, LEFT.i = RIGHT.i AND RIGHT.id>'B', trans(LEFT, RIGHT), LOOKUP);
+j1 := JOIN(ds1, ds2, LEFT.i = RIGHT.i, trans(LEFT, RIGHT, COUNTER), KEEP(1), LEFT OUTER);
+j2 := JOIN(ds1, ds2, LEFT.i = RIGHT.i, trans(LEFT, RIGHT, COUNTER), ATMOST(1), LEFT OUTER);
+j3 := JOIN(ds2, ds1, LEFT.i = RIGHT.i, trans(LEFT, RIGHT, COUNTER), RIGHT OUTER);
+j4 := JOIN(ds1, ds7, (LEFT.i = RIGHT.i), transkip(LEFT, RIGHT, COUNTER));
+j5 := JOIN(ds1, ds2, LEFT.i = RIGHT.i, trans(LEFT, RIGHT, COUNTER), LEFT OUTER);
+j6 := JOIN(ds2, ds1, LEFT.i = RIGHT.i, trans(LEFT, RIGHT, COUNTER), RIGHT OUTER);
+j7 := JOIN(ds4, ds3, (LEFT.i = RIGHT.i)and(RIGHT.id>'E'), trans(LEFT, RIGHT, COUNTER), KEEP(2));
+j8 := JOIN(ds1, ds3, (LEFT.i = RIGHT.i), trans(LEFT, RIGHT, COUNTER), KEEP(2),LEFT OUTER);
+j9 := JOIN(ds6, ds7, (LEFT.i = RIGHT.i), trans(LEFT, RIGHT, COUNTER), KEEP(5));
+j10 := JOIN(ds6, ds7, (LEFT.i = RIGHT.i), trans(LEFT, RIGHT, COUNTER), RIGHT OUTER);
+j11 := JOIN(ds1, ds7, (LEFT.i = RIGHT.i), transkip(LEFT, RIGHT, COUNTER));
+j12 := JOIN(ds1, ds8, LEFT.i = RIGHT.i, transkip(LEFT, RIGHT, COUNTER), RIGHT OUTER);
+j13 := JOIN(ds1, ds3, LEFT.i = RIGHT.i, trans(LEFT, RIGHT, COUNTER), RIGHT OUTER);
+j14 := JOIN(ds1, ds8, (LEFT.i = RIGHT.i)and(RIGHT.id>'E'), transkip(LEFT, RIGHT, COUNTER), KEEP(2));
+j15 := JOIN(ds1, ds1, (LEFT.i = RIGHT.i), trans(LEFT, RIGHT, COUNTER), KEEP(1));
+j16 := JOIN(ds1, ds8, (LEFT.i = RIGHT.i), trans(LEFT, RIGHT, COUNTER), KEEP(1));
+j17 := JOIN(ds1, ds8, (LEFT.i = RIGHT.i), trans(LEFT, RIGHT, COUNTER), LOOKUP);
+j18 := JOIN(ds1, ds8, (LEFT.i = RIGHT.i), trans(LEFT, RIGHT, COUNTER), ATMOST(1));
+j19 := JOIN(ds7, ds6, (LEFT.i = RIGHT.i)and(LEFT.id<=RIGHT.id), trans(LEFT, RIGHT, COUNTER), LEFT OUTER, LOOKUP);
+j20 := JOIN(ds1, ds9, LEFT.i = RIGHT.i, trans(LEFT, RIGHT, COUNTER), KEEP(1), LEFT OUTER);
+j21 := JOIN(ds1, ds9, LEFT.i = RIGHT.i, trans(LEFT, RIGHT, COUNTER), LOOKUP, LEFT OUTER);
+j22 := JOIN(ds1, ds9, LEFT.i = RIGHT.i, trans(LEFT, RIGHT, COUNTER), ATMOST(1), LEFT OUTER);
+j23 := JOIN(ds1, ds9, LEFT.i = RIGHT.i, trans(LEFT, RIGHT, COUNTER), RIGHT OUTER);
+j24 := JOIN(ds1, ds9, (LEFT.i = RIGHT.i)and(RIGHT.id>'E'), trans(LEFT, RIGHT, COUNTER), KEEP(2));
+j25 := JOIN(ds9, ds1, LEFT.i = RIGHT.i, trans(LEFT, RIGHT, COUNTER), KEEP(1), LEFT OUTER);
+j26 := JOIN(ds9, ds1, LEFT.i = RIGHT.i, trans(LEFT, RIGHT, COUNTER), LOOKUP, LEFT OUTER);
+j27 := JOIN(ds9, ds1, LEFT.i = RIGHT.i, trans(LEFT, RIGHT, COUNTER), ATMOST(1), LEFT OUTER);
+j28 := JOIN(ds9, ds1, LEFT.i = RIGHT.i, trans(LEFT, RIGHT, COUNTER), RIGHT OUTER);
+j29 := JOIN(ds9, ds1, (LEFT.i = RIGHT.i)and(RIGHT.id>'E'), trans(LEFT, RIGHT, COUNTER), KEEP(2));
+j30 := JOIN(ds1, ds8, LEFT.i = RIGHT.i, transkip(LEFT, RIGHT, COUNTER), KEEP(1), LEFT OUTER);
+j31 := JOIN(ds1, ds8, LEFT.i = RIGHT.i, trans(LEFT, RIGHT, COUNTER), LOOKUP);
+j32 := JOIN(ds1, ds1, LEFT.i = RIGHT.i, transkip(LEFT, RIGHT, COUNTER), LOOKUP);
+j33 := JOIN(ds1, ds8, LEFT.i = RIGHT.i, trans(LEFT, RIGHT, COUNTER), LOOKUP, LEFT OUTER);
+j34 := JOIN(ds1, ds8, LEFT.i = RIGHT.i, transkip(LEFT, RIGHT, COUNTER), LOOKUP, LEFT OUTER);
+j35 := JOIN(ds1, ds8, (LEFT.i = RIGHT.i)and(RIGHT.id>'E'), transkip(LEFT, RIGHT, COUNTER), LOOKUP);
+j36 := JOIN(ds1, ds8, (LEFT.i = RIGHT.i)and(RIGHT.id>'E'), transkip(LEFT, RIGHT, COUNTER), LOOKUP, LEFT OUTER);
+j37 := JOIN(ds1, ds8, LEFT.i = RIGHT.i, transkip(LEFT, RIGHT, COUNTER), LOOKUP, LEFT OUTER);
+j38 := JOIN(ds1, ds8, LEFT.i = RIGHT.i, transkip(LEFT, RIGHT, COUNTER), KEEP(1), LEFT OUTER);
+j39 := JOIN(ds1, ds8, LEFT.i = RIGHT.i, transkip(LEFT, RIGHT, COUNTER), LOOKUP, LEFT OUTER);
+j40 := JOIN(ds1, ds8, LEFT.i = RIGHT.i, transkip(LEFT, RIGHT, COUNTER), ATMOST(1), LEFT OUTER);
+
+j41 := JOIN(ds1, ds8, (LEFT.i = RIGHT.i), trans(LEFT, RIGHT, COUNTER), MANY LOOKUP);
+j42 := JOIN(ds1, ds8, (LEFT.i = RIGHT.i), trans(LEFT, RIGHT, COUNTER), MANY LOOKUP, KEEP(1));
+j43 := JOIN(ds1, ds8, (LEFT.i = RIGHT.i), trans(LEFT, RIGHT, COUNTER), MANY LOOKUP, ATMOST(1));
+j44 := JOIN(ds1, ds8, (LEFT.i = RIGHT.i), trans(LEFT, RIGHT, COUNTER), MANY LOOKUP, LEFT OUTER);
+j45 := JOIN(ds1, ds8, RIGHT.id<'B', trans(LEFT, RIGHT, COUNTER), ALL);
+j46 := JOIN(ds1, ds8, RIGHT.id<'B', trans(LEFT, RIGHT, COUNTER), ALL, KEEP(1));
+j47 := JOIN(ds1, ds8, RIGHT.id>'E', trans(LEFT, RIGHT, COUNTER), ALL, LEFT OUTER);
+j48 := JOIN(ds1, ds8, LEFT.i = RIGHT.i AND RIGHT.id>'B', trans(LEFT, RIGHT, COUNTER), LOOKUP);
+j49 := JOIN(ds9, ds8, LEFT.i = RIGHT.i, transkip(LEFT, RIGHT, COUNTER), RIGHT ONLY);
 
 sequential(
     output(j1),
@@ -152,6 +156,6 @@ sequential(
     output(j45),
     output(j46),
     output(j47),
-    output(j48)
+    output(j48),
+    output(j49),
 );
-

testing/regress/ecl/key/joinattr2.xml

@@ -1,239 +1,243 @@
 <Dataset name='Result 1'>
- <Row><idl>A</idl><idr>D</idr></Row>
- <Row><idl>B</idl><idr>D</idr></Row>
- <Row><idl>C</idl><idr>D</idr></Row>
+ <Row><idl>A</idl><idr>D</idr><cnt>1</cnt></Row>
+ <Row><idl>B</idl><idr>D</idr><cnt>1</cnt></Row>
+ <Row><idl>C</idl><idr>D</idr><cnt>1</cnt></Row>
 </Dataset>
 <Dataset name='Result 2'>
- <Row><idl>A</idl><idr>D</idr></Row>
- <Row><idl>B</idl><idr>D</idr></Row>
- <Row><idl>C</idl><idr>D</idr></Row>
+ <Row><idl>A</idl><idr>D</idr><cnt>1</cnt></Row>
+ <Row><idl>B</idl><idr>D</idr><cnt>1</cnt></Row>
+ <Row><idl>C</idl><idr>D</idr><cnt>1</cnt></Row>
 </Dataset>
 <Dataset name='Result 3'>
- <Row><idl>D</idl><idr>A</idr></Row>
- <Row><idl>D</idl><idr>B</idr></Row>
- <Row><idl>D</idl><idr>C</idr></Row>
+ <Row><idl>D</idl><idr>A</idr><cnt>1</cnt></Row>
+ <Row><idl>D</idl><idr>B</idr><cnt>2</cnt></Row>
+ <Row><idl>D</idl><idr>C</idr><cnt>3</cnt></Row>
 </Dataset>
 <Dataset name='Result 4'>
- <Row><idl>B</idl><idr>C</idr></Row>
- <Row><idl>B</idl><idr>D</idr></Row>
- <Row><idl>B</idl><idr>E</idr></Row>
- <Row><idl>C</idl><idr>C</idr></Row>
- <Row><idl>C</idl><idr>D</idr></Row>
- <Row><idl>C</idl><idr>E</idr></Row>
+ <Row><idl>B</idl><idr>C</idr><cnt>1</cnt></Row>
+ <Row><idl>B</idl><idr>D</idr><cnt>2</cnt></Row>
+ <Row><idl>B</idl><idr>E</idr><cnt>3</cnt></Row>
+ <Row><idl>C</idl><idr>C</idr><cnt>1</cnt></Row>
+ <Row><idl>C</idl><idr>D</idr><cnt>2</cnt></Row>
+ <Row><idl>C</idl><idr>E</idr><cnt>3</cnt></Row>
 </Dataset>
 <Dataset name='Result 5'>
- <Row><idl>A</idl><idr>D</idr></Row>
- <Row><idl>B</idl><idr>D</idr></Row>
- <Row><idl>C</idl><idr>D</idr></Row>
+ <Row><idl>A</idl><idr>D</idr><cnt>1</cnt></Row>
+ <Row><idl>B</idl><idr>D</idr><cnt>1</cnt></Row>
+ <Row><idl>C</idl><idr>D</idr><cnt>1</cnt></Row>
 </Dataset>
 <Dataset name='Result 6'>
- <Row><idl>D</idl><idr>A</idr></Row>
- <Row><idl>D</idl><idr>B</idr></Row>
- <Row><idl>D</idl><idr>C</idr></Row>
+ <Row><idl>D</idl><idr>A</idr><cnt>1</cnt></Row>
+ <Row><idl>D</idl><idr>B</idr><cnt>2</cnt></Row>
+ <Row><idl>D</idl><idr>C</idr><cnt>3</cnt></Row>
 </Dataset>
 <Dataset name='Result 7'>
- <Row><idl>A</idl><idr>F</idr></Row>
- <Row><idl>A</idl><idr>G</idr></Row>
+ <Row><idl>A</idl><idr>F</idr><cnt>1</cnt></Row>
+ <Row><idl>A</idl><idr>G</idr><cnt>2</cnt></Row>
 </Dataset>
 <Dataset name='Result 8'>
- <Row><idl>A</idl><idr>E</idr></Row>
- <Row><idl>A</idl><idr>F</idr></Row>
- <Row><idl>B</idl><idr>E</idr></Row>
- <Row><idl>B</idl><idr>F</idr></Row>
- <Row><idl>C</idl><idr>E</idr></Row>
- <Row><idl>C</idl><idr>F</idr></Row>
+ <Row><idl>A</idl><idr>E</idr><cnt>1</cnt></Row>
+ <Row><idl>A</idl><idr>F</idr><cnt>2</cnt></Row>
+ <Row><idl>B</idl><idr>E</idr><cnt>1</cnt></Row>
+ <Row><idl>B</idl><idr>F</idr><cnt>2</cnt></Row>
+ <Row><idl>C</idl><idr>E</idr><cnt>1</cnt></Row>
+ <Row><idl>C</idl><idr>F</idr><cnt>2</cnt></Row>
 </Dataset>
 <Dataset name='Result 9'>
- <Row><idl>A</idl><idr>C</idr></Row>
- <Row><idl>A</idl><idr>D</idr></Row>
- <Row><idl>A</idl><idr>E</idr></Row>
- <Row><idl>B</idl><idr>C</idr></Row>
- <Row><idl>B</idl><idr>D</idr></Row>
- <Row><idl>B</idl><idr>E</idr></Row>
+ <Row><idl>A</idl><idr>C</idr><cnt>1</cnt></Row>
+ <Row><idl>A</idl><idr>D</idr><cnt>2</cnt></Row>
+ <Row><idl>A</idl><idr>E</idr><cnt>3</cnt></Row>
+ <Row><idl>B</idl><idr>C</idr><cnt>1</cnt></Row>
+ <Row><idl>B</idl><idr>D</idr><cnt>2</cnt></Row>
+ <Row><idl>B</idl><idr>E</idr><cnt>3</cnt></Row>
 </Dataset>
 <Dataset name='Result 10'>
- <Row><idl>A</idl><idr>C</idr></Row>
- <Row><idl>A</idl><idr>D</idr></Row>
- <Row><idl>A</idl><idr>E</idr></Row>
- <Row><idl>B</idl><idr>C</idr></Row>
- <Row><idl>B</idl><idr>D</idr></Row>
- <Row><idl>B</idl><idr>E</idr></Row>
+ <Row><idl>A</idl><idr>C</idr><cnt>1</cnt></Row>
+ <Row><idl>A</idl><idr>D</idr><cnt>2</cnt></Row>
+ <Row><idl>A</idl><idr>E</idr><cnt>3</cnt></Row>
+ <Row><idl>B</idl><idr>C</idr><cnt>1</cnt></Row>
+ <Row><idl>B</idl><idr>D</idr><cnt>2</cnt></Row>
+ <Row><idl>B</idl><idr>E</idr><cnt>3</cnt></Row>
 </Dataset>
 <Dataset name='Result 11'>
- <Row><idl>B</idl><idr>C</idr></Row>
- <Row><idl>B</idl><idr>D</idr></Row>
- <Row><idl>B</idl><idr>E</idr></Row>
- <Row><idl>C</idl><idr>C</idr></Row>
- <Row><idl>C</idl><idr>D</idr></Row>
- <Row><idl>C</idl><idr>E</idr></Row>
+ <Row><idl>B</idl><idr>C</idr><cnt>1</cnt></Row>
+ <Row><idl>B</idl><idr>D</idr><cnt>2</cnt></Row>
+ <Row><idl>B</idl><idr>E</idr><cnt>3</cnt></Row>
+ <Row><idl>C</idl><idr>C</idr><cnt>1</cnt></Row>
+ <Row><idl>C</idl><idr>D</idr><cnt>2</cnt></Row>
+ <Row><idl>C</idl><idr>E</idr><cnt>3</cnt></Row>
 </Dataset>
 <Dataset name='Result 12'>
- <Row><idl>B</idl><idr>B</idr></Row>
- <Row><idl>B</idl><idr>C</idr></Row>
- <Row><idl>C</idl><idr>B</idr></Row>
- <Row><idl>C</idl><idr>C</idr></Row>
+ <Row><idl>B</idl><idr>B</idr><cnt>2</cnt></Row>
+ <Row><idl>B</idl><idr>C</idr><cnt>3</cnt></Row>
+ <Row><idl>C</idl><idr>B</idr><cnt>2</cnt></Row>
+ <Row><idl>C</idl><idr>C</idr><cnt>3</cnt></Row>
 </Dataset>
 <Dataset name='Result 13'>
- <Row><idl>A</idl><idr>E</idr></Row>
- <Row><idl>A</idl><idr>F</idr></Row>
- <Row><idl>A</idl><idr>G</idr></Row>
- <Row><idl>B</idl><idr>E</idr></Row>
- <Row><idl>B</idl><idr>F</idr></Row>
- <Row><idl>B</idl><idr>G</idr></Row>
- <Row><idl>C</idl><idr>E</idr></Row>
- <Row><idl>C</idl><idr>F</idr></Row>
- <Row><idl>C</idl><idr>G</idr></Row>
+ <Row><idl>A</idl><idr>E</idr><cnt>1</cnt></Row>
+ <Row><idl>A</idl><idr>F</idr><cnt>2</cnt></Row>
+ <Row><idl>A</idl><idr>G</idr><cnt>3</cnt></Row>
+ <Row><idl>B</idl><idr>E</idr><cnt>1</cnt></Row>
+ <Row><idl>B</idl><idr>F</idr><cnt>2</cnt></Row>
+ <Row><idl>B</idl><idr>G</idr><cnt>3</cnt></Row>
+ <Row><idl>C</idl><idr>E</idr><cnt>1</cnt></Row>
+ <Row><idl>C</idl><idr>F</idr><cnt>2</cnt></Row>
+ <Row><idl>C</idl><idr>G</idr><cnt>3</cnt></Row>
 </Dataset>
 <Dataset name='Result 14'>
 </Dataset>
 <Dataset name='Result 15'>
- <Row><idl>A</idl><idr>A</idr></Row>
- <Row><idl>B</idl><idr>A</idr></Row>
- <Row><idl>C</idl><idr>A</idr></Row>
+ <Row><idl>A</idl><idr>A</idr><cnt>1</cnt></Row>
+ <Row><idl>B</idl><idr>A</idr><cnt>1</cnt></Row>
+ <Row><idl>C</idl><idr>A</idr><cnt>1</cnt></Row>
 </Dataset>
 <Dataset name='Result 16'>
- <Row><idl>A</idl><idr>A</idr></Row>
- <Row><idl>B</idl><idr>A</idr></Row>
- <Row><idl>C</idl><idr>A</idr></Row>
+ <Row><idl>A</idl><idr>A</idr><cnt>1</cnt></Row>
+ <Row><idl>B</idl><idr>A</idr><cnt>1</cnt></Row>
+ <Row><idl>C</idl><idr>A</idr><cnt>1</cnt></Row>
 </Dataset>
 <Dataset name='Result 17'>
- <Row><idl>A</idl><idr>A</idr></Row>
- <Row><idl>B</idl><idr>A</idr></Row>
- <Row><idl>C</idl><idr>A</idr></Row>
+ <Row><idl>A</idl><idr>A</idr><cnt>1</cnt></Row>
+ <Row><idl>B</idl><idr>A</idr><cnt>1</cnt></Row>
+ <Row><idl>C</idl><idr>A</idr><cnt>1</cnt></Row>
 </Dataset>
 <Dataset name='Result 18'>
 </Dataset>
 <Dataset name='Result 19'>
- <Row><idl>C</idl><idr> </idr></Row>
- <Row><idl>D</idl><idr> </idr></Row>
- <Row><idl>E</idl><idr> </idr></Row>
+ <Row><idl>C</idl><idr> </idr><cnt>0</cnt></Row>
+ <Row><idl>D</idl><idr> </idr><cnt>0</cnt></Row>
+ <Row><idl>E</idl><idr> </idr><cnt>0</cnt></Row>
 </Dataset>
 <Dataset name='Result 20'>
- <Row><idl>A</idl><idr> </idr></Row>
- <Row><idl>B</idl><idr> </idr></Row>
- <Row><idl>C</idl><idr> </idr></Row>
+ <Row><idl>A</idl><idr> </idr><cnt>0</cnt></Row>
+ <Row><idl>B</idl><idr> </idr><cnt>0</cnt></Row>
+ <Row><idl>C</idl><idr> </idr><cnt>0</cnt></Row>
 </Dataset>
 <Dataset name='Result 21'>
- <Row><idl>A</idl><idr> </idr></Row>
- <Row><idl>B</idl><idr> </idr></Row>
- <Row><idl>C</idl><idr> </idr></Row>
+ <Row><idl>A</idl><idr> </idr><cnt>0</cnt></Row>
+ <Row><idl>B</idl><idr> </idr><cnt>0</cnt></Row>
+ <Row><idl>C</idl><idr> </idr><cnt>0</cnt></Row>
 </Dataset>
 <Dataset name='Result 22'>
- <Row><idl>A</idl><idr> </idr></Row>
- <Row><idl>B</idl><idr> </idr></Row>
- <Row><idl>C</idl><idr> </idr></Row>
+ <Row><idl>A</idl><idr> </idr><cnt>0</cnt></Row>
+ <Row><idl>B</idl><idr> </idr><cnt>0</cnt></Row>
+ <Row><idl>C</idl><idr> </idr><cnt>0</cnt></Row>
 </Dataset>
 <Dataset name='Result 23'>
- <Row><idl> </idl><idr>A</idr></Row>
+ <Row><idl> </idl><idr>A</idr><cnt>0</cnt></Row>
 </Dataset>
 <Dataset name='Result 24'>
 </Dataset>
 <Dataset name='Result 25'>
- <Row><idl>A</idl><idr> </idr></Row>
+ <Row><idl>A</idl><idr> </idr><cnt>0</cnt></Row>
 </Dataset>
 <Dataset name='Result 26'>
- <Row><idl>A</idl><idr> </idr></Row>
+ <Row><idl>A</idl><idr> </idr><cnt>0</cnt></Row>
 </Dataset>
 <Dataset name='Result 27'>
- <Row><idl>A</idl><idr> </idr></Row>
+ <Row><idl>A</idl><idr> </idr><cnt>0</cnt></Row>
 </Dataset>
 <Dataset name='Result 28'>
- <Row><idl> </idl><idr>A</idr></Row>
- <Row><idl> </idl><idr>B</idr></Row>
- <Row><idl> </idl><idr>C</idr></Row>
+ <Row><idl> </idl><idr>A</idr><cnt>0</cnt></Row>
+ <Row><idl> </idl><idr>B</idr><cnt>0</cnt></Row>
+ <Row><idl> </idl><idr>C</idr><cnt>0</cnt></Row>
 </Dataset>
 <Dataset name='Result 29'>
 </Dataset>
 <Dataset name='Result 30'>
- <Row><idl>B</idl><idr>B</idr></Row>
- <Row><idl>C</idl><idr>B</idr></Row>
+ <Row><idl>B</idl><idr>B</idr><cnt>2</cnt></Row>
+ <Row><idl>C</idl><idr>B</idr><cnt>2</cnt></Row>
 </Dataset>
 <Dataset name='Result 31'>
- <Row><idl>A</idl><idr>A</idr></Row>
- <Row><idl>B</idl><idr>A</idr></Row>
- <Row><idl>C</idl><idr>A</idr></Row>
+ <Row><idl>A</idl><idr>A</idr><cnt>1</cnt></Row>
+ <Row><idl>B</idl><idr>A</idr><cnt>1</cnt></Row>
+ <Row><idl>C</idl><idr>A</idr><cnt>1</cnt></Row>
 </Dataset>
 <Dataset name='Result 32'>
- <Row><idl>B</idl><idr>B</idr></Row>
- <Row><idl>C</idl><idr>B</idr></Row>
+ <Row><idl>B</idl><idr>B</idr><cnt>2</cnt></Row>
+ <Row><idl>C</idl><idr>B</idr><cnt>2</cnt></Row>
 </Dataset>
 <Dataset name='Result 33'>
- <Row><idl>A</idl><idr>A</idr></Row>
- <Row><idl>B</idl><idr>A</idr></Row>
- <Row><idl>C</idl><idr>A</idr></Row>
+ <Row><idl>A</idl><idr>A</idr><cnt>1</cnt></Row>
+ <Row><idl>B</idl><idr>A</idr><cnt>1</cnt></Row>
+ <Row><idl>C</idl><idr>A</idr><cnt>1</cnt></Row>
 </Dataset>
 <Dataset name='Result 34'>
- <Row><idl>B</idl><idr>B</idr></Row>
- <Row><idl>C</idl><idr>B</idr></Row>
+ <Row><idl>B</idl><idr>B</idr><cnt>2</cnt></Row>
+ <Row><idl>C</idl><idr>B</idr><cnt>2</cnt></Row>
 </Dataset>
 <Dataset name='Result 35'>
 </Dataset>
 <Dataset name='Result 36'>
- <Row><idl>B</idl><idr> </idr></Row>
- <Row><idl>C</idl><idr> </idr></Row>
+ <Row><idl>B</idl><idr> </idr><cnt>0</cnt></Row>
+ <Row><idl>C</idl><idr> </idr><cnt>0</cnt></Row>
 </Dataset>
 <Dataset name='Result 37'>
- <Row><idl>B</idl><idr>B</idr></Row>
- <Row><idl>C</idl><idr>B</idr></Row>
+ <Row><idl>B</idl><idr>B</idr><cnt>2</cnt></Row>
+ <Row><idl>C</idl><idr>B</idr><cnt>2</cnt></Row>
 </Dataset>
 <Dataset name='Result 38'>
- <Row><idl>B</idl><idr>B</idr></Row>
- <Row><idl>C</idl><idr>B</idr></Row>
+ <Row><idl>B</idl><idr>B</idr><cnt>2</cnt></Row>
+ <Row><idl>C</idl><idr>B</idr><cnt>2</cnt></Row>
 </Dataset>
 <Dataset name='Result 39'>
- <Row><idl>B</idl><idr>B</idr></Row>
- <Row><idl>C</idl><idr>B</idr></Row>
+ <Row><idl>B</idl><idr>B</idr><cnt>2</cnt></Row>
+ <Row><idl>C</idl><idr>B</idr><cnt>2</cnt></Row>
 </Dataset>
 <Dataset name='Result 40'>
- <Row><idl>B</idl><idr> </idr></Row>
- <Row><idl>C</idl><idr> </idr></Row>
+ <Row><idl>B</idl><idr> </idr><cnt>0</cnt></Row>
+ <Row><idl>C</idl><idr> </idr><cnt>0</cnt></Row>
 </Dataset>
 <Dataset name='Result 41'>
- <Row><idl>A</idl><idr>A</idr></Row>
- <Row><idl>A</idl><idr>B</idr></Row>
- <Row><idl>A</idl><idr>C</idr></Row>
- <Row><idl>B</idl><idr>A</idr></Row>
- <Row><idl>B</idl><idr>B</idr></Row>
- <Row><idl>B</idl><idr>C</idr></Row>
- <Row><idl>C</idl><idr>A</idr></Row>
- <Row><idl>C</idl><idr>B</idr></Row>
- <Row><idl>C</idl><idr>C</idr></Row>
+ <Row><idl>A</idl><idr>A</idr><cnt>1</cnt></Row>
+ <Row><idl>A</idl><idr>B</idr><cnt>2</cnt></Row>
+ <Row><idl>A</idl><idr>C</idr><cnt>3</cnt></Row>
+ <Row><idl>B</idl><idr>A</idr><cnt>1</cnt></Row>
+ <Row><idl>B</idl><idr>B</idr><cnt>2</cnt></Row>
+ <Row><idl>B</idl><idr>C</idr><cnt>3</cnt></Row>
+ <Row><idl>C</idl><idr>A</idr><cnt>1</cnt></Row>
+ <Row><idl>C</idl><idr>B</idr><cnt>2</cnt></Row>
+ <Row><idl>C</idl><idr>C</idr><cnt>3</cnt></Row>
 </Dataset>
 <Dataset name='Result 42'>
- <Row><idl>A</idl><idr>A</idr></Row>
- <Row><idl>B</idl><idr>A</idr></Row>
- <Row><idl>C</idl><idr>A</idr></Row>
+ <Row><idl>A</idl><idr>A</idr><cnt>1</cnt></Row>
+ <Row><idl>B</idl><idr>A</idr><cnt>1</cnt></Row>
+ <Row><idl>C</idl><idr>A</idr><cnt>1</cnt></Row>
 </Dataset>
 <Dataset name='Result 43'>
 </Dataset>
 <Dataset name='Result 44'>
- <Row><idl>A</idl><idr>A</idr></Row>
- <Row><idl>A</idl><idr>B</idr></Row>
- <Row><idl>A</idl><idr>C</idr></Row>
- <Row><idl>B</idl><idr>A</idr></Row>
- <Row><idl>B</idl><idr>B</idr></Row>
- <Row><idl>B</idl><idr>C</idr></Row>
- <Row><idl>C</idl><idr>A</idr></Row>
- <Row><idl>C</idl><idr>B</idr></Row>
- <Row><idl>C</idl><idr>C</idr></Row>
+ <Row><idl>A</idl><idr>A</idr><cnt>1</cnt></Row>
+ <Row><idl>A</idl><idr>B</idr><cnt>2</cnt></Row>
+ <Row><idl>A</idl><idr>C</idr><cnt>3</cnt></Row>
+ <Row><idl>B</idl><idr>A</idr><cnt>1</cnt></Row>
+ <Row><idl>B</idl><idr>B</idr><cnt>2</cnt></Row>
+ <Row><idl>B</idl><idr>C</idr><cnt>3</cnt></Row>
+ <Row><idl>C</idl><idr>A</idr><cnt>1</cnt></Row>
+ <Row><idl>C</idl><idr>B</idr><cnt>2</cnt></Row>
+ <Row><idl>C</idl><idr>C</idr><cnt>3</cnt></Row>
 </Dataset>
 <Dataset name='Result 45'>
- <Row><idl>A</idl><idr>A</idr></Row>
- <Row><idl>B</idl><idr>A</idr></Row>
- <Row><idl>C</idl><idr>A</idr></Row>
+ <Row><idl>A</idl><idr>A</idr><cnt>1</cnt></Row>
+ <Row><idl>B</idl><idr>A</idr><cnt>1</cnt></Row>
+ <Row><idl>C</idl><idr>A</idr><cnt>1</cnt></Row>
 </Dataset>
 <Dataset name='Result 46'>
- <Row><idl>A</idl><idr>A</idr></Row>
- <Row><idl>B</idl><idr>A</idr></Row>
- <Row><idl>C</idl><idr>A</idr></Row>
+ <Row><idl>A</idl><idr>A</idr><cnt>1</cnt></Row>
+ <Row><idl>B</idl><idr>A</idr><cnt>1</cnt></Row>
+ <Row><idl>C</idl><idr>A</idr><cnt>1</cnt></Row>
 </Dataset>
 <Dataset name='Result 47'>
- <Row><idl>A</idl><idr> </idr></Row>
- <Row><idl>B</idl><idr> </idr></Row>
- <Row><idl>C</idl><idr> </idr></Row>
+ <Row><idl>A</idl><idr> </idr><cnt>0</cnt></Row>
+ <Row><idl>B</idl><idr> </idr><cnt>0</cnt></Row>
+ <Row><idl>C</idl><idr> </idr><cnt>0</cnt></Row>
 </Dataset>
 <Dataset name='Result 48'>
- <Row><idl>A</idl><idr>C</idr></Row>
- <Row><idl>B</idl><idr>C</idr></Row>
- <Row><idl>C</idl><idr>C</idr></Row>
+ <Row><idl>A</idl><idr>C</idr><cnt>1</cnt></Row>
+ <Row><idl>B</idl><idr>C</idr><cnt>1</cnt></Row>
+ <Row><idl>C</idl><idr>C</idr><cnt>1</cnt></Row>
+</Dataset>
+<Dataset name='Result 49'>
+ <Row><idl> </idl><idr>B</idr><cnt>0</cnt></Row>
+ <Row><idl> </idl><idr>C</idr><cnt>0</cnt></Row>
 </Dataset>

thorlcr/activities/keyedjoin/thkeyedjoinslave.cpp

@@ -2139,7 +2139,7 @@ public:
                             {
                                 case TAKkeyedjoin:
                                 {
-                                    transformedSize = helper->transform(row.ensureRow(), djg->queryLeft(), defaultRight, 0);
+                                    transformedSize = helper->transform(row.ensureRow(), djg->queryLeft(), defaultRight, (__uint64)0, 0U);
                                     break;
                                 }
                                 case TAKkeyeddenormalize:
@@ -2174,7 +2174,7 @@ public:
                                     offset_t fpos;
                                     const void *rhs = djg->queryRow(currentMatchIdx, fpos);
                                     row.ensureRow();
-                                    transformedSize = helper->transform(row, djg->queryLeft(), rhs, fpos);
+                                    transformedSize = helper->transform(row, djg->queryLeft(), rhs, fpos, currentMatchIdx+1);
                                     if (transformedSize)
                                     {
                                         currentAdded++;

thorlcr/activities/lookupjoin/thlookupjoinslave.cpp

@@ -629,10 +629,6 @@ public:
     {
         return helper->match(lhs, rhsrow);
     }
-    inline const size32_t joinTransform(ARowBuilder &rowBuilder, const void *lhs, const void *rhsrow)
-    {
-        return helper->transform(rowBuilder, lhs, rhsrow);
-    }
     inline const size32_t joinTransform(ARowBuilder &rowBuilder, const void *left, const void *right, unsigned numRows, const void **rows)
     {
         return helper->transform(rowBuilder, left, right, numRows, rows);
@@ -776,6 +772,7 @@ protected:
     rowidx_t nextRhsRow;
     unsigned keepLimit;
     unsigned joined;
+    unsigned joinCounter;
     OwnedConstThorRow defaultLeft;
 
     bool leftMatch, grouped;
@@ -979,6 +976,7 @@ protected:
                 if (NULL == rhsNext)
                 {
                     leftRow.setown(left->nextRow());
+                    joinCounter = 0;
                     if (leftRow)
                     {
                         eog = false;
@@ -1018,7 +1016,7 @@ protected:
                             leftMatch = true;
                             if (!exclude)
                             {
-                                size32_t sz = HELPERBASE::joinTransform(rowBuilder, leftRow, rhsNext);
+                                size32_t sz = HELPERBASE::joinTransform(rowBuilder, leftRow, rhsNext, ++joinCounter);
                                 if (sz)
                                 {
                                     OwnedConstThorRow row = rowBuilder.finalizeRowClear(sz);
@@ -1037,7 +1035,7 @@ protected:
                     }
                     if (!leftMatch && NULL == rhsNext && 0!=(flags & JFleftouter))
                     {
-                        size32_t sz = HELPERBASE::joinTransform(rowBuilder, leftRow, defaultRight);
+                        size32_t sz = HELPERBASE::joinTransform(rowBuilder, leftRow, defaultRight, 0);
                         if (sz)
                             ret.setown(rowBuilder.finalizeRowClear(sz));
                     }
@@ -1068,6 +1066,7 @@ public:
         leftITDL = rightITDL = NULL;
 
         joined = 0;
+        joinCounter = 0;
         leftMatch = false;
         returnMany = false;
 
@@ -1116,6 +1115,7 @@ public:
         gotRHS = false;
         nextRhsRow = 0;
         joined = 0;
+        joinCounter = 0;
         leftMatch = false;
         rhsNext = NULL;
         rhsTableLen = 0;

thorlcr/activities/msort/thsortu.cpp

@@ -298,6 +298,7 @@ class CJoinHelper : public CSimpleInterface, implements IJoinHelper
     RtlDynamicRowBuilder denormTmp;
     CThorExpandingRowArray denormRows;
     unsigned denormCount;
+    unsigned joinCounter;
     size32_t outSz;
     unsigned rightidx;
     enum { JScompare, JSmatch, JSrightgrouponly, JSonfail } state;
@@ -363,6 +364,7 @@ public:
         kind = activity.queryContainer().getKind();
         helper = _helper; 
         denormCount = 0;
+        joinCounter = 0;
         outSz = 0;
         lhsProgressCount = rhsProgressCount = 0;
         keepmax = (unsigned)-1;
@@ -492,10 +494,12 @@ public:
                 nextleft.setown(strmL->nextRow());
                 if (!nextleft) 
                     break;
+
                 lhsProgressCount++;
                 if (!firstonlyL || (lhsProgressCount==1) || (compareL->docompare(prevleft,nextleft)!=0)) {
                     denormLhs.set(nextleft.get());
                     nextleftgot = true;
+                    joinCounter = 0;
                     return true;
                 }
             }
@@ -588,7 +592,7 @@ public:
                         case TAKselfjoinlight:
                         case TAKlookupjoin:
                         case TAKsmartjoin:
-                            gotsz = helper->transform(ret, defaultLeft, nextright);
+                            gotsz = helper->transform(ret, defaultLeft, nextright, 0);
                             nextR();
                             break;
                         default:
@@ -645,7 +649,7 @@ public:
                     case TAKlookupjoin:
                     case TAKsmartjoin:
                         if (!rightgroupmatched[rightidx]) 
-                            gotsz = helper->transform(ret, defaultLeft, rightgroup.query(rightidx));
+                            gotsz = helper->transform(ret, defaultLeft, rightgroup.query(rightidx), 0);
                         rightidx++;
                         break;
                     default:
@@ -675,7 +679,7 @@ public:
                     case TAKselfjoinlight:
                     case TAKlookupjoin:
                     case TAKsmartjoin:
-                        gotsz = helper->transform(ret, nextleft, defaultRight);
+                        gotsz = helper->transform(ret, nextleft, defaultRight, 0);
                         break;
                     default:
                         throwUnexpected();
@@ -714,7 +718,7 @@ public:
             if (r==Onext) {
                 // JCSMORE - I can't see when this can happen? if r==Onext, l is always Oouter.
                 if (!exclude) 
-                    gotsz = helper->transform(ret,nextleft,nextright);
+                    gotsz = helper->transform(ret,nextleft,nextright,++joinCounter);
                 rightmatched = true;
             }
             else {
@@ -752,7 +756,7 @@ public:
                         case TAKselfjoinlight:
                         case TAKlookupjoin:
                         case TAKsmartjoin:
-                            gotsz = helper->transform(ret,nextleft,rightgroup.query(rightidx));
+                            gotsz = helper->transform(ret,nextleft,rightgroup.query(rightidx), ++joinCounter);
                             break;
                         default:
                             throwUnexpected();
@@ -988,6 +992,7 @@ class SelfJoinHelper: public CSimpleInterface, implements IJoinHelper
     bool *abort;
     unsigned atmost;
     rowcount_t progressCount;
+    unsigned joinCounter;
     unsigned keepmax;
     unsigned abortlimit;
     unsigned keepremaining;
@@ -1076,6 +1081,7 @@ public:
         keepremaining = keepmax;
         outputmetaL = _outputmeta;
         progressCount = 0;
+        joinCounter = 0;
         return true;
     }
 
@@ -1198,6 +1204,7 @@ retry:
                     }
                     leftidx = 0;
                     rightidx = 0;
+                    joinCounter = 0;
                     leftmatched = false;
                     if (state==JSload) {     // catch atmost above
                         rightmatched = (bool *)rightmatchedbuf.clear().reserve(curgroup.ordinality());
@@ -1217,7 +1224,7 @@ retry:
                                 if (keepremaining>0) {
                                     if (!exclude) {
                                         RtlDynamicRowBuilder rtmp(allocator);
-                                        size32_t sz = helper->transform(rtmp,l,r);
+                                        size32_t sz = helper->transform(rtmp,l,r,++joinCounter);
                                         if (sz)
                                             ret.setown(rtmp.finalizeRowClear(sz));
                                     }
@@ -1233,15 +1240,16 @@ retry:
                             }
                             rightidx++;
                         }
-                        else { // right all done 
+                        else { // right all done
                             if (leftouter&&!leftmatched) {
                                 RtlDynamicRowBuilder rtmp(allocator);
-                                size32_t sz = helper->transform(rtmp, l, defaultRight);
+                                size32_t sz = helper->transform(rtmp, l, defaultRight, 0);
                                 if (sz)
                                     ret.setown(rtmp.finalizeRowClear(sz));
                             }
                             keepremaining = keepmax; // lefts don't count in keep
                             rightidx = 0;
+                            joinCounter = 0;
                             leftidx++;
                             if ((leftidx>=curgroup.ordinality())||(firstonlyL&&(leftidx>0)))
                                 state = JSrightonly;
@@ -1254,14 +1262,14 @@ retry:
                     // must be left outer after atmost to get here
                     if (leftidx<curgroup.ordinality()) {
                         RtlDynamicRowBuilder rtmp(allocator);
-                        size32_t sz = helper->transform(rtmp, curgroup.query(leftidx), defaultRight);
+                        size32_t sz = helper->transform(rtmp, curgroup.query(leftidx), defaultRight, 0);
                         if (sz)
                             ret.setown(rtmp.finalizeRowClear(sz));
                         leftidx++;
                     }
                     else if (getRow() && (compare->docompare(nextrow,curgroup.query(0))==0)) {
                         RtlDynamicRowBuilder rtmp(allocator);
-                        size32_t sz = helper->transform(rtmp, nextrow, defaultRight);
+                        size32_t sz = helper->transform(rtmp, nextrow, defaultRight, 0);
                         if (sz)
                             ret.setown(rtmp.finalizeRowClear(sz));
                         next();
@@ -1274,7 +1282,7 @@ retry:
                     if (rightouter&&(rightidx<curgroup.ordinality())) {
                         if (!rightmatched[rightidx]) {
                             RtlDynamicRowBuilder rtmp(allocator);
-                            size32_t sz = helper->transform(rtmp, defaultLeft,curgroup.query(rightidx));
+                            size32_t sz = helper->transform(rtmp, defaultLeft,curgroup.query(rightidx), 0);
                             if (sz)
                                 ret.setown(rtmp.finalizeRowClear(sz));
                         }
@@ -1528,13 +1536,14 @@ public:
         ForEachItemIn(leftidx,work.lgroup)
         {
             bool lmatched = !leftouter;
+            unsigned joinCounter = 0;
             for (unsigned rightidx=0; rightidx<rgroup.ordinality(); rightidx++) {
                 if (helper->match(work.lgroup.query(leftidx),rgroup.query(rightidx))) {
                     lmatched = true;
                     if (rightouter)
                         rmatched[rightidx] = true;
                     RtlDynamicRowBuilder ret(theAllocator);
-                    size32_t sz = exclude?0:helper->transform(ret,work.lgroup.query(leftidx),rgroup.query(rightidx));
+                    size32_t sz = exclude?0:helper->transform(ret,work.lgroup.query(leftidx),rgroup.query(rightidx),++joinCounter);
                     if (sz)
                         writer.putRow(ret.finalizeRowClear(sz));
 
@@ -1542,7 +1551,7 @@ public:
             }
             if (!lmatched) {
                 RtlDynamicRowBuilder ret(theAllocator);
-                size32_t sz =  helper->transform(ret, work.lgroup.query(leftidx), defaultRight);
+                size32_t sz =  helper->transform(ret, work.lgroup.query(leftidx), defaultRight, 0);
                 if (sz)
                     writer.putRow(ret.finalizeRowClear(sz));
             }
@@ -1551,7 +1560,7 @@ public:
             ForEachItemIn(rightidx2,rgroup) {
                 if (!rmatched[rightidx2]) {
                     RtlDynamicRowBuilder ret(theAllocator);
-                    size32_t sz =  helper->transform(ret, defaultLeft, rgroup.query(rightidx2));
+                    size32_t sz =  helper->transform(ret, defaultLeft, rgroup.query(rightidx2), 0);
                     if (sz)
                         writer.putRow(ret.finalizeRowClear(sz));
                 }