Merge branch 'candidate-5.4.0'

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

common/thorhelper/roxiehelper.cpp

@@ -2112,8 +2112,10 @@ void FlushingStringBuffer::startDataset(const char *elementName, const char *res
 
 void FlushingStringBuffer::startScalar(const char *resultName, unsigned sequence)
 {
+    if (!s.length())
+        throw MakeStringException(0, "Attempt to output scalar ('%s',%d) multiple times", resultName ? resultName : "", (int)sequence);
+
     CriticalBlock b(crit);
-    assertex(!s.length());
     name.clear().append(resultName ? resultName : "Dataset");
 
     sequenceNumber = 0;
@@ -2214,8 +2216,10 @@ void FlushingJsonBuffer::startDataset(const char *elementName, const char *resul
 
 void FlushingJsonBuffer::startScalar(const char *resultName, unsigned sequence)
 {
+    if (!s.length())
+        throw MakeStringException(0, "Attempt to output scalar ('%s',%d) multiple times", resultName ? resultName : "", (int)sequence);
+
     CriticalBlock b(crit);
-    assertex(!s.length());
     name.set(resultName ? resultName : "Dataset");
 
     sequenceNumber = 0;

docs/ECLLanguageReference/ECLR-includer.xml

@@ -711,6 +711,9 @@
                 xpointer="element(/1)"
                 xmlns:xi="http://www.w3.org/2001/XInclude" />
 
+    <xi:include href="ECLLanguageReference/ECLR_mods/BltInFunc-ORDERED.xml"
+                xpointer="element(/1)"
+                xmlns:xi="http://www.w3.org/2001/XInclude" />
     <xi:include href="ECLLanguageReference/ECLR_mods/BltInFunc-OUTPUT.xml"
                 xpointer="element(/1)"
                 xmlns:xi="http://www.w3.org/2001/XInclude" />
@@ -1078,7 +1081,12 @@
     <xi:include href="ECLLanguageReference/ECLR_mods/Templ-WARNING.xml"
                 xpointer="element(/1)"
                 xmlns:xi="http://www.w3.org/2001/XInclude" />
+ 
 
+    <xi:include href="ECLLanguageReference/ECLR_mods/Templ-WEBSERVICE.xml"
+                xpointer="element(/1)"
+                xmlns:xi="http://www.w3.org/2001/XInclude" />
+ 
     <xi:include href="ECLLanguageReference/ECLR_mods/Templ-WORKUNIT.xml"
                 xpointer="element(/1)"
                 xmlns:xi="http://www.w3.org/2001/XInclude" />

docs/ECLLanguageReference/ECLR_mods/BltInFunc-ORDERED.xml

@@ -0,0 +1,72 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE sect1 PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN"
+"http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd">
+<sect1 id="ORDERED">
+  <title>ORDERED</title>
+
+  <para><emphasis role="bold">[</emphasis><emphasis>attributename</emphasis>
+  :=<emphasis role="bold"> ] ORDERED<indexterm>
+      <primary>ORDERED</primary>
+    </indexterm><indexterm>
+      <primary>ORDERED function</primary>
+    </indexterm>(</emphasis><emphasis> actionlist </emphasis><emphasis
+  role="bold">)</emphasis></para>
+
+  <para><informaltable colsep="1" frame="all" rowsep="1">
+      <tgroup cols="2">
+        <colspec colwidth="82.95pt" />
+
+        <colspec />
+
+        <tbody>
+          <row>
+            <entry><emphasis>attributename</emphasis></entry>
+
+            <entry>Optional. The action name, which turns the action into an
+            attribute definition, therefore not executed until the
+            attributename is used as an action.</entry>
+          </row>
+
+          <row>
+            <entry><emphasis>actionlist</emphasis></entry>
+
+            <entry>A comma-delimited list of the actions to execute in order.
+            These may be ECL actions or external actions.</entry>
+          </row>
+        </tbody>
+      </tgroup>
+    </informaltable></para>
+
+  <para>The <emphasis role="bold">ORDERED </emphasis>action executes the items
+  in the <emphasis>actionlist</emphasis> in the order in which they appear in
+  the <emphasis>actionlist</emphasis>. This is useful when a subsequent action
+  requires the output of a precedent action. </para>
+
+  <para>It has the ordering requirements of SEQUENTIAL. This is most useful
+  for ordering actions which do not have anything in common, for example,
+  generating files and then sending email. If there is any chance of a shared
+  value which may change meaning, you should use SEQUENTIAL.</para>
+
+  <para>By definition, PERSIST on an attribute means the attribute is
+  evaluated outside of any given evaluation order. Therefore, ORDERED has no
+  effect on PERSISTed attributes.</para>
+
+  <para>Example:</para>
+
+  <programlisting>Act1 :=
+        OUTPUT(A_People,OutputFormat1,'//hold01/fred.out');
+Act2 :=
+        OUTPUT(Person,{Person.per_first_name,Person.per_last_name})
+Act2 := OUTPUT(Person,{Person.per_last_name})));
+//by naming these actions, they become inactive
+        attributes
+//that only execute when the attribute names are called as
+        actions
+ORDERED(Act1,PARALLEL(Act2,Act3));
+//executes Act1 alone, and then executes Act2 and Act3 together
+</programlisting>
+
+  <para>See Also: <link linkend="PARALLEL_function">PARALLEL</link>, <link
+  linkend="PERSIST">PERSIST</link>, <link
+  linkend="SEQUENTIAL">SEQUENTIAL</link></para>
+</sect1>

docs/ECLLanguageReference/ECLR_mods/BltInFunc-OUTPUT.xml

@@ -959,7 +959,7 @@ OUTPUT(B,,'fred3.xml',XML('MyRow',TRIM,OPT));
 </programlisting>
   </sect2>
 
-  <sect2 id="OUTPUT_XML_Files">
+  <sect2 id="OUTPUT_JSON_Files">
     <title>OUTPUT JSON Files<indexterm>
         <primary>JSON Files</primary>
       </indexterm><indexterm>

docs/ECLLanguageReference/ECLR_mods/Expr-RecordSetOps.xml

@@ -8,8 +8,8 @@
       <primary>Record Set Operators</primary>
     </indexterm></title>
 
-  <para>The following record set Append operators are supported (both require
-  that the files were created using identical RECORD structures<indexterm>
+  <para>The following record set operators are supported (both require that
+  the files were created using identical RECORD structures<indexterm>
       <primary>RECORD structure</primary>
     </indexterm>):</para>
 
@@ -28,6 +28,12 @@
         </row>
 
         <row>
+          <entry>-</entry>
+
+          <entry>Subtract records from a file</entry>
+        </row>
+
+        <row>
           <entry>&amp;</entry>
 
           <entry>Append all records from both files, maintaining record order
@@ -36,4 +42,25 @@
       </tbody>
     </tgroup>
   </informaltable>
+
+  <para>Example:</para>
+
+  <programlisting>MyLayout := RECORD
+  UNSIGNED Num;
+  STRING Number;
+END;
+
+FirstRecSet := DATASET([{1, 'ONE'}, {2, 'Two'}, {3, 'Three'}, {4, 'Four'}], MyLayout);
+SecondRecSet := DATASET([{5, 'FIVE'}, {6, 'SIX'}, {7, 'SEVEN'}, {8, 'EIGHT'}], MyLayout);
+
+ExcludeThese := SecondRecSet(Num &gt; 6);
+
+WholeRecSet := FirstRecSet + SecondRecSet;
+ResultSet:= WholeRecSet-ExcludeThese;
+
+OUTPUT (WholeRecSet);
+OUTPUT(ResultSet);
+</programlisting>
+
+  <para></para>
 </sect1>

docs/ECLLanguageReference/ECLR_mods/Templ-WEBSERVICE.xml

@@ -0,0 +1,86 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE sect1 PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN"
+"http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd">
+<sect1 id="_WEBSERVICE">
+  <title>#WEBSERVICE</title>
+
+  <para><emphasis role="bold">#WEBSERVICE<indexterm>
+      <primary>#WEBSERVICE</primary>
+    </indexterm>( </emphasis><emphasis role="bold">[</emphasis><emphasis
+  role="bold">FIELDS</emphasis>(<emphasis>fieldlist</emphasis>),<emphasis
+  role="bold">][</emphasis><emphasis
+  role="bold">HELP</emphasis>(<emphasis>helptext</emphasis>),<emphasis
+  role="bold">][</emphasis><emphasis
+  role="bold">DESCRIPTION</emphasis>(<emphasis>descriptiontext</emphasis>),<emphasis
+  role="bold">]</emphasis> <emphasis role="bold">);</emphasis></para>
+
+  <para><informaltable colsep="1" frame="all" rowsep="1">
+      <tgroup cols="2">
+        <colspec colwidth="77.25pt" />
+
+        <colspec />
+
+        <tbody>
+          <row>
+            <entry><emphasis role="bold">FIELDS</emphasis></entry>
+
+            <entry>The FIELDS parameter specifies field sequence<indexterm>
+                <primary>field sequence</primary>
+              </indexterm> in WsECL Web forms. This is an exclusive list. If
+            the FIELDS attribute is present, only the fields in the fieldslist
+            are displayed on the Web form in WsECL.</entry>
+          </row>
+
+          <row>
+            <entry><emphasis>fieldlist</emphasis></entry>
+
+            <entry>A comma-separated list of field names in the order in which
+            they should appear on the form.</entry>
+          </row>
+
+          <row>
+            <entry><emphasis role="bold">HELP</emphasis></entry>
+
+            <entry>The HELP Parameter specifies to add help text to the WsECL
+            Web form.</entry>
+          </row>
+
+          <row>
+            <entry><emphasis>helptext</emphasis></entry>
+
+            <entry>The help text to display.</entry>
+          </row>
+
+          <row>
+            <entry><emphasis role="bold">DESCRIPTION</emphasis></entry>
+
+            <entry>The DESCRIPTION Parameter specifies to add descriptive text
+            to the WsECL Web form.</entry>
+          </row>
+
+          <row>
+            <entry><emphasis>descriptiontext</emphasis></entry>
+
+            <entry>The description text to display.</entry>
+          </row>
+        </tbody>
+      </tgroup>
+    </informaltable></para>
+
+  <para>The <emphasis role="bold">#WEBSERVICE</emphasis>statement sets options
+  for the input parameters on a WsECL Web form for a published query.</para>
+
+  <para>Example:</para>
+
+  <programlisting>#WEBSERVICE(FIELDS('Field1','AddThem','Field2'),
+            HELP('Enter Integer Values'),
+            DESCRIPTION('If AddThem is TRUE, this adds the two integers')); 
+Field1 := 1 : Stored('Field1');
+Field2 := 2 :Stored('Field2');
+AddThem := TRUE :STORED ('AddThem');
+HiddenValue := 12 :STORED ('HiddenValue'); //not in fieldlist, won't display on WsECl form
+IF(AddThem,OUTPUT(Field1+Field2),OUTPUT('Not Added'));
+</programlisting>
+
+  <para>See Also: <link linkend="STORED_workflow_service">STORED</link></para>
+</sect1>

docs/ECLLanguageReference/ECLR_mods/WkFlo-STORED.xml

@@ -11,7 +11,14 @@
     </indexterm>( </emphasis><emphasis> storedname </emphasis><emphasis
   role="bold">[, FEW<indexterm>
       <primary>FEW</primary>
-    </indexterm> ] ) </emphasis><indexterm>
+    </indexterm> ][</emphasis>, <emphasis
+  role="bold">FORMAT</emphasis>(<emphasis
+  role="bold">[FIELDWIDTH(</emphasis><emphasis>widthvalue</emphasis>)<emphasis
+  role="bold">]<emphasis
+  role="bold">[,FIELDHEIGHT(</emphasis><emphasis>heightvalue</emphasis>)][,SEQUENCE</emphasis>(<emphasis>sequencevalue</emphasis>)<emphasis
+  role="bold">]<emphasis
+  role="bold">[,NOINPUT</emphasis><emphasis></emphasis>)<emphasis
+  role="bold">]</emphasis>] )</emphasis> <indexterm>
       <primary>STORED workflow service</primary>
     </indexterm>;</para>
 
@@ -42,7 +49,7 @@
           </row>
 
           <row>
-            <entry><emphasis>FEW</emphasis></entry>
+            <entry><emphasis role="bold">FEW</emphasis></entry>
 
             <entry>Optional. When the expression is a dataset or recordset,
             FEW specifies that the dataset is stored completely within the
@@ -51,6 +58,63 @@
             option is required when using STORED in a SOAP-enabled MACRO and
             the expected input is a dataset (such as tns:xmlDataset).</entry>
           </row>
+
+          <row>
+            <entry><emphasis role="bold">FORMAT</emphasis></entry>
+
+            <entry>Optional. FORMAT specifies options for formatting the field
+            on a Web form in WsECL.</entry>
+          </row>
+
+          <row>
+            <entry><emphasis role="bold">FIELDWIDTH</emphasis></entry>
+
+            <entry>Optional. FIELDWIDTH specifies the width of the input box
+            on a Web form in WsECL.</entry>
+          </row>
+
+          <row>
+            <entry><emphasis>widthvalue</emphasis></entry>
+
+            <entry>An integer expression defining the width (number of
+            characters) of the input box</entry>
+          </row>
+
+          <row>
+            <entry><emphasis role="bold">FIELDHEIGHT</emphasis></entry>
+
+            <entry>Optional. FIELDHEIGHT specifies the height of the input box
+            on a Web form in WsECL.</entry>
+          </row>
+
+          <row>
+            <entry><emphasis>heightvalue</emphasis></entry>
+
+            <entry>An integer expression defining the height (number of rows)
+            of the input box</entry>
+          </row>
+
+          <row>
+            <entry><emphasis role="bold">SEQUENCE</emphasis></entry>
+
+            <entry>Optional. SEQUENCE specifies field ordering on a Web form
+            in WsECL.</entry>
+          </row>
+
+          <row>
+            <entry><emphasis>sequencevalue</emphasis></entry>
+
+            <entry>An integer expression defining the sequential location of
+            the input box. These can be sparse values (e.g., 100, 200, 300) to
+            allow insertion of new inputs in the future. </entry>
+          </row>
+
+          <row>
+            <entry><emphasis role="bold">NOINPUT</emphasis></entry>
+
+            <entry>Optional. If NOINPUT is specified, the field is not
+            displayed on the Web form in WsECL. </entry>
+          </row>
         </tbody>
       </tgroup>
     </informaltable></para>
@@ -76,7 +140,18 @@
      // Name in work unit is fred
   fred := COUNT(person) : STORED('mindy');
      // Name in work unit is mindy
+
+//FORMAT options for WsECL form
+  
+  Field1 := 1 : Stored('Field1',FORMAT(SEQUENCE(10)));
+  Field2 := 2 : Stored('Field2',FORMAT(SEQUENCE(20)));
+  AddThem := TRUE :STORED ('AddThem',FORMAT(SEQUENCE(15))); // places field in between Field1 and Field2
+  HiddenValue := 12 :STORED ('HiddenValue',FORMAT(NOINPUT)); // not on form
+  TextField1 :='Fill in description' :Stored('Description',
+                                      FORMAT(FIELDWIDTH(25),FIELDHEIGHT(2),
+                                      SEQUENCE(5))); //Creates 25 character wide, 2 row high input box
 </programlisting>
 
-  <para>See Also: <link linkend="STORED">STORED function</link></para>
+  <para>See Also: <link linkend="STORED">STORED function</link>, <link
+  linkend="_WEBSERVICE">#WEBSERVICE</link></para>
 </sect1>

docs/HPCCClientTools/CT_Mods/ECLCC.xml

@@ -35,7 +35,7 @@
     <releaseinfo>© 2015 HPCC Systems<superscript>®</superscript>. All rights
     reserved</releaseinfo>
 
-    <date><emphasis role="bold">BETA</emphasis></date>
+    <date><emphasis role="bold"></emphasis></date>
 
     <corpname>HPCC Systems<superscript>®</superscript></corpname>
 
@@ -330,6 +330,13 @@
               </row>
 
               <row>
+                <entry>-D<emphasis>name</emphasis>=<emphasis>value</emphasis></entry>
+
+                <entry>Override the definition of a global attribute 'name'
+                </entry>
+              </row>
+
+              <row>
                 <entry>-Wl,xx</entry>
 
                 <entry>Pass option xx to the linker</entry>
@@ -446,6 +453,13 @@
               </row>
 
               <row>
+                <entry>--keywords</entry>
+
+                <entry>Outputs the lists of ECL reserved words to stdout (XML
+                format)</entry>
+              </row>
+
+              <row>
                 <entry>--logfile <emphasis>&lt;file&gt;</emphasis></entry>
 
                 <entry>Write log to specified file</entry>
@@ -652,6 +666,21 @@
         <para>This would result in the output of the following:</para>
 
         <programlisting>&lt;Dataset name="Result 1"&gt;&lt;Row&gt;&lt;Result_1&gt;Hello world&lt;/Result_1&gt;&lt;/Row&gt;&lt;/Dataset&gt;</programlisting>
+
+        <para>The following example provides a defined value passed to the
+        compiler:</para>
+
+        <programlisting>//file named hello2.ecl
+IMPORT ^ as repo;
+OUTPUT(repo.optionXX);</programlisting>
+
+        <para><programlisting>eclcc -Doptionxx='HELLO' hello2.ecl</programlisting></para>
+
+        <para>This would result in the output of the following:</para>
+
+        <programlisting>&lt;Dataset name="Result 1"&gt;&lt;Row&gt;&lt;Result_1&gt;HELLO&lt;/Result_1&gt;&lt;/Row&gt;&lt;/Dataset&gt;</programlisting>
+
+        <para></para>
       </sect2>
     </sect1>
   </chapter>

ecl/eclcc/eclcc.hpp

@@ -81,7 +81,7 @@ const char * const helpText[] = {
     "    -help -v      Display verbose help message",
     "!   -internal     Run internal tests",
     "?!  -legacy       Use legacy import and when semantics (deprecated)",
-    "!   --keywords    Outputs the list of ECL reserved words to ECLKeywords.xml",
+    "!   --keywords    Outputs the list of ECL reserved words to stdout (XML format)",
     "!   -legacyimport Use legacy import semantics (deprecated)",
     "!   -legacywhen   Use legacy when/side-effects semantics (deprecated)",
     "    --logfile <file> Write log to specified file",

ecl/eclcc/reservedwords.cpp

@@ -572,13 +572,6 @@ void printKeywordsToXml()
          }
          buffer.append("  </cat>\n");
      }
-     buffer.append("</xml>\n");
-
-     Owned<IFile> treeFile = createIFile("ECLKeywords.xml");
-     assertex(treeFile);
-     Owned<IFileIO> io = treeFile->open(IFOcreaterw);
-     assertex(io);
-     Owned<IIOStream> out = createIOStream(io);
-     assertex(out);
-     out->write(buffer.length(), buffer.str());
+     buffer.append("</xml>");
+     fprintf(stdout, "%s\n", buffer.str());
 }

ecl/eclcmd/eclcmd_common.hpp

@@ -89,6 +89,8 @@ typedef IEclCommand *(*EclCommandFactory)(const char *cmdname);
 #define ECLOPT_UPDATE_DFS "--update-dfs"
 #define ECLOPT_GLOBAL_SCOPE "--global-scope"
 #define ECLOPT_DELETE_FILES "--delete"
+#define ECLOPT_DELETE_SUBFILES "--delete-subfiles"
+#define ECLOPT_DELETE_RECURSIVE "--delete-recursive"
 
 #define ECLOPT_MAIN "--main"
 #define ECLOPT_MAIN_S "-main"  //eclcc compatible format

ecl/eclcmd/roxie/ecl-roxie.cpp

@@ -363,7 +363,7 @@ private:
 class EclCmdRoxieUnusedFiles : public EclCmdCommon
 {
 public:
-    EclCmdRoxieUnusedFiles() : optCheckPackageMaps(false)
+    EclCmdRoxieUnusedFiles() : optCheckPackageMaps(false), optDeleteFiles(false), optDeleteSubFiles(false), optDeleteRecursive(false)
     {
     }
     virtual bool parseCommandLineOptions(ArgvIterator &iter)
@@ -384,6 +384,10 @@ public:
             }
             if (iter.matchFlag(optCheckPackageMaps, ECLOPT_CHECK_PACKAGEMAPS))
                 continue;
+            if (iter.matchFlag(optDeleteRecursive, ECLOPT_DELETE_RECURSIVE))
+                continue;
+            if (iter.matchFlag(optDeleteSubFiles, ECLOPT_DELETE_SUBFILES))
+                continue;
             if (iter.matchFlag(optDeleteFiles, ECLOPT_DELETE_FILES))
                 continue;
             if (EclCmdCommon::matchCommandLineOption(iter, true)!=EclCmdOptionMatch)
@@ -400,12 +404,16 @@ public:
             fputs("process cluster must be specified.\n\n", stderr);
             return false;
         }
+        if (optDeleteRecursive)
+            optDeleteSubFiles = true; //implied
+        if (optDeleteSubFiles)
+            optDeleteFiles = true; //implied
         return true;
     }
 
     virtual int processCMD()
     {
-        Owned<IClientWsDFUXRef> client = createCmdClient(WsDFUXRef, *this);
+        Owned<IClientWsDFUXRef> client = createCmdClientExt(WsDFUXRef, *this, "?ver_=1.29");
         Owned<IClientDFUXRefUnusedFilesRequest> req = client->createDFUXRefUnusedFilesRequest();
         req->setProcessCluster(optProcess);
         req->setCheckPackageMaps(optCheckPackageMaps);
@@ -437,18 +445,31 @@ public:
         {
             fputs("Deleting...\n", stderr);
 
-            Owned<IClientWsDfu> dfuClient = createCmdClient(WsDfu, *this);
+            Owned<IClientWsDfu> dfuClient = createCmdClientExt(WsDfu, *this, "?ver_=1.29");
             Owned<IClientDFUArrayActionRequest> dfuReq = dfuClient->createDFUArrayActionRequest();
             dfuReq->setType("Delete");
             dfuReq->setLogicalFiles(filesToDelete);
+            dfuReq->setRemoveFromSuperfiles(optDeleteSubFiles);
+            dfuReq->setRemoveRecursively(optDeleteRecursive);
 
             Owned<IClientDFUArrayActionResponse> dfuResp = dfuClient->DFUArrayAction(dfuReq);
             if (dfuResp->getExceptions().ordinality())
                 outputMultiExceptions(dfuResp->getExceptions());
 
             IArrayOf<IConstDFUActionInfo> &results = dfuResp->getActionResults();
-            ForEachItemIn(i, results)
-                fprintf(stdout, "  %s\n", results.item(i).getActionResult()); //result text already has filename
+            ForEachItemIn(i1, results) //list successes first
+            {
+                IConstDFUActionInfo &info = results.item(i1);
+                if (!info.getFailed())
+                    fprintf(stdout, "  %s\n", info.getActionResult()); //result text already has filename
+            }
+            fputs("\n", stdout);
+            ForEachItemIn(i2, results) //then errors
+            {
+                IConstDFUActionInfo &info = results.item(i2);
+                if (info.getFailed())
+                    fprintf(stdout, "  %s\n", info.getActionResult()); //result text already has filename
+            }
             fputs("\n", stdout);
         }
 
@@ -470,6 +491,10 @@ public:
         fputs("\n"
             "   --check-packagemaps    Exclude files referenced in active packagemaps\n"
             "   --delete               Delete unused files from DFS\n"
+            "   --delete-subfiles      Delete unused files from DFS and remove them from\n"
+            "                          superfiles.\n"
+            "   --delete-recursive     Delete unused files from DFS and remove them from\n"
+            "                          superfiles recursively.\n"
             " Common Options:\n",
             stdout);
         EclCmdCommon::usage();
@@ -478,6 +503,8 @@ private:
     StringAttr optProcess;
     bool optCheckPackageMaps;
     bool optDeleteFiles;
+    bool optDeleteSubFiles;
+    bool optDeleteRecursive;
 };
 
 IEclCommand *createEclRoxieCommand(const char *cmdname)

ecl/hql/hqlattr.cpp

@@ -1727,9 +1727,6 @@ bool isLocalActivity(IHqlExpression * expr)
     default:
         {
             assertex(!localChangesActivity(expr));
-            //MORE: What is this test here for??
-            if (!expr->isAction() && queryDistribution(expr))
-                return !isGroupedActivity(expr);
             return false;
         }
     }

ecl/hql/hqlgram.y

@@ -6587,7 +6587,7 @@ primexpr1
                             Owned<IHqlExpression> locale = (ltype->getTypeCode() == type_varstring) ? lexpr.getLink() : createValue(no_implicitcast, makeVarStringType(ltype->getStringLen()), lexpr.getLink());
                             $$.setExpr(createValue(no_unicodeorder, makeIntType(4, true), $3.getExpr(), $5.getExpr(), locale.getLink(), $9.getExpr()));
                         }
-    | '[' beginList sortList ']'
+    | '[' beginList nonDatasetList ']'
                         {
                             HqlExprArray sortItems;
                             parser->endList(sortItems);
@@ -11255,7 +11255,25 @@ sortList
                         }
     ;
 
-sortItem
+nonDatasetList
+    : nonDatasetExpr
+                        {
+                            parser->addListElement($1.getExpr());
+                            $$.clear();
+                        }
+    |   nonDatasetList ',' nonDatasetExpr
+                        {
+                            parser->addListElement($3.getExpr());
+                            $$.clear();
+                        }
+    |   nonDatasetList ';' nonDatasetExpr
+                        {
+                            parser->addListElement($3.getExpr());
+                            $$.clear();
+                        }
+    ;
+
+nonDatasetExpr
     : expression            
                         {
                             node_operator op = $1.getOperator();
@@ -11288,6 +11306,11 @@ sortItem
                             $$.setPosition($1);
                         }
     | dictionary
+    ;
+
+sortItem
+    : nonDatasetExpr
+    | dataSet
     | FEW               {   $$.setExpr(createAttribute(fewAtom));   }
     | MANY              {   $$.setExpr(createAttribute(manyAtom));  }
     | MERGE             {   $$.setExpr(createAttribute(mergeAtom)); }
@@ -11419,6 +11442,7 @@ dedupFlag
                                 row.setown(createAttribute(rightAtom));
                             $$.setExpr(row.getClear(), $1);
                         }
+    | dataSet
     ;
 
 rollupExtra
@@ -11447,6 +11471,7 @@ rollupFlag
                             $$.setExpr(createAttribute(exceptAtom, $2.getExpr())); 
                         }
     | dataRow
+    | dataSet
     ;
 
 conditions

ecl/hql/hqlopt.cpp

@@ -763,7 +763,73 @@ IHqlExpression * CTreeOptimizer::optimizeDatasetIf(IHqlExpression * transformed)
             noteUnused(left);
         return transformed->cloneAllAnnotations(ret);
     }
-    return LINK(transformed);
+    return NULL;
+}
+
+IHqlExpression * CTreeOptimizer::optimizeIfAppend(IHqlExpression * expr, node_operator combineOp)
+{
+    IHqlExpression * trueExpr = expr->queryChild(1);
+    IHqlExpression * falseExpr = expr->queryChild(2);
+    if (!falseExpr)
+        return NULL;
+
+    IHqlExpression * commonExpr = NULL;
+    IHqlExpression * appendExpr = trueExpr;
+    if ((trueExpr->getOperator() == combineOp) && !isShared(trueExpr))
+    {
+        IHqlExpression * trueArg0 = trueExpr->queryChild(0);
+        if (trueArg0->queryBody() == falseExpr->queryBody())
+        {
+            //Convert IF(a, b+c, b) to b + IF(a, DATASET(c))
+            commonExpr = falseExpr;
+        }
+        else if ((falseExpr->getOperator() == combineOp) && !isShared(falseExpr) &&
+                 (trueArg0->queryBody() == falseExpr->queryChild(0)->queryBody()))
+        {
+            //Convert IF(a, b+c, b+d) to b + IF(a, DATASET(c), DATASET(d))
+
+            //Check any other attributes match.  This ensures that + is only combined with +,
+            //& with & and && with &&.
+            if (remainingChildrenMatch(trueExpr, falseExpr, 3))
+                commonExpr = trueArg0;
+        }
+    }
+    else if ((falseExpr->getOperator() == combineOp) && !isShared(falseExpr))
+    {
+        if (trueExpr->queryBody() == falseExpr->queryChild(0)->queryBody())
+        {
+            //Convert IF(a, b, b+c) to b + IF(a, DATASET(), DATASET(c))
+            commonExpr = trueExpr;
+            appendExpr = falseExpr;
+        }
+    }
+
+    if (!commonExpr)
+        return NULL;
+
+    //Create an IF() for the expression that wasn't common between the two branches
+    HqlExprArray ifargs;
+    ifargs.append(*LINK(expr->queryChild(0)));
+    if (trueExpr->queryBody() == commonExpr->queryBody())
+        ifargs.append(*createNullExpr(expr));
+    else
+        ifargs.append(*ensureDataset(trueExpr->queryChild(1)));
+    if (falseExpr->queryBody() == commonExpr->queryBody())
+        ifargs.append(*createNullExpr(expr));
+    else
+        ifargs.append(*ensureDataset(falseExpr->queryChild(1)));
+
+    OwnedHqlExpr newIf = expr->clone(ifargs);
+    incUsage(newIf);
+
+    //Append the common dataset with the new IF()
+    HqlExprArray args;
+    args.append(*LINK(commonExpr));
+    args.append(*newIf.getClear());
+    unwindChildren(args, appendExpr, 2);
+    OwnedHqlExpr ret = appendExpr->clone(args);
+    DBGLOG("Optimizer: Extract common branch - replace %s with %s", queryNode0Text(expr), queryNode1Text(ret));
+    return ret.getClear();
 }
 
 static bool branchesMatch(unsigned options, IHqlExpression * left, IHqlExpression * right)
@@ -2195,19 +2261,27 @@ IHqlExpression * CTreeOptimizer::doCreateTransformed(IHqlExpression * transforme
     IHqlExpression * child = transformed->queryChild(0);
 
     //Any optimizations that remove the current node, or modify the current node don't need to check if the children are shared
-    //Removing child nodes could be included, but it may create more spillers/spliters - which may be significant in thor.
+    //Removing child nodes could be included, but it may create more spillers/splitters - which may be significant in thor.
     switch (op)
     {
     case no_if:
         {
             OwnedHqlExpr ret = optimizeIf(transformed);
+
+            //This won't split shared nodes, but one of the children may be shared - so processed here
+            if (!ret && transformed->isDataset())
+            {
+                //Convert IF(a, f(ds), g(ds)) to ds(IF(a,f,g)) - revisit since may increase dependencies
+                ret.setown(optimizeDatasetIf(transformed));
+
+                //Convert IF(a, b+c, b+d) to b + IF(a, DATASET(c), DATASET(d))
+                if (!ret)
+                    ret.setown(optimizeIfAppend(transformed, no_addfiles));
+            }
+
             if (ret)
                 return ret.getClear();
-
-            //Processed hereThis won't split shared nodes, but one of the children may be shared - so proce
-            if (transformed->isDataset())
-                return optimizeDatasetIf(transformed);
-            break;
+            return LINK(transformed);
         }
     case no_keyedlimit:
         {
@@ -3107,6 +3181,17 @@ IHqlExpression * CTreeOptimizer::doCreateTransformed(IHqlExpression * transforme
             break;
 
         }
+    case no_stepped:
+        {
+            node_operator childOp = child->getOperator();
+            switch(childOp)
+            {
+            case no_limit:
+            case no_keyedlimit:
+                return swapNodeWithChild(transformed);
+            }
+            break;
+        }
     case no_keyedlimit:
         {
             node_operator childOp = child->getOperator();
@@ -3114,7 +3199,6 @@ IHqlExpression * CTreeOptimizer::doCreateTransformed(IHqlExpression * transforme
             {
             case no_distributed:
             case no_sorted:
-            case no_stepped:
             case no_limit:
             case no_choosen:
             case no_compound_indexread:
@@ -3248,13 +3332,9 @@ IHqlExpression * CTreeOptimizer::doCreateTransformed(IHqlExpression * transforme
             case no_stepped:
                 return moveProjectionOverSimple(transformed, true, false);
             case no_keyedlimit:
-                if (isWorthMovingProjectOverLimit(transformed))
-                {
-                    if (child->hasAttribute(onFailAtom))
-                        return moveProjectionOverLimit(transformed);
-                    return swapNodeWithChild(transformed);
-                }
-                break;
+                if (child->hasAttribute(onFailAtom))
+                    return moveProjectionOverLimit(transformed);
+                return swapNodeWithChild(transformed);
             case no_catchds:
                 //could treat like a limit, but not at the moment
                 break;

ecl/hql/hqlopt.ipp

@@ -115,6 +115,7 @@ protected:
     IHqlExpression * optimizeJoinCondition(IHqlExpression * expr);
     IHqlExpression * optimizeDistributeDedup(IHqlExpression * expr);
     IHqlExpression * optimizeIf(IHqlExpression * expr);
+    IHqlExpression * optimizeIfAppend(IHqlExpression * expr, node_operator combineOp);
     IHqlExpression * optimizeProjectInlineTable(IHqlExpression * transformed, bool childrenAreShared);
         
     inline const char * queryNode0Text(IHqlExpression * expr) { return queryChildNodeTraceText(nodeText[0], expr); }

ecl/hql/hqlutil.cpp

@@ -2268,6 +2268,19 @@ IHqlExpression * queryResultName(IHqlExpression * expr)
     return NULL;
 }
 
+
+bool remainingChildrenMatch(IHqlExpression * left, IHqlExpression * right, unsigned first)
+{
+    if (left->numChildren() != right->numChildren())
+        return false;
+    ForEachChildFrom(i, left, first)
+    {
+        if (left->queryChild(i) != right->queryChild(i))
+            return false;
+    }
+    return true;
+}
+
 //---------------------------------------------------------------------------
 
 IHqlExpression * queryConvertChoosenNSort(IHqlExpression * expr, unsigned __int64 topNlimit)

ecl/hql/hqlutil.hpp

@@ -103,6 +103,7 @@ extern HQL_API bool transformContainsSkip(IHqlExpression * transform);
 extern HQL_API bool transformListContainsSkip(IHqlExpression * transforms);
 extern HQL_API bool recordContainsNestedRecord(IHqlExpression * record);
 extern HQL_API IHqlExpression * queryStripCasts(IHqlExpression * expr);
+extern HQL_API bool remainingChildrenMatch(IHqlExpression * left, IHqlExpression * right, unsigned first);
 
 extern HQL_API IHqlExpression * queryInvalidCsvRecordField(IHqlExpression * expr);
 extern HQL_API bool isValidCsvRecord(IHqlExpression * expr);

ecl/hqlcpp/hqlcerrors.hpp

@@ -263,6 +263,7 @@
 #define HQLWRN_NoThorContextDependent           4544
 #define HQLWRN_OnlyLocalMergeJoin               4545
 #define HQLWRN_WorkflowDependParameter          4546
+#define HQLWRN_OutputScalarInsideChildQuery     4547
 
 //Temporary errors
 #define HQLERR_OrderOnVarlengthStrings          4601
@@ -542,6 +543,7 @@
 #define HQLWRN_NoThorContextDependent_Text      "NOTHOR expression%s appears to access a parent dataset - this may cause a dataset not active error"
 #define HQLWRN_OnlyLocalMergeJoin_Text          "Only LOCAL versions of %s are currently supported on THOR"
 #define HQLWRN_WorkflowDependParameter_Text     "Workflow action %s appears to be dependent upon a parameter"
+#define HQLWRN_OutputScalarInsideChildQuery_Text "Output(%s) of single value inside a child query has undefined behaviour"
 
 #define HQLERR_DistributionVariableLengthX_Text "DISTRIBUTION does not support variable length field '%s'"
 #define HQLERR_DistributionUnsupportedTypeXX_Text "DISTRIBUTION does not support field '%s' with type %s"

ecl/hqlcpp/hqlcpp.cpp

@@ -7982,50 +7982,52 @@ void HqlCppTranslator::doBuildAssignCompareTable(BuildCtx & ctx, EvaluateCompare
     // i2; forEachIn(i2); {
     CHqlBoundExpr isValid;
     BuildCtx loopctx(subctx);
-    buildDatasetIterate(loopctx, right, true);
-    bindTableCursor(loopctx, left, leftRow);
- 
-    //     if (!i1.isValid()) { cmp = -1; break; }
-    buildIteratorIsValid(loopctx, leftIter, leftRow, isValid);
-    OwnedHqlExpr test = createValue(no_not, makeBoolType(), isValid.expr.getClear());
-    BuildCtx moreRightCtx(loopctx);
-    moreRightCtx.addFilter(test);
-
-    if (info.actionIfDiffer == return_stmt)
+    if (buildDatasetIterate(loopctx, right, true))
     {
-        if (info.isEqualityCompare())
+        bindTableCursor(loopctx, left, leftRow);
+
+        //     if (!i1.isValid()) { cmp = -1; break; }
+        buildIteratorIsValid(loopctx, leftIter, leftRow, isValid);
+        OwnedHqlExpr test = createValue(no_not, makeBoolType(), isValid.expr.getClear());
+        BuildCtx moreRightCtx(loopctx);
+        moreRightCtx.addFilter(test);
+
+        if (info.actionIfDiffer == return_stmt)
         {
-            OwnedHqlExpr returnValue = info.getEqualityReturnValue();
-            moreRightCtx.addReturn(returnValue);
+            if (info.isEqualityCompare())
+            {
+                OwnedHqlExpr returnValue = info.getEqualityReturnValue();
+                moreRightCtx.addReturn(returnValue);
+            }
+            else
+                moreRightCtx.addReturn(minusOne);
         }
         else
-            moreRightCtx.addReturn(minusOne);
-    }
-    else
-    {
-        buildExprAssign(moreRightCtx, info.target, minusOne);
-        moreRightCtx.addBreak();
-    }
+        {
+            buildExprAssign(moreRightCtx, info.target, minusOne);
+            moreRightCtx.addBreak();
+        }
 
-    //Now do the comparison....
-    {
-        EvaluateCompareInfo childInfo(info);
-        if (childInfo.actionIfDiffer == break_stmt)
-            childInfo.actionIfDiffer = null_stmt;
-        //***childInfo??
-        doBuildAssignCompareRow(loopctx, info, left, right);
-    }
+        //Now do the comparison....
+        {
+            EvaluateCompareInfo childInfo(info);
+            if (childInfo.actionIfDiffer == break_stmt)
+                childInfo.actionIfDiffer = null_stmt;
+            //***childInfo??
+            doBuildAssignCompareRow(loopctx, info, left, right);
+        }
 
-    if (info.actionIfDiffer != return_stmt)
-    {
-        //     if (cmp != 0) break;
-        BuildCtx donectx(loopctx);
-        donectx.addFilter(info.target.expr);
-        donectx.addQuotedLiteral("break;");
-    }
+        if (info.actionIfDiffer != return_stmt)
+        {
+            //     if (cmp != 0) break;
+            BuildCtx donectx(loopctx);
+            donectx.addFilter(info.target.expr);
+            donectx.addQuotedLiteral("break;");
+        }
 
-    //     i1.next();
-    buildIteratorNext(*this, loopctx, leftIter, leftRow);
+        //     i1.next();
+        buildIteratorNext(*this, loopctx, leftIter, leftRow);
+    }
 
     buildIteratorIsValid(subctx, leftIter, leftRow, isValid);
     if (info.actionIfDiffer == return_stmt)

ecl/hqlcpp/hqlhtcpp.cpp

@@ -7632,6 +7632,13 @@ void HqlCppTranslator::doBuildStmtSetResult(BuildCtx & ctx, IHqlExpression * exp
         graphLabel.set(text.str());
     }
 
+    if (insideChildQuery(ctx))
+    {
+        StringBuffer description;
+        getStoredDescription(description, seq, name, true);
+        reportWarning(CategoryUnusual, SeverityError, queryLocation(expr), ECODETEXT(HQLWRN_OutputScalarInsideChildQuery), description.str());
+    }
+
     if (cluster)
         pushCluster(subctx, cluster->queryChild(0));
 
@@ -17848,6 +17855,13 @@ ABoundActivity * HqlCppTranslator::doBuildActivitySetResult(BuildCtx & ctx, IHql
 
     buildInstancePrefix(instance);
 
+    if (insideChildQuery(ctx))
+    {
+        StringBuffer description;
+        getStoredDescription(description, sequence, name, true);
+        reportWarning(CategoryUnusual, SeverityError, queryLocation(expr), ECODETEXT(HQLWRN_OutputScalarInsideChildQuery), description.str());
+    }
+
     noteResultDefined(ctx, instance, sequence, name, isRoot);
     if (attribute->isDatarow())
         attribute.setown(::ensureSerialized(attribute, diskAtom));

ecl/regress/scalarchild.ecl

@@ -0,0 +1,18 @@
+//fail
+
+idRec := {unsigned id };
+
+ds := DATASET(10, TRANSFORM(idRec, SELF.id := HASH64(COUNTER)));
+
+f(unsigned id) := FUNCTION
+    o := OUTPUT(id);
+    RETURN WHEN(id * 2, o);
+END;
+
+idRec t(idRec l) := TRANSFORM
+    SELF.id := f(l.id);
+END;
+
+p := PROJECT(NOFOLD(ds), t(LEFT));
+
+output(p);

esp/bindings/http/platform/httpservice.cpp

@@ -728,11 +728,46 @@ static void httpGetDirectory(CHttpRequest* request, CHttpResponse* response, con
     response->send();
 }
 
+static bool checkHttpPathStaysWithinBounds(const char *path)
+{
+    if (!path || !*path)
+        return true;
+    int depth = 0;
+    StringArray nodes;
+    nodes.appendList(path, "/");
+    ForEachItemIn(i, nodes)
+    {
+        const char *node = nodes.item(i);
+        if (!*node || streq(node, ".")) //empty or "." doesn't advance
+            continue;
+        if (!streq(node, ".."))
+            depth++;
+        else
+        {
+            depth--;
+            if (depth<0)  //only really care that the relative http path doesn't position itself above its own root node
+                return false;
+        }
+    }
+    return true;
+}
+
 int CEspHttpServer::onGetFile(CHttpRequest* request, CHttpResponse* response, const char *urlpath)
 {
         if (!request || !response || !urlpath)
             return -1;
 
+        StringBuffer basedir(getCFD());
+        basedir.append("files/");
+
+        if (!checkHttpPathStaysWithinBounds(urlpath))
+        {
+            DBGLOG("Get File %s: attempted access outside of %s", urlpath, basedir.str());
+            response->setStatus(HTTP_STATUS_NOT_FOUND);
+            response->send();
+            return 0;
+        }
+
         StringBuffer ext;
         StringBuffer tail;
         splitFilename(urlpath, NULL, NULL, &tail, &ext);
@@ -745,19 +780,8 @@ int CEspHttpServer::onGetFile(CHttpRequest* request, CHttpResponse* response, co
         else if (top)
             tail.set("./files");
 
-        StringBuffer basedir(getCFD());
-        basedir.append("files/");
-
         StringBuffer fullpath;
         makeAbsolutePath(urlpath, basedir.str(), fullpath);
-        if (*urlpath && strncmp(basedir, fullpath, basedir.length()))
-        {
-            DBGLOG("Get File %s: attempted access outside of %s", urlpath, basedir.str());
-            response->setStatus(HTTP_STATUS_NOT_FOUND);
-            response->send();
-            return 0;
-        }
-
         if (!checkFileExists(fullpath) && !checkFileExists(fullpath.toUpperCase()) && !checkFileExists(fullpath.toLowerCase()))
         {
             DBGLOG("Get File %s: file not found", urlpath);

esp/scm/ws_dfu.ecm

@@ -240,6 +240,8 @@ DFUArrayActionRequest
     bool NoDelete;
     [min_ver("1.04")] string BackToPage;
     ESParray<string> LogicalFiles;
+    bool removeFromSuperfiles(false);
+    bool removeRecursively(false);
 };
 
 ESPresponse 

esp/services/ws_dfu/ws_dfuService.cpp

@@ -1134,7 +1134,7 @@ bool CWsDfuEx::onAddtoSuperfile(IEspContext &context, IEspAddtoSuperfileRequest
     return true;
 }
 
-void CWsDfuEx::setDeleteFileResults(const char* fileName, const char* nodeGroup, bool failed, const char* actionResult, StringBuffer& resultString,
+void setDeleteFileResults(const char* fileName, const char* nodeGroup, bool failed, const char *start, const char* text, StringBuffer& resultString,
     IArrayOf<IEspDFUActionInfo>& actionResults)
 {
     if (!fileName || !*fileName)
@@ -1144,121 +1144,182 @@ void CWsDfuEx::setDeleteFileResults(const char* fileName, const char* nodeGroup,
     resultObj->setFailed(failed);
     if (nodeGroup && *nodeGroup)
         resultObj->setNodeGroup(nodeGroup);
-    if (actionResult && *actionResult)
-    {
-        resultObj->setActionResult(actionResult);
-        resultString.appendf("<Message><Value>%s</Value></Message>", actionResult);
-    }
+
+    StringBuffer message;
+    if (start)
+        message.append(start).append(' ');
+    message.append(fileName);
+    if (nodeGroup && *nodeGroup)
+        message.append(" on ").append(nodeGroup);
+    if (text && *text)
+        message.append(failed ? ": " : " ").append(text);
+    resultObj->setActionResult(message);
+    resultString.appendf("<Message><Value>%s</Value></Message>", message.str());
+
     actionResults.append(*resultObj.getClear());
-    return;
 }
 
-bool CWsDfuEx::DFUDeleteFiles(IEspContext &context, IEspDFUArrayActionRequest &req, IEspDFUArrayActionResponse &resp)
-{
-    double version = context.getClientVersion();
-    StringBuffer username;
-    context.getUserID(username);
+typedef enum {
+    DeleteActionSuccess,
+    DeleteActionFailure,
+    DeleteActionSkip
+} DeleteActionResult;
 
-    Owned<IUserDescriptor> userdesc;
-    if(username.length() > 0)
-    {
-        const char* passwd = context.queryPassword();
-        userdesc.setown(createUserDescriptor());
-        userdesc->set(username.str(), passwd);
-    }
-    StringBuffer returnStr;
 
-    StringArray superFileNames, filesCannotBeDeleted;
-    IArrayOf<IEspDFUActionInfo> actionResults;
-    for(int j = 0; j < 2; j++) //j=0: delete superfiles first
+DeleteActionResult doDeleteFile(const char *fn, IUserDescriptor *userdesc, StringArray &superFiles, StringArray &failedFiles, StringBuffer& returnStr, IArrayOf<IEspDFUActionInfo>& actionResults, bool superFilesOnly, bool removeFromSuperfiles, bool deleteRecursively);
+
+bool doRemoveFileFromSuperfiles(const char *lfn, IUserDescriptor *userdesc, StringArray &superFiles, StringArray &failedFiles, bool deleteRecursively, StringBuffer& returnStr, IArrayOf<IEspDFUActionInfo>& actionResults)
+{
+    StringArray emptySuperFiles;
+    IDistributedFileDirectory &fdir = queryDistributedFileDirectory();
     {
-        for(unsigned i = 0; i < req.getLogicalFiles().length(); i++)
+        Owned<IDistributedFile> df = fdir.lookup(lfn, userdesc, true);
+        if(!df)
+            return false;
+        Owned<IDistributedSuperFileIterator> supers = df->getOwningSuperFiles();
+        ForEach(*supers)
         {
-            const char* fileNameAndNodeGroup = req.getLogicalFiles().item(i);
-            if(!fileNameAndNodeGroup || !*fileNameAndNodeGroup)
-                continue;
-
-            const char* fileName = NULL;
-            const char* nodeGroup = NULL;
-            StringArray fileNameOrNodeGroup;
-            fileNameOrNodeGroup.appendListUniq(fileNameAndNodeGroup, "@");
-            fileName = fileNameOrNodeGroup.item(0);
-            if (fileNameOrNodeGroup.length() > 1)
-            {
-                nodeGroup = fileNameOrNodeGroup.item(1);
-                if (!*nodeGroup || strieq(nodeGroup, "null")) //null is used by new ECLWatch for a superfile
-                    nodeGroup = NULL;
-            }
-            if (j>0)
-            { // 2nd pass, now we want to skip superfiles and the files which cannot do the lookup.
-                if (superFileNames.contains(fileNameAndNodeGroup) || filesCannotBeDeleted.contains(fileNameAndNodeGroup))
-                    continue;
-            }
-
+            IDistributedSuperFile &super = supers->query();
             try
             {
-                IDistributedFileDirectory &fdir = queryDistributedFileDirectory();
-                {
-                    Owned<IDistributedFile> df = fdir.lookup(fileNameAndNodeGroup, userdesc, true);
-                    if(!df)
-                    {
-                        StringBuffer message;
-                        if (!nodeGroup || !*nodeGroup)
-                            message.appendf("Cannot delete %s: file not found", fileName);
-                        else
-                            message.appendf("Cannot delete %s on %s: file not found", fileName, nodeGroup);
-                        PROGLOG("CWsDfuEx::DFUDeleteFiles: %s", message.str());
-                        setDeleteFileResults(fileName, nodeGroup, true, message, returnStr, actionResults);
-                        filesCannotBeDeleted.append(fileNameAndNodeGroup);
-                        continue;
-                    }
-                    if (0==j) // skip non-super files on 1st pass
-                    {
-                        if(!df->querySuperFile())
-                            continue;
-
-                        superFileNames.append(fileNameAndNodeGroup);
-                    }
-                }
-                fdir.removeEntry(fileNameAndNodeGroup, userdesc, NULL, REMOVE_FILE_SDS_CONNECT_TIMEOUT, true);
-                StringBuffer message;
-                if (!nodeGroup || !*nodeGroup)
-                    message.appendf("File %s deleted", fileName);
-                else
-                    message.appendf("File %s deleted on %s", fileName, nodeGroup);
-                setDeleteFileResults(fileName, nodeGroup, false, message, returnStr, actionResults);
+                super.removeSubFile(lfn, false, false, NULL);
+                VStringBuffer text("from superfile %s", super.queryLogicalName());
+                setDeleteFileResults(lfn, NULL, false, "Removed subfile", text, returnStr, actionResults);
             }
             catch(IException* e)
             {
-                filesCannotBeDeleted.append(fileNameAndNodeGroup);
-
                 StringBuffer emsg;
-                e->errorMessage(emsg);
-                if((e->errorCode() == DFSERR_CreateAccessDenied) && (req.getType() != NULL))
-                    emsg.replaceString("Create ", "Delete ");
-
-                StringBuffer message;
-                if (!nodeGroup || !*nodeGroup)
-                    message.appendf("Cannot delete %s: %s", fileName, emsg.str());
-                else
-                    message.appendf("Cannot delete %s on %s: %s", fileName, nodeGroup, emsg.str());
-                setDeleteFileResults(fileName, nodeGroup, true, message, returnStr, actionResults);
+                VStringBuffer text("from superfile %s: %s", super.queryLogicalName(), e->errorMessage(emsg).str());
+                setDeleteFileResults(lfn, NULL, true, "Could not remove subfile ", text, returnStr, actionResults);
                 e->Release();
+                return false;
             }
             catch(...)
             {
-                StringBuffer message;
-                if (!nodeGroup || !*nodeGroup)
-                    message.appendf("Cannot delete %s: unknown exception.", fileName);
-                else
-                    message.appendf("Cannot delete %s on %s: unknown exception.", fileName, nodeGroup);
-                setDeleteFileResults(fileName, nodeGroup, true, message, returnStr, actionResults);
+                VStringBuffer text("from superfile %s", super.queryLogicalName());
+                setDeleteFileResults(lfn, NULL, true, "Could not remove subfile ", text, returnStr, actionResults);
+                return false;
+            }
+            if (deleteRecursively && super.numSubFiles(false)==0)
+                emptySuperFiles.appendUniq(super.queryLogicalName());
+        }
+    }
+    ForEachItemIn(i, emptySuperFiles)
+        doDeleteFile(emptySuperFiles.item(i), userdesc, superFiles, failedFiles, returnStr, actionResults, false, true, deleteRecursively);
+
+    return true;
+}
+
+DeleteActionResult doDeleteFile(const char *fn, IUserDescriptor *userdesc, StringArray &superFiles, StringArray &failedFiles, StringBuffer& returnStr, IArrayOf<IEspDFUActionInfo>& actionResults,
+        bool superFilesOnly, bool removeFromSuperfiles, bool deleteRecursively)
+{
+    StringArray parsed;
+    parsed.appendListUniq(fn, "@");
+    const char *lfn = parsed.item(0);
+    const char *group = NULL;
+    if (parsed.length() > 1)
+    {
+        group = parsed.item(1);
+        if (group && (!*group || strieq(group, "null"))) //null is used by new ECLWatch for a superfile
+            group = NULL;
+    }
+
+    bool isSuper = false;
+    if (superFiles.contains(fn) || failedFiles.contains(fn))
+        return DeleteActionSkip;
+    try
+    {
+        IDistributedFileDirectory &fdir = queryDistributedFileDirectory();
+        {
+            Owned<IDistributedFile> df = fdir.lookup(lfn, userdesc, true);
+            if(!df)
+            {
+                PROGLOG("CWsDfuEx::DFUDeleteFiles: %s not found", lfn);
+                setDeleteFileResults(lfn, group, true, "File not found", NULL, returnStr, actionResults);
+                return DeleteActionFailure;
+            }
+            isSuper = df->querySuperFile()!=NULL;
+            if (superFilesOnly) // skip non-super files on 1st pass
+            {
+                if(!isSuper)
+                    return DeleteActionSkip;
+                superFiles.append(fn);
             }
         }
+        fdir.removeEntry(fn, userdesc, NULL, REMOVE_FILE_SDS_CONNECT_TIMEOUT, true);
+        setDeleteFileResults(lfn, group, false, isSuper ? "Deleted Superfile" : "Deleted File", NULL, returnStr, actionResults);
+    }
+    catch(IException* e)
+    {
+        StringBuffer emsg;
+        e->errorMessage(emsg);
+        if (removeFromSuperfiles && strstr(emsg, "owned by"))
+        {
+            if (!doRemoveFileFromSuperfiles(lfn, userdesc, superFiles, failedFiles, deleteRecursively, returnStr, actionResults))
+                return DeleteActionFailure;
+            return doDeleteFile(fn, userdesc, superFiles, failedFiles, returnStr, actionResults, superFilesOnly, false, false);
+        }
+        if (e->errorCode() == DFSERR_CreateAccessDenied)
+            emsg.replaceString("Create ", "Delete ");
+
+        setDeleteFileResults(lfn, group, true, "Could not delete", emsg.str(), returnStr, actionResults);
+        e->Release();
+        return DeleteActionFailure;
+    }
+    catch(...)
+    {
+        setDeleteFileResults(lfn, group, true, "Could not delete", "unknown exception", returnStr, actionResults);
+        return DeleteActionFailure;
     }
+    return DeleteActionSuccess;
+}
+
+void doDeleteFiles(StringArray &files, IUserDescriptor *userdesc, StringArray &superFiles, StringArray &failedFiles, StringBuffer &returnStr, IArrayOf<IEspDFUActionInfo> &actionResults,
+        bool superFilesOnly, bool removeFromSuperfiles, bool deleteRecursively)
+{
+    ForEachItemIn(i, files)
+    {
+        const char* fn = files.item(i);
+        if(!fn || !*fn)
+            continue;
+
+        DeleteActionResult ar;
+        if (DeleteActionFailure==doDeleteFile(fn, userdesc, superFiles, failedFiles, returnStr, actionResults, superFilesOnly, removeFromSuperfiles, deleteRecursively))
+            failedFiles.appendUniq(fn);
+    }
+
+}
+
+inline void doDeleteSuperFiles(StringArray &files, IUserDescriptor *userdesc, StringArray &superFiles, StringArray &failedFiles, StringBuffer &returnStr, IArrayOf<IEspDFUActionInfo> &actionResults,
+        bool removeFromSuperfiles, bool deleteRecursively)
+{
+    doDeleteFiles(files, userdesc, superFiles, failedFiles, returnStr, actionResults, true, removeFromSuperfiles, deleteRecursively);
+}
+
+inline void doDeleteSubFiles(StringArray &files, IUserDescriptor *userdesc, StringArray &superFiles, StringArray &failedFiles, StringBuffer &returnStr, IArrayOf<IEspDFUActionInfo> &actionResults,
+        bool removeFromSuperfiles, bool deleteRecursively)
+{
+    doDeleteFiles(files, userdesc, superFiles, failedFiles, returnStr, actionResults, false, removeFromSuperfiles, deleteRecursively);
+}
+
+bool CWsDfuEx::DFUDeleteFiles(IEspContext &context, IEspDFUArrayActionRequest &req, IEspDFUArrayActionResponse &resp)
+{
+    Owned<IUserDescriptor> userdesc;
+    const char *username = context.queryUserId();
+    if(username && *username)
+    {
+        userdesc.setown(createUserDescriptor());
+        userdesc->set(username, context.queryPassword());
+    }
+
+    StringBuffer returnStr;
+    IArrayOf<IEspDFUActionInfo> actionResults;
+
+    StringArray superFiles, failedFiles;
+    doDeleteSuperFiles(req.getLogicalFiles(), userdesc, superFiles, failedFiles, returnStr, actionResults, req.getRemoveFromSuperfiles(), req.getRemoveRecursively());
+    doDeleteSubFiles(req.getLogicalFiles(), userdesc, superFiles, failedFiles, returnStr, actionResults, req.getRemoveFromSuperfiles(), req.getRemoveRecursively());
 
-    if (version >= 1.27)
-        resp.setActionResults(actionResults);
+    resp.setActionResults(actionResults);
     resp.setDFUArrayActionResult(returnStr.str());//Used by legacy
     return true;
 }

esp/services/ws_dfu/ws_dfuService.hpp

@@ -135,8 +135,6 @@ private:
         const char* beforeSubFile, bool existingSuperfile, bool autocreatesuper, bool deleteFile, bool removeSuperfile =  true);
     void getFilePartsOnClusters(IEspContext &context, const char* clusterReq, StringArray& clusters, IDistributedFile* df, IEspDFUFileDetail& FileDetails,
         offset_t& mn, offset_t& mx, offset_t& sum, offset_t& count);
-    void setDeleteFileResults(const char* fileName, const char* nodeGroup, bool failed, const char* message, StringBuffer& resultString,
-        IArrayOf<IEspDFUActionInfo>& actionResults);
 private:
     bool         m_disableUppercaseTranslation;
     StringBuffer m_clusterName;

esp/services/ws_workunits/ws_workunitsService.cpp

@@ -2332,7 +2332,8 @@ INewResultSet* createFilteredResultSet(INewResultSet* result, IArrayOf<IConstNam
 }
 
 void getWsWuResult(IEspContext &context, const char* wuid, const char *name, const char *logical, unsigned index, __int64 start,
-    unsigned& count, __int64& total, IStringVal& resname, bool bin, IArrayOf<IConstNamedValue>* filterBy, MemoryBuffer& mb, bool xsd=true)
+    unsigned& count, __int64& total, IStringVal& resname, bool bin, IArrayOf<IConstNamedValue>* filterBy, MemoryBuffer& mb,
+    WUState& wuState, bool xsd=true)
 {
     Owned<IWorkUnitFactory> factory = getWorkUnitFactory(context.querySecManager(), context.queryUser());
     Owned<IConstWorkUnit> cw = factory->openWorkUnit(wuid, false);
@@ -2381,6 +2382,8 @@ void getWsWuResult(IEspContext &context, const char* wuid, const char *name, con
         Owned<INewResultSet> filteredResult = createFilteredResultSet(rs, filterBy);
         appendResultSet(mb, filteredResult, name, start, count, total, bin, xsd, context.getResponseFormat(), result->queryResultXmlns());
     }
+
+    wuState = cw->getState();
 }
 
 void checkFileSizeLimit(unsigned long xmlSize, unsigned long sizeLimit)
@@ -2626,11 +2629,12 @@ bool CWsWorkunitsEx::onWUResultBin(IEspContext &context,IEspWUResultBinRequest &
         IArrayOf<IConstNamedValue>* filterBy = &req.getFilterBy();
         SCMStringBuffer name;
 
+        WUState wuState = WUStateUnknown;
         bool bin = (req.getFormat() && strieq(req.getFormat(),"raw"));
         if (notEmpty(wuidIn) && notEmpty(req.getResultName()))
-            getWsWuResult(context, wuidIn, req.getResultName(), NULL, 0, start, count, total, name, bin, filterBy, mb);
+            getWsWuResult(context, wuidIn, req.getResultName(), NULL, 0, start, count, total, name, bin, filterBy, mb, wuState);
         else if (notEmpty(wuidIn) && (req.getSequence() >= 0))
-            getWsWuResult(context, wuidIn, NULL, NULL, req.getSequence(), start, count, total, name, bin,filterBy, mb);
+            getWsWuResult(context, wuidIn, NULL, NULL, req.getSequence(), start, count, total, name, bin,filterBy, mb, wuState);
         else if (notEmpty(req.getLogicalName()))
         {
             const char* logicalName = req.getLogicalName();
@@ -2638,7 +2642,7 @@ bool CWsWorkunitsEx::onWUResultBin(IEspContext &context,IEspWUResultBinRequest &
             getWuidFromLogicalFileName(context, logicalName, wuid);
             if (!wuid.length())
                 throw MakeStringException(ECLWATCH_CANNOT_GET_WORKUNIT,"Cannot find the workunit for file %s.",logicalName);
-            getWsWuResult(context, wuid.str(), NULL, logicalName, 0, start, count, total, name, bin, filterBy, mb);
+            getWsWuResult(context, wuid.str(), NULL, logicalName, 0, start, count, total, name, bin, filterBy, mb, wuState);
         }
         else
             throw MakeStringException(ECLWATCH_CANNOT_GET_WU_RESULT,"Cannot open the workunit result.");
@@ -2871,6 +2875,7 @@ bool CWsWorkunitsEx::onWUResult(IEspContext &context, IEspWUResultRequest &req,
         }
         else
         {
+            WUState wuState = WUStateUnknown;
             if(logicalName && *logicalName)
             {
                 StringBuffer lwuid;
@@ -2890,17 +2895,22 @@ bool CWsWorkunitsEx::onWUResult(IEspContext &context, IEspWUResultRequest &req,
                 }
                 else
                     throw MakeStringException(ECLWATCH_INVALID_INPUT,"Need valid target cluster to browse file %s.",logicalName);
+
+                Owned<IWorkUnitFactory> wf = getWorkUnitFactory(context.querySecManager(), context.queryUser());
+                Owned<IConstWorkUnit> cw = wf->openWorkUnit(lwuid.str(), false);
+                if (cw)
+                    wuState = cw->getState();
             }
             else if (notEmpty(wuid) && notEmpty(resultName))
             {
                 name.set(resultName);
-                getWsWuResult(context, wuid, resultName, NULL, 0, start, count, total, name, false, filterBy, mb, inclXsd);
+                getWsWuResult(context, wuid, resultName, NULL, 0, start, count, total, name, false, filterBy, mb, wuState, inclXsd);
                 resp.setWuid(wuid);
                 resp.setSequence(seq);
             }
             else
             {
-                getWsWuResult(context, wuid, NULL, NULL, seq, start, count, total, name, false, filterBy, mb, inclXsd);
+                getWsWuResult(context, wuid, NULL, NULL, seq, start, count, total, name, false, filterBy, mb, wuState, inclXsd);
                 resp.setWuid(wuid);
                 resp.setSequence(seq);
             }
@@ -2909,7 +2919,15 @@ bool CWsWorkunitsEx::onWUResult(IEspContext &context, IEspWUResultRequest &req,
             if (requested > total)
                 requested = (unsigned)total;
 
-            dataCache->add(filter, mb.toByteArray(), name.str(), logicalName, wuid, resultName, seq, start, count, requested, total);
+            switch (wuState)
+            {
+                 case WUStateCompleted:
+                 case WUStateAborted:
+                 case WUStateFailed:
+                 case WUStateArchived:
+                     dataCache->add(filter, mb.toByteArray(), name.str(), logicalName, wuid, resultName, seq, start, count, requested, total);
+                     break;
+            }
         }
 
         resp.setName(name.str());

esp/src/eclwatch/ActivityWidget.js

@@ -23,6 +23,7 @@ define([
 
     "dijit/registry",
     "dijit/form/Button",
+    "dijit/form/ToggleButton",
     "dijit/ToolbarSeparator",
     "dijit/layout/ContentPane",
 
@@ -36,7 +37,7 @@ define([
     "hpcc/ESPUtil"
 
 ], function (declare, lang, i18n, nlsHPCC, arrayUtil, on,
-                registry, Button, ToolbarSeparator, ContentPane,
+                registry, Button, ToggleButton, ToolbarSeparator, ContentPane,
                 selector, tree,
                 GridDetailsWidget, ESPRequest, ESPActivity, DelayLoadWidget, ESPUtil) {
     return declare("ActivityWidget", [GridDetailsWidget], {
@@ -45,6 +46,10 @@ define([
         gridTitle: nlsHPCC.title_Activity,
         idProperty: "__hpcc_id",
 
+        _onAutoRefresh: function (event) {
+            this.activity.disableMonitor(!this.autoRefreshButton.get("checked"));
+        },
+
         _onPause: function (event, params) {
             arrayUtil.forEach(this.grid.getSelected(), function (item, idx) {
                 if (this.activity.isInstanceOfQueue(item)) {
@@ -187,6 +192,8 @@ define([
                 return;
 
             var context = this;
+            this.autoRefreshButton = registry.byId(this.id + "AutoRefresh");
+            this.activity.disableMonitor(true);
             this.activity.watch("__hpcc_changedCount", function (item, oldValue, newValue) {
                 context.grid.set("query", {});
                 context._refreshActionState();
@@ -204,6 +211,17 @@ define([
             var context = this;
 
             this.openButton = registry.byId(this.id + "Open");
+            this.refreshButton = registry.byId(this.id + "Refresh");            
+            this.autoRefreshButton = new ToggleButton({
+                id: this.id + "AutoRefresh",
+                iconClass:'iconAutoRefresh',
+                showLabel: false,
+                checked: false,
+                onClick: function (event) {
+                    context._onAutoRefresh(event);
+                }
+            }).placeAt(this.refreshButton.domNode, "before");
+            tmpSplitter = new ToolbarSeparator().placeAt(this.refreshButton.domNode, "before");
             this.clusterPauseButton = new Button({
                 id: this.id + "PauseButton",
                 label: this.i18n.Pause,

esp/src/eclwatch/DFUQueryWidget.js

@@ -49,6 +49,7 @@ define([
     "hpcc/ESPDFUWorkunit",
     "hpcc/DelayLoadWidget",
     "hpcc/TargetSelectWidget",
+    "hpcc/TargetComboBoxWidget",
     "hpcc/FilterDropDownWidget",
     "hpcc/SelectionGridWidget",
     "hpcc/WsTopology",
@@ -78,7 +79,7 @@ define([
 ], function (declare, lang, i18n, nlsHPCC, arrayUtil, dom, domAttr, domConstruct, domClass, domForm, date, on, topic,
                 registry, Dialog, Menu, MenuItem, MenuSeparator, PopupMenuItem, Textarea, ValidationTextBox,
                 editor, selector, tree,
-                _TabContainerWidget, WsDfu, FileSpray, ESPUtil, ESPLogicalFile, ESPDFUWorkunit, DelayLoadWidget, TargetSelectWidget, FilterDropDownWidget, SelectionGridWidget, WsTopology,
+                _TabContainerWidget, WsDfu, FileSpray, ESPUtil, ESPLogicalFile, ESPDFUWorkunit, DelayLoadWidget, TargetSelectWidget, TargetComboBoxWidget, FilterDropDownWidget, SelectionGridWidget, WsTopology,
                 put,
                 template) {
     return declare("DFUQueryWidget", [_TabContainerWidget, ESPUtil.FormHelper], {
@@ -102,6 +103,26 @@ define([
             this.addToSuperfileGrid = registry.byId(this.id + "AddToSuperfileGrid");
             this.desprayForm = registry.byId(this.id + "DesprayForm");
             this.desprayTargetSelect = registry.byId(this.id + "DesprayTargetSelect");
+            this.desprayTooltiopDialog = registry.byId(this.id + "DesprayTooltipDialog");
+            var context = this;
+            var origOnOpen = this.desprayTooltiopDialog.onOpen;
+            this.desprayTooltiopDialog.onOpen = function () {
+                if (!context.desprayTargetSelect.initalized) {
+                    context.desprayTargetSelect.init({
+                        DropZones: true,
+                        callback: function (value, item) {
+                            registry.byId(context.id + "DesprayTargetIPAddress").set("value", item.machine.Netaddress);
+                            if (context.desprayTargetPath) {
+                                context.desprayTargetPath.reset();
+                                context.desprayTargetPath._dropZoneTarget = item;
+                                context.desprayTargetPath.defaultValue = context.desprayTargetPath.get("value");
+                                context.desprayTargetPath.loadDropZoneFolders();
+                            }
+                        }
+                    });
+                }
+                origOnOpen.apply(context.desprayTooltiopDialog, arguments);
+            }
             this.desprayTargetPath = registry.byId(this.id + "DesprayTargetPath");
             this.desprayGrid = registry.byId(this.id + "DesprayGrid");
             this.remoteCopyReplicateCheckbox = registry.byId(this.id + "RemoteCopyReplicate");
@@ -232,6 +253,7 @@ define([
                 var context = this;
                 arrayUtil.forEach(this.desprayGrid.store.data, function (item, idx) {
                     var request = domForm.toObject(context.id + "DesprayForm");
+                    request.destPath = context.desprayTargetPath.getDropZoneFolder();
                     if (!context.endsWith(request.destPath, "/")) {
                         request.destPath += "/";
                     }
@@ -308,18 +330,7 @@ define([
             this.copyTargetSelect.init({
                 Groups: true
             });
-            this.desprayTargetSelect.init({
-                DropZones: true,
-                callback: function (value, item) {
-                    registry.byId(context.id + "DesprayTargetIPAddress").set("value", item.machine.Netaddress);
-                    if (context.desprayTargetPath) {
-                        context.desprayTargetPath.reset();
-                        context.desprayTargetPath._dropZoneTarget = item;
-                        context.desprayTargetPath.defaultValue = context.desprayTargetPath.get("value");
-                        context.desprayTargetPath.loadDropZoneFolders();
-                    }
-                }
-            });
+
             this.desprayTargetPath.init({
                 DropZoneFolders: true
             });

esp/src/eclwatch/ESPRequest.js

@@ -114,11 +114,37 @@ define([
 
             var handleAs = params.handleAs ? params.handleAs : "json";
             return this._send(service, action, params).then(function (response) {
-                if (!params.suppressExceptionToaster && handleAs == "json") {
+                if (handleAs === "json") {
                     if (lang.exists("Exceptions.Source", response)) {
+                        var severity = params.suppressExceptionToaster ? "Info" : "Error";
+                        var source = service + "." + action;
+                        if (lang.exists("Exceptions.Exception", response) && response.Exceptions.Exception.length === 1) {
+                            switch (source) {
+                                case "WsWorkunits.WUInfo":
+                                    if (response.Exceptions.Exception[0].Code === 20080) {
+                                        severity = "Info";
+                                    }
+                                    break;
+                                case "WsWorkunits.WUQuery":
+                                    if (response.Exceptions.Exception[0].Code === 20081) {
+                                        severity = "Info";
+                                    }
+                                    break;
+                                case "FileSpray.GetDFUWorkunit":
+                                    if (response.Exceptions.Exception[0].Code === 20080) {
+                                        severity = "Info";
+                                    }
+                                    break;
+                                case "WsDfu.DFUInfo":
+                                    if (response.Exceptions.Exception[0].Code === 20038) {
+                                        severity = "Info";
+                                    }
+                                    break;
+                            }
+                        }
                         topic.publish("hpcc/brToaster", {
-                            Severity: "Error",
-                            Source: service + "." + action,
+                            Severity: severity,
+                            Source: source,
                             Exceptions: response.Exceptions.Exception
                         });
                     }

esp/src/eclwatch/ESPResult.js

@@ -32,6 +32,19 @@ define([
             parser, entities,
             ESPBase, ESPRequest, WsWorkunits) {
 
+    var safeEncode = function (item) {
+        switch (Object.prototype.toString.call(item)) {
+            case "[object Boolean]":
+            case "[object Number]":
+                return item;
+            case "[object String]":
+                return entities.encode(item);
+            default:
+                console.log("Unknown cell type.")
+        }
+        return "";
+    }
+
     var Store = declare([ESPRequest.Store, ESPBase], {
         service: "WsWorkunits",
         action: "WUResult",
@@ -156,52 +169,89 @@ define([
             return this.getFirstSchemaNode(complexType, "sequence");
         },
 
+        isChildDataset: function (cell) {
+            if (Object.prototype.toString.call(cell) !== "[object Object]") {
+                return false;
+            }
+            var propCount = 0;
+            var firstPropType = null;
+            for (var key in cell) {
+                if (!firstPropType) {
+                    firstPropType = Object.prototype.toString.call(cell[key]);
+                }
+                propCount++;
+            }
+            return propCount === 1 && firstPropType === "[object Array]";
+        },
+
         rowToTable: function (cell, __row, node) {
-            var table = domConstruct.create("table", { border: 1, cellspacing: 0, width: "100%" }, node);
-            if (Object.prototype.toString.call(cell) === '[object Object]') {
-                //  Set of Scalar or "Row" ---
+            if (this.isChildDataset(cell)) {  //  Don't display "Row" as a header  ---
                 for (var key in cell) {
                     this.rowToTable(cell[key], __row, node);
                 }
-            } else if (Object.prototype.toString.call(cell) === '[object Array]') {
-                for (var i = 0; i < cell.length; ++i) {
-                    switch (Object.prototype.toString.call(cell[i])) {
-                    case "[object Boolean]":
-                    case "[object Number]":
-                    case "[object String]":
-                        //  Item in Scalar  ---
-                        var tr = domConstruct.create("tr", null, table);
-                        domConstruct.create("td", { innerHTML: cell[i] }, tr);
-                        break;
-                    default:
-                        //  Child Dataset  ---
-                        if (i === 0) {
-                            var tr = domConstruct.create("tr", null, table);
-                            for (var key in cell[i]) {
-                                var th = domConstruct.create("th", { innerHTML: entities.encode(key) }, tr);
-                            }
+                return;
+            }
+
+            var table = domConstruct.create("table", { border: 1, cellspacing: 0, width: "100%" }, node);
+            switch(Object.prototype.toString.call(cell)) {
+                case "[object Object]":
+                    var tr = domConstruct.create("tr", null, table);
+                    for (var key in cell) {
+                        domConstruct.create("th", { innerHTML: safeEncode(key) }, tr);
+                    }
+                    tr = domConstruct.create("tr", null, table);
+                    for (var key in cell) {
+                        switch (Object.prototype.toString.call(cell[key])) {
+                            case "[object Object]":
+                            case "[object Array]":
+                                this.rowToTable(cell[key], __row, node);
+                                break;
+                            default:
+                                domConstruct.create("td", { innerHTML: safeEncode(cell[key]) }, tr);
+                                break;
                         }
-                        var tr = domConstruct.create("tr", null, table);
-                        for (var key in cell[i]) {
-                            if (cell[i][key]) {
-                                if (Object.prototype.toString.call(cell[i][key]) === '[object Object]' || Object.prototype.toString.call(cell[i][key]) === '[object Array]') {
-                                    var td = domConstruct.create("td", null, tr);
-                                    this.rowToTable(cell[i][key], cell[i], td);
-                                } else if (key.indexOf("__html", key.length - "__html".length) !== -1) {
-                                    var td = domConstruct.create("td", { innerHTML: cell[i][key] }, tr);
-                                } else if (key.indexOf("__javascript", key.length - "__javascript".length) !== -1) {
-                                    var td = domConstruct.create("td", null, tr);
-                                    this.injectJavascript(cell[i][key], cell[i], td);
-                                } else {
-                                    var val = cell[i][key];
-                                    var td = domConstruct.create("td", { innerHTML: Object.prototype.toString.call(val) === '[object String]' ? entities.encode(val) : val }, tr);
+                    }
+                    break;
+                case "[object Array]":
+                    for (var i = 0; i < cell.length; ++i) {
+                        switch (Object.prototype.toString.call(cell[i])) {
+                            case "[object Boolean]":
+                            case "[object Number]":
+                            case "[object String]":
+                                //  Item in Scalar  ---
+                                var tr = domConstruct.create("tr", null, table);
+                                domConstruct.create("td", { innerHTML: safeEncode(cell[i]) }, tr);
+                                break;
+                            default:
+                                //  Child Dataset  ---
+                                if (i === 0) {
+                                    var tr = domConstruct.create("tr", null, table);
+                                    for (var key in cell[i]) {
+                                        var th = domConstruct.create("th", { innerHTML: safeEncode(key) }, tr);
+                                    }
+                                }
+                                var tr = domConstruct.create("tr", null, table);
+                                for (var key in cell[i]) {
+                                    if (cell[i][key]) {
+                                        if (Object.prototype.toString.call(cell[i][key]) === '[object Object]' || Object.prototype.toString.call(cell[i][key]) === '[object Array]') {
+                                            var td = domConstruct.create("td", null, tr);
+                                            this.rowToTable(cell[i][key], cell[i], td);
+                                        } else if (key.indexOf("__html", key.length - "__html".length) !== -1) {
+                                            var td = domConstruct.create("td", { innerHTML: cell[i][key] }, tr);
+                                        } else if (key.indexOf("__javascript", key.length - "__javascript".length) !== -1) {
+                                            var td = domConstruct.create("td", null, tr);
+                                            this.injectJavascript(cell[i][key], cell[i], td);
+                                        } else {
+                                            var val = cell[i][key];
+                                            var td = domConstruct.create("td", { innerHTML: safeEncode(val) }, tr);
+                                        }
+                                    } else {
+                                        var td = domConstruct.create("td", { innerHTML: "" }, tr);
+                                    }
                                 }
-                            } else {
-                                var td = domConstruct.create("td", { innerHTML: "" }, tr);
-                            }
                         }
                     }
-                }
+                    break;
             }
         },
         injectJavascript : function(__cellContent, __row, __cell, __width) {
@@ -215,7 +265,7 @@ define([
             try {
                 eval(__cellContent);
             } catch (e) {
-                __cell.innerHTML = "<b>Error:</b>&nbsp;&nbsp;" + entities.encode(e.message) + "<br>" + entities.encode(__cellContent);
+                __cell.innerHTML = "<b>Error:</b>&nbsp;&nbsp;" + safeEncode(e.message) + "<br>" + safeEncode(__cellContent);
             }
         },
         parseName: function (nameObj) {

esp/src/eclwatch/FileSpray.js

@@ -91,10 +91,13 @@ define([
         preProcessRow: function (row) {
             var partialPath = this.parent.partialPath + row.name + (row.isDir ? "/" : "");
             var fullPath = this.parent.fullPath + row.name + (row.isDir ? "/" : "");
+            var fullFolderPathParts = fullPath.split("/");
+            fullFolderPathParts.pop();
             lang.mixin(row, {
                 calculatedID: this.parent.DropZone.NetAddress + fullPath,
                 partialPath: partialPath,
                 fullPath: fullPath,
+                fullFolderPath: fullFolderPathParts.join("/"),
                 DropZone: this.parent.DropZone,
                 displayName: row.name,
                 type: row.isDir ? "folder" : "file"

esp/src/eclwatch/FilterDropDownWidget.js

@@ -46,7 +46,7 @@ define([
         baseClass: "FilterDropDownWidget",
         i18n: nlsHPCC,
 
-        _width:"460px",
+        _width:"100%",
         iconFilter: null,
         filterDropDown: null,
         filterForm: null,

esp/src/eclwatch/GraphWidget.js

@@ -42,6 +42,7 @@ define([
     "dijit/Toolbar", 
     "dijit/ToolbarSeparator", 
     "dijit/form/Button",
+    "dijit/form/ComboBox",
     "dijit/form/NumberSpinner"
     
 ], function (declare, lang, i18n, nlsHPCC, arrayUtil, Deferred, has, dom, domConstruct, domClass, domStyle, Memory, Observable, QueryResults, Evented,
@@ -422,17 +423,22 @@ define([
                 this.graphViewHistory.navigateNext();
             },
 
-            _onClickZoomOrig: function (args) {
-                this.setScale(100);
-                this.centerOnItem(0);
-            },
-
-            _onClickZoomAll: function (args) {
-                this.centerOnItem(0, true);
-            },
-
-            _onClickZoomWidth: function (args) {
-                this.centerOnItem(0, true, true);
+            _onChangeZoom: function (args) {
+                var selection = this.zoomDropCombo.get("value");
+                switch (selection) {
+                    case this.i18n.All:
+                        this.centerOnItem(0, true);
+                        break;
+                    case this.i18n.Width:
+                        this.centerOnItem(0, true, true);
+                        break;
+                    default:
+                        var scale = parseFloat(selection);
+                        if (!isNaN(scale)) {
+                            this.setScale(scale);
+                        }
+                        break;
+                }
             },
 
             _onDepthChange: function (value) {
@@ -482,6 +488,7 @@ define([
                 this.graphContentPane = registry.byId(this.id + "GraphContentPane");
                 this.next = registry.byId(this.id + "Next");
                 this.previous = registry.byId(this.id + "Previous");
+                this.zoomDropCombo = registry.byId(this.id + "ZoomDropCombo");
                 this.depth = registry.byId(this.id + "Depth");
                 this.distance = registry.byId(this.id + "Distance");
                 this.syncSelectionSplitter = registry.byId(this.id + "SyncSelectionSplitter");
@@ -493,6 +500,7 @@ define([
                 this._isPluginInstalled = dojoConfig.isPluginInstalled();
                 this.createPlugin();
                 this.watchStyleChange();
+                this.watchSelect(this.zoomDropCombo);
             },
 
             resize: function (args) {

esp/src/eclwatch/InfoGridWidget.js

@@ -324,29 +324,40 @@ define([
                 var errorChecked = this.errorsCheck.get("checked");
                 var warningChecked = this.warningsCheck.get("checked");
                 var infoChecked = this.infoCheck.get("checked");
+                this._counts = {
+                    error: 0,
+                    warning: 0,
+                    errorWarning: 0,
+                    info: 0
+                };
                 arrayUtil.forEach(this.infoData, function (item, idx) {
                     lang.mixin(item, {
                         id: idx
                     });
                     switch(item.Severity) {
                         case "Error":
+                            this._counts.error++;
+                            this._counts.errorWarning++;
                             if (errorChecked) {
                                 data.push(item);
                             }
                             break;
                         case "Warning":
+                            this._counts.warning++;
+                            this._counts.errorWarning++;
                             if (warningChecked) {
                                 data.push(item);
                             }
                             break;
                         case "Message":
                         case "Info":
+                            this._counts.info++;
                             if (infoChecked) {
                                 data.push(item);
                             }
                             break;
                     }
-                });
+                }, this);
                 this.infoStore.setData(data);
                 this.infoGrid.refresh();
             },
@@ -401,10 +412,10 @@ define([
             refreshTopics: function() {
                 this.refreshFilter();
                 if (this.errWarnCount) {
-                    this.errWarnCount.innerHTML = this.infoData.length > 0 ? this.infoData.length : "";
+                    this.errWarnCount.innerHTML = this._counts.errorWarning > 0 ? this._counts.errorWarning : "";
                 }
                 if (this.errWarnMenuItem) {
-                    this.errWarnMenuItem.set("label", this.i18n.ErrorWarnings + (this.infoData.length > 0 ? " (" + this.infoData.length + ")" : ""));
+                    this.errWarnMenuItem.set("label", this.i18n.ErrorWarnings + (this._counts.errorWarning > 0 ? " (" + this._counts.errorWarning + ")" : ""));
                 }
             }
         });

esp/src/eclwatch/LFDetailsWidget.js

@@ -42,6 +42,7 @@ define([
     "hpcc/_TabContainerWidget",
     "hpcc/DelayLoadWidget",
     "hpcc/TargetSelectWidget",
+    "hpcc/TargetComboBoxWidget",
     "hpcc/ESPLogicalFile",
     "hpcc/ESPDFUWorkunit",
     "hpcc/FileBelongsToWidget",
@@ -58,7 +59,7 @@ define([
 
 ], function (exports, declare, lang, i18n, nlsHPCC, arrayUtil, dom, domAttr, domClass, domForm, query,
                 BorderContainer, TabContainer, ContentPane, Toolbar, TooltipDialog, Form, SimpleTextarea, TextBox, Button, DropDownButton, TitlePane, registry,
-                _TabContainerWidget, DelayLoadWidget, TargetSelectWidget, ESPLogicalFile, ESPDFUWorkunit, FileBelongsToWidget,
+                _TabContainerWidget, DelayLoadWidget, TargetSelectWidget, TargetComboBoxWidget, ESPLogicalFile, ESPDFUWorkunit, FileBelongsToWidget,
                 template) {
     exports.fixCircularDependency = declare("LFDetailsWidget", [_TabContainerWidget], {
         templateString: template,
@@ -180,6 +181,7 @@ define([
             if (this.desprayForm.validate()) {
                 var context = this;
                 var request = domForm.toObject(this.id + "DesprayForm");
+                request.destPath = this.desprayTargetPath.getDropZoneFolder();
                 if (!context.endsWith(request.destPath, "/")) {
                     request.destPath += "/";
                 }

esp/src/eclwatch/LZBrowseWidget.js

@@ -44,6 +44,7 @@ define([
     "hpcc/ESPDFUWorkunit",
     "hpcc/DelayLoadWidget",
     "hpcc/TargetSelectWidget",
+    "hpcc/TargetComboBoxWidget",
     "hpcc/SelectionGridWidget",
 
     "dojo/text!../templates/LZBrowseWidget.html",
@@ -71,7 +72,7 @@ define([
 ], function (declare, lang, i18n, nlsHPCC, arrayUtil, dom, domForm, domClass, iframe, on, topic,
                 registry, Dialog, Menu, MenuItem, MenuSeparator, PopupMenuItem,
                 tree, editor, selector,
-                _TabContainerWidget, FileSpray, ESPUtil, ESPRequest, ESPDFUWorkunit, DelayLoadWidget, TargetSelectWidget, SelectionGridWidget,
+                _TabContainerWidget, FileSpray, ESPUtil, ESPRequest, ESPDFUWorkunit, DelayLoadWidget, TargetSelectWidget, TargetComboBoxWidget, SelectionGridWidget,
                 template) {
     return declare("LZBrowseWidget", [_TabContainerWidget, ESPUtil.FormHelper], {
         templateString: template,
@@ -152,6 +153,20 @@ define([
         },
 
         _onUpload: function (event) {
+            var context = this;
+            if (!this.dropZoneTargetSelect.initalized) {
+                this.dropZoneTargetSelect.init({
+                    DropZones: true,
+                    callback: function (value, row) {
+                        if (context.dropZoneFolderSelect) {
+                            context.dropZoneFolderSelect.reset();
+                            context.dropZoneFolderSelect._dropZoneTarget = row;
+                            context.dropZoneFolderSelect.defaultValue = context.dropZoneFolderSelect.get("value");
+                            context.dropZoneFolderSelect.loadDropZoneFolders();
+                        }
+                    }
+                });
+            }
             var fileList = registry.byId(this.id + "Upload").getFileList();
             var totalFileSize = 0;
 
@@ -226,7 +241,7 @@ define([
         },
 
         getUploadPath: function () {
-            return this.dropZoneFolderSelect.get("row").value;
+            return this.dropZoneFolderSelect.getDropZoneFolder();
         },
 
         _onUploadSubmit: function (event) {
@@ -244,7 +259,7 @@ define([
             arrayUtil.forEach(this.landingZonesGrid.getSelected(), function (item, idx) {
                 var downloadIframeName = "downloadIframe_" + item.calculatedID;
                 var frame = iframe.create(downloadIframeName);
-                var url = ESPRequest.getBaseURL("FileSpray") + "/DownloadFile?Name=" + encodeURIComponent(item.partialPath) + "&NetAddress=" + item.DropZone.NetAddress + "&Path=" + encodeURIComponent(item.DropZone.Path) + "&OS=" + item.DropZone.OS;
+                var url = ESPRequest.getBaseURL("FileSpray") + "/DownloadFile?Name=" + encodeURIComponent(item.name) + "&NetAddress=" + item.DropZone.NetAddress + "&Path=" + encodeURIComponent(item.fullFolderPath) + "&OS=" + item.DropZone.OS;
                 iframe.setSrc(frame, url, true);
             });
         },
@@ -455,17 +470,6 @@ define([
                 SprayTargets: true
             });
             var context = this;
-            this.dropZoneTargetSelect.init({
-                DropZones: true,
-                callback: function (value, row) {
-                    if (context.dropZoneFolderSelect) {
-                        context.dropZoneFolderSelect.reset();
-                        context.dropZoneFolderSelect._dropZoneTarget = row;
-                        context.dropZoneFolderSelect.defaultValue = context.dropZoneFolderSelect.get("value");
-                        context.dropZoneFolderSelect.loadDropZoneFolders();
-                    }
-                }
-            });
             this.dropZoneFolderSelect.init({
                 DropZoneFolders: true,
                 includeBlank: true

esp/src/eclwatch/TargetComboBoxWidget.js

@@ -0,0 +1,27 @@
+/*##############################################################################
+#    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+############################################################################## */
+define([
+    "dojo/_base/declare",
+
+    "dijit/form/ComboBox",
+
+    "hpcc/TargetSelectClass"
+], function (declare,
+    ComboBox,
+    TargetSelectClass) {
+
+    return declare("TargetComboBoxWidget", [ComboBox], TargetSelectClass);
+});

esp/src/eclwatch/TargetSelectClass.js

@@ -0,0 +1,380 @@
+/*##############################################################################
+#    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+############################################################################## */
+define([
+    "dojo/_base/declare",
+    "dojo/_base/lang",
+    "dojo/i18n",
+    "dojo/i18n!./nls/hpcc",
+    "dojo/_base/array",
+    "dojo/_base/xhr",
+    "dojo/_base/Deferred",
+    "dojo/data/ItemFileReadStore",
+    "dojo/promise/all",
+    "dojo/store/Memory",
+    "dojo/on",
+
+    "dijit/registry",
+
+    "hpcc/WsTopology",
+    "hpcc/WsWorkunits",
+    "hpcc/FileSpray"
+], function (declare, lang, i18n, nlsHPCC, arrayUtil, xhr, Deferred, ItemFileReadStore, all, Memory, on,
+    registry,
+    WsTopology, WsWorkunits, FileSpray) {
+
+    return {
+        i18n: nlsHPCC,
+
+        loading: false,
+        defaultValue: "",
+
+        //  Implementation  ---
+        reset: function () {
+            this.initalized = false;
+            this.loading = false;
+            this.defaultValue = "";
+            this.options = [];
+        },
+
+        init: function (params) {
+            if (this.initalized)
+                return;
+            this.initalized = true;
+            this.loading = true;
+            this.options = [];
+
+            if (params.Target) {
+                this.defaultValue = params.Target;
+                this.set("value", params.Target);
+            }
+
+            if (params.includeBlank) {
+                this.includeBlank = params.includeBlank;
+                this.options.push({
+                    label: " ",
+                    value: ""
+                });
+            }
+            if (params.Groups === true) {
+                this.loadClusterGroups();
+            } else if (params.SprayTargets === true) {
+                this.loadSprayTargets();
+            } else if (params.DropZones === true) {
+                this.loadDropZones();
+            } else if (params.DropZoneFolders === true) {
+                this.defaultValue = ".";
+                this.set("value", ".");
+                this.loadDropZoneFolders();
+            } else if (params.WUState === true) {
+                this.loadWUState();
+            } else if (params.DFUState === true) {
+                this.loadDFUState();
+            } else if (params.ECLSamples === true) {
+                this.loadECLSamples();
+            } else if (params.Logs === true) {
+                this.loadLogs(params);
+            } else {
+                this.loadTargets();
+            }
+            if (params.callback) {
+                this.callback = params.callback;
+            }
+        },
+
+        _setValueAttr: function (target) {
+            if (target === null)
+                target = "";
+            this.inherited(arguments);
+            if (this.callback) {
+                this.callback(this.value, this._getRowAttr());
+            }
+        },
+
+        _getValueAttr: function () {
+            if (this.loading)
+                return this.defaultValue;
+
+            if (this.textbox)
+                return this.textbox.value;
+
+            return this.value;
+        },
+
+        _getRowAttr: function () {
+            var context = this;
+            var retVal = null;
+            arrayUtil.forEach(this.options, function (item, idx) {
+                if (context.value === item.value) {
+                    retVal = item;
+                    return false;
+                }
+            });
+            return retVal;
+        },
+
+        _postLoad: function () {
+            if (this.defaultValue === "" && this.options.length) {
+                this.defaultValue = this.options[0].value;
+            }
+            this.set("value", this.defaultValue);
+            this.loading = false;
+        },
+
+        loadDropZones: function () {
+            var context = this;
+            WsTopology.TpServiceQuery({
+                load: function (response) {
+                    if (lang.exists("TpServiceQueryResponse.ServiceList.TpDropZones.TpDropZone", response)) {
+                        var targetData = response.TpServiceQueryResponse.ServiceList.TpDropZones.TpDropZone;
+                        for (var i = 0; i < targetData.length; ++i) {
+                            context.options.push({
+                                label: targetData[i].Name,
+                                value: targetData[i].Name,
+                                machine: targetData[i].TpMachines.TpMachine[0]
+                            });
+                        }
+                        context._postLoad();
+                    }
+                }
+            });
+        },
+
+        _loadDropZoneFolders: function (Netaddr, Path, OS, depth) {
+            depth = depth || 0;
+            var retVal = [];
+            retVal.push(Path);
+            var deferred = new Deferred();
+            if (depth > 2) {
+                setTimeout(function () {
+                    deferred.resolve(retVal);
+                }, 20);
+            } else {
+                var context = this;
+                FileSpray.FileList({
+                    request: {
+                        Netaddr: Netaddr,
+                        Path: Path,
+                        OS: OS
+                    },
+                    suppressExceptionToaster: true
+                }).then(function (response) {
+                    var requests = [];
+                    if (lang.exists("FileListResponse.files.PhysicalFileStruct", response)) {
+                        var files = response.FileListResponse.files.PhysicalFileStruct;
+                        for (var i = 0; i < files.length; ++i) {
+                            if (files[i].isDir) {
+                                requests.push(context._loadDropZoneFolders(Netaddr, Path + "/" + files[i].name, OS, ++depth));
+                            }
+                        }
+                    }
+                    all(requests).then(function (responses) {
+                        arrayUtil.forEach(responses, function (response) {
+                            retVal = retVal.concat(response);
+                        });
+                        deferred.resolve(retVal);
+                    });
+                });
+            }
+            return deferred.promise;
+        },
+
+        endsWith: function (str, suffix) {
+            return str.indexOf(suffix, str.length - suffix.length) !== -1;
+        },
+
+        loadDropZoneFolders: function () {
+            var context = this;
+            this.getDropZoneFolder = function () {
+                var baseFolder = this._dropZoneTarget.machine.Directory + (this.endsWith(this._dropZoneTarget.machine.Directory, "/") ? "" : "/");
+                var selectedFolder = this.get("value");
+                return baseFolder + selectedFolder;
+            }
+            if (this._dropZoneTarget) {
+                this._loadDropZoneFolders(this._dropZoneTarget.machine.Netaddress, this._dropZoneTarget.machine.Directory, this._dropZoneTarget.machine.OS).then(function (results) {
+                    results.sort();
+                    var store = new Memory({
+                        data: arrayUtil.map(results, function (_path) {
+                            var path = _path.substring(context._dropZoneTarget.machine.Directory.length);
+                            return {
+                                name: "." + path,
+                                id: _path
+                            };
+                        })
+                    });
+                    context.set("store", store);
+                    context._postLoad();
+                });
+            }
+        },
+
+        loadClusterGroups: function () {
+            var context = this;
+            WsTopology.TpGroupQuery({
+                load: function (response) {
+                    if (lang.exists("TpGroupQueryResponse.TpGroups.TpGroup", response)) {
+                        var targetData = response.TpGroupQueryResponse.TpGroups.TpGroup;
+                        for (var i = 0; i < targetData.length; ++i) {
+                            switch(targetData[i].Kind) {
+                                case "Thor":
+                                case "hthor":
+                                case "Roxie":
+                                    context.options.push({
+                                        label: targetData[i].Name,
+                                        value: targetData[i].Name
+                                    });
+                                    break;
+                            }
+                        }
+                        context._postLoad();
+                    }
+                }
+            });
+        },
+
+        loadSprayTargets: function () {
+            var context = this;
+            FileSpray.GetSprayTargets({
+                load: function (response) {
+                    if (lang.exists("GetSprayTargetsResponse.GroupNodes.GroupNode", response)) {
+                        var targetData = response.GetSprayTargetsResponse.GroupNodes.GroupNode;
+                        for (var i = 0; i < targetData.length; ++i) {
+                            context.options.push({
+                                label: targetData[i].Name,
+                                value: targetData[i].Name
+                            });
+                        }
+                        context._postLoad();
+                    }
+                }
+            });
+        },
+
+        loadWUState: function() {
+            for (var key in WsWorkunits.States) {
+                this.options.push({
+                    label: WsWorkunits.States[key],
+                    value: WsWorkunits.States[key]
+                });
+            }
+            this._postLoad();
+        },
+
+        loadDFUState: function () {
+            for (var key in FileSpray.States) {
+                this.options.push({
+                    label: FileSpray.States[key],
+                    value: FileSpray.States[key]
+                });
+            }
+            this._postLoad();
+        },
+
+        LogicalFileSearchType: function() {
+            this.options.push({
+                label: "Created",
+                value: "Created"
+            });
+            this.options.push({
+                label: "Used",
+                value: "Referenced"
+            });
+            this._postLoad();
+        },
+
+        loadTargets: function () {
+            var context = this;
+            WsTopology.TpLogicalClusterQuery({
+            }).then(function (response) {
+                if (lang.exists("TpLogicalClusterQueryResponse.TpLogicalClusters.TpLogicalCluster", response)) {
+                    var targetData = response.TpLogicalClusterQueryResponse.TpLogicalClusters.TpLogicalCluster;
+                    for (var i = 0; i < targetData.length; ++i) {
+                        context.options.push({
+                            label: targetData[i].Name,
+                            value: targetData[i].Name
+                        });
+                    }
+
+                    if (!context.includeBlank && context._value == "") {
+                        if (response.TpLogicalClusterQueryResponse["default"]) {
+                            context._value = response.TpLogicalClusterQueryResponse["default"].Name;
+                        } else {
+                            context._value = context.options[0].value;
+                        }
+                    }
+                }
+                context._postLoad();
+            });
+        },
+
+        loadECLSamples: function () {
+            var sampleStore = new ItemFileReadStore({
+                url: dojoConfig.getURL("ecl/ECLPlaygroundSamples.json")
+            });
+            this.setStore(sampleStore);
+            var context = this;
+            this.on("change", function (evt) {
+                var filename = this.get("value");
+                xhr.get({
+                    url: dojoConfig.getURL("ecl/" + filename),
+                    handleAs: "text",
+                    load: function (eclText) {
+                        context.onNewSelection(eclText);
+                    },
+                    error: function () {
+                    }
+                });
+            });
+            context._postLoad();
+        },
+
+        loadLogs: function (params) {
+            var context = this;
+            this.set("options", []);
+            FileSpray.FileList({
+                request: {
+                    Mask: "*.log",
+                    Netaddr: params.params.Netaddress,
+                    OS: params.params.OS,
+                    Path: params.params.getLogDirectory()
+                }
+            }).then(function (response) {
+                if (lang.exists("FileListResponse.files.PhysicalFileStruct", response)) {
+                    var options = [];
+                    var targetData = response.FileListResponse.files.PhysicalFileStruct;
+                    var shortestLabelLen = 9999;
+                    var shortestLabel = "";
+                    for (var i = 0; i < targetData.length; ++i) {
+                        options.push({
+                            label: targetData[i].name,// + " " + targetData[i].filesize + " " + targetData[i].modifiedtime,
+                            value: targetData[i].name
+                        });
+                        if (shortestLabelLen > targetData[i].name.length) {
+                            shortestLabelLen = targetData[i].name.length;
+                            shortestLabel = targetData[i].name;
+                        }
+                    }
+                    options.sort(function (l, r) {
+                        return -l.label.localeCompare(r.label);
+                    });
+                    context.set("options", options);
+                    context.defaultValue = shortestLabel;
+                    context._value = shortestLabel;
+                }
+                context._postLoad();
+            });
+        }
+    };
+});

esp/src/eclwatch/TargetSelectWidget.js

@@ -15,342 +15,13 @@
 ############################################################################## */
 define([
     "dojo/_base/declare",
-    "dojo/_base/lang",
-    "dojo/i18n",
-    "dojo/i18n!./nls/hpcc",
-    "dojo/_base/array",
-    "dojo/_base/xhr",
-    "dojo/_base/Deferred",
-    "dojo/data/ItemFileReadStore",
-    "dojo/promise/all",
-    "dojo/on",
 
     "dijit/form/Select",
-    "dijit/registry",
 
-    "hpcc/WsTopology",
-    "hpcc/WsWorkunits",
-    "hpcc/FileSpray"
-], function (declare, lang, i18n, nlsHPCC, arrayUtil, xhr, Deferred, ItemFileReadStore, all, on,
-    Select, registry,
-    WsTopology, WsWorkunits, FileSpray) {
-    return declare("TargetSelectWidget", [Select], {
-        i18n: nlsHPCC,
+    "hpcc/TargetSelectClass"
+], function (declare,
+    Select,
+    TargetSelectClass) {
 
-        loading: false,
-        defaultValue: "",
-
-        //  Implementation  ---
-        reset: function () {
-            this.initalized = false;
-            this.loading = false;
-            this.defaultValue = "";
-            this.options = [];
-        },
-
-        init: function (params) {
-            if (this.initalized)
-                return;
-            this.initalized = true;
-            this.loading = true;
-            this.options = [];
-
-            if (params.Target) {
-                this.defaultValue = params.Target;
-                this.set("value", params.Target);
-            }
-
-            if (params.includeBlank) {
-                this.includeBlank = params.includeBlank;
-                this.options.push({
-                    label: " ",
-                    value: ""
-                });
-            }
-            if (params.Groups === true) {
-                this.loadClusterGroups();
-            } else if (params.SprayTargets === true) {
-                this.loadSprayTargets();
-            } else if (params.DropZones === true) {
-                this.loadDropZones();
-            } else if (params.DropZoneFolders === true) {
-                this.loadDropZoneFolders();
-            } else if (params.WUState === true) {
-                this.loadWUState();
-            } else if (params.DFUState === true) {
-                this.loadDFUState();
-            } else if (params.ECLSamples === true) {
-                this.loadECLSamples();
-            } else if (params.Logs === true) {
-                this.loadLogs(params);
-            } else {
-                this.loadTargets();
-            }
-            if (params.callback) {
-                this.callback = params.callback;
-            }
-        },
-
-        _setValueAttr: function (target) {
-            if (target === null)
-                target = "";
-            this.inherited(arguments);
-            if (this.callback) {
-                this.callback(this.value, this._getRowAttr());
-            }
-        },
-
-        _getValueAttr: function () {
-            if (this.loading)
-                return this.defaultValue;
-
-            return this.value;
-        },
-
-        _getRowAttr: function () {
-            var context = this;
-            var retVal = null;
-            arrayUtil.forEach(this.options, function (item, idx) {
-                if (context.value === item.value) {
-                    retVal = item;
-                    return false;
-                }
-            });
-            return retVal;
-        },
-
-        _postLoad: function () {
-            if (this.defaultValue === "" && this.options.length) {
-                this.defaultValue = this.options[0].value;
-            }
-            this.set("value", this.defaultValue);
-            this.loading = false;
-        },
-
-        loadDropZones: function () {
-            var context = this;
-            WsTopology.TpServiceQuery({
-                load: function (response) {
-                    if (lang.exists("TpServiceQueryResponse.ServiceList.TpDropZones.TpDropZone", response)) {
-                        var targetData = response.TpServiceQueryResponse.ServiceList.TpDropZones.TpDropZone;
-                        for (var i = 0; i < targetData.length; ++i) {
-                            context.options.push({
-                                label: targetData[i].Name,
-                                value: targetData[i].Name,
-                                machine: targetData[i].TpMachines.TpMachine[0]
-                            });
-                        }
-                        context._postLoad();
-                    }
-                }
-            });
-        },
-
-        _loadDropZoneFolders: function (Netaddr, Path, OS) {
-            var retVal = [];
-            retVal.push(Path);
-            var deferred = new Deferred();
-
-            var context = this;
-            FileSpray.FileList({
-                request: {
-                    Netaddr: Netaddr,
-                    Path: Path,
-                    OS: OS
-                },
-                suppressExceptionToaster: true
-            }).then(function (response) {
-                var requests = [];
-                if (lang.exists("FileListResponse.files.PhysicalFileStruct", response)) {
-                    var files = response.FileListResponse.files.PhysicalFileStruct;
-                    for (var i = 0; i < files.length; ++i) {
-                        if (files[i].isDir) {
-                            requests.push(context._loadDropZoneFolders(Netaddr, Path + "/" + files[i].name, OS));
-                        }
-                    }
-                }
-                all(requests).then(function (responses) {
-                    arrayUtil.forEach(responses, function (response) {
-                        retVal = retVal.concat(response);
-                    });
-                    deferred.resolve(retVal);
-                });
-            });
-            return deferred.promise;
-        },
-
-        loadDropZoneFolders: function () {
-            var context = this;
-            if (this._dropZoneTarget) {
-                this._loadDropZoneFolders(this._dropZoneTarget.machine.Netaddress, this._dropZoneTarget.machine.Directory, this._dropZoneTarget.machine.OS).then(function (results) {
-                    results.sort();
-                    context.options = arrayUtil.map(results, function (_path) {
-                        var path = _path.substring(context._dropZoneTarget.machine.Directory.length);
-                        return {
-                            label: "..." + path,
-                            value: _path
-                        };
-                    });
-                    context._postLoad();
-                });
-            }
-        },
-
-        loadClusterGroups: function () {
-            var context = this;
-            WsTopology.TpGroupQuery({
-                load: function (response) {
-                    if (lang.exists("TpGroupQueryResponse.TpGroups.TpGroup", response)) {
-                        var targetData = response.TpGroupQueryResponse.TpGroups.TpGroup;
-                        for (var i = 0; i < targetData.length; ++i) {
-                            switch(targetData[i].Kind) {
-                                case "Thor":
-                                case "hthor":
-                                case "Roxie":
-                                    context.options.push({
-                                        label: targetData[i].Name,
-                                        value: targetData[i].Name
-                                    });
-                                break;
-                            }
-                        }
-                        context._postLoad();
-                    }
-                }
-            });
-        },
-
-        loadSprayTargets: function () {
-            var context = this;
-            FileSpray.GetSprayTargets({
-                load: function (response) {
-                    if (lang.exists("GetSprayTargetsResponse.GroupNodes.GroupNode", response)) {
-                        var targetData = response.GetSprayTargetsResponse.GroupNodes.GroupNode;
-                        for (var i = 0; i < targetData.length; ++i) {
-                            context.options.push({
-                                label: targetData[i].Name,
-                                value: targetData[i].Name
-                            });
-                        }
-                        context._postLoad();
-                    }
-                }
-            });
-        },
-
-        loadWUState: function() {
-            for (var key in WsWorkunits.States) {
-                this.options.push({
-                    label: WsWorkunits.States[key],
-                    value: WsWorkunits.States[key]
-                });
-            }
-            this._postLoad();
-        },
-
-        loadDFUState: function () {
-            for (var key in FileSpray.States) {
-                this.options.push({
-                    label: FileSpray.States[key],
-                    value: FileSpray.States[key]
-                });
-            }
-            this._postLoad();
-        },
-
-        LogicalFileSearchType: function() {
-            this.options.push({
-                label: "Created",
-                value: "Created"
-            });
-            this.options.push({
-                label: "Used",
-                value: "Referenced"
-            });
-            this._postLoad();
-        },
-
-        loadTargets: function () {
-            var context = this;
-            WsTopology.TpLogicalClusterQuery({
-            }).then(function (response) {
-                if (lang.exists("TpLogicalClusterQueryResponse.TpLogicalClusters.TpLogicalCluster", response)) {
-                    var targetData = response.TpLogicalClusterQueryResponse.TpLogicalClusters.TpLogicalCluster;
-                    for (var i = 0; i < targetData.length; ++i) {
-                        context.options.push({
-                            label: targetData[i].Name,
-                            value: targetData[i].Name
-                        });
-                    }
-
-                    if (!context.includeBlank && context._value == "") {
-                        if (response.TpLogicalClusterQueryResponse["default"]) {
-                            context._value = response.TpLogicalClusterQueryResponse["default"].Name;
-                        } else {
-                            context._value = context.options[0].value;
-                        }
-                    }
-                }
-                context._postLoad();
-            });
-        },
-
-        loadECLSamples: function () {
-            var sampleStore = new ItemFileReadStore({
-                url: dojoConfig.getURL("ecl/ECLPlaygroundSamples.json")
-            });
-            this.setStore(sampleStore);
-            var context = this;
-            this.on("change", function (evt) {
-                var filename = this.get("value");
-                xhr.get({
-                    url: dojoConfig.getURL("ecl/" + filename),
-                    handleAs: "text",
-                    load: function (eclText) {
-                        context.onNewSelection(eclText);
-                    },
-                    error: function () {
-                    }
-                });
-            });
-            context._postLoad();
-        },
-
-        loadLogs: function (params) {
-            var context = this;
-            this.set("options", []);
-            FileSpray.FileList({
-                request: {
-                    Mask: "*.log",
-                    Netaddr: params.params.Netaddress,
-                    OS: params.params.OS,
-                    Path: params.params.getLogDirectory()
-                }
-            }).then(function (response) {
-                if (lang.exists("FileListResponse.files.PhysicalFileStruct", response)) {
-                    var options = [];
-                    var targetData = response.FileListResponse.files.PhysicalFileStruct;
-                    var shortestLabelLen = 9999;
-                    var shortestLabel = "";
-                    for (var i = 0; i < targetData.length; ++i) {
-                        options.push({
-                            label: targetData[i].name,// + " " + targetData[i].filesize + " " + targetData[i].modifiedtime,
-                            value: targetData[i].name
-                        });
-                        if (shortestLabelLen > targetData[i].name.length) {
-                            shortestLabelLen = targetData[i].name.length;
-                            shortestLabel = targetData[i].name;
-                        }
-                    }
-                    options.sort(function (l, r) {
-                        return -l.label.localeCompare(r.label);
-                    });
-                    context.set("options", options);
-                    context.defaultValue = shortestLabel;
-                    context._value = shortestLabel;
-                }
-                context._postLoad();
-            });
-        }
-    });
+    return declare("TargetSelectWidget", [Select], TargetSelectClass);
 });

esp/src/eclwatch/_Widget.js

@@ -43,7 +43,9 @@ define([
 
         _onNewPage: function (event) {
             var win = window.open(this.getURL(), "_blank");
-            win.focus();
+            if (win) {
+                win.focus();
+            }
         },
 
         init: function (params) {

esp/src/eclwatch/nls/hpcc.js

@@ -600,6 +600,7 @@ define({root:
     ZeroLogicalFilesCheckFilter: "Zero Logical Files(check filter)",
     Zip: "Zip",
     ZippedAnalysisPackage: "Zipped Analysis Package",
+    Zoom: "Zoom",
     Zoom100Pct: "Zoom 100%",
     ZoomAll: "Zoom All",
     ZoomWidth: "Zoom Width"

esp/src/eclwatch/stub.js

@@ -74,11 +74,7 @@ define([
                                 }
                             }
 
-                            if ((item.Source === "WsWorkunits.WUInfo" && item.Code === 20080) ||
-                                (item.Source === "WsWorkunits.WUQuery" && item.Code === 20081) ||
-                                (item.Source === "FileSpray.GetDFUWorkunit" && item.Code === 20080) ||
-                                (item.Source === "WsDfu.DFUInfo" && item.Code === 20038)) {
-                            } else {
+                            if (topic.Severity !== "Info") {
                                 var message = "<h4>" + entities.encode(item.Source) + "</h4><p>" + entities.encode(item.Message) + (clipped ? "<br>...  ...  ..." : "") + "</p>";
                                 myToaster.setContent(message, item.Severity, item.Severity === "Error" ? -1 : null);
                                 myToaster.show();

esp/src/eclwatch/templates/DFUQueryWidget.html

@@ -121,14 +121,14 @@
                     </div>
                     <div id="${id}DesprayDropDown" data-dojo-type="dijit.form.DropDownButton">
                         <span>${i18n.Despray}</span>
-                        <div data-dojo-type="dijit.TooltipDialog">
+                        <div id="${id}DesprayTooltipDialog" data-dojo-type="dijit.TooltipDialog">
                             <div id="${id}DesprayForm" style="width: 460px;" onsubmit="return false;" data-dojo-type="dijit.form.Form">
                                 <div data-dojo-type="dijit.Fieldset">
                                     <legend>${i18n.Target}</legend>
                                     <div data-dojo-type="hpcc.TableContainer">
                                         <input id="${id}DesprayTargetSelect" title="${i18n.DropZone}:" name="destGroup" style="width: 100%;" data-dojo-type="TargetSelectWidget" />
                                         <input id="${id}DesprayTargetIPAddress" title="${i18n.IPAddress}:" name="destIP" style="width: 100%;" required="true" data-dojo-props="trim: true" data-dojo-type="dijit.form.ValidationTextBox" />
-                                        <input id="${id}DesprayTargetPath" title="${i18n.Path}:" name="destPath" required="true" style="width: 100%;" data-dojo-props="trim: true" data-dojo-type="TargetSelectWidget" />
+                                        <input id="${id}DesprayTargetPath" title="${i18n.Path}:" name="destPath" required="true" style="width: 100%;" data-dojo-props="trim: true" data-dojo-type="TargetComboBoxWidget" />
                                         <input id="${id}DesprayTargetSplitPrefix" title="${i18n.SplitPrefix}:" name="splitprefix" style="width: 100%;" data-dojo-props="trim: true" data-dojo-type="dijit.form.ValidationTextBox" />
                                     </div>
                                     <div id="${id}DesprayGrid" data-dojo-type="SelectionGridWidget">

esp/src/eclwatch/templates/GraphWidget.html

@@ -6,9 +6,18 @@
                 <div id="${id}Previous" data-dojo-attach-event="onClick:_onClickPrevious" data-dojo-props="iconClass:'iconLeft', showLabel:false" data-dojo-type="dijit.form.Button">${i18n.Back}</div>
                 <div id="${id}Next" data-dojo-attach-event="onClick:_onClickNext" data-dojo-props="iconClass:'iconRight', showLabel:false" data-dojo-type="dijit.form.Button">${i18n.Forward}</div>
                 <span data-dojo-type="dijit.ToolbarSeparator"></span>
-                <div data-dojo-attach-event="onClick:_onClickZoomOrig" data-dojo-props="iconClass:'iconZoomOrig', showLabel:false" data-dojo-type="dijit.form.Button">${i18n.Zoom100Pct}</div>
-                <div data-dojo-attach-event="onClick:_onClickZoomAll" data-dojo-props="iconClass:'iconZoomAll', showLabel:false" data-dojo-type="dijit.form.Button">${i18n.ZoomAll}</div>
-                <div data-dojo-attach-event="onClick:_onClickZoomWidth" data-dojo-props="iconClass:'iconZoomWidth', showLabel:false" data-dojo-type="dijit.form.Button">${i18n.ZoomWidth}</div>
+                <select id="${id}ZoomDropCombo" style="width: 60px" data-dojo-props="placeHolder:'${i18n.Zoom}'" data-dojo-attach-event="onChange:_onChangeZoom" data-dojo-type="dijit.form.ComboBox">
+                    <option selected></option>
+                    <option>${i18n.All}</option>
+                    <option>${i18n.Width}</option>
+                    <option>100%</option>
+                    <option>90%</option>
+                    <option>75%</option>
+                    <option>50%</option>
+                    <option>25%</option>
+                    <option>10%</option>
+                    <option>5%</option>
+                </select>
                 <span data-dojo-type="dijit.ToolbarSeparator"></span>
                 <span data-dojo-attach-point="containerNode"></span>
                 <div title="${i18n.DepthTooltip}" data-dojo-props="iconClass:'iconDepth', showLabel:false, disabled: true" data-dojo-type="dijit.form.Button"></div>

esp/src/eclwatch/templates/LFDetailsWidget.html

@@ -67,7 +67,7 @@
                                     <div data-dojo-props="cols:2" data-dojo-type="hpcc.TableContainer">
                                         <input id="${id}DesprayTargetSelect" title="${i18n.DropZone}:" name="destGroup" colspan="2" style="width: 100%;" data-dojo-type="TargetSelectWidget" />
                                         <input id="${id}DesprayTargetIPAddress" title="${i18n.IPAddress}:" name="destIP" colspan="2" required="true" style="width: 100%;" data-dojo-props="trim: true" data-dojo-type="dijit.form.ValidationTextBox" />
-                                        <input id="${id}DesprayTargetPath" title="${i18n.Path}:" name="destPath" colspan="2" required="true" style="width: 100%;" data-dojo-props="trim: true" data-dojo-type="TargetSelectWidget" />
+                                        <input id="${id}DesprayTargetPath" title="${i18n.Path}:" name="destPath" colspan="2" required="true" style="width: 100%;" data-dojo-props="trim: true" data-dojo-type="TargetComboBoxWidget" />
                                         <input id="${id}DesprayTargetName" title="${i18n.TargetName}:" colspan="2" required="true" style="width: 100%;" data-dojo-props="trim: true" data-dojo-type="dijit.form.ValidationTextBox" />
                                         <input id="${id}DesprayTargetSplitPrefix" title="${i18n.SplitPrefix}:" name="splitprefix" colspan="2" style="width: 100%;" data-dojo-props="trim: true" data-dojo-type="dijit.form.TextBox" />
                                     </div>

esp/src/eclwatch/templates/LZBrowseWidget.html

@@ -264,7 +264,7 @@
         <div class="dijitDialogPaneContentArea">
             <div data-dojo-props="cols:1" data-dojo-type="hpcc.TableContainer">
                 <input id="${id}DropZoneTargetSelect" title="${i18n.LandingZone}:" style="width: 95%;" data-dojo-type="TargetSelectWidget" />
-                <input id="${id}DropZoneFolderSelect" title="${i18n.Folder}:" style="width: 95%;" data-dojo-type="TargetSelectWidget" />
+                <input id="${id}DropZoneFolderSelect" title="${i18n.Folder}:" style="width: 95%;" data-dojo-type="TargetComboBoxWidget" />
             </div>
             <p>
             <div id="${id}UploadFileList" class="dijitDialogPaneContentArea" data-dojo-props="uploaderId: '${id}Upload'" data-dojo-type="dojox.form.uploader.FileList"></div>

initfiles/etc/bash_completion/ecl

@@ -112,12 +112,16 @@ _ecl_opts_roxie()
     local subcmdname="${COMP_WORDS[subcmdpos]}"
     case "${subcmdname}" in
         "" | ecl)
-            echo "--help attach detach check reload"
+            echo "--help attach detach check reload unused-files"
             ;;
         attach | detach | check | reload)
             echo -n "--wait= "
             _ecl_opts_common
             ;;
+        "unused-files")
+            echo -n "--delete-recursive --delete-subfiles --delete --check-packagemaps --wait="
+            _ecl_opts_common
+            ;;
          *)
             ;;
     esac

roxie/ccd/ccdlistener.cpp

@@ -1881,7 +1881,7 @@ readAnother:
             }
             else
             {
-                fprintf(stderr, "EXCEPTION: %s", error.str());
+                fprintf(stderr, "EXCEPTION: %s\n", error.str());
             }
             E->Release();
         }

roxie/ccd/ccdserver.cpp

@@ -10248,7 +10248,7 @@ public:
             fileProps.setPropInt64("@size", uncompressedBytesWritten);
             partProps.setPropInt64("@size", uncompressedBytesWritten);
         }
-        else if (tallycrc)
+        else if (tallycrc && crc.get())
             partProps.setPropInt64("@fileCrc", crc.get());
 
         if (encrypted)

roxie/roxiemem/roxiemem.cpp

@@ -96,6 +96,7 @@ static unsigned *heapBitmap;
 static unsigned heapBitmapSize;
 static unsigned heapTotalPages; // derived from heapBitmapSize - here for code clarity
 static unsigned heapLWM;
+static unsigned heapHWM;
 static unsigned heapLargeBlocks;
 static unsigned heapLargeBlockGranularity;
 static ILargeMemCallback * heapLargeBlockCallback;
@@ -298,7 +299,8 @@ static void initializeHeap(bool allowHugePages, bool allowTransparentHugePages,
     {
         DBGLOG("MEMORY WILL NOT BE RELEASED TO OS");
         if (!retainMemory)
-            DBGLOG("Increase HEAP_ALIGNMENT_SIZE so HEAP_ALIGNMENT_SIZE*32 is a multiple of system huge page size");
+            DBGLOG("Increase HEAP_ALIGNMENT_SIZE so HEAP_ALIGNMENT_SIZE*32 (0x%" I64F "x) is a multiple of system huge page size (0x%" I64F "x)",
+                    (unsigned __int64)HEAP_ALIGNMENT_SIZE * 32, (unsigned __int64) getHugePageSize());
     }
 
     assertex(((memsize_t)heapBase & (HEAP_ALIGNMENT_SIZE-1)) == 0);
@@ -308,6 +310,7 @@ static void initializeHeap(bool allowHugePages, bool allowTransparentHugePages,
     memset(heapBitmap, 0xff, heapBitmapSize*sizeof(unsigned));
     heapLargeBlocks = 1;
     heapLWM = 0;
+    heapHWM = heapBitmapSize;
 
     if (memTraceLevel)
         DBGLOG("RoxieMemMgr: %u Pages successfully allocated for the pool - memsize=%" I64F "u base=%p alignment=%" I64F "u bitmapSize=%u", 
@@ -533,7 +536,7 @@ static void *suballoc_aligned(size32_t pages, bool returnNullWhenExhausted)
             lastStatsCycles = cyclesNow;
             StringBuffer s;
             memstats(s);
-            s.appendf(", heapLWM %u, dataBuffersActive=%d, dataBufferPages=%d", heapLWM, atomic_read(&dataBuffersActive), atomic_read(&dataBufferPages));
+            s.appendf(", heapLWM %u, heapHWM %u, dataBuffersActive=%d, dataBufferPages=%d", heapLWM, heapHWM, atomic_read(&dataBuffersActive), atomic_read(&dataBufferPages));
             DBGLOG("RoxieMemMgr: %s", s.str());
         }
     }
@@ -578,7 +581,11 @@ static void *suballoc_aligned(size32_t pages, bool returnNullWhenExhausted)
                 const unsigned mask = 1U << pos;
                 const unsigned match = i*UNSIGNED_BITS + pos;
                 char *ret = heapBase + match*HEAP_ALIGNMENT_SIZE;
-                heapBitmap[i] = (hbi & ~mask);
+                hbi &= ~mask;
+                heapBitmap[i] = hbi;
+                //If no more free pages in this mask increment the low water mark
+                if (hbi == 0)
+                    i++;
                 heapLWM = i;
                 heapAllocated++;
                 if (memTraceLevel >= 2)
@@ -590,7 +597,21 @@ static void *suballoc_aligned(size32_t pages, bool returnNullWhenExhausted)
     else
     {
         // Usage pattern is such that we expect normally to succeed immediately.
-        unsigned i = heapBitmapSize;
+        dbgassertex(heapHWM != 0); // should have been caught much earlier
+
+        unsigned i = heapHWM;
+        //Check if the last page allocated created a new run of completely allocated memory
+        if (heapBitmap[i-1] == 0)
+        {
+            loop
+            {
+                i--;
+                dbgassertex(i != 0); // should never occur - memory must be full, and should have been caught much earlier
+                if (heapBitmap[i-1])
+                    break;
+            }
+            heapHWM = i;
+        }
         unsigned matches = 0;
         while (i)
         {
@@ -605,6 +626,8 @@ static void *suballoc_aligned(size32_t pages, bool returnNullWhenExhausted)
                         matches++;
                         if (matches==pages)
                         {
+                            unsigned start = i;
+                            unsigned startHbi = hbi;
                             char *ret = heapBase + (i*UNSIGNED_BITS+b-1)*HEAP_ALIGNMENT_SIZE;
                             loop
                             {
@@ -622,6 +645,16 @@ static void *suballoc_aligned(size32_t pages, bool returnNullWhenExhausted)
                                     mask <<= 1;
                             }
                             heapBitmap[i] = hbi;
+
+                            //Check if all memory above this allocation is allocated - if so extend the HWM
+                            if ((hbi == 0) && (i+1 == heapHWM))
+                            {
+                                if (startHbi != 0)
+                                    heapHWM = start+1;
+                                else
+                                    heapHWM = start;
+                            }
+
                             if (memTraceLevel >= 2)
                                 DBGLOG("RoxieMemMgr: suballoc_aligned() %u pages ok - addr=%p", pages, ret);
                             heapAllocated += pages;
@@ -719,12 +752,15 @@ static void subfree_aligned(void *ptr, unsigned pages = 1)
             }
         }
 
+        if (wordOffset >= heapHWM)
+            heapHWM = wordOffset+1;
+
         if (firstReleaseBlock)
             notifyMemoryUnused(firstReleaseBlock, (lastReleaseBlock - firstReleaseBlock) + heapBlockSize);
     }
 
     if (memTraceLevel >= 2)
-        DBGLOG("RoxieMemMgr: subfree_aligned() %u pages ok - addr=%p heapLWM=%u totalPages=%u", _pages, ptr, heapLWM, heapTotalPages);
+        DBGLOG("RoxieMemMgr: subfree_aligned() %u pages ok - addr=%p heapLWM=%u heapHWM=%u totalPages=%u", _pages, ptr, heapLWM, heapHWM, heapTotalPages);
 }
 
 static void clearBits(unsigned start, unsigned len)
@@ -761,6 +797,9 @@ static void clearBits(unsigned start, unsigned len)
 
 static void *subrealloc_aligned(void *ptr, unsigned pages, unsigned newPages)
 {
+    //If this function shrinks the size of the block, then the call to subfree_aligned() may increase heapHWM.
+    //If this function increases the size of the block, it is possible heapHWM could decrease - it will be
+    //updated on the next multi-page allocation
     assertex(newPages > 0);
     unsigned _pages = pages;
     memsize_t offset = (char *)ptr - heapBase;
@@ -5113,6 +5152,7 @@ protected:
             _heapBitmapSize = heapBitmapSize;
             _heapTotalPages = heapTotalPages;
             _heapLWM = heapLWM;
+            _heapHWM = heapHWM;
             _heapAllocated = heapAllocated;
             _heapUseHugePages = heapUseHugePages;
             _heapNotifyUnusedEachFree = heapNotifyUnusedEachFree;
@@ -5126,6 +5166,7 @@ protected:
             heapBitmapSize = _heapBitmapSize;
             heapTotalPages = _heapTotalPages;
             heapLWM = _heapLWM;
+            heapHWM = _heapHWM;
             heapAllocated = _heapAllocated;
             heapUseHugePages = _heapUseHugePages;
             heapNotifyUnusedEachFree = _heapNotifyUnusedEachFree;
@@ -5137,6 +5178,7 @@ protected:
         unsigned _heapBitmapSize;
         unsigned _heapTotalPages;
         unsigned _heapLWM;
+        unsigned _heapHWM;
         unsigned _heapAllocated;
         bool _heapUseHugePages;
         bool _heapNotifyUnusedEachFree;
@@ -5150,6 +5192,7 @@ protected:
         heapBitmapSize = size;
         heapTotalPages = heapBitmapSize * UNSIGNED_BITS;
         heapLWM = 0;
+        heapHWM = heapBitmapSize;
         heapAllocated = 0;
     }
 

system/jlib/jlog.cpp

@@ -1165,15 +1165,6 @@ void LogMsgComponentReporter::report(const LogMsg & msg)
 
 // LogMsgPrepender
 
-LogMsgPrepender * LogMsgPrepender::setContext(LogMsgComponentReporter * r, char const * f, unsigned l)
-{
-    crit.enter();
-    reporter = r;
-    file = f;
-    line = l;
-    return this;
-}
-
 void LogMsgPrepender::report(const LogMsgCategory & cat, const char * format, ...)
 {
     StringBuffer buff;
@@ -1185,7 +1176,6 @@ void LogMsgPrepender::report(const LogMsgCategory & cat, const char * format, ..
     else
         queryLogMsgManager()->report_va(cat, unknownJob, buff.str(), args);
     va_end(args);
-    crit.leave();
 }
 
 void LogMsgPrepender::report_va(const LogMsgCategory & cat, const char * format, va_list args)
@@ -1196,7 +1186,6 @@ void LogMsgPrepender::report_va(const LogMsgCategory & cat, const char * format,
         reporter->report_va(cat, unknownJob, buff.str(), args);
     else
         queryLogMsgManager()->report_va(cat, unknownJob, buff.str(), args);
-    crit.leave();
 }
 
 void LogMsgPrepender::report(const LogMsgCategory & cat, LogMsgCode code, const char * format, ...)
@@ -1210,7 +1199,6 @@ void LogMsgPrepender::report(const LogMsgCategory & cat, LogMsgCode code, const
     else
         queryLogMsgManager()->report_va(cat, unknownJob, buff.str(), args);
     va_end(args);
-    crit.leave();
 }
 
 void LogMsgPrepender::report_va(const LogMsgCategory & cat, LogMsgCode code, const char * format, va_list args)
@@ -1221,7 +1209,6 @@ void LogMsgPrepender::report_va(const LogMsgCategory & cat, LogMsgCode code, con
         reporter->report_va(cat, unknownJob, buff.str(), args);
     else
         queryLogMsgManager()->report_va(cat, unknownJob, buff.str(), args);
-    crit.leave();
 }
 
 void LogMsgPrepender::report(const LogMsgCategory & cat, const IException * exception, const char * prefix)
@@ -1234,7 +1221,6 @@ void LogMsgPrepender::report(const LogMsgCategory & cat, const IException * exce
         reporter->report(cat, unknownJob, exception->errorCode(), "%s",  buff.str());
     else
         queryLogMsgManager()->report(cat, unknownJob, exception->errorCode(), "%s", buff.str());
-    crit.leave();
 }
 
 void LogMsgPrepender::report(const LogMsgCategory & cat, const LogMsgJobInfo & job, const char * format, ...)
@@ -1248,7 +1234,6 @@ void LogMsgPrepender::report(const LogMsgCategory & cat, const LogMsgJobInfo & j
     else
         queryLogMsgManager()->report_va(cat, job, buff.str(), args);
     va_end(args);
-    crit.leave();
 }
 
 void LogMsgPrepender::report_va(const LogMsgCategory & cat, const LogMsgJobInfo & job, const char * format, va_list args)
@@ -1259,7 +1244,6 @@ void LogMsgPrepender::report_va(const LogMsgCategory & cat, const LogMsgJobInfo
         reporter->report_va(cat, job, buff.str(), args);
     else
         queryLogMsgManager()->report_va(cat, job, buff.str(), args);
-    crit.leave();
 }
 
 void LogMsgPrepender::report(const LogMsgCategory & cat, const LogMsgJobInfo & job, LogMsgCode code, const char * format, ...)
@@ -1273,7 +1257,6 @@ void LogMsgPrepender::report(const LogMsgCategory & cat, const LogMsgJobInfo & j
     else
         queryLogMsgManager()->report_va(cat, job, buff.str(), args);
     va_end(args);
-    crit.leave();
 }
 
 void LogMsgPrepender::report_va(const LogMsgCategory & cat, const LogMsgJobInfo & job, LogMsgCode code, const char * format, va_list args)
@@ -1284,7 +1267,6 @@ void LogMsgPrepender::report_va(const LogMsgCategory & cat, const LogMsgJobInfo
         reporter->report_va(cat, job, buff.str(), args);
     else
         queryLogMsgManager()->report_va(cat, job, buff.str(), args);
-    crit.leave();
 }
 
 void LogMsgPrepender::report(const LogMsgCategory & cat, const LogMsgJobInfo & job, const IException * exception, const char * prefix)
@@ -1297,7 +1279,6 @@ void LogMsgPrepender::report(const LogMsgCategory & cat, const LogMsgJobInfo & j
         reporter->report(cat, job, exception->errorCode(), "%s(%d) : %s", file, line, txt.str());
     else
         queryLogMsgManager()->report(cat, job, exception->errorCode(), "%s(%d) : %s", file, line, txt.str());
-    crit.leave();
 }
 
 IException * LogMsgPrepender::report(IException * e, const char * prefix, LogMsgClass cls)
@@ -2217,7 +2198,6 @@ HandleLogMsgHandlerTable * theStderrHandler;
 CLogMsgManager * theManager;
 CSysLogEventLogger * theSysLogEventLogger;
 LogMsgComponentReporter * theReporters[MSGCOMP_NUMBER];
-LogMsgPrepender * thePrepender;
 
 MODULE_INIT(INIT_PRIORITY_JLOG)
 {
@@ -2230,12 +2210,10 @@ MODULE_INIT(INIT_PRIORITY_JLOG)
     theManager->resetMonitors();
     for(unsigned compo = 0; compo<MSGCOMP_NUMBER; compo++)
         theReporters[compo] = new LogMsgComponentReporter(compo);
-    thePrepender = new LogMsgPrepender;
     return true;
 }
 MODULE_EXIT()
 {
-    delete thePrepender;
     for(unsigned compo = 0; compo<MSGCOMP_NUMBER; compo++)
         delete theReporters[compo];
     delete theManager;
@@ -2261,11 +2239,6 @@ LogMsgComponentReporter * queryLogMsgComponentReporter(unsigned compo)
     return theReporters[compo];
 }
 
-LogMsgPrepender * queryLogMsgPrepender()
-{
-    return thePrepender;
-}
-
 ILogMsgManager * createLogMsgManager() // use with care! (needed by mplog listener facility)
 {
     return new CLogMsgManager();

system/jlib/jlog.hpp

@@ -600,9 +600,7 @@ private:
 class jlib_decl LogMsgPrepender
 {
 public:
-    // N.B. This method locks the critical section...
-    LogMsgPrepender * setContext(LogMsgComponentReporter * r, char const * f, unsigned l);
-    // ... and these ones unlock it. So they should be used in pairs.
+    LogMsgPrepender(LogMsgComponentReporter * r, char const * f, unsigned l) : reporter(r), file(f), line(l) { }
     void                      report(const LogMsgCategory & cat, const char * format, ...) __attribute__((format(printf, 3, 4)));
     void                      report_va(const LogMsgCategory & cat, const char * format, va_list args);
     void                      report(const LogMsgCategory & cat, LogMsgCode code, const char * format, ...) __attribute__((format(printf, 4, 5)));
@@ -618,7 +616,6 @@ private:
     LogMsgComponentReporter * reporter;
     char const *              file;
     unsigned                  line;
-    CriticalSection           crit;
 };
 
 // FUNCTIONS, DATA, AND MACROS
@@ -706,7 +703,6 @@ extern jlib_decl const LogMsgJobInfo unknownJob;
 extern jlib_decl ILogMsgManager * queryLogMsgManager();
 extern jlib_decl ILogMsgHandler * queryStderrLogMsgHandler();
 extern jlib_decl LogMsgComponentReporter * queryLogMsgComponentReporter(unsigned compo);
-extern jlib_decl LogMsgPrepender * queryLogMsgPrepender();
 
 extern jlib_decl ILogMsgManager * createLogMsgManager(); // use with care! (needed by mplog listener facility)
 
@@ -714,10 +710,10 @@ extern jlib_decl ILogMsgManager * createLogMsgManager(); // use with care! (need
 
 #ifdef LOGMSGCOMPONENT
 #define LOGMSGREPORTER queryLogMsgComponentReporter(LOGMSGCOMPONENT)
-#define FLLOG queryLogMsgPrepender()->setContext(LOGMSGREPORTER, __FILE__, __LINE__)->report
+#define FLLOG LogMsgPrepender(LOGMSGREPORTER, __FILE__, __LINE__).report
 #else // LOGMSGCOMPONENT
 #define LOGMSGREPORTER queryLogMsgManager()
-#define FLLOG queryLogMsgPrepender()->setContext(0, __FILE__, __LINE__)->report
+#define FLLOG LogMsgPrepender(NULL, __FILE__, __LINE__).report
 #endif // LOGMSGCOMPONENT
 
 #ifdef LOGMSGCOMPONENT

testing/regress/ecl/appendoptimize.ecl

@@ -0,0 +1,38 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2015 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+r := { unsigned id; };
+
+
+mkId(unsigned num) := TRANSFORM(r, SELF.id := num);
+
+ds(unsigned num, unsigned from) := NOFOLD(DATASET(num, mkId(from + (COUNTER-1))));
+
+ds11 := ds(1,1);
+ds22 := ds(2,2);
+
+boolean true1 := true : stored('true1');
+boolean false1 := false : stored('false1');
+
+SEQUENTIAL(
+    output(IF(true1, ds11 & ROW(mkId(12)), ds11));
+    output(IF(true1, ds11, ds11 & ROW(mkId(12))));
+    output(IF(true1, ds11 & ds22, ds11 & ROW(mkId(12))));
+    output(IF(false1, ds11 & ROW(mkId(12)), ds11));
+    output(IF(false1, ds11, ds11 & ROW(mkId(12))));
+    output(IF(false1, ds11 & ds22, ds11 & ROW(mkId(12))))
+);

testing/regress/ecl/childds8.ecl

@@ -0,0 +1,43 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2015 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//This is needed to prevent cntBad being hoisted before the resourcing, and becoming unconditional
+//The tests are part of the work aiming to remove this code.
+#option ('workunitTemporaries', false);
+
+idRec := { unsigned id; };
+mainRec := { unsigned seq, dataset(idRec) ids };
+
+idRec createId(unsigned id) := TRANSFORM
+    SELF.id := id;
+END;
+
+mainRec createMain(unsigned c, unsigned num) := TRANSFORM
+    SELF.seq := c;
+    SELF.ids := DATASET(num, createId(c DIV 2 + (COUNTER-1)));
+END;
+
+emptyIds := DATASET([], idRec);
+
+ds := NOFOLD(DATASET(4, createMain(COUNTER, 3)));
+
+ds1 := SORT(ds, ids);
+ds2 := DEDUP(ds1, ids);
+ds3 := ds2(ids != emptyIds);
+output(ds3);
+ds4 := ROLLUP(ds1, TRANSFORM(mainRec, SELF.ids := LEFT.ids + RIGHT.ids, SELF := LEFT), ids);
+output(ds4);

testing/regress/ecl/key/appendoptimize.xml

@@ -0,0 +1,23 @@
+<Dataset name='Result 1'>
+ <Row><id>1</id></Row>
+ <Row><id>12</id></Row>
+</Dataset>
+<Dataset name='Result 2'>
+ <Row><id>1</id></Row>
+</Dataset>
+<Dataset name='Result 3'>
+ <Row><id>1</id></Row>
+ <Row><id>2</id></Row>
+ <Row><id>3</id></Row>
+</Dataset>
+<Dataset name='Result 4'>
+ <Row><id>1</id></Row>
+</Dataset>
+<Dataset name='Result 5'>
+ <Row><id>1</id></Row>
+ <Row><id>12</id></Row>
+</Dataset>
+<Dataset name='Result 6'>
+ <Row><id>1</id></Row>
+ <Row><id>12</id></Row>
+</Dataset>

testing/regress/ecl/key/childds8.xml

@@ -0,0 +1,10 @@
+<Dataset name='Result 1'>
+ <Row><seq>1</seq><ids><Row><id>0</id></Row><Row><id>1</id></Row><Row><id>2</id></Row></ids></Row>
+ <Row><seq>2</seq><ids><Row><id>1</id></Row><Row><id>2</id></Row><Row><id>3</id></Row></ids></Row>
+ <Row><seq>4</seq><ids><Row><id>2</id></Row><Row><id>3</id></Row><Row><id>4</id></Row></ids></Row>
+</Dataset>
+<Dataset name='Result 2'>
+ <Row><seq>1</seq><ids><Row><id>0</id></Row><Row><id>1</id></Row><Row><id>2</id></Row></ids></Row>
+ <Row><seq>2</seq><ids><Row><id>1</id></Row><Row><id>2</id></Row><Row><id>3</id></Row><Row><id>1</id></Row><Row><id>2</id></Row><Row><id>3</id></Row></ids></Row>
+ <Row><seq>4</seq><ids><Row><id>2</id></Row><Row><id>3</id></Row><Row><id>4</id></Row></ids></Row>
+</Dataset>

thorlcr/graph/thgraphslave.cpp

@@ -920,22 +920,27 @@ class CThorCodeContextSlave : public CThorCodeContextBase, implements IEngineCon
     mptag_t mptag;
     Owned<IDistributedFileTransaction> superfiletransaction;
 
+    void invalidSetResult(const char * name, unsigned seq)
+    {
+        throw MakeStringException(0, "Attempt to output result ('%s',%d) from a child query", name ? name : "", (int)seq);
+    }
+
 public:
     CThorCodeContextSlave(CJobBase &job, ILoadedDllEntry &querySo, IUserDescriptor &userDesc, mptag_t _mptag) : CThorCodeContextBase(job, querySo, userDesc), mptag(_mptag)
     {
     }
-    virtual void setResultBool(const char *name, unsigned sequence, bool value) { throwUnexpected(); }
-    virtual void setResultData(const char *name, unsigned sequence, int len, const void * data) { throwUnexpected(); }
-    virtual void setResultDecimal(const char * stepname, unsigned sequence, int len, int precision, bool isSigned, const void *val) { throwUnexpected(); } 
-    virtual void setResultInt(const char *name, unsigned sequence, __int64 value, unsigned size) { throwUnexpected(); }
-    virtual void setResultRaw(const char *name, unsigned sequence, int len, const void * data) { throwUnexpected(); }
-    virtual void setResultReal(const char * stepname, unsigned sequence, double value) { throwUnexpected(); }
-    virtual void setResultSet(const char *name, unsigned sequence, bool isAll, size32_t len, const void * data, ISetToXmlTransformer * transformer) { throwUnexpected(); }
-    virtual void setResultString(const char *name, unsigned sequence, int len, const char * str) { throwUnexpected(); }
-    virtual void setResultUInt(const char *name, unsigned sequence, unsigned __int64 value, unsigned size) { throwUnexpected(); }
-    virtual void setResultUnicode(const char *name, unsigned sequence, int len, UChar const * str) { throwUnexpected(); }
-    virtual void setResultVarString(const char * name, unsigned sequence, const char * value) { throwUnexpected(); }
-    virtual void setResultVarUnicode(const char * name, unsigned sequence, UChar const * value) { throwUnexpected(); }
+    virtual void setResultBool(const char *name, unsigned sequence, bool value) { invalidSetResult(name, sequence); }
+    virtual void setResultData(const char *name, unsigned sequence, int len, const void * data) { invalidSetResult(name, sequence); }
+    virtual void setResultDecimal(const char * stepname, unsigned sequence, int len, int precision, bool isSigned, const void *val) { invalidSetResult(stepname, sequence); }
+    virtual void setResultInt(const char *name, unsigned sequence, __int64 value, unsigned size) { invalidSetResult(name, sequence); }
+    virtual void setResultRaw(const char *name, unsigned sequence, int len, const void * data) { invalidSetResult(name, sequence); }
+    virtual void setResultReal(const char * stepname, unsigned sequence, double value) { invalidSetResult(stepname, sequence); }
+    virtual void setResultSet(const char *name, unsigned sequence, bool isAll, size32_t len, const void * data, ISetToXmlTransformer * transformer) { invalidSetResult(name, sequence); }
+    virtual void setResultString(const char *name, unsigned sequence, int len, const char * str) { invalidSetResult(name, sequence); }
+    virtual void setResultUInt(const char *name, unsigned sequence, unsigned __int64 value, unsigned size) { invalidSetResult(name, sequence); }
+    virtual void setResultUnicode(const char *name, unsigned sequence, int len, UChar const * str) { invalidSetResult(name, sequence); }
+    virtual void setResultVarString(const char * name, unsigned sequence, const char * value) { invalidSetResult(name, sequence); }
+    virtual void setResultVarUnicode(const char * name, unsigned sequence, UChar const * value) { invalidSetResult(name, sequence); }
 
     virtual bool getResultBool(const char * name, unsigned sequence) { throwUnexpected(); }
     virtual void getResultData(unsigned & tlen, void * & tgt, const char * name, unsigned sequence) { throwUnexpected(); }