Merge branch 'candidate-5.4.0'

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

docs/ECLWatch/TheECLWatchMan.xml

@@ -341,6 +341,14 @@
       <xi:include href="HPCCDataHandling/DH-Mods/DH-Mod1.xml"
                   xpointer="Spray_XML"
                   xmlns:xi="http://www.w3.org/2001/XInclude" />
+ 
+
+ 
+
+      <xi:include href="HPCCDataHandling/DH-Mods/DH-Mod1.xml"
+                  xpointer="Spray_JSON"
+                  xmlns:xi="http://www.w3.org/2001/XInclude" />
+               
 
       <xi:include href="HPCCDataHandling/DH-Mods/DH-Mod1.xml"
                   xpointer="Spray_Variable"
@@ -576,7 +584,7 @@
 
         <mediaobject>
           <imageobject>
-            <imagedata fileref="images/ECLWA460.jpg" />
+            <imagedata fileref="images/ECLWA460.jpg" vendor="eclwatchSS" />
           </imageobject>
         </mediaobject>
       </figure></para>
@@ -591,6 +599,99 @@
     system administration tasks through ECL watch.</para>
 
     <sect1>
+      <title>Topology</title>
+
+      <para>The Topology page provides a visual tree display with information
+      about your clusters, services, and nodes.</para>
+
+      <para>Click on the <emphasis role="bold">Topology</emphasis> link from
+      the Operations navigation sub-menu to access the topology page.</para>
+
+      <para><figure>
+          <title>Topology Page</title>
+
+          <mediaobject>
+            <imageobject>
+              <imagedata fileref="images/ECLWA463.jpg" vendor="ECLWatchSS" />
+            </imageobject>
+          </mediaobject>
+        </figure></para>
+
+      <sect2 id="TopologyTargets">
+        <title>Targets</title>
+
+        <para>The <emphasis role="bold">Targets</emphasis> Action button
+        displays your clusters by type.</para>
+
+        <para>Click on the arrow to the left of the Cluster folder/object to
+        expand. The expanded view displays.</para>
+
+        <para><figure>
+            <title>Expanded View</title>
+
+            <mediaobject>
+              <imageobject>
+                <imagedata fileref="images/ECLWA464.jpg" vendor="ECLWatchSS" />
+              </imageobject>
+            </mediaobject>
+          </figure>The expanded view displays the objects and nodes in the
+        selected container. Select the node or object to display more
+        information or to access the logs. The Summary, Configuration, and Log
+        tabs on the left side of the page display the relevant information for
+        the selected component.</para>
+      </sect2>
+
+      <sect2 id="TopologyServices">
+        <title>Services</title>
+
+        <para>Press the <emphasis role="bold">Services</emphasis> Action
+        button to display information on the various services running on your
+        cluster.</para>
+
+        <para>Click on the arrow to the left of the service you wish to
+        expand. The expanded view displays.</para>
+
+        <para><figure>
+            <title>Expanded Services</title>
+
+            <mediaobject>
+              <imageobject>
+                <imagedata fileref="images/ECLWA465.jpg" vendor="ECLWatchSS" />
+              </imageobject>
+            </mediaobject>
+          </figure></para>
+
+        <para>The services view provides a service oriented tree view that
+        provides access to the services. Expand the tree, and select the
+        component to view the Summary, Configuration, or Logs tabs for the
+        selected component.</para>
+      </sect2>
+
+      <sect2>
+        <title>Machines</title>
+
+        <para>Press the <emphasis role="bold">Machines</emphasis> Action
+        button for more information on the various machines or nodes running
+        in your cluster(s).</para>
+
+        <para><figure>
+            <title>Machines View</title>
+
+            <mediaobject>
+              <imageobject>
+                <imagedata fileref="images/ECLWA466.jpg" vendor="ECLWatchSS" />
+              </imageobject>
+            </mediaobject>
+          </figure></para>
+
+        <para>The <emphasis role="bold">Machines</emphasis> action button
+        opens a node oriented view of the tree. Expand the nodes to see the
+        services on each node. Select the component to view the Summary,
+        Configuration, or Logs tabs for that selected component.</para>
+      </sect2>
+    </sect1>
+
+    <sect1>
       <title>Disk Usage</title>
 
       <para>When you click on the Operations link, it opens the Disk Usage
@@ -708,7 +809,7 @@
 
         <para>The default Ganglia page has a tab for Custom Monitoring where
         you can easily add custom monitoring components. <figure>
-            <title>Ganglia Custom Monitoring </title>
+            <title>Ganglia Custom Monitoring</title>
 
             <mediaobject>
               <imageobject>
@@ -744,7 +845,7 @@
                 <imagedata fileref="images/ECLWa482.jpg" vendor="VM_welcome" />
               </imageobject>
             </mediaobject>
-          </figure> </para>
+          </figure></para>
       </sect2>
 
       <sect2>
@@ -753,7 +854,7 @@
         <para>In order to use Ganglia in ECL Watch, you need to have Ganglia
         installed on your HPCC System. For details on installing Ganglia for
         ECL Watch, refer to the <emphasis>HPCC Monitoring and
-        Reporting</emphasis> manual. </para>
+        Reporting</emphasis> manual.</para>
       </sect2>
     </sect1>
   </chapter>

docs/HPCCDataHandling/DH-Mods/DH-Mod1.xml

@@ -748,6 +748,150 @@
       </itemizedlist>
     </sect2>
 
+    <sect2 id="Spray_JSON" role="brk">
+      <title><emphasis role="bold">Spray JSON</emphasis></title>
+
+      <itemizedlist>
+        <listitem>
+          <para>Click on the <emphasis role="bold">Files</emphasis> icon, then
+          click the <emphasis role="bold">Landing Zones</emphasis> link from
+          the navigation sub-menu.</para>
+        </listitem>
+
+        <listitem>
+          <para>Click on the arrow next to your dropzone to expand the
+          list.</para>
+
+          <para>The files on your drop zone display.</para>
+        </listitem>
+
+        <listitem>
+          <para>Check the checkboxes for the file(s) you want to spray, then
+          press the Spray:<emphasis role="bold"> JSON </emphasis> action
+          button.</para>
+
+          <para>The dialog displays.</para>
+        </listitem>
+
+        <listitem>
+          <para>Fill in relevant details:</para>
+
+          <para><informaltable colsep="0" frame="none" rowsep="0">
+              <tgroup cols="2">
+                <colspec colwidth="122.40pt" />
+
+                <colspec colwidth="333.00pt" />
+
+                <tbody>
+                  <row>
+                    <entry align="right"><emphasis
+                    role="bold">Target</emphasis></entry>
+                  </row>
+
+                  <row>
+                    <entry><emphasis role="bold">Group</emphasis></entry>
+
+                    <entry>Select the name of cluster to spray to. You can
+                    only select a cluster in your environment.</entry>
+                  </row>
+
+                  <row>
+                    <entry><emphasis role="bold">Name
+                    Prefix</emphasis></entry>
+
+                    <entry>The prefix for the logical file</entry>
+                  </row>
+
+                  <row>
+                    <entry><emphasis role="bold">Target
+                    Name</emphasis></entry>
+
+                    <entry>The logical filename to create. This is pre-filled
+                    with the name of the source file on the landing zone, but
+                    can be changed.</entry>
+                  </row>
+
+                  <row>
+                    <entry><emphasis role="bold">Row Path </emphasis></entry>
+
+                    <entry>The path specifier to the JSON content. The default
+                    takes the root level content as an array of objects to be
+                    treated as rows.</entry>
+                  </row>
+
+                  <row>
+                    <entry align="right"><emphasis
+                    role="bold">Options:</emphasis></entry>
+                  </row>
+
+                  <row>
+                    <entry><emphasis role="bold">Format</emphasis></entry>
+
+                    <entry>Select the format from the droplist</entry>
+                  </row>
+
+                  <row>
+                    <entry><emphasis role="bold">Max Record
+                    Length</emphasis></entry>
+
+                    <entry>The length of longest record in the file.</entry>
+                  </row>
+
+                  <row>
+                    <entry><emphasis role="bold">Overwrite</emphasis></entry>
+
+                    <entry>Check this box to overwrite files of the same
+                    name.</entry>
+                  </row>
+
+                  <row>
+                    <entry><emphasis role="bold">No Split</emphasis></entry>
+
+                    <entry>Check this box to prevent splitting file parts to
+                    multiple target parts.</entry>
+                  </row>
+
+                  <row>
+                    <entry><emphasis role="bold">Fail if no source
+                    file</emphasis></entry>
+
+                    <entry>Check this box to allow the spray to fail if no
+                    source file is found.</entry>
+                  </row>
+
+                  <row>
+                    <entry><emphasis role="bold">Replicate</emphasis></entry>
+
+                    <entry><para>Check this box to create backup copies of all
+                    file parts in the backup directory (by convention on the
+                    secondary drive of the node following in the
+                    cluster).</para><para><emphasis role="bold">This option is
+                    only available on systems where replication has been
+                    enabled.</emphasis></para></entry>
+                  </row>
+
+                  <row>
+                    <entry><emphasis role="bold">Compress</emphasis></entry>
+
+                    <entry>Check this box to compress the files.</entry>
+                  </row>
+                </tbody>
+              </tgroup>
+            </informaltable></para>
+        </listitem>
+
+        <listitem>
+          <para>Press the <emphasis role="bold">Spray</emphasis>
+          button.</para>
+
+          <para>A <emphasis role="bold">DFU Workunit</emphasis> tab displays
+          for each job. You can see the progress of each spray operation on
+          the tab. If a job fails, information related to the cause of the
+          failure also displays.</para>
+        </listitem>
+      </itemizedlist>
+    </sect2>
+
     <sect2 id="Spray_Variable" role="brk">
       <title><emphasis role="bold">Spray Variable</emphasis></title>
 

ecl/eclcc/eclcc.cpp

@@ -819,6 +819,11 @@ void EclCC::instantECL(EclCompileInstance & instance, IWorkUnit *wu, const char
                 }
             }
         }
+        catch (IError * _e)
+        {
+            Owned<IError> e = _e;
+            errorProcessor.report(e);
+        }
         catch (IException * _e)
         {
             Owned<IException> e = _e;

ecl/hql/hqlgram.y

@@ -2274,9 +2274,11 @@ action
                         {
                             $1.annotateExprWithLocation();
                             $$.inherit($1);
-                            $$.setPosition($1);
                         }
-    | setMetaCommand
+    | setMetaCommand    {
+                            $1.annotateExprWithLocation();
+                            $$.inherit($1);
+                        }
     ;
 
 actionStmt

ecl/hqlcpp/hqlcpp.cpp

@@ -2184,19 +2184,16 @@ void HqlCppTranslator::ThrowStringException(int code,const char *format, ...)
     throw ret;
 }
 
-void HqlCppTranslator::reportErrorDirect(IHqlExpression * location, int code,const char *msg, bool alwaysAbort)
+void HqlCppTranslator::reportErrorDirect(IHqlExpression * exprOrLocation, int code,const char *msg, bool alwaysAbort)
 {
-    if (location)
-    {
-        ECLlocation loc;
-        loc.extractLocationAttr(location);
-        if (alwaysAbort)
-            throw createError(code, msg, loc.sourcePath->str(), loc.lineno, loc.column, loc.position);
-        errorProcessor->reportError(code, msg, loc.sourcePath->str(), loc.lineno, loc.column, loc.position);
-    }
-    else
-//        errorProcessor->reportError(code, msg, NULL, 0, 0, 0);
-        throw MakeStringExceptionDirect(code, msg);
+    ECLlocation loc;
+    if (!loc.extractLocationAttr(exprOrLocation))
+        loc.extractLocationAttr(queryActiveActivityLocation());
+    const char * sourcePath = loc.sourcePath->str();
+
+    if (alwaysAbort)
+        throw createError(code, msg, sourcePath, loc.lineno, loc.column, loc.position);
+    errorProcessor->reportError(code, msg, sourcePath, loc.lineno, loc.column, loc.position);
 }
 
 void HqlCppTranslator::reportError(IHqlExpression * location, int code,const char *format, ...)
@@ -4476,6 +4473,14 @@ void HqlCppTranslator::buildTempExpr(BuildCtx & ctx, IHqlExpression * expr, CHql
             }
             break;
         }
+    case no_getresult:
+    case no_deserialize:
+        if (expr->isDatarow())
+        {
+            buildAnyExpr(ctx, expr, tgt);
+            return;
+        }
+        break;
     case no_id2blob:
         buildExpr(ctx, expr, tgt);
         return;
@@ -5732,7 +5737,7 @@ void HqlCppTranslator::doBuildCall(BuildCtx & ctx, const CHqlBoundTarget * tgt,
 
     IHqlExpression * external = funcdef->queryChild(0);
     IHqlExpression * formals = funcdef->queryChild(1);
-    if (external->hasAttribute(ctxmethodAtom))
+    if (external->hasAttribute(ctxmethodAtom) || external->hasAttribute(contextAtom))
         ensureContextAvailable(ctx);
     if (external->hasAttribute(gctxmethodAtom) || external->hasAttribute(globalContextAtom))
     {

ecl/hqlcpp/hqlcppds.cpp

@@ -4373,6 +4373,14 @@ void HqlCppTranslator::buildRowAssign(BuildCtx & ctx, BoundRow * targetRow, IHql
             buildStmt(ctx, expr->queryChild(0));
             buildRowAssign(ctx, targetRow, expr->queryChild(1));
             return;
+        case no_selectnth:
+            {
+                Owned<IReferenceSelector> src = buildNewRow(ctx, expr);
+                Owned<BoundRow> srcRow = src->getRow(ctx);
+                BoundRow * linkedRow = ensureLinkCountedRow(ctx, srcRow);
+                ctx.addAssignLink(targetRow->queryExpr(), linkedRow->queryExpr());
+                return;
+            }
         }
     }
 

ecl/hqlcpp/hqlcse.cpp

@@ -120,6 +120,7 @@ bool canCreateTemporary(IHqlExpression * expr)
     case type_rule:
     case type_pattern:
     case type_token:
+    case type_event:
         return false;
     default:
         return true;

ecl/hqlcpp/hqlcset.cpp

@@ -73,7 +73,7 @@ BoundRow * BaseDatasetCursor::buildIterateLoop(BuildCtx & ctx, bool needToBreak)
     buildIterateClass(ctx, iterName, NULL);
 
     StringBuffer s, rowName;
-    OwnedHqlExpr row = createRow(ctx, "row", rowName, false);
+    OwnedHqlExpr row = createRow(ctx, "row", rowName);
 
     //row = iter.first()
     s.clear().append(rowName).append(" = ").append(iterName).append(".first();");
@@ -96,7 +96,7 @@ void BaseDatasetCursor::buildIterateClass(BuildCtx & ctx, SharedHqlExpr & iter,
     buildIterateClass(ctx, cursorName, NULL);
 
     iter.setown(createVariable(cursorName.str(), makeBoolType()));
-    row.setown(createRow(ctx, "row", rowName, false));
+    row.setown(createRow(ctx, "row", rowName));
 }
 
 void BaseDatasetCursor::buildIterateMembers(BuildCtx & declarectx, BuildCtx & initctx)
@@ -105,7 +105,7 @@ void BaseDatasetCursor::buildIterateMembers(BuildCtx & declarectx, BuildCtx & in
     buildIterateClass(declarectx, iterName, &initctx);
 
     StringBuffer s, rowName;
-    OwnedHqlExpr row = createRow(declarectx, "row", rowName, false);
+    OwnedHqlExpr row = createRow(declarectx, "row", rowName);
 
     //row = iter.first()
     BuildCtx firstctx(declarectx);
@@ -163,7 +163,7 @@ BoundRow * BaseDatasetCursor::buildSelectNth(BuildCtx & ctx, IHqlExpression * in
     //create a unique dataset and associate it with a call to select
     //set value to be the field selection from the dataset
     StringBuffer s, rowName;
-    OwnedHqlExpr row = createRow(ctx, "row", rowName, conditional && CREATE_DEAULT_ROW_IF_NULL_VALUE);
+    OwnedHqlExpr row = createRow(ctx, "row", rowName);
 
     CHqlBoundExpr boundIndex;
     OwnedHqlExpr index = adjustIndexBaseToZero(indexExpr->queryChild(1));
@@ -197,7 +197,7 @@ BoundRow * BaseDatasetCursor::buildSelectNth(BuildCtx & ctx, IHqlExpression * in
 }
 
 
-IHqlExpression * BaseDatasetCursor::createRow(BuildCtx & ctx, const char * prefix, StringBuffer & rowName, bool conditional)
+IHqlExpression * BaseDatasetCursor::createRow(BuildCtx & ctx, const char * prefix, StringBuffer & rowName)
 {
     translator.getUniqueId(rowName.append(prefix));
     OwnedITypeInfo type;
@@ -205,8 +205,6 @@ IHqlExpression * BaseDatasetCursor::createRow(BuildCtx & ctx, const char * prefi
         type.setown(makeConstantModifier(makeRowReferenceType(boundDs)));
     else
         type.setown(makeConstantModifier(makeRowReferenceType(ds)));
-    if (conditional)
-        type.setown(setLinkCountedAttr(type, false));
 
     OwnedHqlExpr row = createVariable(rowName, type.getClear());
     ctx.addDeclare(row);
@@ -293,7 +291,7 @@ InlineBlockDatasetCursor::InlineBlockDatasetCursor(HqlCppTranslator & _translato
 BoundRow * InlineBlockDatasetCursor::buildIterateLoop(BuildCtx & ctx, bool needToBreak)
 {
     StringBuffer rowName;
-    OwnedHqlExpr row = createRow(ctx, "row", rowName, false);
+    OwnedHqlExpr row = createRow(ctx, "row", rowName);
     if (isEmptyDataset(boundDs))
     {
         ctx.addFilter(queryBoolExpr(false));
@@ -312,7 +310,7 @@ BoundRow * InlineBlockDatasetCursor::buildIterateLoop(BuildCtx & ctx, bool needT
     {
         OwnedHqlExpr length = translator.getBoundLength(boundDs);
         StringBuffer endName;
-        OwnedHqlExpr end = createRow(ctx, "end", endName, false);
+        OwnedHqlExpr end = createRow(ctx, "end", endName);
 
         //end = row+length;
         s.clear().append(endName).append(" = ").append(rowName).append("+");
@@ -366,7 +364,7 @@ BoundRow * InlineBlockDatasetCursor::buildSelectFirst(BuildCtx & ctx, IHqlExpres
 {
     StringBuffer s, rowName;
     bool conditional = !indexExpr->hasAttribute(noBoundCheckAtom);
-    OwnedHqlExpr row = createRow(ctx, "row", rowName, (conditional && createDefaultRowIfNull));
+    OwnedHqlExpr row = createRow(ctx, "row", rowName);
 
     BuildCtx subctx(ctx);
     if (conditional)
@@ -449,7 +447,7 @@ BoundRow * InlineBlockDatasetCursor::buildSelectNth(BuildCtx & ctx, IHqlExpressi
     bool conditional = !indexExpr->hasAttribute(noBoundCheckAtom);
     //row = NULL
     StringBuffer s, rowName;
-    OwnedHqlExpr row = createRow(ctx, "row", rowName, (conditional && CREATE_DEAULT_ROW_IF_NULL_VALUE));
+    OwnedHqlExpr row = createRow(ctx, "row", rowName);
 
     //if (index > 0 && (index <= count) or (index * fixedSize <= size)
     //MORE: Need to be very careful about the types...
@@ -589,7 +587,7 @@ void InlineLinkedDatasetCursor::buildIterateClass(BuildCtx & ctx, StringBuffer &
 BoundRow * InlineLinkedDatasetCursor::doBuildIterateLoop(BuildCtx & ctx, bool needToBreak, bool checkForNull)
 {
     StringBuffer rowName;
-    OwnedHqlExpr row = createRow(ctx, "row", rowName, false);
+    OwnedHqlExpr row = createRow(ctx, "row", rowName);
     if (isEmptyDataset(boundDs))
     {
         ctx.addFilter(queryBoolExpr(false));
@@ -642,7 +640,7 @@ BoundRow * InlineLinkedDatasetCursor::buildSelectNth(BuildCtx & ctx, IHqlExpress
 
     //row = NULL
     StringBuffer s, rowName;
-    OwnedHqlExpr row = createRow(ctx, "row", rowName, (conditional && CREATE_DEAULT_ROW_IF_NULL_VALUE));
+    OwnedHqlExpr row = createRow(ctx, "row", rowName);
 
     //if (index > 0 && (index <= count)
     //MORE: Need to be very careful about the types...

ecl/hqlcpp/hqlcset.ipp

@@ -34,7 +34,7 @@ public:
 
     using IHqlCppDatasetCursor::buildIterateClass;
 protected:
-    IHqlExpression * createRow(BuildCtx & ctx, const char * prefix, StringBuffer & rowName, bool conditional);
+    IHqlExpression * createRow(BuildCtx & ctx, const char * prefix, StringBuffer & rowName);
 
 protected:
     HqlCppTranslator & translator;

ecl/hqlcpp/hqlhtcpp.cpp

@@ -6757,10 +6757,12 @@ ABoundActivity * HqlCppTranslator::buildActivity(BuildCtx & ctx, IHqlExpression
                 }
         }
     }
+    catch (IError * e)
+    {
+        throw;
+    }
     catch (IException * e)
     {
-        if (dynamic_cast<IError *>(e))
-            throw;
         IHqlExpression * location = queryActiveActivityLocation();
         if (location)
         {

ecl/hqlcpp/hqlstmt.cpp

@@ -127,6 +127,18 @@ IHqlStmt * BuildCtx::addAssign(IHqlExpression * target, IHqlExpression * value)
 }
 
 
+IHqlStmt * BuildCtx::addAssignLink(IHqlExpression * target, IHqlExpression * value)
+{
+    if (ignoreInput)
+        return NULL;
+
+    HqlStmt * next = new HqlStmt(assign_link_stmt, curStmts);
+    next->addExpr(LINK(target));
+    next->addExpr(LINK(value));
+    return appendSimple(next);
+}
+
+
 IHqlStmt * BuildCtx::addAssignIncrement(IHqlExpression * target, IHqlExpression * value)
 {
     if (ignoreInput)

ecl/hqlcpp/hqlstmt.hpp

@@ -91,6 +91,7 @@ public:
     ~BuildCtx();
 
     IHqlStmt *                  addAssign(IHqlExpression * target, IHqlExpression * value);
+    IHqlStmt *                  addAssignLink(IHqlExpression * target, IHqlExpression * value);
     IHqlStmt *                  addAssignIncrement(IHqlExpression * target, IHqlExpression * value);
     IHqlStmt *                  addAssignDecrement(IHqlExpression * target, IHqlExpression * value);
     IHqlStmt *                  addAlias(IHqlStmt * stmt);
@@ -195,6 +196,7 @@ enum StmtKind {
              line_stmt,
              continue_stmt,
              function_stmt,
+             assign_link_stmt,
 };
 
 

ecl/hqlcpp/hqlttcpp.cpp

@@ -271,7 +271,7 @@ NewThorStoredReplacer::NewThorStoredReplacer(HqlCppTranslator & _translator, IWo
 }
 
 
-void NewThorStoredReplacer::doAnalyseBody(IHqlExpression * expr)
+void NewThorStoredReplacer::doAnalyse(IHqlExpression * expr)
 {
     //NOTE: This is called very early before no_assertconstant has been processed, so we need to explicitly
     //constant fold, and check it is constant (bug 26963)
@@ -286,7 +286,7 @@ void NewThorStoredReplacer::doAnalyseBody(IHqlExpression * expr)
         {
             Owned<IWUWebServicesInfo> wsi = wu->updateWebServicesInfo(false);
             if (wsi)
-                throwError(HQLERR_MultipleHashWebserviceCalls);
+                translator.ERRORAT(expr, HQLERR_MultipleHashWebserviceCalls);
             wsi.setown(wu->updateWebServicesInfo(true));
 
             IHqlExpression *wsExpr = expr->queryChild(0);
@@ -306,9 +306,9 @@ void NewThorStoredReplacer::doAnalyseBody(IHqlExpression * expr)
             IValue * value = foldedValue->queryValue();
 
             if (!name)
-                throwError1(HQLERR_ExpectedConstantDebug, getExprECL(foldedName, errorTemp).str());
+                translator.reportError(expr, ECODETEXT(HQLERR_ExpectedConstantDebug), getExprECL(foldedName, errorTemp).str());
             if (!value)
-                throwError1(HQLERR_ExpectedConstantDebug, getExprECL(foldedValue, errorTemp).str());
+                translator.reportError(expr, ECODETEXT(HQLERR_ExpectedConstantDebug), getExprECL(foldedValue, errorTemp).str());
 
             StringBuffer nameText,valueText;
             name->getStringValue(nameText);
@@ -329,9 +329,9 @@ void NewThorStoredReplacer::doAnalyseBody(IHqlExpression * expr)
             IValue * value = foldedValue->queryValue();
 
             if (!name)
-                throwError1(HQLERR_ExpectedConstantWorkunit, getExprECL(foldedName, errorTemp).str());
+                translator.reportError(expr, ECODETEXT(HQLERR_ExpectedConstantWorkunit), getExprECL(foldedName, errorTemp).str());
             if (!value)
-                throwError1(HQLERR_ExpectedConstantWorkunit, getExprECL(foldedValue, errorTemp).str());
+                translator.reportError(expr, ECODETEXT(HQLERR_ExpectedConstantWorkunit), getExprECL(foldedValue, errorTemp).str());
 
             StringBuffer nameText,valueText;
             name->getStringValue(nameText);
@@ -368,7 +368,7 @@ void NewThorStoredReplacer::doAnalyseBody(IHqlExpression * expr)
                 wu->setWuScope(valueText.str());
             }
             else
-                throwError1(HQLERR_UnsupportedHashWorkunit, nameText.str());
+                translator.reportError(expr, ECODETEXT(HQLERR_UnsupportedHashWorkunit), nameText.str());
         }
         else if (kind == linkAtom)
         {
@@ -395,7 +395,9 @@ void NewThorStoredReplacer::doAnalyseBody(IHqlExpression * expr)
             seenMeta = true;
     }
 
-    QuickHqlTransformer::doAnalyseBody(expr);
+    //Walk the body rather than expr so that the no_setmeta isn't processed again.
+    //If the code above was in doAnalyseBody() the error location information would not be available.
+    QuickHqlTransformer::doAnalyseBody(expr->queryBody());
 }
 
 bool NewThorStoredReplacer::needToTransform()
@@ -479,10 +481,10 @@ IHqlExpression * NewThorStoredReplacer::createTransformed(IHqlExpression * expr)
                             othersText.append(",");
                             getExprECL(&activeReplacements.item(i), othersText);
                         }
-                        throwError3(HQLERR_RecursiveStoredOther,  forceConstant ? "CONSTANT" : "STORED", nameText.str(), othersText.str()+1);
+                        translator.reportError(expr, ECODETEXT(HQLERR_RecursiveStoredOther),  forceConstant ? "CONSTANT" : "STORED", nameText.str(), othersText.str()+1);
                     }
                     else
-                        throwError2(HQLERR_RecursiveStored,  forceConstant ? "CONSTANT" : "STORED", nameText.str());
+                        translator.reportError(expr, ECODETEXT(HQLERR_RecursiveStored),  forceConstant ? "CONSTANT" : "STORED", nameText.str());
                 }
 
                 ITypeInfo * exprType = expr->queryType();
@@ -505,12 +507,12 @@ IHqlExpression * NewThorStoredReplacer::createTransformed(IHqlExpression * expr)
                             getExprECL(matchedName, nameText);
                             getFriendlyTypeStr(exprType, exprTypeText);
                             getFriendlyTypeStr(replacementType, replacementTypeText);
-                            throwError3(HQLERR_HashStoredTypeMismatch, nameText.str(), exprTypeText.str(), replacementTypeText.str());
+                            translator.reportError(expr, ECODETEXT(HQLERR_HashStoredTypeMismatch), nameText.str(), exprTypeText.str(), replacementTypeText.str());
                         }
                         else if (expr->queryRecord() != replacement->queryRecord())
                         {
                             StringBuffer s;
-                            throwError1(HQLERR_HashStoredRecordMismatch, getExprECL(matchedName, s).str());
+                            translator.reportError(expr, ECODETEXT(HQLERR_HashStoredRecordMismatch), getExprECL(matchedName, s).str());
                         }
                     }
                     break;
@@ -522,7 +524,7 @@ IHqlExpression * NewThorStoredReplacer::createTransformed(IHqlExpression * expr)
                             getExprECL(matchedName, nameText);
                             getFriendlyTypeStr(exprType, exprTypeText);
                             getFriendlyTypeStr(replacementType, replacementTypeText);
-                            throwError3(HQLERR_HashStoredTypeMismatch, nameText.str(), exprTypeText.str(), replacementTypeText.str());
+                            translator.reportError(expr, ECODETEXT(HQLERR_HashStoredTypeMismatch), nameText.str(), exprTypeText.str(), replacementTypeText.str());
                         }
                         replacement.setown(ensureExprType(replacement, exprType));
                         break;
@@ -541,7 +543,7 @@ IHqlExpression * NewThorStoredReplacer::createTransformed(IHqlExpression * expr)
                                 getExprECL(matchedName, nameText);
                                 getFriendlyTypeStr(exprType, exprTypeText);
                                 getFriendlyTypeStr(replacementType, replacementTypeText);
-                                throwError3(HQLERR_HashStoredTypeMismatch, nameText.str(), exprTypeText.str(), replacementTypeText.str());
+                                translator.reportError(expr, ECODETEXT(HQLERR_HashStoredTypeMismatch), nameText.str(), exprTypeText.str(), replacementTypeText.str());
                             }
                         default:
                             replacement.setown(ensureExprType(replacement, exprType));

ecl/hqlcpp/hqlttcpp.ipp

@@ -30,7 +30,7 @@ class NewThorStoredReplacer : public QuickHqlTransformer
 public:
     NewThorStoredReplacer(HqlCppTranslator & _translator, IWorkUnit * _wu, ICodegenContextCallback * _logger);
 
-    virtual void doAnalyseBody(IHqlExpression * expr);
+    virtual void doAnalyse(IHqlExpression * expr);
     virtual IHqlExpression * createTransformed(IHqlExpression * expr);
 
     bool needToTransform();

ecl/hqlcpp/hqlwcpp.cpp

@@ -1641,7 +1641,10 @@ void HqlCppWriter::generateStmt(IHqlStmt * stmt)
     switch (kind)
     {
         case assign_stmt:
-            generateStmtAssign(stmt);
+            generateStmtAssign(stmt, false);
+            break;
+        case assign_link_stmt:
+            generateStmtAssign(stmt, true);
             break;
         case block_stmt:
             generateChildren(stmt, true);
@@ -1753,12 +1756,13 @@ void HqlCppWriter::generateSimpleAssign(IHqlExpression * target, IHqlExpression
     generateExprCpp(source).append(";");
 }
 
-void HqlCppWriter::generateStmtAssign(IHqlStmt * assign)
+void HqlCppWriter::generateStmtAssign(IHqlStmt * assign, bool link)
 {
     IHqlExpression * target = assign->queryExpr(0);
     IHqlExpression * source = assign->queryExpr(1);
 
     ITypeInfo * type = target->queryType();
+    const char * setFunction = link ? ".set(" : ".setown(";
 
     switch (type->getTypeCode())
     {
@@ -1781,7 +1785,7 @@ void HqlCppWriter::generateStmtAssign(IHqlStmt * assign)
             else if (type->getSize() == UNKNOWN_LENGTH)
             {
                 indent();
-                generateExprCpp(target).append(".setown(");
+                generateExprCpp(target).append(setFunction);
                 generateExprCpp(source).append(");");
             }
             else
@@ -1796,14 +1800,14 @@ void HqlCppWriter::generateStmtAssign(IHqlStmt * assign)
                 {
                     assertex(source->getOperator() == no_complex);
                     indent();
-                    generateExprCpp(target).append(".setown(");
+                    generateExprCpp(target).append(setFunction);
                     generateExprCpp(source->queryChild(0)).append(",");
                     generateExprCpp(source->queryChild(1)).append(");");
                 }
                 else
                 {
                     indent();
-                    generateExprCpp(target).append(".setown(");
+                    generateExprCpp(target).append(setFunction);
                     generateExprCpp(source).append(");");
                 }
             }
@@ -1815,7 +1819,7 @@ void HqlCppWriter::generateStmtAssign(IHqlStmt * assign)
             if (hasWrapperModifier(type))
             {
                 indent();
-                generateExprCpp(target).append(".setown(");
+                generateExprCpp(target).append(setFunction);
                 generateExprCpp(source).append(");");
             }
             else

ecl/hqlcpp/hqlwcpp.ipp

@@ -88,7 +88,7 @@ protected:
     void generateParamCpp(IHqlExpression * expr, IHqlExpression * attrs);
     void generateSimpleAssign(IHqlExpression * target, IHqlExpression * source);
     void generateStmt(IHqlStmt * stmt);
-    void generateStmtAssign(IHqlStmt * assign);
+    void generateStmtAssign(IHqlStmt * assign, bool link);
     void generateStmtAssignModify(IHqlStmt * assign);
     void generateStmtCase(IHqlStmt * stmt);
     void generateStmtDeclare(IHqlStmt * declare);

esp/scm/ws_access.ecm

@@ -691,7 +691,7 @@ ESPresponse [nil_remove] UserAccountExportResponse
 };
 
 
-ESPservice [version("1.08"), default_client_version("1.08"), exceptions_inline("./smc_xslt/exceptions.xslt")] ws_access
+ESPservice [version("1.08"), exceptions_inline("./smc_xslt/exceptions.xslt")] ws_access
 {
     ESPmethod [client_xslt("/esp/xslt/access_users.xslt")] Users(UserRequest, UserResponse);
     ESPmethod [client_xslt("/esp/xslt/access_useredit.xslt")] UserEdit(UserEditRequest, UserEditResponse);

esp/scm/ws_dfu.ecm

@@ -679,7 +679,7 @@ ESPresponse [exceptions_inline, nil_remove, http_encode(0)] DFUGetFileMetaDataRe
 
 //  ===========================================================================
 ESPservice [
-    version("1.29"), default_client_version("1.29"),
+    version("1.29"),
     noforms, 
     exceptions_inline("./smc_xslt/exceptions.xslt")] WsDfu
 {

esp/scm/ws_esdlconfig.ecm

@@ -166,7 +166,7 @@ ESPresponse [exceptions_inline] ListESDLBindingsResponse
     ESParray<ESPstruct ESDLBinding, Binding> Bindings;
 };
 
-ESPservice [version("1.0"), default_client_version("1.0"), exceptions_inline("./smc_xslt/exceptions.xslt")] WsESDLConfig
+ESPservice [version("1.0"), exceptions_inline("./smc_xslt/exceptions.xslt")] WsESDLConfig
 {
     ESPmethod Echo(EchoRequest, EchoResponse);
     ESPmethod PublishESDLDefinition(PublishESDLDefinitionRequest, PublishESDLDefinitionResponse);

esp/scm/ws_fs.ecm

@@ -629,7 +629,7 @@ ESPresponse [exceptions_inline, nil_remove] GetSprayTargetsResponse
 };
 
 ESPservice [
-    version("1.11"), default_client_version("1.10"),
+    version("1.11"),
     exceptions_inline("./smc_xslt/exceptions.xslt")] FileSpray
 {
     ESPuses ESPstruct DFUWorkunit;

esp/scm/ws_loggingservice.ecm

@@ -51,7 +51,7 @@ ESPresponse [exceptions_inline] UpdateLogResponse
     string StatusMessage;
 };
 
-ESPService [version("1.0"), default_client_version("1.0"), noforms, use_method_name] WsLoggingService
+ESPService [version("1.0"), noforms, use_method_name] WsLoggingService
 {
     ESPmethod GetTransactionSeed(GetTransactionSeedRequest, GetTransactionSeedResponse);
     ESPmethod UpdateLog(UpdateLogRequest, UpdateLogResponse);

esp/scm/ws_machine.ecm

@@ -310,7 +310,7 @@ ESPresponse [encode(0), exceptions_inline] GetTargetClusterInfoResponse
     [min_ver("1.12")] string AcceptLanguage;
 };
 //-------- service ---------
-ESPservice [version("1.13"), default_client_version("1.13")] ws_machine
+ESPservice [version("1.13")] ws_machine
 {
     ESPuses ESPstruct RequestInfoStruct;
     ESPuses ESPstruct MachineInfoEx;

esp/scm/ws_packageprocess.ecm

@@ -232,7 +232,7 @@ ESPresponse [exceptions_inline] GetPackageMapSelectOptionsResponse
     ESParray<string> ProcessFilters;
 };
 
-ESPservice [version("1.01"), default_client_version("1.01"), exceptions_inline("./smc_xslt/exceptions.xslt")] WsPackageProcess
+ESPservice [version("1.01"), exceptions_inline("./smc_xslt/exceptions.xslt")] WsPackageProcess
 {
     ESPmethod Echo(EchoRequest, EchoResponse);
     ESPmethod AddPackage(AddPackageRequest, AddPackageResponse);

esp/scm/ws_smc.ecm

@@ -357,7 +357,7 @@ GetStatusServerInfoResponse
     ESPstruct StatusServerInfo StatusServerInfo;
 };
 
-ESPservice [noforms, version("1.19"), default_client_version("1.19"), exceptions_inline("./smc_xslt/exceptions.xslt"), use_method_name] WsSMC
+ESPservice [noforms, version("1.19"), exceptions_inline("./smc_xslt/exceptions.xslt"), use_method_name] WsSMC
 {
     ESPmethod Index(SMCIndexRequest, SMCIndexResponse);
     ESPmethod [resp_xsl_default("/esp/xslt/index.xslt")] Activity(ActivityRequest, ActivityResponse);

esp/scm/ws_topology.ecm

@@ -583,7 +583,7 @@ ESPresponse [exceptions_inline,encode(0)] TpGetServicePluginsResponse
     ESParray<ESPstruct TpEspServicePlugin, Plugin> Plugins;
 };
 
-ESPservice [noforms, version("1.21"), default_client_version("1.21"), exceptions_inline("./smc_xslt/exceptions.xslt")] WsTopology
+ESPservice [noforms, version("1.21"), exceptions_inline("./smc_xslt/exceptions.xslt")] WsTopology
 {
     ESPuses ESPStruct TpBinding;
     ESPuses ESPstruct TpCluster;

esp/scm/ws_workunits.ecm

@@ -1609,7 +1609,7 @@ ESPresponse [exceptions_inline] WUGetStatsResponse
 };
 
 ESPservice [
-    version("1.54"), default_client_version("1.54"),
+    version("1.54"),
     noforms,exceptions_inline("./smc_xslt/exceptions.xslt"),use_method_name] WsWorkunits
 {
     ESPmethod [resp_xsl_default("/esp/xslt/workunits.xslt")]     WUQuery(WUQueryRequest, WUQueryResponse);

esp/services/ws_dfu/ws_dfuService.cpp

@@ -1701,7 +1701,7 @@ void CWsDfuEx::getFilePartsOnClusters(IEspContext &context, const char* clusterR
                 partSizeStr.set("<N/A>");
             else
             {
-                __uint64 size = partPropertyTree->getPropInt("@size");
+                __uint64 size = partPropertyTree->getPropInt64("@size");
                 comma c4(size);
                 partSizeStr<<c4;
 

esp/src/eclwatch/nls/es/hpcc.js

@@ -33,6 +33,7 @@ define(
     ArchivedOnly: "Solo archivado",
     ArchivedWarning: "Advertencia: Por favor especifique rango de fechas corto. O si no, la recuperacion de workunits puede ser demorada.",
     AutoRefresh: "Actualizar Automaticamente",
+    Back: "Atras",
     BannerColor: "Color de Banner",
     BannerMessage: "Mensaje del Banner",
     BannerScroll: "Desplazamiento del Banner",
@@ -60,9 +61,11 @@ define(
     Command: "Comando",
     Comment: "Comentario",
     Completed: "Completado",
+    ComplexityWarning: "Mas de {threshold} actividades ({activityCount}) - ¿suprimir primer visualización?",
     Component: "Componente",
     Compress: "Comprimir",
     Compressed: "Comprimido",
+    CompressedFileSize: "Tamaño de Archivo Comprimido",
     Configuration: "Configuracion",
     ConfirmPassword: "Confirme la Contraseña",
     ConfirmRemoval: "¿Seguro que quieres hacer esto?",
@@ -80,12 +83,14 @@ define(
     Debug: "Depurar",
     DEF: "DEF",
     Delete: "Eliminar",
+    Deleted: "Borrado",
     DeleteSelectedFiles: "¿Borrar archivos seleccionados?",
     DeleteSelectedGroups: "Eliminar  grupo(s) escojido(s)?",
     DeleteSelectedPermissions: "Borrar permiso(s) escojido(s)?",
     DeleteSelectedQueries: "¿Borrar queries seleccionados?",
     DeleteSelectedUsers: "Eliminar Usuario(s) escojido(s)?",
     DeleteSelectedWorkunits: "¿Borrar unidades de trabajo seleccionadas?",
+    DeleteSuperfile2: "¿Borrar Super-Archivo?",
     DeleteSuperfile: "¿Borrar super-archivo?",
     DeleteThisPackage: "Eliminar este paquete?",
     Delimited: "Delimitado",
@@ -94,17 +99,20 @@ define(
     DenyRead: "<center>Negar<br>Leer</center>",
     DenyWrite: "<center>Negar<br>Escribir</center>",
     Depth: "Profundidad",
+    DepthTooltip: "Máxima Profundidad de Subgrafo",
     Deschedule: "Eliminar del plan de ejecución",
     DescheduleSelectedWorkunits: "Desprogramar Workunit(s) escojida(s)?",
     Description: "Descripción",
     Despray: "Despray",
     Details: "Detalles",
     DFUServerName: "Nombre del servidor de DFU",
+    DFUWorkunit: "Unidad de Trabajo de DFU",
     Directory: "Directorio",
     DisableScopeScanConfirm: "Esta seguro que desea inhabilitar Scope Scans? Cuando DALI se reinicie, las opciones cambiadas se revierten a las opciones de la configuración.",
     DisableScopeScans: "Inhabilitar Scope Scans",
     DiskUsage: "Utilización del disco",
     Distance: "Distancia",
+    DistanceTooltip: "Maxima Actividad Distancia de Vecindario",
     Dll: "Dll",
     DOT: "DOT",
     DOTAttributes: "Atributos DOT",
@@ -114,6 +122,7 @@ define(
     Duration: "Duración",
     EBCDIC: "EBCDIC",
     ECL: "ECL",
+    ECLWorkunit: "Unidad de Trabajo de ECL",
     Edges: "Bordes",
     Edit: "Editar",
     EditDOT: "Editar DOT",
@@ -160,11 +169,16 @@ define(
     Find: "Encontrar",
     FindNext: "Encontrar próximo",
     FindPrevious: "Encontrar previo",
+    Finished: "Terminado",
     FirstName: "Primer Nombre",
+    FirstNRows: "Primeras N Filas",
     Fixed: "Fijo",
+    Folder: "Carpeta",
     Format: "Formato",
+    Forward: "Adelante",
     FromDate: "Desde Fecha",
     FromSizes: "Desde Tamaños",
+    FromTime: "Tiempo de Comenzar",
     FullName: "Nombre Entero",
     Graph: "Gráfico",
     Graphs: "Gráficos",
@@ -207,6 +221,8 @@ define(
     LargestSize: "Tomaño mas Grande",
     LastName: "Apellido",
     LastNDays: "ULtimos N días",
+    LastNHours: "Ultimas N Horas",
+    LastNRows: "Ultimas N Filas",
     LDAPWarning: "<b>Error en Servicios de LDAP:</b>  'Demasiados Usuarios' - Por favor aplique un Filtro.",
     LegacyForm: "Forma histórica",
     LibrariesUsed: "Bibliotecas usadas",
@@ -221,6 +237,7 @@ define(
     LoadPackageContentHere: "(Cargar aqui el contenido del paquete)",
     LoadPackageFromFile: "Cargar paquete con contenido de archivo",
     Local: "Local",
+    Log: "Registro",
     LogFile: "Archivo de registro",
     LoggedInAs: "Conectado como",
     LogicalFile: "Archivo Lógico",
@@ -229,9 +246,12 @@ define(
     LogicalFilesOnly: "Archivos logicos Solamente",
     LogicalFileType: "Tipo de archivo lógico",
     LogicalName: "Nombre lógico",
+    Logs: "Registros",
     log_analysis_1: "log_analysis_1*",
     Low: "Bajo",
     ManualCopy: "Oprima Crtl_C",
+    ManualOverviewSelection: "Seleccion manual sera requirida",
+    ManualTreeSelection: "Seleccion manual de arbol sera requirida",
     Mappings: "Mapeados",
     Mask: "Máscara",
     Max: "Máx",
@@ -250,6 +270,7 @@ define(
     MonitorShotLimit: "Límite de disparos del monitor",
     MonitorSub: "Sub monitor",
     Month: "Mez",
+    More: "Mas",
     MustContainUppercaseAndSymbol: "Debe Contener mayúscula y símbolo",
     NA: "No Applica",
     Name: "Nombre",
@@ -306,10 +327,12 @@ define(
     PasswordExpired: "Su contraseña expiro. Por favor cambiela.",
     PasswordExpirePostfix: " dia(s). Desea cambiarla?",
     PasswordExpirePrefix: "Su contraseña expira en",
+    PasswordOpenZAP: "Contraseña para abrir ZAP (opcional)",
     PasswordsDoNotMatch: "Las Contraseñas No Coinciden",
     Path: "Camino",
     PathMask: "Máscara del camino",
     Pause: "Pausa",
+    PauseNow: "Pausar",
     PctComplete: "% completado",
     PercentDone: "Porcentaje completado",
     PerformingLayout: "Generando disposición...",
@@ -319,6 +342,7 @@ define(
     PlaceholderFirstName: "John",
     PlaceholderLastName: "Smith",
     Playground: "Patio de juegos",
+    Plugins: "Plugins",
     Port: "Puerto",
     Prefix: "Prefijo",
     PrefixPlaceholder: "filename{:length}, filesize{:[B|L][1-8]}",
@@ -339,13 +363,17 @@ define(
     Quarter: "Cuarto",
     Queries: "Consultas",
     QueriesNoPackage: "Consultas sin correspondiente paquete",
+    Query: "Consulta",
     QueryDetailsfor: "Detalles de Query por",
+    QueryID: "ID de la Consulta",
     QueryIDPlaceholder: "som?q*ry.1",
+    QueryName: "Nombre de la Consulta",
     QueryNamePlaceholder: "My?Su?erQ*ry",
     QuerySet: "Grupo de Query",
     Queue: "Cola",
     Quote: "Comilla",
     QuotedTerminator: "Terminador Encomillado",
+    RawTextPage: "Texto (Pagina Actual)",
     RecordCount: "Número de registros",
     RecordLength: "Largo de registro",
     Records: "Registros",
@@ -360,6 +388,7 @@ define(
     RemoteDali: "Dali remoto",
     RemoteDaliIP: "Remote&nbsp;Dali&nbsp;IP&nbsp;Address",
     Remove: "Remover",
+    RemoveSubfiles: "Borrar Sub-Archivo(s)",
     RemoveUser: "Esta apunto de excluirse del grupo:",
     Rename: "Renombrar",
     RenderedSVG: "SVG creado",
@@ -368,6 +397,8 @@ define(
     RequestSchema: "Esquema del requerimiento",
     Reschedule: "Re-planear ejecución",
     Reset: "Origen",
+    ResetThisQuery: "Reajustar esta Consulat",
+    ResetViewToSelection: "Reajustar Vista a la Seleccion",
     Resource: "Recurso",
     Resources: "Recursos",
     ResponseSchema: "Esquema de la respuesta",
@@ -380,6 +411,8 @@ define(
     Resume: "Continuar",
     RetainSuperfileStructure: "Retener estructura del super-archivo",
     RetypePassword: "Confirme la contraseña",
+    Reverse: "Marchar Atrás",
+    RowPath: "Ruta de Fila",
     Rows: "Líneas",
     RowTag: "Etiqueta de línea",
     RoxieCluster: "Sistema Roxie",
@@ -418,6 +451,7 @@ define(
     Start: "Comienzo",
     Started: "Comenzado",
     State: "Estado",
+    Stats: "Estatísticas",
     Status: "Estado",
     Stopped: "Detenido",
     Subgraph: "Subgrafo",
@@ -428,7 +462,9 @@ define(
     SummaryMessage: "Mensaje resumido",
     Superfile: "Superfile",
     SuperFile: "Super File",
+    Superfiles: "Super-Archivos",
     SuperFiles: "Superfiles",
+    SuperFilesBelongsTo: "Miembro de Super-Archivo(s)",
     SuperfilesOnly: "Solo Superfiles",
     Suspend: "Suspender",
     Suspended: "Suspendido",
@@ -448,6 +484,7 @@ define(
     TargetWuid: "Wuid del Objecto",
     Terminators: "Terminadores",
     TestPages: "Páginas de prueba",
+    Text: "Texto",
     ThorMasterAddress: "Direccion del Thor Master",
     ThorNetworkAddress: "Dirección de red de Thor",
     Time: "Tiempo",
@@ -474,8 +511,11 @@ define(
     title_HPCCPlatformMain: "ECL Watch - Hogar",
     title_HPCCPlatformOps: "ECL Watch - Operaciones",
     title_HPCCPlatformRoxie: "ECL Watch - Roxie",
+    title_HPCCPlatformServicesPlugin: "ECL Watch - Plugins",
     title_Inputs: "Entradas",
     title_LFDetails: "Detalles de archivos lógicos",
+    title_LibrariesUsed: "Librerias Usadas",
+    title_Log: "Archivo de Registro",
     title_LZBrowse: "Zonas de descarga",
     title_MemberOf: "Miembros de",
     title_Members: "Miembros",
@@ -489,15 +529,20 @@ define(
     title_Results: "Salidas",
     title_SearchResults: "Resultados de búsqueda ",
     title_SourceFiles: "",
+    title_SourceFiles: "",
+    title_Topology: "Topología",
     title_TpThorStatus: "Estatus de Thor",
     title_UserPermissions: "Permisos del Usuario",
     title_UserQuery: "Permisos",
     title_WUDetails: "Detalles de las unidades de trabajo ECL",
     title_WUQuery: "Unidades de trabajo ECL",
+    To: "A:",
     ToDate: "Hasta fecha",
     Toenablegraphviews: "Para habilitar vista de gráficos, instale el plugin Graph View Control",
     Top: "Arriva",
+    Topology: "Topología",
     ToSizes: "A tamaño",
+    TotalClusterTime: "Tiempo Total del Cluster",
     TotalSize: "Tamaño Total",
     TotalThorTime: "Tiempo total de Thor",
     TransitionGuide: "Guía de transición",

esp/src/eclwatch/nls/hu/hpcc.js

@@ -33,6 +33,7 @@ define(
     ArchivedOnly: "Csak tárolt.",
     ArchivedWarning: "Figyelem: Adjon meg rövidebb időintervallumot! Ellenkező esetben a feladatok lekérdezése hosszabb ideig tarthat, mint amit az Ön bőngésző programja engedélyez.",
     AutoRefresh: "Automatikus frissítés",
+    Back: "Vissza",
     BannerColor: "Fejléc szín",
     BannerMessage: "Fejléc szövege",
     BannerScroll: "Fejléc görgetés",
@@ -59,11 +60,14 @@ define(
     Command: "Parancs",
     Comment: "Megjegyzés",
     Completed: "Kész",
+    ComplexityWarning: "Több mint {threshold} aktivitás ({activityCount}) - elnyomjuk a kezdeti megjelenítést?",
     Component: "Komponens",
     Compress: "Tömörít",
     Compressed: "Tömörített",
+    CompressedFileSize: "Tömörített fájl méret",
     Configuration: "Beállítások",
     ConfirmPassword: "Jelszó jóváhagyás",
+    ConfirmRemoval: "Biztos, hogy ezt akarja csinálni?",
     Content: "Tartalom",
     Contents: "Tartalom",
     ContentType: "Tartalom típusa",
@@ -78,12 +82,14 @@ define(
     Debug: "Hibakeresés",
     DEF: "DEF",
     Delete: "Törlés",
+    Deleted: "Törölt",
     DeleteSelectedFiles: "Töröljük a kiválasztott fájlokat?",
     DeleteSelectedGroups: "Törölni akarja a kijelölt csoporto(ka)t?",
     DeleteSelectedPermissions: "Törli a kiválasztott hozzáférési engedélyeket?",
     DeleteSelectedQueries: "Töröljük a kiválasztott lekérdezéseket?",
     DeleteSelectedUsers: "Törölni akarja a kijelölt felhasználó(ka)t?",
     DeleteSelectedWorkunits: "Törölni akarja a kiválasztott feladatokat?",
+    DeleteSuperfile2: "Szuper-fájl törlése",
     DeleteSuperfile: "Töröljük a Superfile-t?",
     DeleteThisPackage: "Törölni akarja ezt a csomagot?",
     Delimited: "Határolt",
@@ -92,17 +98,20 @@ define(
     DenyRead: "<center>Olvasás<br>Tiltása</center>",
     DenyWrite: "<center>Írás<br>Tiltása</center>",
     Depth: "Mélység",
+    DepthTooltip: "Maximális al-gráf mélység",
     Deschedule: "Ütemezés törlése",
     DescheduleSelectedWorkunits: "Megszűnteti a kijelőlt feladat ütemezését?",
     Description: "Leírás",
     Despray: "Összegyűjt",
     Details: "Részletek",
     DFUServerName: "DFU Szerver név",
+    DFUWorkunit: "DFU-munkaegység",
     Directory: "Könyvtár",
     DisableScopeScanConfirm: "Biztos, hogy letíltja a hatókör vizsgálatot? DALI újraíndításakor ezek a változtatások elvesznek.",
     DisableScopeScans: "Hatókör vizsgálat letiltása",
     DiskUsage: "Lemez használat",
     Distance: "Távolság",
+    DistanceTooltip: "Legnagyobb aktivitás-szomszéd távolság",
     Dll: "Dll",
     DOT: "DOT",
     DOTAttributes: "DOT Attribútumok",
@@ -112,6 +121,7 @@ define(
     Duration: "Időtartam",
     EBCDIC: "EBCDIC",
     ECL: "ECL",
+    ECLWorkunit: "ECL-munkaegység",
     Edges: "Élek",
     Edit: "Szerkesztés",
     EditDOT: "DOT szerkesztés",
@@ -159,11 +169,16 @@ define(
     Find: "Keresés",
     FindNext: "Következőt",
     FindPrevious: "Előzőt",
+    Finished: "Befejezett",
     FirstName: "Keresztnév",
+    FirstNRows: "Első N rekord",
     Fixed: "Állandó rekordméret",
+    Folder: "Könyvtár",
     Format: "Formátum",
+    Forward: "Előre",
     FromDate: "Dátumtól",
     FromSizes: "Mérettől",
+    FromTime: "Időponttól",
     FullName: "Teljes név",
     Graph: "Gráf",
     Graphs: "Gráfok",
@@ -206,6 +221,8 @@ define(
     LargestSize: "Legnagyobb méret",
     LastName: "Vezetéknév",
     LastNDays: "Az utolsó N nap",
+    LastNHours: "Utolsó N óra",
+    LastNRows: "Utolsó N rekord",
     LDAPWarning: "<b>LDAP szolgáltatás hiba:</b>  'Túl sok felhasználó' - Kérem, használjon szűrést!",
     LegacyForm: "Örökölt űrlap (Legacy Form)",
     LibrariesUsed: "Használt könyvtárak",
@@ -220,6 +237,7 @@ define(
     LoadPackageContentHere: "(Csomag tartalmának betöltése erre a helyre)",
     LoadPackageFromFile: "Csomag betöltése fájlból",
     Local: "Helyi",
+    Log: "Log",
     LogFile: "Log fájl",
     LoggedInAs: "Bejelentkezve mint ",
     LogicalFile: "Logikai fájl",
@@ -229,9 +247,12 @@ define(
     LogicalFilesOnly: "Csak a logikai fájlokat",
     LogicalFileType: "Logikai fájl típusa",
     LogicalName: "Logikai név",
+    Logs: "Logok",
     log_analysis_1: "log_elemzés_1*",
     Low: "Alacsony",
     ManualCopy: "Használja a Ctr+C-t",
+    ManualOverviewSelection: "Gyorsnézet manuális kiválasztása szükséges",
+    ManualTreeSelection: "Manuális fa kiválasztás szükséges",
     Mappings: "Leképezések",
     Mask: "Maszk",
     Max: "Maximum",
@@ -250,6 +271,7 @@ define(
     MonitorShotLimit: "Monitor találatok limitje",
     MonitorSub: "Almonitor",
     Month: "Hónap",
+    More: "Tovább",
     MustContainUppercaseAndSymbol: "Csak nagybetűket és jeleket tartalmazhat",
     NA: "N/A",
     Name: "Név",
@@ -305,10 +327,12 @@ define(
     PasswordExpired: "A jelszava lejárt! Adjon meg újat!",
     PasswordExpirePostfix: "napon bellül. Meg akarja változtatni?",
     PasswordExpirePrefix: "Jelszava lejár",
+    PasswordOpenZAP: "Jelszó a ZAP megnyitásához (opciónális)",
     PasswordsDoNotMatch: "Jelszó nem egyezik",
     Path: "Útvonal",
     PathMask: "Útvonal maszk",
     Pause: "Szüneteltetés",
+    PauseNow: "Azonnali felfüggesztés",
     PctComplete: "% kész",
     PercentDone: "% kész",
     PerformingLayout: "Elrendezés generálása...",
@@ -318,6 +342,7 @@ define(
     PlaceholderFirstName: "József",
     PlaceholderLastName: "Kiss",
     Playground: "ECL teszt labor",
+    Plugins: "Betölthető modulok",
     Port: "Port",
     Prefix: "Előtag",
     PrefixPlaceholder: "fájlnév{:hossz}, fájlméret{:[B|L][1-8]}",
@@ -338,13 +363,17 @@ define(
     Quarter: "Negyed",
     Queries: "Lekérdezések",
     QueriesNoPackage: "Lekérdezések hozzátartozó csomag nélkül",
+    Query: "Lekérdezés",
     QueryDetailsfor: "A lekérdezés paraméterei",
+    QueryID: "Lekérdezés azonosító",
     QueryIDPlaceholder: "som?q*ry.1'",
+    QueryName: "Lekérdezés név",
     QueryNamePlaceholder: "My?Su?erQ*ry",
     QuerySet: "Lekérdezés készlet",
     Queue: "Sor",
     Quote: "Idézőjel",
     QuotedTerminator: "A fájl idézőjelek közötti rekordhatároló karaktert tartalmaz",
+    RawTextPage: "Formázatlan szöveg (aktuális lap)",
     RecordCount: "Rekord szám",
     RecordLength: "Rekord méret",
     Records: "Rekordok száma",
@@ -359,6 +388,8 @@ define(
     RemoteDali: "Távoli Dali hálózati címe",
     RemoteDaliIP: "Remote&nbsp;Dali&nbsp;IP&nbsp;Address",
     Remove: "Eltávolít",
+    RemoveSubfiles: "Al-fájl(ok) eltávolítása",
+    RemoveUser: "Arra készül, hogy eltávolítsa a felhasználói nevét a csoportból!",
     Rename: "Átnevez",
     RenderedSVG: "Generált SVG",
     RenderSVG: "SVG generálás",
@@ -366,6 +397,8 @@ define(
     RequestSchema: "Igénylő séma (Request Schema)",
     Reschedule: "Újraütemez",
     Reset: "Forrás",
+    ResetThisQuery: "Vissza akarja állítani ezt a lekérdezést az allapállapotába?",
+    ResetViewToSelection: "Vissza akarja állítani ezt a nézetet a kiválasztott elemekre?",
     Resource: "Erőforrás",
     Resources: "Erőforrások",
     ResponseSchema: "Válasz séma (Response Schema)",
@@ -378,6 +411,8 @@ define(
     Resume: "Tovább folytat",
     RetainSuperfileStructure: "Superfile szerkezetének megőrzése",
     RetypePassword: "Jelszó mégegyszer",
+    Reverse: "Visszafelé",
+    RowPath: "Rekord útvonal",
     Rows: "Sor",
     RowTag: "Sor azonositó",
     RoxieCluster: "Roxie klaszter",
@@ -417,6 +452,7 @@ define(
     Start: "Indítás",
     Started: "Elindítva",
     State: "Állapot",
+    Stats: "Statisztika",
     Status: "Állapot",
     Stopped: "Megállítva",
     Subgraph: "Al-gráf",
@@ -425,10 +461,11 @@ define(
     Subtype: "Altípus",
     Summary: "Összegzés",
     SummaryMessage: "Összegzés",
-    SuperFile: "Super File",
-    SuperFile: "Szuperfájl",
-    Superfile: "Szuperfálj",
+    SuperFile: "Szuper-fájl",
+    Superfile: "Szuper-fájl",
+    Superfiles: "Szuper-fájlok",
     SuperFiles: "Szuperfájlok",
+    SuperFilesBelongsTo: "Szuperfájl(ok)hoz tartozik",
     SuperfilesOnly: "Kizárólag szuperfájl",
     Suspend: "Felfüggeszt",
     Suspended: "Felfüggesztve",
@@ -448,6 +485,7 @@ define(
     TargetWuid: "Cél/WUID",
     Terminators: "Rekord lezáró jelek",
     TestPages: "Tesztlapok",
+    Text: "Szöveg",
     ThorMasterAddress: "Thor Master hálózati címe",
     ThorNetworkAddress: "Thor hálózati címe",
     Time: "Idő",
@@ -474,8 +512,11 @@ define(
     title_HPCCPlatformMain: "ECL Watch - Kezdőlap",
     title_HPCCPlatformOps: "ECL Watch - Műveletek",
     title_HPCCPlatformRoxie: "ECL Watch - Roxie",
+    title_HPCCPlatformServicesPlugin: "ECL Watch betölthető modulok",
     title_Inputs: "Bementek",
     title_LFDetails: "Logikai fájl információk",
+    title_LibrariesUsed: "Felhasznált kódkönyvtárak",
+    title_Log: "Log fájl",
     title_LZBrowse: "Lerakatok",
     title_MemberOf: "Tagja a",
     title_Members: "Tagok",
@@ -489,15 +530,19 @@ define(
     title_Results: "Kimenet",
     title_SearchResults: "Keresés eredménye",
     title_SourceFiles: "Forrás fájlok",
+    title_Topology: "Topológia",
     title_TpThorStatus: "Thor állapot",
     title_UserPermissions: "Felhasználó engedélyei",
     title_UserQuery: "Engedélyek",
     title_WUDetails: "ECL feladat jellemzők",
     title_WUQuery: "ECL feladatok",
+    To: "-ig",
     ToDate: "Dátumig",
     Toenablegraphviews: "A gráfok megjelenítéshez telepiteni kell a 'Graph View Control' bővítményt.",
     Top: "Felső",
+    Topology: "Topológia",
     ToSizes: "Méretig",
+    TotalClusterTime: "Teljes cluster idő",
     TotalSize: "Összméret",
     TotalThorTime: "Összes Thor idő",
     TransitionGuide: "Átmenet útmutató",

esp/src/eclwatch/nls/zh/hpcc.js

@@ -33,6 +33,7 @@
     ArchivedOnly: "仅限已存档的工作单元",
     ArchivedWarning: "警告:请指定一个小的日期范围. 否则, 检索时间可能较长,网页浏览器可能超时",
     AutoRefresh: "自动更新",
+    Back: "返回",
     BannerColor: "标语的颜色",
     BannerMessage: "标语的文字",
     BannerScroll: "标语滚动",
@@ -59,11 +60,14 @@
     Command: "指令",
     Comment: "注释",
     Completed: "完成",
+    ComplexityWarning: "超过{}活动（{}）- 重新显示？",
     Component: "组成部分",
     Compress: "压缩",
     Compressed: "已压缩过的",
+    CompressedFileSize: "压缩后的文件长度",
     Configuration: "设置",
     ConfirmPassword: "确认密码",
+    ConfirmRemoval: "确认",
     Content: "内容",
     Contents: "内容",
     ContentType: "内容类型",
@@ -78,12 +82,14 @@
     Debug: "故障诊断",
     DEF: "DEF",
     Delete: "删除",
+    Deleted: "已删除",
     DeleteSelectedFiles: "删除所选择的文件?",
     DeleteSelectedGroups: "删除所选择的用户组?",
     DeleteSelectedPermissions: "删除所选的权限?",
     DeleteSelectedQueries: "删除所选择的查询程序?",
     DeleteSelectedUsers: "删除所选择的用户?",
     DeleteSelectedWorkunits: "删除所选择的工作单元?",
+    DeleteSuperfile2: "删除文件集",
     DeleteSuperfile: "删除文件集?",
     DeleteThisPackage: "删除文件包?",
     Delimited: "定界的",
@@ -92,17 +98,20 @@
     DenyRead: "<center>拒绝<br>读</center>",
     DenyWrite: "<center>拒绝<br>写</center>",
     Depth: "深度",
+    DepthTooltip: "最大子图深度",
     Deschedule: "取消运行计划",
     DescheduleSelectedWorkunits: "取消所选工作单元的计划?",
     Description: "说明",
     Despray: "复合原文件",
     Details: "细节",
     DFUServerName: "DFU服务器名",
+    DFUWorkunit: "DFU工作单元",
     Directory: "文件目录",
     DisableScopeScanConfirm: "你确认要禁用范围扫描吗?在DALI重新启动后, 将复归原始的系统设置.",
     DisableScopeScans: "禁用范围扫描",
     DiskUsage: "硬盘使用率",
     Distance: "距离",
+    DistanceTooltip: "最大相邻活动的距离",
     Dll: "动态联接库",
     DOT: "DOT",
     DOTAttributes: "DOT属性",
@@ -112,6 +121,7 @@
     Duration: "时间段",
     EBCDIC: "EBCDIC",
     ECL: "ECL",
+    ECLWorkunit: "ECL工作单元",
     Edges: "连接",
     Edit: "编辑",
     EditDOT: "编辑DOT",
@@ -158,11 +168,16 @@
     Find: "查找",
     FindNext: "查找下一个",
     FindPrevious: "查找前一个",
+    Finished: "已完成",
     FirstName: "名",
+    FirstNRows: "初始N行",
     Fixed: "定长的",
+    Folder: "文件夹",
     Format: "格式",
+    Forward: "向前",
     FromDate: "起始日期",
     FromSizes: "最小文件长度",
+    FromTime: "起始时间",
     FullName: "姓名",
     Graph: "图形",
     Graphs: "图形",
@@ -191,8 +206,8 @@
     IP: "IP",
     IPAddress: "IP地址",
     IsLibrary: "程序库",
-    JobName: "任务名",
     Jobname: "任务名",
+    JobName: "任务名",
     jsmi: "李名*",
     JSmith: "李名*",
     JSON: "JSON",
@@ -205,6 +220,8 @@
     LargestSize: "最大尺寸",
     LastName: "姓",
     LastNDays: "过去若干天以内",
+    LastNHours: "最后N小时",
+    LastNRows: "最后N行",
     LDAPWarning: "<b>LDAP服务错误：</b>用户太多无法显示,请选用过滤器.",
     LegacyForm: "旧式表格",
     LibrariesUsed: "使用程序库",
@@ -219,6 +236,7 @@
     LoadPackageContentHere: "（把文件包内容上载到这里）",
     LoadPackageFromFile: "从文件中上载文件包内容",
     Local: "局部",
+    Log: "日志",
     LogFile: "日志文件",
     LoggedInAs: "登录用户",
     LogicalFile: "逻辑文件",
@@ -227,9 +245,12 @@
     LogicalFilesOnly: "逻辑文件",
     LogicalFileType: "逻辑文件类型",
     LogicalName: "逻辑文件名",
+    Logs: "日志",
     log_analysis_1: "日志_分析_1*",
     Low: "低",
     ManualCopy: "按Ctrl+C键",
+    ManualOverviewSelection: "要求人工检查选项",
+    ManualTreeSelection: "要求人工选择树",
     Mappings: "映像",
     Mask: "掩码",
     Max: "最大",
@@ -248,6 +269,7 @@
     MonitorShotLimit: "监视发射限制",
     MonitorSub: "监视子文件",
     Month: "月",
+    More: "更多",
     MustContainUppercaseAndSymbol: "需含大写字符和符号",
     NA: "不适用",
     Name: "名称",
@@ -303,10 +325,12 @@
     PasswordExpired: "你的密码已过期。请立刻更新。",
     PasswordExpirePostfix: "天过期. 现在更新吗?",
     PasswordExpirePrefix: "你的登录密码将在",
+    PasswordOpenZAP: "打开ZAP的口令（选项）",
     PasswordsDoNotMatch: "密码不匹配",
     Path: "路径",
     PathMask: "路径掩码",
     Pause: "暂停",
+    PauseNow: "马上暂停",
     PctComplete: "任务完成%",
     PercentDone: "任务完成%",
     PerformingLayout: "图形布局中...",
@@ -316,6 +340,7 @@
     PlaceholderFirstName: "民",
     PlaceholderLastName: "张",
     Playground: "操作平台",
+    Plugins: "插件",
     Port: "端口",
     Prefix: "前缀",
     PrefixPlaceholder: "文件名{:长度}, 文件大小{:[B|L][1-8]}",
@@ -336,13 +361,17 @@
     Quarter: "季",
     Queries: "查询程序",
     QueriesNoPackage: "与文件包不匹配的查询程序",
+    Query: "查询",
     QueryDetailsfor: "查询程序的详细说明",
+    QueryID: "查询程序标识",
     QueryIDPlaceholder: "某查询程序标识",
+    QueryName: "查询程序名",
     QueryNamePlaceholder: "某查询程序名",
     QuerySet: "查询程序集",
     Queue: "队列",
     Quote: "引用",
     QuotedTerminator: "引号里的终止符",
+    RawTextPage: "原始文本（当前页）",
     RecordCount: "记录数量",
     RecordLength: "记录长度",
     Records: "记录",
@@ -357,6 +386,8 @@
     RemoteDali: "远程Dali",
     RemoteDaliIP: "远程&nbsp;Dali&nbsp;IP&nbsp;地址",
     Remove: "删除",
+    RemoveSubfiles: "删除子文件",
+    RemoveUser: "将把你的用户从用户组里删除：",
     Rename: "更名",
     RenderedSVG: "渲染的SVG",
     RenderSVG: "渲染SVG",
@@ -364,6 +395,8 @@
     RequestSchema: "请求格式",
     Reschedule: "再次加入运行计划",
     Reset: "设置更新",
+    ResetThisQuery: "清零当前查询程序",
+    ResetViewToSelection: "清除",
     Resource: "资源",
     Resources: "资源",
     ResponseSchema: "响应格式",
@@ -376,6 +409,8 @@
     Resume: "恢复",
     RetainSuperfileStructure: "保留文件集的结构",
     RetypePassword: "验证密码",
+    Reverse: "反转",
+    RowPath: "行路径",
     Rows: "行",
     RowTag: "行标记",
     RoxieCluster: "Roxie集群",
@@ -414,6 +449,7 @@
     Start: "开始",
     Started: "已开始",
     State: "状态",
+    Stats: "统计",
     Status: "状态",
     Stopped: "已结束",
     Subgraph: "子图",
@@ -422,9 +458,11 @@
     Subtype: "子类",
     Summary: "总结",
     SummaryMessage: "总结信息",
-    Superfile: "文件集",
     SuperFile: "文件集",
+    Superfile: "文件集",
+    Superfiles: "文件集",
     SuperFiles: "文件集",
+    SuperFilesBelongsTo: "文件集里的文件",
     SuperfilesOnly: "仅含文件集",
     Suspend: "暂停使用",
     Suspended: "已暂停使用",
@@ -443,6 +481,7 @@
     TargetWuid: "系统/工作单元标识",
     Terminators: "终止符",
     TestPages: "测试页面",
+    Text: "文本",
     ThorMasterAddress: "THOR主服务器的网址",
     ThorNetworkAddress: "Thor网址",
     Time: "时间",
@@ -469,8 +508,11 @@
     title_HPCCPlatformMain: "ECL Watch - 首页",
     title_HPCCPlatformOps: "ECL Watch - 运行",
     title_HPCCPlatformRoxie: "ECL Watch - Roxie",
+    title_HPCCPlatformServicesPlugin: "ECL Watch插件",
     title_Inputs: "输入",
     title_LFDetails: "逻辑文件的详细说明",
+    title_LibrariesUsed: "使用的库文件",
+    title_Log: "日志文件",
     title_LZBrowse: "文件停放区",
     title_MemberOf: "隶属",
     title_Members: "成员",
@@ -486,15 +528,19 @@
     title_SearchResults: "查询结果",
     title_SourceFiles: "",
     title_SourceFiles: "源文件",
+    title_Topology: "系统结构",
     title_TpThorStatus: "状态",
     title_UserPermissions: "用户使用权限",
     title_UserQuery: "使用权限",
     title_WUDetails: "ECL工作单元的详细说明",
     title_WUQuery: "ECL工作单元查询程序",
+    To: "到",
     ToDate: "截止日期",
     Toenablegraphviews: "在使用图形阅读器前,请先安装图形控制器",
     Top: "上部",
+    Topology: "系统结构",
     ToSizes: "最大文件长度",
+    TotalClusterTime: "集群服务器累积时间",
     TotalSize: "总尺寸",
     TotalThorTime: "在Thor上的时间",
     TransitionGuide: "转换指南",

initfiles/sbin/install-hpcc.exp

@@ -200,6 +200,10 @@ proc runPayload {} {
       }
    }
 
+   expect  -re $
+   send "${cmd_prefix} rm ~/remote_install.tgz\r"
+   expect  -re "${prompt}" {}
+
    if { $remotelog == 1 } {
       expect -re $
       send "cat ${remote_install}/remote-install.log\r"

initfiles/sbin/rm_conf_settings.sh.in

@@ -26,7 +26,7 @@ PREFIX_PATH=`cat ${HPCC_CONFIG} | sed -n "/\[${SECTION}\]/,/\[/p" | grep "^path=
 
 source $PREFIX_PATH/sbin/alter_confs.sh
 
-alter_file /etc/sudoers "^Cmnd_Alias HPCC_|^${USER_NAME} ALL =" < /dev/null
+alter_file /etc/sudoers "^Cmnd_Alias HPCC_|^${USER_NAME} ALL =|Defaults:${USER_NAME}" < /dev/null
 
 alter_file /etc/security/limits.conf "^${USER_NAME}" < /dev/null
 

plugins/memcached/README.md

@@ -0,0 +1,135 @@
+ECL Memcached Plugin
+================
+
+This is the ECL plugin to utilize the volatile key-value cache [Memcached](http://memcached.org).
+It utilises the C API [libmemcached](http://libmemcached.org/libMemcached.html).
+
+Installation and Dependencies
+----------------------------
+
+To build the memcached plugin with the HPCC-Platform, libmemcached-dev is required.
+```
+sudo apt-get install libmemcached-dev
+```
+
+The memcached daemon can be obtained via - [source](http://memcached.org/downloads) or the preferred method:
+```
+sudo apt-get install memcached
+```
+
+*Note:* **libmemcached** 0.53 or greater is required to use this plugin as intended. It is advised to use the newest version of **memcached** possible to you.
+
+
+Getting started
+---------------
+
+The daemon can be started by typing `memcached -d` within a terminal. To run with with a non-default configuration, for example to listen on another **IP** and **port** -
+`memcached -d -l <ip> -p <port>`. When wishing to use a pool of memcached servers, each instance must be started, bound to the **IP** that makes it visible to the other
+instances, e.g. the default `memcached -d` on all machines will not work as they will all be bound to the localhost loopback, 127.0.0.1.
+
+This plugin forces the **_binary-only_** communication protocol and therefore **memcached** cannot be started in its ASCII mode, i.e. `memcached -b ascii`.
+
+*Note:* The default memcached **port** is 11211 and that if multiple and individual caches are required then they are by definition memacached instances with different ports.
+
+Further documentation is available [here](https://code.google.com/p/memcached/wiki/NewStart).
+
+The Actual Plugin
+-----------------
+
+The bulk of this memcached plugin for **ECL** is made up of the various `SET` and `GET` commands e.g. `GetString` or `SetReal`. They are accessible via the module `memcached`
+from the memcached plugin **ECL** library `lib-memcached`. i.e.
+```
+IMPORT memcached FROM lib_memcached;
+```
+Here is a list of the core plugin **functions**.
+
+###Set
+```
+SetUnicode (CONST VARSTRING key, CONST UNICODE value, CONST VARSTRING options, CONST VARSTRING partitionKey = '', UNSIGNED expire = 0)
+SetString  (CONST VARSTRING key, CONST STRING value,  CONST VARSTRING options, CONST VARSTRING partitionKey = '', UNSIGNED expire = 0)
+SetUtf8    (CONST VARSTRING key, CONST UTF8 value,    CONST VARSTRING options, CONST VARSTRING partitionKey = '', UNSIGNED expire = 0)
+SetBoolean (CONST VARSTRING key, BOOLEAN value,       CONST VARSTRING options, CONST VARSTRING partitionKey = '', UNSIGNED expire = 0)
+SetReal    (CONST VARSTRING key, REAL value,          CONST VARSTRING options, CONST VARSTRING partitionKey = '', UNSIGNED expire = 0)
+SetInteger (CONST VARSTRING key, INTEGER value,       CONST VARSTRING options, CONST VARSTRING partitionKey = '', UNSIGNED expire = 0)
+SetUnsigned(CONST VARSTRING key, UNSIGNED value,      CONST VARSTRING options, CONST VARSTRING partitionKey = '', UNSIGNED expire = 0)
+SetData    (CONST VARSTRING key, CONST DATA value,    CONST VARSTRING options, CONST VARSTRING partitionKey = '', UNSIGNED expire = 0)
+```
+
+###Get
+```
+INTEGER8   GetInteger(CONST VARSTRING key, CONST VARSTRING options, CONST VARSTRING partitionKey = '')
+UNSIGNED8 GetUnsigned(CONST VARSTRING key, CONST VARSTRING options, CONST VARSTRING partitionKey = '')
+STRING      GetString(CONST VARSTRING key, CONST VARSTRING options, CONST VARSTRING partitionKey = '')
+UNICODE    GetUnicode(CONST VARSTRING key, CONST VARSTRING options, CONST VARSTRING partitionKey = '')
+UTF8          GetUtf8(CONST VARSTRING key, CONST VARSTRING options, CONST VARSTRING partitionKey = '')
+BOOLEAN    GetBoolean(CONST VARSTRING key, CONST VARSTRING options, CONST VARSTRING partitionKey = '')
+REAL          GetReal(CONST VARSTRING key, CONST VARSTRING options, CONST VARSTRING partitionKey = '')
+DATA          GetData(CONST VARSTRING key, CONST VARSTRING options, CONST VARSTRING partitionKey = '')
+```
+
+###Utility
+```
+BOOLEAN Exists(CONST VARSTRING key, CONST VARSTRING options, CONST VARSTRING partitionKey = '')
+CONST VARSTRING KeyType(CONST VARSTRING key, CONST VARSTRING options, CONST VARSTRING partitionKey = '')
+Clear(CONST VARSTRING options)
+Delete(CONST VARSTRING key, CONST VARSTRING options, CONST VARSTRING partitionKey = '')
+```
+
+The core points to note here are:
+   * There is a **SET** and **GET** function associated with each fundamental **ECL** type. These must be used for and with their correct *value* types! Miss-use *should* result
+   in an runtime exception, however, this is only conditional on having the value retrieved from the server fitting into memory of the requested type. E.g. it is possible for a
+   STRING of length 8, set with SetString, being successfully retrieved from the cache via GetInteger without an **ECL** exception being thrown. A warning is added to the
+   local log file when this occurs.
+   * `CONST VARSTRING options` passes the server **IP** and **port** to the plugin in the *strict* format - `--SERVER=<ip>:<port>`. Multiple server use simply requires all
+   to be specified e.g. `--SERVER=192.168.1.98:11211 --SERVER=192.168.1.97:11211`. In addition a variety of options are passed in with this string e.g **_timeout_ _values_**.
+   A full list of possible options exists [here](http://docs.libmemcached.org/libmemcached_configuration.html).
+   * `UNSIGNED expire` has units seconds and a default of **0**, i.e. *forever*. *Note:* Anything above 30 days is treated as a unix timestamp.
+   * The default timeout is 1 second.
+   * `KeyType()` returns the **ECL** type as an **ECL** `VARSTRING`, i.e. `STRING` or `UNSIGNED`, or `UNKNOWN` if the value was set externally from this plugin
+   without a type specifier. *c.f.* Behaviour and Implementation Details below for further details.
+
+
+###An Memcached 'Partition Key'
+A *partition key* supplied to a `SET` will be hashed with the key and thus distribute the key-value pair according to this hash, such that all keys with the same
+*partition key* are on the same server. The notion of a partition is physical rather than logical. A *partition key* should therefore **_not_** be used with only
+a single memcached server. Observer the following two examples with servers started as `memcached -d -l 192.168.1.97` and `memcached -d -l 192.168.1.98`.
+
+```
+IMPORT memcached FROM lib_memcached;
+
+STRING server1 := '--SERVER=192.168.1.97:11211';
+REAL pi := 3.14159265359;
+
+SEQUENTIAL(
+    memcached.SetReal('pi', pi, server1);
+    memcached.SetReal('pi', pi*pi, server1, 'pi again');
+
+    memcached.GetReal('pi', server1);                                 //returns 9.869604401090658
+    memcached.GetReal('pi', server1, 'pi again');                     //returns 9.869604401090658
+    memcached.GetReal('pi', server1, 'you\'d think this would fail'); //returns 9.869604401090658
+    memcached.Clear(server1);
+    );
+
+STRING servers := server1 + ' --SERVER=192.168.1.98:11211';
+SEQUENTIAL(
+    memcached.SetReal('pi', pi, servers);
+    memcached.SetReal('pi', pi*pi, servers, 'pi again');
+
+    memcached.GetReal('pi', servers);                                 //returns 3.14159265359
+    memcached.GetReal('pi', servers, 'pi again');                     //returns 9.869604401090658
+    memcached.GetReal('pi', servers, 'you\'d think this would fail'); //returns 3.14159265359
+    memcached.Clear(servers);
+    );
+```
+
+*NOTE:* **libmemcached** uses a separate set of functions when using a *partition key*. When this plugin's default is used, the empty string is not hashed with the key,
+instead the non-partition-key functions are used.
+
+Behaviour and Implementation Details
+------------------------------------
+A few notes to point out here:
+   * When a key and value are stored with Set<type>, memcached also allows for a 4byte flag to be stored. This plugin utilizes this space to store an enumeration specifying the
+   **ECL** type that is being stored. Care should therefore be taken when using KeyType(<key>) when the key was set from a client other than this plugin.
+   * The following libmemcached settings are invoked by default for this plugin, all of which take precedence over any passed in via the `options` string:
+   **MEMCACHED_BEHAVIOR_KETAMA** = 1, **MEMCACHED_BEHAVIOR_USE_UDP** = 0, **MEMCACHED_BEHAVIOR_NO_BLOCK** = 0, **MEMCACHED_BEHAVIOR_BUFFER_REQUESTS** = 0,
+   **MEMCACHED_BEHAVIOR_BINARY_PROTOCOL** = 1.

plugins/memcached/lib_memcached.ecllib

@@ -37,4 +37,5 @@ export memcached := SERVICE : plugin('memcached'), namespace('MemCachedPlugin')
   BOOLEAN Exists(CONST VARSTRING key, CONST VARSTRING options, CONST VARSTRING partitionKey = '') : cpp,action,context,entrypoint='MExists';
   CONST VARSTRING KeyType(CONST VARSTRING key, CONST VARSTRING options, CONST VARSTRING partitionKey = '') : cpp,action,context,entrypoint='MKeyType'; //NOTE: calls get
   Clear(CONST VARSTRING options) : cpp,action,context,entrypoint='MClear';
+  Delete(CONST VARSTRING key, CONST VARSTRING options, CONST VARSTRING partionKey = '') : cpp,action,context,entrypoint='MDelete';
 END;

plugins/memcached/memcachedplugin.cpp

@@ -97,6 +97,7 @@ public :
 
     void clear(ICodeContext * ctx, unsigned when);
     bool exists(ICodeContext * ctx, const char * key, const char * partitionKey);
+    void deleteKey(ICodeContext * ctx, const char * key, const char * partitionKey);
     eclDataType getKeyType(const char * key, const char * partitionKey);
 
     bool isSameConnection(const char * _options) const;
@@ -109,10 +110,6 @@ private :
     bool logErrorOnFail(ICodeContext * ctx, memcached_return_t rc, const char * _msg);
     void reportKeyTypeMismatch(ICodeContext * ctx, const char * key, uint32_t flag, eclDataType eclType);
     void * cpy(const char * src, size_t length);
-    void logServerStats(ICodeContext * ctx);
-    void init(ICodeContext * ctx);
-    void invokePoolSecurity(ICodeContext * ctx);
-    void invokeConnectionSecurity(ICodeContext * ctx);
     void setPoolSettings();
     void assertPool();//For internal purposes to insure correct order of the above processes and instantiation.
 
@@ -120,7 +117,6 @@ private :
     memcached_st * connection;
     memcached_pool_st * pool;
     StringAttr options;
-    bool alreadyInitialized;
     unsigned typeMismatchCount;
 };
 
@@ -298,13 +294,9 @@ void MemCachedPlugin::MCached::getVoidPtrLenPair(ICodeContext * ctx, const char
     returnValue = reinterpret_cast<void*>(cpy(value, returnLength));
 }
 
-MemCachedPlugin::MCached::MCached(ICodeContext * ctx, const char * _options)
+MemCachedPlugin::MCached::MCached(ICodeContext * ctx, const char * _options) : connection(NULL), pool(NULL), typeMismatchCount(0)
 {
-    alreadyInitialized = false;
-    connection = NULL;
-    pool = NULL;
     options.set(_options);
-    typeMismatchCount = 0;
 
 #if (LIBMEMCACHED_VERSION_HEX<0x53000)
     memcached_st *memc = memcached_create(NULL);
@@ -363,7 +355,6 @@ MemCachedPlugin::MCached::MCached(ICodeContext * ctx, const char * _options)
     assertPool();
 
     setPoolSettings();
-    invokePoolSecurity(ctx);
     connect(ctx);
     checkServersUp(ctx);
 }
@@ -496,7 +487,16 @@ bool MemCachedPlugin::MCached::exists(ICodeContext * ctx, const char * key, cons
     }
 #endif
 }
-
+void MemCachedPlugin::MCached::deleteKey(ICodeContext * ctx, const char * key, const char * partitionKey)
+{
+    memcached_return_t rc;
+    size_t partitionKeyLength = strlen(partitionKey);
+    if (partitionKeyLength)
+        rc = memcached_delete_by_key(connection, partitionKey, partitionKeyLength, key, strlen(key), (time_t)0);
+    else
+        rc = memcached_delete(connection, key, strlen(key), (time_t)0);
+    assertOnError(rc, "'Delete' request failed - ");
+}
 MemCachedPlugin::eclDataType MemCachedPlugin::MCached::getKeyType(const char * key, const char * partitionKey)
 {
     size_t returnValueLength;
@@ -530,72 +530,20 @@ void MemCachedPlugin::MCached::reportKeyTypeMismatch(ICodeContext * ctx, const c
     }
 }
 
-void MemCachedPlugin::MCached::logServerStats(ICodeContext * ctx)
-{
-    //NOTE: errors are ignored here so that at least some info is reported, such as non-connection related libmemcached version numbers
-    memcached_return_t rc;
-    char * args = NULL;
-
-    OwnedMalloc<memcached_stat_st> stats;
-    stats.setown(memcached_stat(connection, args, &rc));
-
-    OwnedMalloc<char*> keys;
-    keys.setown(memcached_stat_get_keys(connection, stats, &rc));
-
-    unsigned int numberOfServers = memcached_server_count(connection);
-    for (unsigned int i = 0; i < numberOfServers; ++i)
-    {
-        StringBuffer statsStr;
-        unsigned j = 0;
-        do
-        {
-            OwnedMalloc<char> value;
-            value.setown(memcached_stat_get_value(connection, &stats[i], keys[j], &rc));
-            statsStr.newline().append("libmemcached server stat - ").append(keys[j]).append(":").append(value);
-        } while (keys[++j]);
-        statsStr.newline().append("libmemcached client stat - libmemcached version:").append(memcached_lib_version());
-        ctx->logString(statsStr.str());
-    }
-}
-
-void MemCachedPlugin::MCached::init(ICodeContext * ctx)
-{
-    logServerStats(ctx);
-}
-
 void MemCachedPlugin::MCached::setPoolSettings()
 {
     assertPool();
     const char * msg = "memcached_pool_behavior_set failed - ";
-    assertOnError(memcached_pool_behavior_set(pool, MEMCACHED_BEHAVIOR_HASH_WITH_PREFIX_KEY, 1), msg);//key set in invokeConnectionSecurity. Only hashed with keys and not partitionKeys
     assertOnError(memcached_pool_behavior_set(pool, MEMCACHED_BEHAVIOR_KETAMA, 1), msg);//NOTE: alias of MEMCACHED_DISTRIBUTION_CONSISTENT_KETAMA amongst others.
     memcached_pool_behavior_set(pool, MEMCACHED_BEHAVIOR_USE_UDP, 0);  // Note that this fails on early versions of libmemcached, so ignore result
-    assertOnError(memcached_pool_behavior_set(pool, MEMCACHED_BEHAVIOR_SERVER_FAILURE_LIMIT, 1), msg);
-#if (LIBMEMCACHED_VERSION_HEX>=0x50000)
-    assertOnError(memcached_pool_behavior_set(pool, MEMCACHED_BEHAVIOR_REMOVE_FAILED_SERVERS, 1), msg);
-#endif
     assertOnError(memcached_pool_behavior_set(pool, MEMCACHED_BEHAVIOR_NO_BLOCK, 0), msg);
     assertOnError(memcached_pool_behavior_set(pool, MEMCACHED_BEHAVIOR_CONNECT_TIMEOUT, 1000), msg);//units of ms.
     assertOnError(memcached_pool_behavior_set(pool, MEMCACHED_BEHAVIOR_SND_TIMEOUT, 1000000), msg);//units of mu-s.
     assertOnError(memcached_pool_behavior_set(pool, MEMCACHED_BEHAVIOR_RCV_TIMEOUT, 1000000), msg);//units of mu-s.
-    assertOnError(memcached_pool_behavior_set(pool, MEMCACHED_BEHAVIOR_BUFFER_REQUESTS, 0), msg);// Buffering does not work with the ecl runtime paradigm
-}
-
-void MemCachedPlugin::MCached::invokePoolSecurity(ICodeContext * ctx)
-{
-    assertPool();
+    assertOnError(memcached_pool_behavior_set(pool, MEMCACHED_BEHAVIOR_BUFFER_REQUESTS, 0), msg);
     assertOnError(memcached_pool_behavior_set(pool, MEMCACHED_BEHAVIOR_BINARY_PROTOCOL, 1), "memcached_pool_behavior_set failed - ");
 }
 
-void MemCachedPlugin::MCached::invokeConnectionSecurity(ICodeContext * ctx)
-{
-    //NOTE: Whether to assert or just report? This depends on when this is called. If before checkServersUp() and
-    //a server is down, it will cause the following to fail if asserted with only a 'poor' libmemcached error message.
-    //Reporting means that these 'security' measures may not be carried out. Moving checkServersUp() to here is probably the best
-    //soln. however, this comes with extra overhead.
-    logErrorOnFail(ctx, memcached_verbosity(connection, (uint32_t)(0)), "memcached_verbosity=0 failed - ");
-}
-
 void MemCachedPlugin::MCached::connect(ICodeContext * ctx)
 {
     assertPool();
@@ -611,13 +559,6 @@ void MemCachedPlugin::MCached::connect(ICodeContext * ctx)
 #else
     connection = memcached_pool_fetch(pool, (struct timespec *)0 , &rc);
 #endif
-    invokeConnectionSecurity(ctx);
-
-    if (!alreadyInitialized)//Do this now rather than after assert. Better to have something even if it could be jiberish.
-    {
-        init(ctx);//doesn't necessarily initialize anything, instead outputs specs etc for debugging
-        alreadyInitialized = true;
-    }
     assertOnError(rc, "memcached_pool_pop failed - ");
 }
 
@@ -640,6 +581,11 @@ ECL_MEMCACHED_API const char * ECL_MEMCACHED_CALL MKeyType(ICodeContext * ctx, c
     const char * keyType = enumToStr(serverPool->getKeyType(key, partitionKey));
     return keyType;
 }
+ECL_MEMCACHED_API void ECL_MEMCACHED_CALL MDelete(ICodeContext * ctx, const char * key, const char * options, const char * partitionKey)
+{
+    OwnedMCached serverPool = MemCachedPlugin::createConnection(ctx, options);
+    serverPool->deleteKey(ctx, key, partitionKey);
+}
 //-----------------------------------SET------------------------------------------
 //NOTE: These were all overloaded by 'value' type, however; this caused problems since ecl implicitly casts and doesn't type check.
 ECL_MEMCACHED_API void ECL_MEMCACHED_CALL MSet(ICodeContext * ctx, const char * key, size32_t valueLength, const char * value, const char * options, const char * partitionKey, unsigned __int64 expire /* = 0 (ECL default)*/)

plugins/memcached/memcachedplugin.hpp

@@ -65,6 +65,7 @@ namespace MemCachedPlugin {
     ECL_MEMCACHED_API bool             ECL_MEMCACHED_CALL MExists (ICodeContext * _ctx, const char * key, const char * options, const char * partitionKey);
     ECL_MEMCACHED_API const char *     ECL_MEMCACHED_CALL MKeyType(ICodeContext * _ctx, const char * key, const char * options, const char * partitionKey);
     ECL_MEMCACHED_API void             ECL_MEMCACHED_CALL MClear  (ICodeContext * _ctx, const char * options);
+    ECL_MEMCACHED_API void             ECL_MEMCACHED_CALL MDelete (ICodeContext * _ctx, const char * key, const char * options, const char * partitionKey);
 }//close namespace
 }
 #endif

testing/regress/ecl/key/memcachedtest.xml

@@ -46,3 +46,15 @@
 <Dataset name='Result 16'>
  <Row><Result_16>false</Result_16></Row>
 </Dataset>
+<Dataset name='Result 17'>
+ <Row><Result_17>true</Result_17></Row>
+</Dataset>
+<Dataset name='Result 18'>
+ <Row><Result_18>false</Result_18></Row>
+</Dataset>
+<Dataset name='Result 19'>
+ <Row><Result_19>true</Result_19></Row>
+</Dataset>
+<Dataset name='Result 20'>
+ <Row><Result_20>false</Result_20></Row>
+</Dataset>

testing/regress/ecl/memcachedtest.ecl

@@ -87,6 +87,18 @@ SEQUENTIAL(
     memcached.Exists('testExpire', servers);
     Std.System.Debug.Sleep(9 * 1000);
     memcached.Exists('testExpire', servers);
-    Std.System.Debug.Sleep(1 * 1000);
+    Std.System.Debug.Sleep(2 * 1000);
     memcached.Exists('testExpire', servers);
     );
+
+SEQUENTIAL(
+    memcached.SetString('testDelete', 'foobar', servers);
+    memcached.Exists('testDelete', servers);
+    memcached.Delete('testDelete', servers);
+    memcached.Exists('testDelete', servers);
+
+    memcached.SetString('testDelete', 'foobar', servers, 'hashWithThis');
+    memcached.Exists('testDelete', servers, 'hashWithThis');
+    memcached.Delete('testDelete', servers, 'hashWithThis');
+    memcached.Exists('testDelete', servers, 'hashWithThis');
+    );

thorlcr/activities/hashdistrib/thhashdistribslave.cpp

@@ -147,8 +147,9 @@ protected:
                 OwnedConstThorRow row = dedupList.getClear(--i);
                 if ((NULL != prev.get()) && (0 == iCompare->docompare(prev, row)))
                 {
-                    size32_t rsz = owner.rowMemSize(row);
-                    total -= rsz;
+                    /* NB: do not alter 'total' size. It represents the amount originally added to the bucket
+                     * which will be deducted when sent.
+                     */
                 }
                 else
                 {
@@ -159,11 +160,12 @@ protected:
             dedupList.clearRows();
             return true; // attempted
         }
-        void add(const void *row, size32_t &rs)
+        size32_t add(const void *row)
         {
-            rs = owner.rowMemSize(row);
+            size32_t rs = owner.rowMemSize(row);
             total += rs;
             rows.enqueue(row);
+            return rs;
         }
         unsigned queryDestination() const { return destination; }
         size32_t querySize() const { return total; }
@@ -298,13 +300,12 @@ protected:
             }
             void send(CMessageBuffer &mb)
             {
-                CriticalBlock b(crit);
+                CriticalBlock b(crit); // protects against multiple senders to the same target
                 if (!atomic_read(&senderFinished))
                 {
                     if (owner.selfPush(target))
                         assertex(target != owner.self);
-                    if (!owner.sendBlock(target, mb))
-                        atomic_set(&senderFinished, 1);
+                    owner.sendBlock(target, mb);
                 }
             }
             inline unsigned getNumPendingBuckets() const
@@ -337,15 +338,6 @@ protected:
             {
                 return atomic_read(&senderFinished) != 0;
             }
-            inline void checkSenderFinished()
-            {
-                CriticalBlock b(crit);
-                if (!atomic_read(&senderFinished))
-                {
-                    atomic_set(&senderFinished, 1);
-                    atomic_inc(&owner.numFinished);
-                }
-            }
             inline CSendBucket *queryBucket()
             {
                 return bucket;
@@ -360,6 +352,10 @@ protected:
             {
                 return bucket.getClear();
             }
+            bool queryMarkSenderFinished()
+            {
+                return atomic_cas(&senderFinished, 1, 0);
+            }
         };
         /*
          * CWriterHandler, a per thread class and member of the writerPool
@@ -400,17 +396,16 @@ protected:
                 size32_t writerTotalSz = 0;
                 size32_t sendSz = 0;
                 CMessageBuffer msg;
-                size32_t rawSz = 0;
                 while (!owner.aborted)
                 {
-                    writerTotalSz += sendBucket->querySize();
-                    rawSz += sendBucket->querySize();
+                    writerTotalSz += sendBucket->querySize(); // NB: This size is pre-dedup, and is the correct amount to pass to decTotal
                     owner.dedup(sendBucket); // conditional
 
                     if (owner.selfPush(dest))
                     {
-                        HDSendPrintLog2("CWriteHandler, sending raw=%d to LOCAL", rawSz);
-                        distributor.addLocal(sendBucket);
+                        HDSendPrintLog2("CWriteHandler, sending raw=%d to LOCAL", writerTotalSz);
+                        if (!target->getSenderFinished())
+                            distributor.addLocal(sendBucket);
                     }
                     else // remote
                     {
@@ -422,10 +417,7 @@ protected:
                         if (sendSz < distributor.bucketSendSize)
                         {
                             // more added to dest I'm processing?
-                            {
-                                CriticalBlock b(owner.activeWritersLock);
-                                sendBucket.setown(target->dequeuePendingBucket());
-                            }
+                            sendBucket.setown(target->dequeuePendingBucket());
                             if (sendBucket)
                             {
                                 HDSendPrintLog3("CWriteHandler, pending(b=%d) rolled, size=%d", sendBucket->queryDestination(), sendBucket->querySize());
@@ -433,9 +425,9 @@ protected:
                                 continue; // NB: it will flow into else "remote" arm
                             }
                         }
-                        target->send(msg);
+                        if (!target->getSenderFinished())
+                            target->send(msg);
                         sendSz = 0;
-                        rawSz = 0;
                         msg.clear();
                     }
                     // see if others to process
@@ -471,6 +463,7 @@ protected:
         Semaphore senderFullSem;
         Linked<IException> exception;
         atomic_t numFinished;
+        atomic_t stoppedTargets;
         unsigned dedupSamples, dedupSuccesses, self;
         Owned<IThreadPool> writerPool;
         unsigned totalActiveWriters;
@@ -481,6 +474,7 @@ protected:
             totalSz = 0;
             senderFull = false;
             atomic_set(&numFinished, 0);
+            atomic_set(&stoppedTargets, 0);
             dedupSamples = dedupSuccesses = 0;
             doDedup = owner.doDedup;
             writerPool.setown(createThreadPool("HashDist writer pool", this, this, owner.writerPoolSize, 5*60*1000));
@@ -503,15 +497,9 @@ protected:
             totalSz = 0;
             senderFull = false;
             atomic_set(&numFinished, 0);
+            atomic_set(&stoppedTargets, 0);
             aborted = false;
         }
-        void reinit()
-        {
-            if (initialized)
-                reset();
-            else
-                init();
-        }
         unsigned queryInactiveWriters() const
         {
             CriticalBlock b(activeWritersLock);
@@ -561,13 +549,12 @@ protected:
             SpinBlock b(totalSzLock);
             return totalSz;
         }
-        inline bool sendBlock(unsigned i, CMessageBuffer &msg)
+        inline void sendBlock(unsigned target, CMessageBuffer &msg)
         {
-            if (owner.sendBlock(i, msg))
-                return true;
-            atomic_inc(&numFinished);
-            owner.ActPrintLog("CSender::sendBlock stopped slave %d (finished=%d)", i+1, atomic_read(&numFinished));
-            return false;
+            if (owner.sendBlock(target, msg))
+                return;
+            markStopped(target); // Probably a bit pointless if target is 'self' - process loop will have done already
+            owner.ActPrintLog("CSender::sendBlock stopped slave %d (finished=%d)", target+1, atomic_read(&numFinished));
         }
         inline bool selfPush(unsigned i) const
         {
@@ -610,6 +597,13 @@ protected:
                     delete targets.item(n);
             }
         }
+        void reinit()
+        {
+            if (initialized)
+                reset();
+            else
+                init();
+        }
         CSendBucket *getAnotherBucket(unsigned &next)
         {
             // NB: called inside activeWritersLock
@@ -632,31 +626,50 @@ protected:
         }
         void add(CSendBucket *bucket)
         {
-            if (owner.selfstopped)
-                targets.item(self)->checkSenderFinished();
             unsigned dest = bucket->queryDestination();
             CTarget *target = targets.item(dest);
-            if (target->getSenderFinished())
+            CriticalBlock b(activeWritersLock);
+            if ((totalActiveWriters < owner.writerPoolSize) && (!owner.targetWriterLimit || (target->getActiveWriters() < owner.targetWriterLimit)))
             {
-                HDSendPrintLog2("CSender::add disposing of bucket [finished(%d)]", dest);
-                bucket->Release();
+                HDSendPrintLog3("CSender::add (new thread), dest=%d, active=%d", dest, totalActiveWriters);
+                writerPool->start(bucket);
             }
-            else
+            else // an existing writer will pick up
+                target->enqueuePendingBucket(bucket);
+        }
+        void checkSendersFinished()
+        {
+            // check if any target has stopped and clear out partial now defunct buckets taking space.
+            if (atomic_read(&stoppedTargets) == 0) // cheap compared to atomic_xchg, so saves a few cycles in common case.
+               return;
+            int numStopped = atomic_xchg(0, &stoppedTargets);
+            if (numStopped)
             {
-                CriticalBlock b(activeWritersLock);
-                if ((totalActiveWriters < owner.writerPoolSize) && (!owner.targetWriterLimit || (target->getActiveWriters() < owner.targetWriterLimit)))
+                /* this will be infrequent, scan all.
+                 * NB: This may pick up / clear more than 'numStopped', but that's okay, all it will mean is that another call to checkSendersFinished() will enter here.
+                 */
+                ForEachItemIn(t, targets)
                 {
-                    HDSendPrintLog3("CSender::add (new thread), dest=%d, active=%d", dest, totalActiveWriters);
-                    writerPool->start(bucket);
+                    CTarget *target = targets.item(t);
+                    if (target->getSenderFinished())
+                    {
+                        Owned<CSendBucket> bucket = target->getBucketClear();
+                        if (bucket)
+                            decTotal(bucket->querySize());
+                        loop
+                        {
+                            bucket.setown(target->dequeuePendingBucket());
+                            if (!bucket)
+                                break;
+                            decTotal(bucket->querySize());
+                        }
+                    }
                 }
-                else // an existing writer will pick up
-                    target->enqueuePendingBucket(bucket);
             }
         }
         void process(IRowStream *input)
         {
             owner.ActPrintLog("Distribute send start");
-            reinit();
             CCycleTimer timer;
             rowcount_t totalSent = 0;
             try
@@ -781,13 +794,12 @@ protected:
                         break;
                     unsigned dest = owner.ihash->hash(row)%owner.numnodes;
                     CTarget *target = targets.item(dest);
-                    if (target->getSenderFinished()) // does this need to be thread safe?
+                    if (target->getSenderFinished())
                         ReleaseThorRow(row);
                     else
                     {
                         CSendBucket *bucket = target->queryBucketCreate();
-                        size32_t rs;
-                        bucket->add(row, rs);
+                        size32_t rs = bucket->add(row);
                         totalSent++;
                         {
                             SpinBlock b(totalSzLock);
@@ -799,6 +811,7 @@ protected:
                             add(target->getBucketClear());
                         }
                     }
+                    checkSendersFinished(); // clears out defunct target buckets if any have stopped
                 }
             }
             catch (IException *e)
@@ -839,6 +852,15 @@ protected:
             aborted = true;
             senderFullSem.signal();
         }
+        void markStopped(unsigned target)
+        {
+            if (targets.item(target)->queryMarkSenderFinished())
+            {
+                atomic_inc(&numFinished);
+                atomic_inc(&stoppedTargets);
+            }
+        }
+        void markSelfStopped() { markStopped(self); }
     // IThreadFactory impl.
         virtual IPooledThread *createNew()
         {
@@ -1104,6 +1126,7 @@ public:
             {
                 recvthread.stop();
                 selfstopped = true;
+                sender.markSelfStopped();
             }
             distribDoneSem.wait();
             if (sendException.get())
@@ -1259,6 +1282,7 @@ public:
     virtual void startTX()=0;
     void start()
     {
+        sender.reinit();
         startTX();
         recvthread.start();
         sendthread.start();

thorlcr/activities/thdiskbaseslave.cpp

@@ -315,9 +315,12 @@ void CDiskWriteSlaveActivityBase::open()
     if (extend)
         ActPrintLog("Extending file %s", fName.get());
 
+    /* Fixed length record size is used when outputting compressed stream to determine run-length compression vs default LZW compression.
+     * NB: only for FLAT files, not CSV or XML
+     */
     size32_t diskRowMinSz = 0;
     IOutputMetaData *diskRowMeta = diskHelperBase->queryDiskRecordSize()->querySerializedDiskMeta();
-    if (diskRowMeta->isFixedSize())
+    if (diskRowMeta->isFixedSize() && (TAKdiskwrite == container.getKind()))
     {
         diskRowMinSz = diskRowMeta->getMinRecordSize();
         if (grouped)

thorlcr/thorutil/thbuf.cpp

@@ -284,6 +284,11 @@ public:
         while (out->ordinality()) 
             ReleaseThorRow(out->dequeue());
         delete out;
+        if (fileio)
+        {
+            fileio.clear();
+            file->remove();
+        }
     }
 
     void putRow(const void *row)

tools/hidl/hidlcomp.cpp

@@ -5920,6 +5920,8 @@ void EspServInfo::write_esp_binding()
     outs("{\n");
     StrBuffer defVer;
     bool hasDefVer = getMetaVerInfo(tags,"default_client_version",defVer);
+    if (!hasDefVer)
+        hasDefVer = getMetaVerInfo(tags,"version",defVer);
     if (hasDefVer)
     {
         outf("\tif (context.getClientVersion()<=0)\n");