Merge branch 'candidate-7.4.x'

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

common/thorhelper/thorcommon.cpp

@@ -2089,6 +2089,8 @@ static bool getTranslators(Owned<const IDynamicTransform> &translator, Owned<con
         if ((projectedFormat != sourceFormat) && (projectedCrc != sourceCrc))
         {
             translator.setown(createRecordTranslator(projectedFormat->queryRecordAccessor(true), sourceFormat->queryRecordAccessor(true)));
+            DBGLOG("Record layout translator created for %s", tracing);
+            translator->describe();
 
             if (!translator->canTranslate())
                 throw MakeStringException(0, "Untranslatable record layout mismatch detected for file %s", tracing);

common/thorhelper/thorsoapcall.cpp

@@ -1568,7 +1568,7 @@ private:
             if (url.userPasswordPair.length() > 0)
             {
                 StringBuffer authToken;
-                JBASE64_Encode(url.userPasswordPair.str(), url.userPasswordPair.length(), authToken);
+                JBASE64_Encode(url.userPasswordPair.str(), url.userPasswordPair.length(), authToken, false);
                 request.append("Authorization: Basic ").append(authToken).append("\r\n");
             }
             else if (master->authToken.length() > 0)

dali/sasha/saxref.cpp

@@ -1400,7 +1400,7 @@ public:
                         dt->addProp("MinIP",s1.str());
                     }
                     if (d->minsize[drv]<d->maxsize[drv]) {
-                        __int64 av = d->totalsize[drv]/(__int64)grp->ordinality();
+                        __int64 av = d->totalsize[drv]/(__int64)rawgrp->ordinality();
                         if (av) {
                             unsigned pcp = (unsigned)(d->maxsize[drv]*100/av);
                             unsigned pcn = (unsigned)(d->minsize[drv]*100/av);

docs/EN_US/ConfiguringHPCC/ConfiguringHPCC.xml

@@ -1696,6 +1696,8 @@ sudo -u hpcc cp /etc/HPCCSystems/source/NewEnvironment.xml /etc/HPCCSystems/envi
                         xmlns:xi="http://www.w3.org/2001/XInclude" />
           </para>
 
+          <?hard-pagebreak ?>
+
           <para>Additional information about the available Authentication
           methods:</para>
 
@@ -1714,13 +1716,6 @@ sudo -u hpcc cp /etc/HPCCSystems/source/NewEnvironment.xml /etc/HPCCSystems/envi
                   </row>
 
                   <row>
-                    <entry>local</entry>
-
-                    <entry>uses the local credentials for the server running
-                    the ESP</entry>
-                  </row>
-
-                  <row>
                     <entry>ldap</entry>
 
                     <entry>uses Lightweight Directory Access Protocol for

docs/EN_US/ECLLanguageReference/ECLR_mods/SpecStruc-BeginC++.xml

@@ -192,6 +192,117 @@ ENDC++;
 isUpper('JIM');
 
 </programlisting>
+
+    <para>Parameters can also include streamed datasets. </para>
+
+    <para>If stream is specified on the dataset then the parameter is passed
+    as an IRowStream. The next row from the dataset is obtained by
+    calling:</para>
+
+    <para><programlisting>dataset-&gt;nextRow(); </programlisting></para>
+
+    <para>After it has been processed the row must be freed by calling </para>
+
+    <para><programlisting>rtlReleaseRow(next); </programlisting></para>
+
+    <para>For example: </para>
+
+    <para><programlisting>traceDataset(STREAMED DATASET(r) ds, BOOLEAN isLocal = FALSE) := EMBED(C++)
+#include &lt;stdio.h&gt;
+#body
+  for(;;)
+  {
+    const byte * next = (const byte *)ds-&gt;nextRow();
+    if (!next)
+      return;
+    unsigned __int64 id = *(__uint64 *)(next);
+    size32_t lenName = *(size32_t *)(next + sizeof(__uint64));
+    const char * name = (char *)(next + sizeof(__uint64) + sizeof(size32_t));
+    printf("id(%u) name(%.*s)\n", (unsigned)id, lenName, name);
+    rtlReleaseRow(next);
+  }
+ENDEMBED;</programlisting></para>
+
+    <para>If the result of a c++ function is a streamed dataset, then it needs
+    to return an instance of an IRowStream interface. The function will also
+    be passed an extra implicit parameter:</para>
+
+    <para><programlisting>IEngineRowAllocator * _resultAllocator</programlisting></para>
+
+    <para>which is used to allocate the rows that are returned from the
+    function. </para>
+
+    <para>For example:</para>
+
+    <para><programlisting>// This function takes two streamed inputs and outputs the result of two values 
+// from the left multiplied together and added to a row from the right.
+
+STREAMED DATASET(r) myDataset(STREAMED DATASET(r) ds1, STREAMED DATASET(r) ds2)
+  := EMBED(C++ : activity)
+#include &lt;stdio.h&gt;
+#body
+    class MyStreamInlineDataset : public RtlCInterface, implements IRowStream
+    {
+    public:
+
+        MyStreamInlineDataset(IEngineRowAllocator * _resultAllocator, IRowStream * _ds1, 
+                              IRowStream * _ds2)
+          : resultAllocator(_resultAllocator), ds1(_ds1), ds2(_ds2)
+        {
+        }
+        RTLIMPLEMENT_IINTERFACE        virtual const void *nextRow() override
+        {
+            const byte * next1a = (const byte *)ds1-&gt;nextRow();
+            if (!next1a)
+                return nullptr;
+            const byte * next1b = (const byte *)ds1-&gt;nextRow();
+            const byte * next2 = (const byte *)ds2-&gt;nextRow();
+            if (!next1b || !next2)
+                rtlFailUnexpected();
+            unsigned __int64 value1a = *(const unsigned __int64 *)next1a;
+            unsigned __int64 value1b = *(const unsigned __int64 *)next1b;
+            unsigned __int64 value2 = *(const unsigned __int64 *)next2;
+            rtlReleaseRow(next1a);
+            rtlReleaseRow(next1b);
+            rtlReleaseRow(next2);
+            
+            unsigned __int64 result = value1a * value1b + value2;
+            RtlDynamicRowBuilder rowBuilder(resultAllocator);
+            byte * row = rowBuilder.getSelf();
+            *(__uint64 *)(row) = result;
+            return rowBuilder.finalizeRowClear(sizeof(unsigned __int64));
+        }
+        virtual void stop() override
+        {
+            ds1-&gt;stop();
+            ds2-&gt;stop();
+        }
+    protected:
+        Linked&lt;IEngineRowAllocator&gt; resultAllocator;
+        IRowStream * ds1;
+        IRowStream * ds2;
+    };    return new MyStreamInlineDataset(_resultAllocator, ds1, ds2);
+ENDEMBED;
+</programlisting></para>
+
+    <para>Note: If the resulting row does not have a fixed size, you should
+    call: </para>
+
+    <para><programlisting>byte * row = rowBuilder.ensureCapacity(&lt;totalSize&gt;, nullptr); </programlisting></para>
+
+    <para>instead of: </para>
+
+    <para><programlisting>byte * row = rowBuilder.getSelf(); </programlisting></para>
+
+    <para>This code uses a RtlDynamicRowBuilder which is a class used by the
+    code generator. Instead of using the RtlDynamicRowBuilder class, you could
+    directly call resultAllocator-&gt;createRow().</para>
+
+    <para>When a data type is included in an input row, rather than being
+    passed as a parameter, the format is the same as the parameters, except
+    that instead of having a pointer to the string etc., the string follows
+    the 4-byte length. The data in the row is not aligned; that is, it has
+    packing of 1. </para>
   </sect2>
 
   <sect2 id="BeginCPP_Available_Options">

docs/EN_US/HPCCSystemAdmin/HPCCSystemAdministratorsGuide.xml

@@ -39,7 +39,7 @@
       similarity to actual persons, living or dead, is purely
       coincidental.</para>
 
-      <para />
+      <para></para>
     </legalnotice>
 
     <xi:include href="common/Version.xml"
@@ -268,7 +268,7 @@
           <para>ECLCC Server uses a queue to convert workunits one at a time,
           however you can have ECLCC Servers deployed in the system to
           increase throughput and they will automatically load balance as
-          required.</para>
+          required. </para>
         </sect3>
 
         <sect3 id="SysAdm_ECLAgent">
@@ -282,6 +282,17 @@
           spawned on-demand when you submit a workunit.</para>
         </sect3>
 
+        <sect3 id="ECLScheduler">
+          <title>ECL Scheduler</title>
+
+          <para>The ECL Scheduler provides a means of automating processes
+          within ECL code or to chain processes together to work in sequence.
+          ECL Scheduling is event-based. The ECL Scheduler monitors a Schedule
+          list containing registered Workunits and Events and executes any
+          Workunits associated with an Event when that Event is triggered.
+          </para>
+        </sect3>
+
         <sect3 id="SysAdm_ESPServer">
           <title>ESP Server</title>
 
@@ -1115,7 +1126,8 @@
     desired you can edit the environment files using any xml or text editor
     however the file structure must remain valid.</para>
 
-    <para><figure>
+    <para>
+      <figure>
         <title>Sample Production Configuration</title>
 
         <mediaobject>
@@ -1123,7 +1135,8 @@
             <imagedata fileref="images/SA008.jpg" />
           </imageobject>
         </mediaobject>
-      </figure></para>
+      </figure>
+    </para>
 
     <!--/*Including special SysAdmin Config Module -paras- */-->
 
@@ -1169,7 +1182,9 @@
 
             <tbody>
               <row>
-                <entry><inlinegraphic fileref="images/caution.png" /></entry>
+                <entry>
+                  <inlinegraphic fileref="images/caution.png" />
+                </entry>
 
                 <entry><emphasis role="bold">WARNING</emphasis>: These
                 settings are essential to proper system operation. Only expert
@@ -1199,7 +1214,8 @@
 
       <para>The default envrionment.conf:</para>
 
-      <para><programlisting>## Default environment configuration file for OpenHPCC
+      <para>
+        <programlisting>## Default environment configuration file for OpenHPCC
 
 [DEFAULT]
 configs=/etc/HPCCSystems
@@ -1276,7 +1292,8 @@ useDropZoneRestriction=true
 # NB: for now this only applies to the last cached server 
 #dafsConnectFailRetrySeconds=10
 
-</programlisting></para>
+</programlisting>
+      </para>
 
       <para>The default environment.conf file includes several comments and
       explanations for many of the values defined in it.</para>
@@ -1311,7 +1328,7 @@ lock=/var/lock/HPCCSystems</programlisting>
           <para>The <emphasis role="bold">logfields</emphasis> setting
           declares the fields to include in the component logs. You can
           customize which fields appear in your logs based on your business
-          needs. </para>
+          needs.</para>
 
           <para>The syntax to use for logfields is to include the desired
           columns with a plus (+) sign, and use the minus (-) to specify any
@@ -1446,7 +1463,8 @@ lock=/var/lock/HPCCSystems</programlisting>
           <para>The following are logfield macros which provide a bundled
           group of columns:</para>
 
-          <para><informaltable>
+          <para>
+            <informaltable>
               <tgroup cols="2">
                 <tbody>
                   <row>
@@ -1463,7 +1481,8 @@ lock=/var/lock/HPCCSystems</programlisting>
                   </row>
                 </tbody>
               </tgroup>
-            </informaltable></para>
+            </informaltable>
+          </para>
         </sect3>
       </sect2>
 
@@ -1734,8 +1753,7 @@ dfsSSLPrivateKeyFile=/keyfilepath/keyfile</programlisting>Set the <emphasis
     <xi:include href="HPCCSystemAdmin/SA-Mods/WUTool.xml"
                 xpointer="xpointer(//*[@id='wutool'])"
                 xmlns:xi="http://www.w3.org/2001/XInclude" />
-				
-				
+
     <sect1 id="Redefining_Thor_Nodes">
       <title>Redefining nodes in a Thor Cluster</title>
 

ecl/hql/hqlexpr.cpp

@@ -4663,6 +4663,9 @@ switch (op)
     case no_countdict:
         assertex(queryChild(0)->isDictionary());
         break;
+    case no_newusertable:
+        assertex(queryChild(2)->getOperator() == no_newtransform);
+        break;
     }
 
 #ifdef _DEBUG

ecl/hql/hqlgram2.cpp

@@ -7115,7 +7115,17 @@ IHqlExpression * HqlGram::createBuildIndexFromIndex(attribute & indexAttr, attri
     }
 
     if (sourceDataset)
+    {
         transform.setown(createDefaultAssignTransform(record, sourceDataset->queryNormalizedSelector(), indexAttr));
+        //The transform operator must be changed to no_newtransform since it will be an argument to no_newusertable
+        if (transform->getOperator() == no_transform)
+        {
+            HqlExprArray args;
+            unwindChildren(args, transform);
+            OwnedHqlExpr newTransform = createValue(no_newtransform, transform->getType(), args);
+            transform.swap(newTransform);
+        }
+    }
 
     //need to tag record scope in this case so it generates no_activetable as top selector
     OwnedHqlExpr distribution, hash;

ecl/hql/hqlutil.cpp

@@ -4005,13 +4005,13 @@ IHqlExpression * convertRecordToTransform(IHqlExpression * record, bool canOmit)
 }
 
 
-IHqlExpression * createTransformForField(IHqlExpression * field, IHqlExpression * value)
+IHqlExpression * createTransformForField(node_operator op, IHqlExpression * field, IHqlExpression * value)
 {
     OwnedHqlExpr record = createRecord(field);
     OwnedHqlExpr self = getSelf(record);
     OwnedHqlExpr target = createSelectExpr(LINK(self), LINK(field));
     OwnedHqlExpr assign = createAssign(LINK(target), LINK(value));
-    return createValue(no_transform, makeTransformType(record->getType()), assign.getClear());
+    return createValue(op, makeTransformType(record->getType()), assign.getClear());
 }
 
 IHqlExpression * convertScalarToRow(IHqlExpression * value, ITypeInfo * fieldType)
@@ -4025,12 +4025,12 @@ IHqlExpression * convertScalarToRow(IHqlExpression * value, ITypeInfo * fieldTyp
     OwnedHqlExpr attribute;
     if (splitResultValue(dataset, attribute, value))
     {
-        OwnedHqlExpr transform = createTransformForField(field, attribute);
+        OwnedHqlExpr transform = createTransformForField(no_newtransform, field, attribute);
         OwnedHqlExpr ds = createDataset(no_newusertable, LINK(dataset), createComma(LINK(record), LINK(transform)));
         return createRow(no_selectnth, LINK(ds), getSizetConstant(1));
     }
 
-    OwnedHqlExpr transform = createTransformForField(field, value);
+    OwnedHqlExpr transform = createTransformForField(no_transform, field, value);
     return createRow(no_createrow, transform.getClear());
 }
 
@@ -5775,7 +5775,7 @@ bool SplitDatasetAttributeTransformer::split(SharedHqlExpr & dataset, SharedHqlE
     case 2:
         {
             OwnedHqlExpr field = createField(unnamedId, value->getType(), NULL);
-            OwnedHqlExpr transform = createTransformForField(field, value);
+            OwnedHqlExpr transform = createTransformForField(no_transform, field, value);
             OwnedHqlExpr combine = createDatasetF(no_combine, LINK(&newDatasets.item(0)), LINK(&newDatasets.item(1)), LINK(transform), LINK(selSeq), NULL);
             OwnedHqlExpr first = createRowF(no_selectnth, LINK(combine), getSizetConstant(1), createAttribute(noBoundCheckAtom), NULL);
             dataset.setown(createDatasetFromRow(first.getClear()));

ecl/hqlcpp/hqlttcpp.cpp

@@ -9822,7 +9822,7 @@ IHqlExpression * HqlLinkedChildRowTransformer::ensureInputSerialized(IHqlExpress
     //so create a mapping <unserialized> := f(serialized)
     //and then use it to expand references to the unserialized format
     IHqlExpression * selector = dataset->queryNormalizedSelector();
-    OwnedHqlExpr mapTransform = createRecordMappingTransform(no_transform, serializedRecord, selector);
+    OwnedHqlExpr mapTransform = createRecordMappingTransform(no_newtransform, serializedRecord, selector);
     OwnedHqlExpr newDataset = createDatasetF(no_newusertable, LINK(dataset), LINK(serializedRecord), LINK(mapTransform), LINK(selSeq), NULL);
 
     NewProjectMapper2 mapper;

ecl/hthor/hthorkey.cpp

@@ -699,6 +699,8 @@ const IDynamicTransform * CHThorIndexReadActivityBase::getLayoutTranslator(IDist
 
             //MORE: We could introduce a more efficient way of checking this that does not create a translator
             Owned<const IDynamicTransform> actualTranslator = createRecordTranslator(expectedFormat->queryRecordAccessor(true), actualFormat->queryRecordAccessor(true));
+            DBGLOG("Record layout translator created for %s",  f->queryLogicalName());
+            actualTranslator->describe();
             if (actualTranslator->keyedTranslated())
                 throw MakeStringException(0, "Untranslatable key layout mismatch reading index %s - keyed fields do not match", f->queryLogicalName());
 
@@ -2540,6 +2542,8 @@ protected:
                 if (actualDiskMeta)
                 {
                     translator.setown(createRecordTranslator(helper.queryProjectedDiskRecordSize()->queryRecordAccessor(true), actualDiskMeta->queryRecordAccessor(true)));
+                    DBGLOG("Record layout translator created for %s",  f->queryLogicalName());
+                    translator->describe();
                     if (translator->canTranslate())
                     {
                         if (getLayoutTranslationMode()==RecordTranslationMode::None)

esp/src/eclwatch/HexViewWidget.js

@@ -4,26 +4,23 @@ define([
     "dojo/i18n",
     "dojo/i18n!./nls/hpcc",
     "dojo/_base/array",
-    "dojo/store/Memory",
-    "dojo/store/Observable",
-    "dojo/request/iframe",
 
     "dijit/registry",
 
     "hpcc/_Widget",
-    "src/ESPWorkunit",
-    "hpcc/ECLSourceWidget",
+    "@hpcc-js/comms",
 
     "dojo/text!../templates/HexViewWidget.html",
 
+    "hpcc/ECLSourceWidget",
     "dijit/form/NumberSpinner",
     "dijit/form/Button",
     "dijit/form/ToggleButton",
     "dijit/form/CheckBox"
 ],
-    function (declare, lang, i18n, nlsHPCC, arrayUtil, Memory, Observable, iframe,
+    function (declare, lang, i18n, nlsHPCC, arrayUtil,
         registry,
-        _Widget, ESPWorkunit, ECLSourceWidget,
+        _Widget, hpccComms,
         template) {
         return declare("HexViewWidget", [_Widget], {
             templateString: template,
@@ -87,74 +84,44 @@ define([
                     sourceMode: ""
                 });
                 var context = this;
-                this.wu = ESPWorkunit.Create({
-                    onCreate: function () {
-                        context.wu.update({
-                            QueryText: context.getQuery()
-                        });
-                        context.watchWU();
-                    },
-                    onUpdate: function () {
-                        context.wu.submit();
-                    },
-                    onSubmit: function () {
-                    }
-                });
-            },
-
-            watchWU: function () {
-                if (this.watchHandle) {
-                    this.watchHandle.unwatch();
-                }
-                var context = this;
-                this.watchHandle = this.wu.watch(function (name, oldValue, newValue) {
-                    switch (name) {
-                        case "hasCompleted":
-                            if (newValue === true) {
-                                this.wu.getInfo({
-                                    onGetWUExceptions: function (exceptions) {
-                                        if (exceptions.length) {
-                                            var msg = "";
-                                            arrayUtil.forEach(exceptions, function (exception) {
-                                                if (exception.Severity === "Error") {
-                                                    if (msg) {
-                                                        msg += "\n";
-                                                    }
-                                                    msg += exception.Message;
-                                                }
-                                            });
-                                            if (msg) {
-                                                dojo.publish("hpcc/brToaster", {
-                                                    Severity: "Error",
-                                                    Source: "HexViewWidget.remoteRead",
-                                                    Exceptions: [{ Source: context.wu.Wuid, Message: msg }]
-                                                });
-                                            }
-                                        }
-                                    }
-                                });
-                                context.wu.fetchResults(function (results) {
-                                    context.cachedResponse = "";
-                                    arrayUtil.forEach(results, function (result, idx) {
-                                        var store = result.getStore();
-                                        var result = store.query({
-                                        }, {
-                                                start: 0,
-                                                count: context.bufferLength
-                                            }).then(function (response) {
-                                                context.watchHandle.unwatch();
-                                                context.cachedResponse = response;
-                                                context.displayHex();
-                                                context.wu.doDelete();
-                                            });
-                                    });
-                                });
+                this.wu = hpccComms.Workunit.submit({ baseUrl: "" }, "hthor", context.getQuery()).then(function (wu) {
+                    wu.on("changed", function () {
+                        if (!wu.isComplete()) {
+                            context.hexView.setText("..." + wu.State + "...");
+                        }
+                    });
+                    return wu.watchUntilComplete();
+                }).then(function (wu) {
+                    return wu.fetchECLExceptions().then(function () { return wu; });
+                }).then(function (wu) {
+                    var exceptions = wu.Exceptions && wu.Exceptions.ECLException ? wu.Exceptions.ECLException : [];
+                    if (exceptions.length) {
+                        var msg = "";
+                        arrayUtil.forEach(exceptions, function (exception) {
+                            if (exception.Severity === "Error") {
+                                if (msg) {
+                                    msg += "\n";
+                                }
+                                msg += exception.Message;
                             }
-                            break;
-                        case "State":
-                            context.hexView.setText("..." + (context.wu.isComplete() ? context.i18n.fetchingresults : newValue) + "...");
-                            break;
+                        });
+                        if (msg) {
+                            dojo.publish("hpcc/brToaster", {
+                                Severity: "Error",
+                                Source: "HexViewWidget.remoteRead",
+                                Exceptions: [{ Source: context.wu.Wuid, Message: msg }]
+                            });
+                        }
                     }
+                    return wu.fetchResults().then(function (results) {
+                        return results.length ? results[0].fetchRows() : [];
+                    }).then(function (rows) {
+                        context.cachedResponse = rows;
+                        context.displayHex();
+                        return wu;
+                    });
+                }).then(function (wu) {
+                    return wu.delete();
                 });
             },
 

esp/src/eclwatch/templates/DFUQueryWidget.html

@@ -35,7 +35,7 @@
                                         <input title="${i18n.NoSplit}:" name="nosplit" data-dojo-type="dijit.form.CheckBox" />
                                         <input title="${i18n.Compress}:" name="compress" data-dojo-type="dijit.form.CheckBox" />
                                         <input title="${i18n.Wrap}:" name="Wrap" data-dojo-type="dijit.form.CheckBox" />
-                                        <input id="${id}RemoteCopyReplicate"title="${i18n.Replicate}:" name="replicate" data-dojo-type="dijit.form.CheckBox" />
+                                        <input id="${id}RemoteCopyReplicate" title="${i18n.Replicate}:" name="replicate" data-dojo-type="dijit.form.CheckBox" />
                                         <input title="${i18n.RetainSuperfileStructure}:" name="superCopy" data-dojo-type="dijit.form.CheckBox" />
                                     </div>
                                 </div>
@@ -53,7 +53,7 @@
                                 <div data-dojo-type="dijit.Fieldset">
                                     <legend>${i18n.Target}</legend>
                                     <div data-dojo-type="hpcc.TableContainer">
-                                        <input id="${id}CopyTargetSelect" title="${i18n.Group}:" name="destGroup" style="width:100%;" data-dojo-type="TargetSelectWidget" style="display: inline-block; vertical-align: middle" />
+                                        <input id="${id}CopyTargetSelect" title="${i18n.Group}:" name="destGroup" style="width:100%;display: inline-block; vertical-align: middle" data-dojo-type="TargetSelectWidget" />
                                     </div>
                                     <div id="${id}CopyGrid" data-dojo-type="SelectionGridWidget">
                                     </div>
@@ -101,13 +101,13 @@
                     <div id="${id}AddtoDropDown" data-dojo-type="dijit.form.DropDownButton">
                         <span>${i18n.AddToSuperfile}</span>
                         <div data-dojo-type="dijit.TooltipDialog">
-                            <div id="${id}AddToSuperfileForm" style="width:680px" onsubmit="return false;" data-dojo-type="dijit.form.Form" >
+                            <div id="${id}AddToSuperfileForm" style="width:680px" onsubmit="return false;" data-dojo-type="dijit.form.Form">
                                 <div data-dojo-type="dijit.Fieldset">
                                     <legend>${i18n.Target}</legend>
                                     <div class="dijitDialogPaneContentArea" data-dojo-props="cols:2" data-dojo-type="hpcc.TableContainer">
-                                        <input id="${id}CreateNewSuperRadio" title="${i18n.CreateANewFile}" checked="true" data-dojo-type="dijit.form.RadioButton"></input>
+                                        <input id="${id}CreateNewSuperRadio" title="${i18n.CreateANewFile}" checked="true" data-dojo-type="dijit.form.RadioButton" />
                                         <input id="${id}AddToSuperfileTargetName" style="width:100%;" name="Superfile" required="true" data-dojo-props="trim: true" data-dojo-type="dijit.form.ValidationTextBox" />
-                                        <input id="${id}AddToSuperfileTargetAppend" name="ExistingFile" title="${i18n.AddToExistingSuperfile}" data-dojo-type="dijit.form.RadioButton"></input>
+                                        <input id="${id}AddToSuperfileTargetAppend" name="ExistingFile" title="${i18n.AddToExistingSuperfile}" data-dojo-type="dijit.form.RadioButton" />
                                     </div>
                                     <div id="${id}AddToSuperfileGrid" data-dojo-type="SelectionGridWidget"></div>
                                 </div>
@@ -177,10 +177,10 @@
                     <span data-dojo-type="dijit.ToolbarSeparator"></span>
                     <img src="${dojoConfig.urlInfo.resourcePath}/img/person.png" style="vertical-align: middle" alt="${i18n.Mine}">
                     <label for="Mine" class="bold" style="vertical-align: middle;">${i18n.Mine}</label>
-                    <input id="${id}Mine" name="Owner" title="${i18n.Mine}" data-dojo-attach-event="onChange:_onMine" data-dojo-type="dijit.form.CheckBox"/>
+                    <input id="${id}Mine" name="Owner" title="${i18n.Mine}" data-dojo-attach-event="onChange:_onMine" data-dojo-type="dijit.form.CheckBox" />
                     <div class="right" data-dojo-attach-event="onChange:_onMaximize" data-dojo-props="iconClass:'iconMaximize', showLabel:false" checked=false data-dojo-type="dijit.form.ToggleButton">${i18n.MaximizeRestore}</div>
                     <div id="${id}NewPage" class="right" data-dojo-attach-event="onClick:_onNewPage" data-dojo-props="iconClass:'iconNewPage', showLabel:false" data-dojo-type="dijit.form.Button">${i18n.OpenInNewPage}</div>
-                     <div id="${id}DownloadToList" class="right" data-dojo-attach-event="onClick:_onDownloadToList" data-dojo-type="dijit.form.Button">
+                    <div id="${id}DownloadToList" class="right" data-dojo-attach-event="onClick:_onDownloadToList" data-dojo-type="dijit.form.Button">
                         <span>${i18n.DownloadToCSV}</span>
                     </div>
                 </div>

esp/src/eclwatch/templates/DFUSearchWidget.html

@@ -2,82 +2,79 @@
     <div id="${id}BorderContainer" class="${baseClass}BorderContainer" style="width: 100%; height: 100%" data-dojo-type="dijit.layout.BorderContainer">
         <div id="${id}ContentPane" data-dojo-props="region: 'center', tabPosition: 'top'" style="width: 100%; height: 100%" data-dojo-type="dijit.layout.ContentPane">
             <div style="width: 100%; height: 100%" data-dojo-type="dijit.layout.BorderContainer">
-                <div data-dojo-props="region: 'center'" data-dojo-type="dijit.layout.ContentPane">                	                    
+                <div data-dojo-props="region: 'center'" data-dojo-type="dijit.layout.ContentPane">
                     <form>
-                    	<ul>
-                    		<li>
-		                    	<h2>Open DFU Workunit</h2>
-			                    <label for="WorkUnitId" class="Prompt">Workunit ID:</label>
-			                    <div><input name="FindDfuWu" type="text" data-dojo-type="dijit.form.TextBox" placeholder="enter workunit id"
-			                    data-dojo-props="trim:true, propercase:true"/>
-								</div>
-							</li>
-							<li>
-								<div>
-							<button data-dojo-type="dijit.form.Button" type="button">Load Below
-			                </button>
-			                	</div>
-			                </li>
-						</ul>	
-						<ul>
-							<li>			
-						<h2>Search DFU Workunits</h2>
-						<label for="Type" class="Prompt">Type:</label>
-						<div>
-							<select name="SelectType" data-dojo-type="dijit.form.Select">
-    							<option value="NA">Non&ndash;Archived Workunits</option>
-    							<option value="A" selected="Selected">Archived Workunits</option>    
-							</select>
-						</div>
-						</li>
-						<li>
-						<label for="Username" class="Prompt">Username:</label>
-						<div>
-							<input id="Username" data-dojo-type="dijit.form.TextBox" placeholder="enter username"
-                                data-dojo-props="trim:true, propercase:true">
-                        </div>
-                        </li>
-                        <li>
-                        <label for="Cluster" class="Prompt">Cluster:</label>
-                        <div>
-                        	<select name="SelectCluster" data-dojo-type="TargetSelectWidget">    								  
-							</select>
-						</div>
-						</li>
-						<li>
-						<label for="State" class="Prompt">State:</label>
-						<div>
-							<select name="SelectState" data-dojo-type="dijit.form.Select">
-	                            	<option>unknown</option>
-									<option>scheduled</option>
-									<option>compiled</option>
-									<option>running</option>
-									<option>finished</option>
-									<option>failed</option>
-									<option>aborting</option>
-									<option>aborted</option>
-									<option>blocked</option>
-									<option>monitoring</option>
-							</select>
-						</div>
-					</li>
-					<li>
-						<label for="Jobname" class="Prompt">Jobname:</label>
-						<div>
-							<input id="FindJobName" data-dojo-type="dijit.form.TextBox" placeholder="enter jobname"
-                                data-dojo-props="trim:true, propercase:true">
-                        </div>
-                    </li>
-                     <li>
-                        <div>
-                        	<button data-dojo-type="dijit.form.Button" type="button">Search
-                            </button>
-                        </div>
-                       </li>
-					</form>
-				</div>
-             	<div id="${id}InfoContainer" style="height: 35%" data-dojo-props="region: 'bottom', splitter: true, minSize: 120" data-dojo-type="InfoGridWidget">  </div>
-     		
+                        <ul>
+                            <li>
+                                <h2>Open DFU Workunit</h2>
+                                <label for="WorkUnitId" class="Prompt">Workunit ID:</label>
+                                <div><input name="FindDfuWu" type="text" data-dojo-type="dijit.form.TextBox" placeholder="enter workunit id" data-dojo-props="trim:true, propercase:true" />
+                                </div>
+                            </li>
+                            <li>
+                                <div>
+                                    <button data-dojo-type="dijit.form.Button" type="button">Load Below
+                                    </button>
+                                </div>
+                            </li>
+                        </ul>
+                        <ul>
+                            <li>
+                                <h2>Search DFU Workunits</h2>
+                                <label for="Type" class="Prompt">Type:</label>
+                                <div>
+                                    <select name="SelectType" data-dojo-type="dijit.form.Select">
+                                        <option value="NA">Non&ndash;Archived Workunits</option>
+                                        <option value="A" selected="Selected">Archived Workunits</option>
+                                    </select>
+                                </div>
+                            </li>
+                            <li>
+                                <label for="Username" class="Prompt">Username:</label>
+                                <div>
+                                    <input id="Username" data-dojo-type="dijit.form.TextBox" placeholder="enter username" data-dojo-props="trim:true, propercase:true">
+                                </div>
+                            </li>
+                            <li>
+                                <label for="Cluster" class="Prompt">Cluster:</label>
+                                <div>
+                                    <select name="SelectCluster" data-dojo-type="TargetSelectWidget">
+                                    </select>
+                                </div>
+                            </li>
+                            <li>
+                                <label for="State" class="Prompt">State:</label>
+                                <div>
+                                    <select name="SelectState" data-dojo-type="dijit.form.Select">
+                                        <option>unknown</option>
+                                        <option>scheduled</option>
+                                        <option>compiled</option>
+                                        <option>running</option>
+                                        <option>finished</option>
+                                        <option>failed</option>
+                                        <option>aborting</option>
+                                        <option>aborted</option>
+                                        <option>blocked</option>
+                                        <option>monitoring</option>
+                                    </select>
+                                </div>
+                            </li>
+                            <li>
+                                <label for="Jobname" class="Prompt">Jobname:</label>
+                                <div>
+                                    <input id="FindJobName" data-dojo-type="dijit.form.TextBox" placeholder="enter jobname" data-dojo-props="trim:true, propercase:true">
+                                </div>
+                            </li>
+                            <li>
+                                <div>
+                                    <button data-dojo-type="dijit.form.Button" type="button">Search</button>
+                                </div>
+                            </li>
+                        </ul>
+                    </form>
+                </div>
+                <div id="${id}InfoContainer" style="height: 35%" data-dojo-props="region: 'bottom', splitter: true, minSize: 120" data-dojo-type="InfoGridWidget"></div>
+            </div>
         </div>
-	</div>
-</div>
+    </div>
+</div>

esp/src/eclwatch/templates/HPCCPlatformWidget.html

@@ -12,10 +12,10 @@
                 </form>
                 <div class="seperator grey"></div>
                 <div id="userAccount">
-                    <span class="navBarLoggedin">${i18n.LoggedInAs}:  </span>
+                    <span class="navBarLoggedin">${i18n.LoggedInAs}: </span>
                     <a id="${id}UserID" href="#" data-dojo-attach-event="onClick:_onUserID"></a>
                     <span id="UserDivider"></span>
-                    <a id="Lock" href="#" data-dojo-attach-event="onClick:_onLock"/></a>
+                    <a id="Lock" href="#" data-dojo-attach-event="onClick:_onLock"></a>
                 </div>
                 <div class="seperator grey"></div>
                 <div id="${id}More" class="left glow" data-dojo-props="iconClass:'iconAdvanced', showLabel:false" data-dojo-type="dijit.form.DropDownButton">

esp/src/eclwatch/templates/LFDetailsWidget.html

@@ -15,7 +15,7 @@
                                 <div data-dojo-type="dijit.Fieldset">
                                     <legend>${i18n.Target}</legend>
                                     <div data-dojo-props="cols:2" data-dojo-type="hpcc.TableContainer">
-                                        <input id="${id}CopyTargetSelect" title="${i18n.Group}:" name="destGroup" colspan="2" style="width:100%;" data-dojo-type="TargetSelectWidget" style="display: inline-block; vertical-align: middle" />
+                                        <input id="${id}CopyTargetSelect" title="${i18n.Group}:" name="destGroup" colspan="2" style="width:100%;display: inline-block; vertical-align: middle" data-dojo-type="TargetSelectWidget" />
                                         <input id="${id}CopyTargetName" title="${i18n.TargetName}:" name="destLogicalName" colspan="2" style="width:100%;" required="true" data-dojo-props="trim: true" data-dojo-type="dijit.form.ValidationTextBox" />
                                     </div>
                                 </div>
@@ -29,7 +29,7 @@
                                         <input id="${id}CopyTargetWrap" title="${i18n.Wrap}:" name="Wrap" data-dojo-type="dijit.form.CheckBox" />
                                         <input id="${id}CopyTargetRetainSuperfileStructure" title="${i18n.RetainSuperfileStructure}:" name="superCopy" data-dojo-type="dijit.form.CheckBox" />
                                         <input id="${id}CopyPreserveCompression" title="${i18n.PreserveCompression}:" checked="true" name="preserveCompression" data-dojo-type="dijit.form.CheckBox" />
-                                        <input id="${id}CopyExpireDays" title="${i18n.ExpireDays}:" name="ExpireDays" data-dojo-type="dijit.form.NumberTextBox", />
+                                        <input id="${id}CopyExpireDays" title="${i18n.ExpireDays}:" name="ExpireDays" data-dojo-type="dijit.form.NumberTextBox" />
                                     </div>
                                 </div>
                                 <div class="dijitDialogPaneActionBar">
@@ -112,7 +112,7 @@
                 </div>
                 <div data-dojo-props="region: 'center'" data-dojo-type="dijit.layout.ContentPane">
                     <h2>
-                        <img id="${id}CompressedImage"/>&nbsp;
+                        <img id="${id}CompressedImage" />&nbsp;
                         <img id="${id}ProtectedImage" src="${dojoConfig.urlInfo.resourcePath}/img/unlocked.png" />&nbsp;<img id="${id}StateIdImage" class="iconLogicalFile" />&nbsp;<span id="${id}Name" class="bold"></span>
                         <button id="${id}ClippyButton" class="clippy" data-clipboard-target="#${id}Name"><img src="${dojoConfig.urlInfo.resourcePath}/img/clippy.png" alt="${i18n.CopyToClipboard}"></button>
                     </h2>
@@ -144,7 +144,7 @@
                             </li>
                             <li>
                                 <label class="Prompt" for="${id}isProtected">${i18n.Protected}:</label>
-                                <div><input id="${id}isProtected" data-dojo-type="dijit.form.CheckBox"/></div>
+                                <div><input id="${id}isProtected" data-dojo-type="dijit.form.CheckBox" /></div>
                             </li>
                             <li>
                                 <label for="${id}ContentType">${i18n.ContentType}: </label>

esp/src/eclwatch/templates/LogVisualizationWidget.html

@@ -8,8 +8,7 @@
                     <div class="right" data-dojo-attach-event="onChange:_onMaximize" data-dojo-props="iconClass:'iconMaximize', showLabel:false" checked=false data-dojo-type="dijit.form.ToggleButton">${i18n.MaximizeRestore}</div>
                     <div id="${id}NewPage" class="right" data-dojo-attach-event="onClick:_onNewPage" data-dojo-props="iconClass:'iconNewPage', showLabel:false" data-dojo-type="dijit.form.Button">${i18n.OpenInNewPage}</div>
                 </div>
-                </div>
             </div>
         </div>
     </div>
-</div>
+</div>

esp/src/eclwatch/templates/UserQueryWidget.html

@@ -120,7 +120,7 @@
                     <div id="${id}AdvancedPermissions" data-dojo-type="dijit.form.DropDownButton">
                         <span>${i18n.Advanced}</span>
                         <div data-dojo-type="dijit.DropDownMenu">
-                            <div data-dojo-attach-event="onClick:_onEnableScopeScans "id="${id}EnableScopeScans" data-dojo-type="dijit.MenuItem">${i18n.EnableScopeScans}</div>
+                            <div data-dojo-attach-event="onClick:_onEnableScopeScans" id="${id}EnableScopeScans" data-dojo-type="dijit.MenuItem">${i18n.EnableScopeScans}</div>
                             <div data-dojo-attach-event="onClick:_onDisableScopeScans" id="${id}DisableScopeScans" data-dojo-type="dijit.MenuItem">${i18n.DisableScopeScans}</div>
                             <span data-dojo-type="dijit.MenuSeparator"></span>
                             <div data-dojo-attach-event="onClick:_onFileScopeDefaultPermissions" id="${id}FileScopeDefaultPermissions" data-dojo-type="dijit.MenuItem">${i18n.FileScopeDefaultPermissions}</div>
@@ -145,7 +145,7 @@
         <div id="${id}FilePermissionForm" style="width:460px" data-dojo-type="dijit.form.Form">
             <div data-dojo-props="cols:2" data-dojo-type="hpcc.TableContainer">
                 <p>${i18n.PleaseSelectAUserOrGroup}</p>
-                <input id="${id}NameSelect" title="${i18n.Name}:" name="FileName" required="true" colspan="2" data-dojo-props="trim: true, required: true"  data-dojo-type="dijit.form.ValidationTextBox" />
+                <input id="${id}NameSelect" title="${i18n.Name}:" name="FileName" required="true" colspan="2" data-dojo-props="trim: true, required: true" data-dojo-type="dijit.form.ValidationTextBox" />
                 <input id="${id}UsersSelect" title="${i18n.Users}:" name="UserName" colspan="2" data-dojo-type="TargetSelectWidget" />
                 <input id="${id}GroupsSelect" title="${i18n.Groups}:" name="GroupName" colspan="2" data-dojo-type="TargetSelectWidget" />
             </div>

esp/src/eclwatch/templates/WUDetailsWidget.html

@@ -28,11 +28,11 @@
                         <div data-dojo-type="dijit.TooltipDialog">
                             <div id="${id}PublishForm" style="width:460px" onsubmit="return false;" data-dojo-type="dijit.form.Form">
                                 <div class="dijitDialogPaneContentArea" data-dojo-props="cols:2" data-dojo-type="hpcc.TableContainer">
-                                    <input id="${id}Jobname2" title="${i18n.JobName}:" colspan="2" style="width:100%" required="true" data-dojo-props="trim: true" data-dojo-type="dijit.form.ValidationTextBox"/>
-                                     <input id="${id}RemoteDali" title="${i18n.RemoteDali}:" colspan="2" style="width:100%" required="false" data-dojo-props="trim: true" data-dojo-type="dijit.form.ValidationTextBox"/>
-                                     <input id="${id}SourceProcess" title="${i18n.SourceProcess}:" colspan="2" style="width:100%" required="false" data-dojo-props="trim: true" data-dojo-type="dijit.form.ValidationTextBox"/>
-                                     <input id="${id}Comment" title="${i18n.Comment}:" colspan="2" style="width:100%" required="false" data-dojo-props="trim: true" data-dojo-type="dijit.form.ValidationTextBox"/>
-                                     <select id="${id}Priority" title="${i18n.Priority}:" colspan="2" data-dojo-type="dijit.form.Select">
+                                    <input id="${id}Jobname2" title="${i18n.JobName}:" colspan="2" style="width:100%" required="true" data-dojo-props="trim: true" data-dojo-type="dijit.form.ValidationTextBox" />
+                                    <input id="${id}RemoteDali" title="${i18n.RemoteDali}:" colspan="2" style="width:100%" required="false" data-dojo-props="trim: true" data-dojo-type="dijit.form.ValidationTextBox" />
+                                    <input id="${id}SourceProcess" title="${i18n.SourceProcess}:" colspan="2" style="width:100%" required="false" data-dojo-props="trim: true" data-dojo-type="dijit.form.ValidationTextBox" />
+                                    <input id="${id}Comment" title="${i18n.Comment}:" colspan="2" style="width:100%" required="false" data-dojo-props="trim: true" data-dojo-type="dijit.form.ValidationTextBox" />
+                                    <select id="${id}Priority" title="${i18n.Priority}:" colspan="2" data-dojo-type="dijit.form.Select">
                                         <option value="" selected="selected">${i18n.None}</option>
                                         <option value="SLA">${i18n.SLA}</option>
                                         <option value="Low">${i18n.Low}</option>
@@ -48,29 +48,29 @@
                         </div>
                     </div>
                     <span data-dojo-type="dijit.ToolbarSeparator"></span>
-                    <div id="${id}ZapReport"  title="${i18n.ZippedAnalysisPackage}" data-dojo-attach-event="onClick:_onZapReport" data-dojo-props="iconClass:'iconZap'" data-dojo-type="dijit.form.Button">${i18n.ZAP}</div>
+                    <div id="${id}ZapReport" title="${i18n.ZippedAnalysisPackage}" data-dojo-attach-event="onClick:_onZapReport" data-dojo-props="iconClass:'iconZap'" data-dojo-type="dijit.form.Button">${i18n.ZAP}</div>
                     <span data-dojo-type="dijit.ToolbarSeparator"></span>
                     <div id="${id}SlaveLogs" data-dojo-type="dijit.form.DropDownButton">
                         <span>${i18n.SlaveLogs}</span>
                         <div data-dojo-type="dijit.TooltipDialog">
-                        <div id="${id}LogsForm" style="width:460px" onsubmit="return false;" data-dojo-type="dijit.form.Form">
-                            <div class="dijitDialogPaneContentArea" data-dojo-props="cols:2" data-dojo-type="hpcc.TableContainer">
-                                <select id="${id}ThorProcess" title="${i18n.ThorProcess}:" name="ThorProcess" colspan="2" data-dojo-type="dijit.form.Select" /></select>
-                                <input id="${id}SlaveNumber" maxlength="" title="${i18n.SlaveNumber}:" name="SlaveNumber" value="1" required="true" data-dojo-props="trim: true, placeHolder:'1'" data-dojo-type="dijit.form.NumberTextBox"/>
-                                <select id="${id}FileFormat" title="${i18n.File}:" name="ThorProcess" colspan="2" data-dojo-type="dijit.form.Select" />
-                                    <option value="1">${i18n.OriginalFile}</option>
-                                    <option value="2">${i18n.Zip}</option>
-                                    <option value="3">${i18n.GZip}</option>
-                                </select>
-                            </div>
-                            </br>
-                            <div><span id="SlavesMaxNumber" class="bold"></span></div>
-                            <div><span id="AllowOnlyNumber" class="boldRed"></span></div>
-                            <div class="dijitDialogPaneActionBar">
-                                <button type="submit" data-dojo-attach-event="onClick:_getDownload" data-dojo-type="dijit.form.Button">${i18n.Download}</button>
+                            <div id="${id}LogsForm" style="width:460px" onsubmit="return false;" data-dojo-type="dijit.form.Form">
+                                <div class="dijitDialogPaneContentArea" data-dojo-props="cols:2" data-dojo-type="hpcc.TableContainer">
+                                    <select id="${id}ThorProcess" title="${i18n.ThorProcess}:" name="ThorProcess" colspan="2" data-dojo-type="dijit.form.Select"></select>
+                                    <input id="${id}SlaveNumber" maxlength="" title="${i18n.SlaveNumber}:" name="SlaveNumber" value="1" required="true" data-dojo-props="trim: true, placeHolder:'1'" data-dojo-type="dijit.form.NumberTextBox" />
+                                    <select id="${id}FileFormat" title="${i18n.File}:" name="ThorProcess" colspan="2" data-dojo-type="dijit.form.Select">
+                                        <option value="1">${i18n.OriginalFile}</option>
+                                        <option value="2">${i18n.Zip}</option>
+                                        <option value="3">${i18n.GZip}</option>
+                                    </select>
+                                </div>
+                                <br />
+                                <div><span id="SlavesMaxNumber" class="bold"></span></div>
+                                <div><span id="AllowOnlyNumber" class="boldRed"></span></div>
+                                <div class="dijitDialogPaneActionBar">
+                                    <button type="submit" data-dojo-attach-event="onClick:_getDownload" data-dojo-type="dijit.form.Button">${i18n.Download}</button>
+                                </div>
                             </div>
                         </div>
-                        </div>
                     </div>
                     <span data-dojo-type="dijit.ToolbarSeparator"></span>
                     <div class="right" data-dojo-attach-event="onChange:_onMaximize" data-dojo-props="iconClass:'iconMaximize', showLabel:false" checked=false data-dojo-type="dijit.form.ToggleButton">${i18n.MaximizeRestore}</div>
@@ -79,7 +79,7 @@
                 <div data-dojo-props="region: 'center'" data-dojo-type="dijit.layout.ContentPane">
                     <div style="display:inline-block">
                         <h2>
-                            <img id="${id}ProtectedImage" src="${dojoConfig.urlInfo.resourcePath}/img/locked.png" />&nbsp;<div id="${id}StateIdImage" class="iconWorkunit" ></div>&nbsp;<span id="${id}Wuid" class="bold">${i18n.WUID}</span>
+                            <img id="${id}ProtectedImage" src="${dojoConfig.urlInfo.resourcePath}/img/locked.png" />&nbsp;<div id="${id}StateIdImage" class="iconWorkunit"></div>&nbsp;<span id="${id}Wuid" class="bold">${i18n.WUID}</span>
                             <button id="${id}ClippyButton" class="clippy" data-clipboard-target="#${id}Wuid"><img src="${dojoConfig.urlInfo.resourcePath}/img/clippy.png" alt="${i18n.CopyToClipboard}"></button>
                         </h2>
                     </div>
@@ -101,24 +101,24 @@
                             </li>
                             <li id="scopeOptional" class="hidden">
                                 <label class="Prompt" for="${id}Scope">${i18n.Scope}:</label>
-                                <div><input id="${id}Scope" data-dojo-props="trim:true" data-dojo-type="dijit.form.TextBox"/></div>
+                                <div><input id="${id}Scope" data-dojo-props="trim:true" data-dojo-type="dijit.form.TextBox" /></div>
                             </li>
                             <li>
                                 <label class="Prompt" for="${id}Jobname">${i18n.JobName}:</label>
-                                <div><input id="${id}Jobname" data-dojo-props="trim:true" data-dojo-type="dijit.form.TextBox"/></div>
+                                <div><input id="${id}Jobname" data-dojo-props="trim:true" data-dojo-type="dijit.form.TextBox" /></div>
                             </li>
                             <li>
                                 <label class="Prompt" for="${id}Description">${i18n.Description}:</label>
-                                <div><input id="${id}Description" data-dojo-props="trim:true" data-dojo-type="dijit.form.TextBox"/></div>
+                                <div><input id="${id}Description" data-dojo-props="trim:true" data-dojo-type="dijit.form.TextBox" /></div>
                             </li>
                             <li>
                                 <label class="Prompt" for="${id}Protected">${i18n.Protected}:</label>
-                                <div><input id="${id}Protected" data-dojo-type="dijit.form.CheckBox"/></div>
+                                <div><input id="${id}Protected" data-dojo-type="dijit.form.CheckBox" /></div>
                             </li>
                             <li>
                                 <label class="Prompt" for="${id}Cluster">${i18n.Cluster}:</label>
                                 <div id="${id}Cluster"></div>
-                                <div id="${id}AllowedClusters" data-dojo-type="dijit.form.Select"/></div>
+                                <div id="${id}AllowedClusters" data-dojo-type="dijit.form.Select"></div>
                             </li>
                             <li>
                                 <label class="Prompt" for="${id}TotalClusterTime">${i18n.TotalClusterTime}:</label>
@@ -134,9 +134,9 @@
                             </li>
                         </ul>
                     </form>
-            </div>
-            <div id="${id}InfoContainer" class="wrap" style="height: 33%" data-dojo-props="region: 'bottom', splitter: true, minSize: 120, showToolbar: true" data-dojo-type="InfoGridWidget">
-            </div>
+                </div>
+                <div id="${id}InfoContainer" class="wrap" style="height: 33%" data-dojo-props="region: 'bottom', splitter: true, minSize: 120, showToolbar: true" data-dojo-type="InfoGridWidget">
+                </div>
             </div>
             <div id="${id}_Variables" title="${i18n.Variables}" data-dojo-props="delayWidget: 'VariablesWidget', disabled: true" data-dojo-type="DelayLoadWidget">
             </div>
@@ -163,23 +163,23 @@
         </div>
     </div>
     <div id="${id}ZapDialog" data-dojo-type="dijit.Dialog" title="${i18n.ZippedAnalysisPackage}">
-        <div id="${id}ZapForm" style="width:460px;" method="post" encType="application/x-www-form-urlencoded" data-dojo-type="dijit.form.Form">
+        <div id="${id}ZapForm" style="width:460px;" method="post" enctype="application/x-www-form-urlencoded" data-dojo-type="dijit.form.Form">
             <div data-dojo-props="cols:2" data-dojo-type="hpcc.TableContainer">
                 <input id="${id}ZapName" title="${i18n.FileName}:" name="ZAPFileName" colspan="2" data-dojo-props="trim: true," data-dojo-type="dijit.form.TextBox" />
                 <input id="${id}ZapWUID" title="${i18n.WUID}:" name="Wuid" colspan="2" data-dojo-props="trim: true, readonly: true," data-dojo-type="dijit.form.TextBox" />
                 <input id="${id}BuildVersion" title="${i18n.ESPBuildVersion}:" name="BuildVersion" colspan="2" data-dojo-props="trim: true, readonly: true," data-dojo-type="dijit.form.TextBox" />
                 <input id="${id}ESPIPAddress" title="${i18n.ESPNetworkAddress}:" name="ESPIPAddress" colspan="2" data-dojo-props="trim: true, readonly: true," data-dojo-type="dijit.form.TextBox" />
                 <input id="${id}ThorIPAddress" title="${i18n.ThorNetworkAddress}:" name="ThorIPAddress" colspan="2" data-dojo-props="trim: true, readonly: true," data-dojo-type="dijit.form.TextBox" />
-                <input id="${id}ZapDescription" title="${i18n.Description}:" name="ProblemDescription" cols="22" colspan="2" data-dojo-type="dijit.form.SimpleTextarea"/>
-                <input id="${id}WarnHistory" title="${i18n.History}:" name="WhatChanged" cols="22" colspan="2" data-dojo-type="dijit.form.SimpleTextarea"/>
-                <input id="${id}WarnTimings" title="${i18n.Timings}:" name="WhereSlow" cols="22" colspan="2" data-dojo-type="dijit.form.SimpleTextarea"/>
+                <input id="${id}ZapDescription" title="${i18n.Description}:" name="ProblemDescription" cols="22" colspan="2" data-dojo-type="dijit.form.SimpleTextarea" />
+                <input id="${id}WarnHistory" title="${i18n.History}:" name="WhatChanged" cols="22" colspan="2" data-dojo-type="dijit.form.SimpleTextarea" />
+                <input id="${id}WarnTimings" title="${i18n.Timings}:" name="WhereSlow" cols="22" colspan="2" data-dojo-type="dijit.form.SimpleTextarea" />
                 <input id="${id}Password" title="${i18n.PasswordOpenZAP}:" name="Password" cols="22" colspan="2" type="password" data-dojo-props="trim: true" data-dojo-type="dijit.form.ValidationTextBox" />
                 <input id="${id}IncludeSlaveLogsCheckbox" title="${i18n.IncludeSlaveLogs}:" name="IncludeThorSlaveLog" cols="22" colspan="2" type="checkbox" data-dojo-type="dijit.form.CheckBox" />
                 <input id="${id}EmailCheckbox" title="${i18n.SendEmail}:" name="SendEmail" cols="22" colspan="2" type="checkbox" data-dojo-type="dijit.form.CheckBox" />
-                <input id="${id}EmailTo" title="${i18n.EmailTo}:" name="EmailTo" colspan="2" data-dojo-props="trim:true, readonly:true, placeHolder:'See Configuration Manager.'" data-dojo-type="dijit.form.TextBox"/>
-                <input id="${id}EmailFrom" title="${i18n.EmailFrom}:" name="EmailFrom" colspan="2" data-dojo-props="trim:true, placeHolder:'See Configuration Manager.'" data-dojo-type="dijit.form.TextBox"/>
-                <input id="${id}EmailSubject" title="${i18n.EmailSubject}:" name="EmailSubject" colspan="2" data-dojo-props="trim:true" data-dojo-type="dijit.form.ValidationTextBox" required="false"/>
-                <input id="${id}EmailBody" title="${i18n.EmailBody}:" name="EmailBody" cols="22" colspan="2" data-dojo-type="dijit.form.SimpleTextarea"/>
+                <input id="${id}EmailTo" title="${i18n.EmailTo}:" name="EmailTo" colspan="2" data-dojo-props="trim:true, readonly:true, placeHolder:'See Configuration Manager.'" data-dojo-type="dijit.form.TextBox" />
+                <input id="${id}EmailFrom" title="${i18n.EmailFrom}:" name="EmailFrom" colspan="2" data-dojo-props="trim:true, placeHolder:'See Configuration Manager.'" data-dojo-type="dijit.form.TextBox" />
+                <input id="${id}EmailSubject" title="${i18n.EmailSubject}:" name="EmailSubject" colspan="2" data-dojo-props="trim:true" data-dojo-type="dijit.form.ValidationTextBox" required="false" />
+                <input id="${id}EmailBody" title="${i18n.EmailBody}:" name="EmailBody" cols="22" colspan="2" data-dojo-type="dijit.form.SimpleTextarea" />
             </div>
             <div class="dijitDialogPaneActionBar">
                 <button id="${id}onZapSubmit" data-dojo-attach-event="onClick:_onSubmitDialog" type="submit" data-dojo-type="dijit.form.Button">${i18n.Apply}</button>

esp/src/lws.config.js

@@ -16,7 +16,7 @@ let rewrite = [
     { from: "/esp/files/esp/logout", to: "http://" + debugServerIP + ":8010/esp/logout" },
     { from: "/ws_elk/*", to: "http://" + debugServerIP + ":8010/ws_elk/$1" },
     { from: "/esp/files/esp/reset_session_timeout", to: "http://" + debugServerIP + ":8010/esp/reset_session_timeout" },
-    { from: "/esp/files/node_modules/@hpcc-js/*/dist/index.min.js", to: "/node_modules/@hpcc-js/$1/dist/index.js" },
+    { from: "/esp/files/node_modules/@hpcc-js/*/dist/index.min.js", to: debugServerIP !== CLUSTER_GJS ? "/node_modules/@hpcc-js/$1/dist/index.js" : "/hpcc-js/$1/dist/index.js" },
     { from: "/esp/files/dist/*", to: "/build/dist/$1" },
     { from: "/esp/files/*", to: "/$1" },
     { from: "/ws_elk/*", to: "http://" + debugServerIP + ":8010/ws_elk/$1" },

esp/src/package.json

@@ -30,13 +30,13 @@
   },
   "main": "src/stub.js",
   "dependencies": {
-    "@hpcc-js/chart": "2.19.0",
+    "@hpcc-js/chart": "2.19.1",
     "@hpcc-js/comms": "2.9.6",
-    "@hpcc-js/eclwatch": "2.5.18",
-    "@hpcc-js/html": "2.6.15",
-    "@hpcc-js/map": "2.9.1",
-    "@hpcc-js/other": "2.12.16",
-    "@hpcc-js/tree": "2.7.15",
+    "@hpcc-js/eclwatch": "2.5.20",
+    "@hpcc-js/html": "2.6.16",
+    "@hpcc-js/map": "2.9.3",
+    "@hpcc-js/other": "2.12.18",
+    "@hpcc-js/tree": "2.7.16",
     "clipboard": "2.0.4",
     "codemirror": "5.46.0",
     "crossfilter2": "1.4.7",
@@ -60,16 +60,15 @@
     "file-loader": "3.0.1",
     "jshint": "2.10.2",
     "local-web-server": "2.6.1",
-    "lodash": "4.17.11",
     "npm-run-all": "4.1.5",
     "rimraf": "2.6.3",
     "style-loader": "0.23.1",
     "tslib": "1.9.3",
     "typescript": "3.4.5",
     "url-loader": "1.1.2",
-    "webpack": "4.31.0",
-    "webpack-bundle-analyzer": "3.3.2",
-    "webpack-cli": "3.3.2"
+    "webpack": "4.39.1",
+    "webpack-bundle-analyzer": "3.4.1",
+    "webpack-cli": "3.3.6"
   },
   "author": "HPCC Systems",
   "license": "Apache-2.0",

initfiles/componentfiles/configschema/xsd/roxie.xsd

@@ -277,6 +277,9 @@
                     <xs:attribute name="soapTraceLevel" type="xs:nonNegativeInteger" hpcc:displayName="Soap trace Level"
                                   hpcc:presetValue="1"
                                   hpcc:tooltip="Level of detail in reporting SOAPCALL information(set to 0 for none, 1 for normal, >1 or more for extended)"/>
+                    <xs:attribute name="traceTranslations" type="xs:boolean" hpcc:displayName="Trace record layout translations"
+                                  hpcc:presetValue="true"
+                                  hpcc:tooltip="Trace record layout translations to log file"/>
                     <xs:attribute name="traceEnabled" type="xs:boolean" hpcc:displayName="Trace Enabled"
                                   hpcc:presetValue="false"
                                   hpcc:tooltip="TRACE activity output enabled by default (can be overridden in workunit or query)"/>

initfiles/componentfiles/configxml/roxie.xsd.in

@@ -1011,6 +1011,13 @@
         </xs:appinfo>
       </xs:annotation>
     </xs:attribute>
+    <xs:attribute name="traceTranslations" type="xs:boolean" use="optional" default="true">
+      <xs:annotation>
+        <xs:appinfo>
+          <tooltip>Trace record layout translations to log file"</tooltip>
+        </xs:appinfo>
+      </xs:annotation>
+    </xs:attribute>
     <xs:attribute name="traceEnabled" type="xs:boolean" use="optional" default="false">
       <xs:annotation>
         <xs:appinfo>

roxie/ccd/ccd.hpp

@@ -272,6 +272,7 @@ extern unsigned preabortKeyedJoinsThreshold;
 extern unsigned preabortIndexReadsThreshold;
 extern bool traceStartStop;
 extern bool traceServerSideCache;
+extern bool traceTranslations;
 extern bool defaultTimeActivities;
 extern bool defaultTraceEnabled;
 extern unsigned defaultTraceLimit;

roxie/ccd/ccdfile.cpp

@@ -2141,8 +2141,11 @@ public:
                     else
                     {
                         translator.setown(createRecordTranslator(projected->queryRecordAccessor(true), actual->queryRecordAccessor(true)));
-                        if (traceLevel > 5)
+                        if (traceLevel>0 && traceTranslations)
+                        {
+                            DBGLOG("Record layout translator created for %s", subname);
                             translator->describe();
+                        }
                         if (!translator || !translator->canTranslate())
                             throw MakeStringException(ROXIE_MISMATCH, "Untranslatable record layout mismatch detected for file %s", subname);
                         else if (translator->needsTranslate())

roxie/ccd/ccdmain.cpp

@@ -84,6 +84,7 @@ bool traceStartStop = false;
 bool traceServerSideCache = false;
 bool defaultTimeActivities = true;
 bool defaultTraceEnabled = false;
+bool traceTranslations = true;
 unsigned defaultTraceLimit = 10;
 unsigned watchActivityId = 0;
 unsigned testSlaveFailure = 0;
@@ -1001,6 +1002,7 @@ int STARTQUERY_API start_query(int argc, const char *argv[])
 
         traceStartStop = topology->getPropBool("@traceStartStop", false);
         traceServerSideCache = topology->getPropBool("@traceServerSideCache", false);
+        traceTranslations = topology->getPropBool("@traceTranslations", true);
         defaultTimeActivities = topology->getPropBool("@timeActivities", true);
         defaultTraceEnabled = topology->getPropBool("@traceEnabled", false);
         defaultTraceLimit = topology->getPropInt("@traceLimit", 10);

rtl/eclrtl/rtldynfield.cpp

@@ -988,7 +988,7 @@ enum FieldMatchType {
     match_recurse     = 0x80,    // Use recursive translator for child records/datasets
     match_fail        = 0x100,   // no translation possible
     match_keychange   = 0x200,   // at least one affected field not marked as payload (set on translator)
-    match_virtual     = 0x800,   // at least one affected field not marked as payload (set on translator)
+    match_virtual     = 0x800,   // at least one affected field is a virtual field (set on translator)
 
     // This flag may be set in conjunction with the others
     match_inifblock   = 0x400,   // matching to a field in an ifblock - may not be present
@@ -1078,6 +1078,8 @@ public:
 private:
     void doDescribe(unsigned indent) const
     {
+        unsigned perfect=0;
+        unsigned reported=0;
         for (unsigned idx = 0; idx <  destRecInfo.getNumFields(); idx++)
         {
             const char *source = destRecInfo.queryName(idx);
@@ -1088,17 +1090,41 @@ private:
                 DBGLOG("%*sUse virtual value for field %s", indent, "", source);
             else
             {
-                StringBuffer matchStr;
-                DBGLOG("%*sMatch (%s) to field %d for field %s (%x)", indent, "", describeFlags(matchStr, match.matchType).str(), match.matchIdx, source, destRecInfo.queryType(idx)->fieldType);
-                if (match.subTrans)
-                    match.subTrans->doDescribe(indent+2);
+                if (match.matchType != match_perfect)
+                {
+                    reported++;
+                    StringBuffer matchStr;
+                    DBGLOG("%*sMatch (%s) to field %d for field %s (typecode %x)", indent, "", describeFlags(matchStr, match.matchType).str(), match.matchIdx, source, destRecInfo.queryType(idx)->fieldType);
+                    if (match.subTrans)
+                        match.subTrans->doDescribe(indent+2);
+                }
+                else
+                    perfect++;
+            }
+        }
+        if (allUnmatched.ordinality())
+        {
+            VStringBuffer msg("%*sDropped field", indent, "");
+            if (allUnmatched.ordinality()>1)
+                msg.append('s');
+            for (unsigned idx = 0; idx < allUnmatched.ordinality() && idx < 5; idx++)
+            {
+
+                if (idx)
+                    msg.append(',');
+                msg.appendf(" %s", sourceRecInfo.queryName(allUnmatched.item(idx)));
             }
+            if (allUnmatched.ordinality() > 5)
+                msg.appendf(" and %u other fields", allUnmatched.ordinality() - 5);
+            DBGLOG("%s", msg.str());
         }
         if (!canTranslate())
             DBGLOG("%*sTranslation is NOT possible", indent, "");
         else if (needsTranslate())
         {
             StringBuffer matchStr;
+            if (perfect)
+                DBGLOG("%u %sfield%s matched perfectly", perfect, reported ? "other " : "", perfect==1 ? "" : "s");
             DBGLOG("%*sTranslation is possible (%s)", indent, "", describeFlags(matchStr, matchFlags).str());
         }
         else
@@ -1432,7 +1458,8 @@ private:
     const RtlRecord &sourceRecInfo;
     bool binarySource = true;
     unsigned fixedDelta = 0;  // total size of all fixed-size source fields that are not matched
-    UnsignedArray unmatched;  // List of all variable-size source fields that are unmatched
+    UnsignedArray allUnmatched;  // List of all source fields that are unmatched (so that we can trace them)
+    UnsignedArray variableUnmatched;  // List of all variable-size source fields that are unmatched
     FieldMatchType matchFlags = match_perfect;
 
     struct MatchInfo
@@ -1506,6 +1533,7 @@ private:
     }
     void createMatchInfo()
     {
+        unsigned defaulted = 0;
         for (unsigned idx = 0; idx < destRecInfo.getNumFields(); idx++)
         {
             const RtlFieldInfo *field = destRecInfo.queryField(idx);
@@ -1520,6 +1548,7 @@ private:
                 fixedDelta -= defaultSize;
                 if ((field->flags & RFTMispayloadfield) == 0)
                     matchFlags |= match_keychange;
+                defaulted++;
                 //DBGLOG("Decreasing fixedDelta size by %d to %d for defaulted field %d (%s)", defaultSize, fixedDelta, idx, destRecInfo.queryName(idx));
             }
             else
@@ -1669,10 +1698,9 @@ private:
             }
             matchFlags |= info.matchType;
         }
-        if (sourceRecInfo.getNumFields() > destRecInfo.getNumFields())
-            matchFlags |= match_remove;
-        if (matchFlags && !destRecInfo.getFixedSize())
+        if (sourceRecInfo.getNumFields() > destRecInfo.getNumFields()-defaulted)
         {
+            matchFlags |= match_remove;
             for (unsigned idx = 0; idx < sourceRecInfo.getNumFields(); idx++)
             {
                 const RtlFieldInfo *field = sourceRecInfo.queryField(idx);
@@ -1691,8 +1719,9 @@ private:
                             fixedDelta += type->getMinSize();
                         }
                         else
-                            unmatched.append(idx);
+                            variableUnmatched.append(idx);
                     }
+                    allUnmatched.append(idx);
                 }
             }
             //DBGLOG("Source record contains %d bytes of omitted fixed size fields", fixedDelta);
@@ -1703,9 +1732,9 @@ private:
         //DBGLOG("Source record size is %d", (int) sourceRow.getRecordSize());
         size32_t expectedSize = sourceRow.getRecordSize() - fixedDelta;
         //DBGLOG("Source record size without omitted fixed size fields is %d", expectedSize);
-        ForEachItemIn(i, unmatched)
+        ForEachItemIn(i, variableUnmatched)
         {
-            unsigned fieldNo = unmatched.item(i);
+            unsigned fieldNo = variableUnmatched.item(i);
             expectedSize -= sourceRow.getSize(fieldNo);
             //DBGLOG("Reducing estimated size by %d to %d for omitted field %d (%s)", (int) sourceRow.getSize(fieldNo), expectedSize, fieldNo, sourceRecInfo.queryName(fieldNo));
         }

thorlcr/activities/keyedjoin/thkeyedjoinslave.cpp

@@ -2149,8 +2149,8 @@ class CKeyedJoinSlave : public CSlaveActivity, implements IJoinProcessor
         if (partToSlaveMap.size())
         {
             slave = partToSlaveMap[partNo];
-            if (NotFound == slave) // part not local to cluster, part is handled locally. 'slave' only used for max (see below).
-                slave = 0;
+            if (NotFound == slave) // part not local to cluster, part is handled locally/directly.
+                slave = handlerCounts.size()-1; // last one reserved for out of cluster part handling.
         }
         unsigned max = queryMaxHandlers(hType);
         unsigned &handlerCount = handlerCounts[slave];
@@ -2222,12 +2222,13 @@ class CKeyedJoinSlave : public CSlaveActivity, implements IJoinProcessor
             std::vector<unsigned> slaveHandlersRR;
             bool remoteLookup = partToSlaveMap.size()>0;
             unsigned slaves = remoteLookup ? queryJob().querySlaves() : 1; // if no map, all parts are treated as if local
-            for (unsigned s=0; s<slaves; s++)
+            unsigned numHandlers = slaves+1; // +1 is for off cluster parts, which will be handled by local/direct handlers
+            for (unsigned s=0; s<numHandlers; s++)
             {
                 handlerCounts.push_back(0);
                 slaveHandlersRR.push_back(0);
             }
-            slaveHandlers.resize(slaves);
+            slaveHandlers.resize(numHandlers);
 
             unsigned currentPart = 0;
             unsigned p = 0;

thorlcr/activities/soapcall/thsoapcallslave.cpp

@@ -28,7 +28,7 @@ static StringBuffer &buildAuthToken(IUserDescriptor *userDesc, StringBuffer &aut
     userDesc->getUserName(uidpair);
     uidpair.append(":");
     userDesc->getPassword(uidpair);
-    JBASE64_Encode(uidpair.str(), uidpair.length(), authToken);
+    JBASE64_Encode(uidpair.str(), uidpair.length(), authToken, false);
     return authToken;
 }
 

thorlcr/slave/slavmain.cpp

@@ -360,6 +360,8 @@ class CKJService : public CSimpleInterfaceOf<IKJService>, implements IThreaded,
                     if (!translator->canTranslate())
                         throw MakeStringException(0, "Untranslatable record layout mismatch detected for: %s", tracing);
                 }
+                DBGLOG("Record layout translator created for %s", tracing);
+                translator->describe();
                 dbgassertex(translator->canTranslate());
             }
             return translator;