Merge branch 'candidate-5.2.4' into candidate-5.4.0

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

dali/base/dasds.cpp

@@ -4469,6 +4469,7 @@ void CSDSTransactionServer::processMessage(CMessageBuffer &mb)
                 if (queryTransactionLogging())
                     transactionLog.log("xpath='%s'", xpath.get());
                 mb.clear();
+                CHECKEDDALIREADLOCKBLOCK(manager.dataRWLock, readWriteTimeout);
                 Owned<IPropertyTree> matchTree = SDSManager->getXPaths(serverId, xpath, DAMP_SDSCMD_GETXPATHSPLUSIDS==action);
                 if (matchTree)
                 {
@@ -4499,6 +4500,7 @@ void CSDSTransactionServer::processMessage(CMessageBuffer &mb)
                         ascending?"true":"false", from, limit);
                 }
                 mb.clear();
+                CHECKEDDALIREADLOCKBLOCK(manager.dataRWLock, readWriteTimeout);
                 Owned<IPropertyTree> matchTree = SDSManager->getXPathsSortLimitMatchTree(xpath, matchXPath, sortBy, caseinsensitive, ascending, from, limit);
                 if (matchTree)
                 {

docs/HPCCSystemAdmin/HPCCSystemAdministratorsGuide.xml

@@ -525,7 +525,7 @@
       clusters. The number of Roxie nodes should never exceed the number of
       Thor nodes. In addition, the number of Thor nodes should be evenly
       divisible by the number of Roxie nodes. This ensures an efficient
-      distribution of file parts from Thor to Roxie. </para>
+      distribution of file parts from Thor to Roxie.</para>
     </sect1>
 
     <sect1>
@@ -1238,9 +1238,9 @@ lock=/var/lock/HPCCSystems</programlisting>
 
         <para>If you are adding or removing Thor cluster nodes but
         <emphasis>all previous nodes remain part of the environment and
-        accessible</emphasis>, you can <emphasis role="bold">rename</emphasis>
-        the group that is associated with the Thor cluster (or the Cluster
-        name if there is no group name).</para>
+        accessible</emphasis>, you must <emphasis
+        role="bold">rename</emphasis> the group that is associated with the
+        Thor cluster (or the Cluster name if there is no group name).</para>
 
         <para>This will ensure all previously existing files, continue to use
         the old group structure, while new files use the new group

esp/eclwatch/ws_XSLT/wuidcommon.xslt

@@ -44,14 +44,8 @@
                 <xsl:otherwise>
                   <xsl:value-of select="$wuid"/>
                   &nbsp;
-                  <xsl:choose>
-                    <xsl:when test="WUXMLSize &lt; 5000000">
-                      <a href="/esp/iframe?esp_iframe_title=ECL Workunit XML - {$wuid}&amp;inner=/WsWorkunits/WUFile%3fWuid%3d{$wuid}%26Type%3dXML%26Option%3d0" >XML</a><xsl:value-of select="WUXMLSize"/>
-                    </xsl:when>
-                    <xsl:otherwise>
-                      <a href="/esp/iframe?esp_iframe_title=Download ECL Workunit XML - {$wuid}&amp;inner=/WsWorkunits/WUFile%3fWuid%3d{$wuid}%26Type%3dXML%26Option%3d2" >Download XML</a>
-                    </xsl:otherwise>
-                  </xsl:choose>
+                  <a href="/esp/iframe?esp_iframe_title=ECL Workunit XML - {$wuid}&amp;inner=/WsWorkunits/WUFile%3fWuid%3d{$wuid}%26Type%3dXML%26Option%3d0%26SizeLimit%3d5000000" >XML</a>
+                  <a href="/esp/iframe?esp_iframe_title=Download ECL Workunit XML - {$wuid}&amp;inner=/WsWorkunits/WUFile%3fWuid%3d{$wuid}%26Type%3dXML%26Option%3d2" >Download XML</a>
                   &nbsp;
                   <a href="/esp/iframe?esp_iframe_title=ECL Playground - {$wuid}&amp;inner=/esp/files/stub.htm%3fWidget%3dECLPlaygroundWidget%26Wuid%3d{$wuid}%26Target%3d{Cluster}" >ECL Playground</a>
                 </xsl:otherwise>

esp/files/gen_form_wsecl.js

@@ -0,0 +1,499 @@
+/*##############################################################################
+#    HPCC SYSTEMS software Copyright (C) 2015 HPCC Systems.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+############################################################################## */
+
+// for test purpose
+function getAttributes(ctrl)
+{
+    var results = "";
+    var attrs = ctrl.attributes;
+    for (var i = 0; i < attrs.length; i++) {
+       var attr = attrs[i];
+       results += attr.nodeName + '=' + attr.nodeValue + ' (' + attr.specified + ')<BR>';
+    }
+    return results;
+}
+
+//==================================================================
+// Restore dynamically generate content, and user input values(non-IE browsers only)
+
+function restoreDataFromCache()
+{
+    var vals = document.getElementById("esp_vals_").value;
+    if (vals && vals!="")
+    {
+        // alert("esp_vals_ = "+vals);
+        var end = vals.indexOf('|');
+        while (end>0)
+        {
+            var name = vals.substring(0,end);
+            vals = vals.substring(end+1);
+            end = vals.indexOf("|");
+            if (end<0) break;
+            var ctrl = document.getElementsByName(name)[0];
+            if (ctrl)
+            {
+                if (ctrl.type == 'checkbox')
+                {
+                    ctrl.checked = vals.substring(0,end)=='1';
+                    //  alert("Name = "+name+", string = "+ vals.substring(0,end) + ", ctrl.checked = "+ ctrl.checked);
+                }
+                else if (ctrl.type == 'text' || ctrl.type == 'textarea')
+                    ctrl.value = decodeURI(vals.substring(0,end));
+                else if(ctrl.type == 'radio')
+                {
+                    //alert("Name = "+name+", string = "+ vals.substring(0,end) + ", ctrl.checked = "+ ctrl.checked);
+                    if(vals.substring(0,end) == '0')
+                    {
+                         ctrl = document.getElementsByName(name)[1];
+                    }
+
+                    if(ctrl)
+                        ctrl.checked = true;
+                }
+                else if(ctrl.type == 'select-one')
+            {
+                    //alert("Select: name="+ctrl.name+"; value="+vals.substring(0,end));
+                    ctrl.options[vals.substring(0,end)].selected = true;
+                }
+                //TODO: more types
+            }
+            vals = vals.substring(end+1);
+            end = vals.indexOf("|");
+        }
+    }
+}
+
+function disableAllInputs(self)
+{
+    var toEnable = self.checked ? 1 : 0;
+    var form = document.forms['esp_form'];
+    var ctrls = form.elements;
+    for (var idx=0; idx<ctrls.length; idx++)
+    {
+        var c = ctrls[idx];
+        if ( (c.id!='') && (c.id.substr(0,3) == '$V.'))
+        {
+            if ( (c.checked && !toEnable) || (!c.checked && toEnable))
+                c.click();
+        }
+    }
+}
+
+function disableInputControls(form)
+{
+    var ctrls = form.elements;
+    for (var idx=0; idx<ctrls.length; idx++)
+    {
+        var c = ctrls[idx];
+        if ( (c.id!='') && (c.id.substr(0,3) == '$V.') && !c.checked)
+        {
+             var id = c.id.substring(3);
+             var ctrl = document.getElementById(id);
+             if (ctrl) // struct name has no id
+                disableInputControl(ctrl,true);
+             var label = document.getElementById('$L.'+id)
+             disableInputLabel(label,true);
+        }
+    }
+}
+
+
+function onPageLoad()
+{
+   var ctrl = document.getElementById('esp_html_');
+   //alert("onPageLoad(): ctrl="+ctrl+"; length="+ctrl.value.length+";value='"+ctrl.value+"'");
+   if (ctrl && ctrl.value != undefined && ctrl.value!="")
+   {
+      //alert("Restore ctrol value: " + ctrl.value);
+        document.forms['esp_form'].innerHTML = ctrl.value;
+   }
+
+   var form = document.forms['esp_form'];
+   initFormValues(form, getUrlFormValues(top.location.href));
+
+   disableInputControls(form);
+
+   // IE seems need this now too
+   //if (isIE) return true;
+   // FF 1.5 history cache works, but seems to stop working afterwards
+   restoreDataFromCache();
+
+   return true;
+}
+
+function getUrlFormValues(url)
+{
+    var idx = url.indexOf('?');
+    if (idx>0)
+        url = url.substring(idx+1);
+    var a = url.split('&');
+
+    var ps = new Hashtable();
+    for (var i=0; i<a.length; i++)
+    {
+         idx = a[i].indexOf('=');
+         if (a[i].charAt(0) == '.' && idx>0)
+         {
+            var key = a[i].substring(0,idx);
+            var val =  a[i].substring(idx+1);
+            if (val != '')
+                ps.put(key, val);
+        }
+    }
+
+    return ps;
+}
+
+function getUrlEspFlags(url)
+{
+    var idx = url.indexOf('?');
+    if (idx>0)
+        url = url.substring(idx+1);
+    var a = url.split('&');
+
+    var ps = new Hashtable();
+    for (var i=0; i<a.length; i++)
+    {
+        if (a[i].charAt(0) != '.')
+        {
+            idx = a[i].indexOf('=');
+            if (idx>0)
+            {
+                var key = a[i].substring(0,idx);
+                var val =   a[i].substring(idx+1);
+                ps.put(key,val);
+            } else
+                ps.put(a[i],"");
+        }
+    }
+
+    return ps;
+}
+
+function createArray(ps)
+{
+    var remains = new Hashtable();
+    ps.moveFirst();
+    while (ps.next())
+    {
+         var name = ps.getKey();
+         var val = ps.getValue();
+         // alert(name + ": " + val);
+         if (val > 0 && name.substring(name.length-11)==".itemcount!")
+         {
+             var id = name.substring(1, name.length-10) + '_AddBtn';
+             var ctrl = document.getElementById(id);
+             if (ctrl)
+             {
+                //   alert("name: " + ctrl.tagName + ", type " + ctrl.type);
+                for (var i=0; i<val; i++)
+                    ctrl.click();
+             }
+             else {
+                //alert("Can not find control: " + id);
+                remains.put(name,val);
+             }
+         }
+    }
+
+    if (remains.size()>0)
+      return remains;
+    else
+      return null;
+}
+
+function initFormValues(form, ps)
+{
+    // create array controls
+    // Implementation NOTE: The Add order is important: if array A contains array B, item in A must be created first before B can be created.
+
+    var working = ps;
+    do
+    {
+        working  = createArray(working);
+        //alert("Left: " + working);
+    } while (working!=null);
+
+    // init values
+    ps.moveFirst();
+    while (ps.next())
+    {
+        var name = ps.getKey();
+        if (name.charAt(0) != '.')
+        {
+            //alert("Skip " + name);
+            continue;
+        }
+        name = name.substring(1);
+        var val = ps.getValue();
+        ctrl = document.getElementsByName(name)[0];
+
+        // alert("Set value for " + name + ": " + val + ". Ctrl type: " + ctrl.type);
+        if (ctrl)
+        {
+            if (ctrl.type == 'checkbox') {
+                ctrl.checked = val =='1';
+            }
+            else if (ctrl.type == 'text') {
+                ctrl.value =  decodeURIComponent(val); //    decodeURI(vals.substring(0,end)); //TODO: do we need encoding
+            }
+            else if (ctrl.type == 'textarea') {
+                ctrl.value =  decodeURIComponent(val);
+            }
+            else if(ctrl.type == 'radio')  {
+                if(val == '0')
+                    ctrl = document.getElementsByName(name)[1];
+                if(ctrl)
+                    ctrl.checked = true;
+            }
+            else if (ctrl.type=='select-one')  {
+                //alert("Set select value: " + val);
+                ctrl.options[val].selected=true;
+            }
+        }
+        else
+            alert("failed to find contrl: " + name);
+    }
+}
+
+function doBookmark(form)
+{
+    var ps = getUrlEspFlags(form.action);
+
+    var ctrls = form.elements;
+    for (var idx=0; idx<ctrls.length; idx++)
+    {
+        var c = ctrls[idx];
+        if ( (c.name!='') && (c.value != '') )
+        {
+            if (c.tagName == 'TEXTAREA') {
+                ps.put('.'+c.name,encodeURIComponent(c.value));
+            } else if (c.tagName == "SELECT") {
+                ps.put('.'+c.name, c.selectedIndex); // use the index
+            } else if (c.tagName == 'INPUT')  {
+                if ( c.type == 'text' || c.type=='password')  {
+                    ps.put('.'+c.name,encodeURIComponent(c.value)); // existing one is overwrotten
+                } else if (c.type == 'radio' && c.checked) {
+                    if (c.id.substring(c.id.length-5) == '.true')
+                        ps.put('.'+c.name,"1");
+                    else if (c.id.substring(c.id.length-6) == '.false')
+                        ps.put('.'+c.name,"0");
+                } else if ( c.type=='hidden') {
+                    // alert("hidden:"+c.name+", value " + c.value + ", sub = " +  c.name.substring(c.name.length-10));
+                    if (c.value!='0' && c.name.substring(c.name.length-11)=='.itemcount!')
+                        ps.put('.'+c.name,c.value);
+                }
+            }
+        }
+    }
+
+    var idx = form.action.indexOf('?');
+    var action = (idx>0) ? form.action.substring(0,idx) : form.action;
+
+    action += "?form";
+
+    var parm = "";
+    ps.moveFirst();
+    while (ps.next())
+       parm += '&' + ps.getKey() + '=' + ps.getValue();
+    //alert("parm="+parm);
+
+    /*
+    // TODO: make inner frame work
+    var url = "/?inner=.." + path + "%3Fform";
+    top.location.href = url + parm;
+    */
+    top.location.href = action + parm;
+}
+
+//==================================================================
+// Save dynamically generate content, and user input values(non-IE browsers only)
+function onSubmit(reqType)  // reqType: 0: regular form, 1: soap, 2: form param passing
+{
+    var form = document.forms['esp_form'];
+    if (!form)  return false;
+
+    // remove "soap_builder_" (somehow FF (not IE) remembers this changed form.action )
+    if (reqType != 1)
+    {
+        var action = form.action;
+        var idx = action.indexOf('soap_builder_');
+        if (idx>0)
+        {
+            if (action.length <= idx + 13) // no more char after 'soap_builder_'
+            {
+                var ch = action.charAt(idx-1);
+                if (ch == '&' || ch == '?')
+                    action = action.substring(0,idx-1);
+            } else {
+                var ch = action.charAt(idx+13) // the char after 'soap_builder_';
+                if (ch == '&')
+                   action = action.substring(0,idx) + action.substring(idx+13);
+            }
+
+            // alert("Old action: " + form.action + "\nNew action: " + action);
+            form.action = action;
+        }
+    }
+
+    // --  change action if user wants to
+    var dest = document.getElementById('esp_dest');
+    if (dest && dest.checked)
+    {
+         form.action = document.getElementById('dest_url').value;
+    }
+    if (reqType==1)
+    {
+         if (form.action.indexOf('soap_builder_')<0) // add only if does not exist already
+         {
+                var c =  (form.action.indexOf('?')>0) ? '&' : '?';
+                form.action += c + "soap_builder_";
+         }
+    }
+    else if (reqType==2)
+    {
+         doBookmark(form);
+    }
+    if (reqType==3)
+    {
+         if (form.action.indexOf('roxie_builder_')<0) // add only if does not exist already
+         {
+                var c =  (form.action.indexOf('?')>0) ? '&' : '?';
+                form.action += c + "roxie_builder_";
+         }
+    }
+    if (reqType==4)
+    {
+         if (form.action.indexOf('json_builder_')<0) // add only if does not exist already
+         {
+                var c =  (form.action.indexOf('?')>0) ? '&' : '?';
+                form.action += c + "json_builder_";
+         }
+    }
+    // alert("Form action = " + form.action);
+
+    // firefox now save input values (version 1.5)
+    saveInputValues(form);
+
+    return true;
+}
+
+//==================================================================
+// Save dynamically generate content, and user input values(non-IE browsers only)
+function onWsEcl2Submit(path)  // reqType: 0: regular form, 1: soap, 2: form param passing, 3: roxiexml
+{
+    var form = document.forms['esp_form'];
+    if (!form)  return false;
+
+    var dest = document.getElementById('esp_dest');
+    if (dest && dest.checked)
+    {
+         form.action = document.getElementById('dest_url').value;
+    }
+    else if (path=="bookmark")
+    {
+         doBookmark(form);
+    }
+    else
+    {
+        form.action = path;
+    }
+
+    alert("Form action = " + form.action);
+
+    // firefox now save input values (version 1.5)
+    saveInputValues(form);
+
+    return true;
+}
+
+
+function saveInputValues(form)
+{
+    // -- save values in input for browser
+    var ctrl = document.getElementById('esp_html_');
+    // IE seems to need this too
+    //if (isIE || !ctrl)  return true;
+
+    ctrl.value=form.innerHTML;
+
+    // save all user input
+    var ctrls = form.elements;
+    var items = ctrls.length;
+    var inputValues = "";
+
+    for (var idx=0; idx<ctrls.length; idx++)
+    {
+        var item = ctrls[idx];
+        var name = item.name;
+
+        if (!name) continue;
+
+        //NOTE: we can not omit empty value since it can be different from the default
+        if (item.type == 'checkbox')
+            inputValues += name+"|"+(item.checked ? "1" : "0")+"|";
+        else if (item.type =='text' || item.type=='textarea')
+        {
+           inputValues += name+"|" + encodeURI(item.value) +"|";
+           //alert("value added: "+inputValues[inputValues.length-1] +", input items: " + inputValues.length+", values="+inputValues.toString());
+        }
+        else if (item.type == 'radio')
+        {
+            //if(item.checked) // the unchecked value can be different from the default
+            inputValues += name+"|"+item.value+"|";
+        }
+        else if (item.type == 'select-one')
+        {
+        inputValues += name+"|"+item.selectedIndex+"|";
+        //alert("inputValues=" + inputValues + "; index="+item.selectedIndex);
+        }
+        // TODO: other control types
+    }
+
+    document.getElementById("esp_vals_").value = inputValues;
+}
+
+//==================================================================
+// Reset all values to orginal (all dynamically generated arrays are removed)
+function onClearAll()
+{
+    // reset dynamic generated content
+    var reqCtrl = document.getElementById('esp_dyn');
+    reqCtrl.innerHTML = getRequestFormHtml();;
+
+    // clear cache
+    var ctrl = document.getElementById('esp_html_');
+    if (ctrl)
+        ctrl.value = "";
+    ctrl = document.getElementById("esp_vals_");
+    if (ctrl)
+        ctrl.value = "";
+}
+
+//  Exclusive selectable
+function  onClickSort(chked)
+{
+    if (chked) {
+         document.getElementById("esp_validate").checked = false;
+    }
+}
+
+function  onClickValidate(chked)
+{
+    if (chked) {
+        document.getElementById("esp_sort_result").checked = false;
+    }
+}

esp/scm/ws_workunits.ecm

@@ -236,7 +236,6 @@ ESPStruct [nil_remove] ECLWorkunit
     [min_ver("1.29")] string ApplicationValuesDesc;
     [min_ver("1.29")] string WorkflowsDesc;
     [min_ver("1.31")] bool HasArchiveQuery(false);
-    [min_ver("1.49")] int64 WUXMLSize;
     [min_ver("1.38")] ESParray<ESPstruct ThorLogInfo> ThorLogList;
     [min_ver("1.47")] ESParray<string, URL> ResourceURLs;
     [min_ver("1.50")] int ResultViewCount;
@@ -655,6 +654,7 @@ ESPrequest WULogFileRequest
     [min_ver("1.38")] string ClusterGroup;
     [min_ver("1.38")] string LogDate;
     [min_ver("1.38")] int SlaveNumber(1);
+    [min_ver("1.55")] int64 SizeLimit(0);
     string PlainText;
 };
 
@@ -1609,7 +1609,7 @@ ESPresponse [exceptions_inline] WUGetStatsResponse
 };
 
 ESPservice [
-    version("1.54"),
+    version("1.55"), default_client_version("1.55"),
     noforms,exceptions_inline("./smc_xslt/exceptions.xslt"),use_method_name] WsWorkunits
 {
     ESPmethod [resp_xsl_default("/esp/xslt/workunits.xslt")]     WUQuery(WUQueryRequest, WUQueryResponse);

esp/services/common/jsonhelpers.hpp

@@ -111,7 +111,9 @@ namespace HttpParamHelpers
         Owned<IPropertyIterator> props = parameters->getIterator();
         ForEach(*props)
         {
-            const char *key = props->getPropKey();
+            StringBuffer key = props->getPropKey();
+            if (!key.length() || key.charAt(key.length()-1)=='!')
+                continue;
             const char *value = parameters->queryProp(key);
             if (value && *value)
                 ensureParameter(pt, key, value);

esp/services/ws_workunits/ws_workunitsHelpers.cpp

@@ -1024,12 +1024,6 @@ void WsWuInfo::getInfo(IEspECLWorkunit &info, unsigned flags)
     info.setDescription(cw->getDebugValue("description", s).str());
     if (version > 1.21)
         info.setXmlParams(cw->getXmlParams(s).str());
-    if (version >= 1.49)
-    {
-        SCMStringBuffer xml;
-        exportWorkUnitToXML(cw, xml, true, false);
-        info.setWUXMLSize(xml.length());
-    }
 
     info.setResultLimit(cw->getResultLimit());
     info.setArchived(false);

esp/services/ws_workunits/ws_workunitsService.cpp

@@ -2385,15 +2385,25 @@ void getWsWuResult(IEspContext &context, const char* wuid, const char *name, con
     }
 }
 
-void openSaveFile(IEspContext &context, int opt, const char* filename, const char* origMimeType, MemoryBuffer& buf, IEspWULogFileResponse &resp)
+void checkFileSizeLimit(unsigned long xmlSize, unsigned long sizeLimit)
+{
+    if ((sizeLimit > 0) && (xmlSize > sizeLimit))
+        throw MakeStringException(ECLWATCH_CANNOT_OPEN_FILE,
+            "The file size (%ld bytes) exceeds the size limit (%ld bytes). You may set 'Option > 1' or use 'Download_XML' link to get compressed file.",
+            xmlSize, sizeLimit);
+}
+
+void openSaveFile(IEspContext &context, int opt, __int64 sizeLimit, const char* filename, const char* origMimeType, MemoryBuffer& buf, IEspWULogFileResponse &resp)
 {
     if (opt < 1)
     {
+        checkFileSizeLimit(buf.length(), sizeLimit);
         resp.setThefile(buf);
         resp.setThefile_mimetype(origMimeType);
     }
     else if (opt < 2)
     {
+        checkFileSizeLimit(buf.length(), sizeLimit);
         StringBuffer headerStr("attachment;");
         if (filename && *filename)
         {
@@ -2509,12 +2519,12 @@ bool CWsWorkunitsEx::onWUFile(IEspContext &context,IEspWULogFileRequest &req, IE
             if (strieq(File_ArchiveQuery, req.getType()))
             {
                 winfo.getWorkunitArchiveQuery(mb);
-                openSaveFile(context, opt, "ArchiveQuery.xml", HTTP_TYPE_APPLICATION_XML, mb, resp);
+                openSaveFile(context, opt, req.getSizeLimit(), "ArchiveQuery.xml", HTTP_TYPE_APPLICATION_XML, mb, resp);
             }
             else if (strieq(File_Cpp,req.getType()) && notEmpty(req.getName()))
             {
                 winfo.getWorkunitCpp(req.getName(), req.getDescription(), req.getIPAddress(),mb, opt > 0);
-                openSaveFile(context, opt, req.getName(), HTTP_TYPE_TEXT_PLAIN, mb, resp);
+                openSaveFile(context, opt, req.getSizeLimit(), req.getName(), HTTP_TYPE_TEXT_PLAIN, mb, resp);
             }
             else if (strieq(File_DLL,req.getType()))
             {
@@ -2522,17 +2532,17 @@ bool CWsWorkunitsEx::onWUFile(IEspContext &context,IEspWULogFileRequest &req, IE
                 winfo.getWorkunitDll(name, mb);
                 resp.setFileName(name.str());
                 resp.setDaliServer(daliServers.get());
-                openSaveFile(context, opt, req.getName(), HTTP_TYPE_OCTET_STREAM, mb, resp);
+                openSaveFile(context, opt, req.getSizeLimit(), req.getName(), HTTP_TYPE_OCTET_STREAM, mb, resp);
             }
             else if (strieq(File_Res,req.getType()))
             {
                 winfo.getWorkunitResTxt(mb);
-                openSaveFile(context, opt, "res.txt", HTTP_TYPE_TEXT_PLAIN, mb, resp);
+                openSaveFile(context, opt, req.getSizeLimit(), "res.txt", HTTP_TYPE_TEXT_PLAIN, mb, resp);
             }
             else if (strncmp(req.getType(), File_ThorLog, 7) == 0)
             {
                 winfo.getWorkunitThorLog(req.getName(), mb);
-                openSaveFile(context, opt, "thormaster.log", HTTP_TYPE_TEXT_PLAIN, mb, resp);
+                openSaveFile(context, opt, req.getSizeLimit(), "thormaster.log", HTTP_TYPE_TEXT_PLAIN, mb, resp);
             }
             else if (strieq(File_ThorSlaveLog,req.getType()))
             {
@@ -2540,12 +2550,12 @@ bool CWsWorkunitsEx::onWUFile(IEspContext &context,IEspWULogFileRequest &req, IE
                 getConfigurationDirectory(directories, "log", "thor", req.getProcess(), logDir);
 
                 winfo.getWorkunitThorSlaveLog(req.getClusterGroup(), req.getIPAddress(), req.getLogDate(), logDir.str(), req.getSlaveNumber(), mb, false);
-                openSaveFile(context, opt, "ThorSlave.log", HTTP_TYPE_TEXT_PLAIN, mb, resp);
+                openSaveFile(context, opt, req.getSizeLimit(), "ThorSlave.log", HTTP_TYPE_TEXT_PLAIN, mb, resp);
             }
             else if (strieq(File_EclAgentLog,req.getType()))
             {
                 winfo.getWorkunitEclAgentLog(req.getName(), req.getProcess(), mb);
-                openSaveFile(context, opt, "eclagent.log", HTTP_TYPE_TEXT_PLAIN, mb, resp);
+                openSaveFile(context, opt, req.getSizeLimit(), "eclagent.log", HTTP_TYPE_TEXT_PLAIN, mb, resp);
             }
             else if (strieq(File_XML,req.getType()) && notEmpty(req.getName()))
             {
@@ -2557,7 +2567,7 @@ bool CWsWorkunitsEx::onWUFile(IEspContext &context,IEspWULogFileRequest &req, IE
                     ptr = name;
 
                 winfo.getWorkunitAssociatedXml(name, req.getIPAddress(), req.getPlainText(), req.getDescription(), opt > 0, mb);
-                openSaveFile(context, opt, ptr, HTTP_TYPE_APPLICATION_XML, mb, resp);
+                openSaveFile(context, opt, req.getSizeLimit(), ptr, HTTP_TYPE_APPLICATION_XML, mb, resp);
             }
             else if (strieq(File_XML,req.getType()) || strieq(File_WUECL,req.getType()))
             {
@@ -2585,7 +2595,7 @@ bool CWsWorkunitsEx::onWUFile(IEspContext &context,IEspWULogFileRequest &req, IE
                         mimeType.set(HTTP_TYPE_APPLICATION_XML);
                     }
                 }
-                openSaveFile(context, opt, fileName.str(), mimeType.str(), mb, resp);
+                openSaveFile(context, opt, req.getSizeLimit(), fileName.str(), mimeType.str(), mb, resp);
             }
         }
     }

esp/src/eclwatch/ESPWorkunit.js

@@ -122,7 +122,12 @@ define([
         _SourceFilesSetter: function (SourceFiles) {
             var sourceFiles = [];
             for (var i = 0; i < SourceFiles.ECLSourceFile.length; ++i) {
-                sourceFiles.push(ESPResult.Get(lang.mixin({ wu: this.wu, Wuid: this.Wuid }, SourceFiles.ECLSourceFile[i])));
+                sourceFiles.push(ESPResult.Get(lang.mixin({ wu: this.wu, Wuid: this.Wuid, __hpcc_parentName: "" }, SourceFiles.ECLSourceFile[i])));
+                if (lang.exists("ECLSourceFiles.ECLSourceFile", SourceFiles.ECLSourceFile[i])) {
+                    for (var j = 0; j < SourceFiles.ECLSourceFile[i].ECLSourceFiles.ECLSourceFile.length; ++j) {
+                        sourceFiles.push(ESPResult.Get(lang.mixin({ wu: this.wu, Wuid: this.Wuid, __hpcc_parentName: SourceFiles.ECLSourceFile[i].Name }, SourceFiles.ECLSourceFile[i].ECLSourceFiles.ECLSourceFile[j])));
+                    }
+                }
             }
             this.set("sourceFiles", sourceFiles);
         },

esp/src/eclwatch/SFDetailsWidget.js

@@ -75,6 +75,7 @@ define([
         postCreate: function (args) {
             this.inherited(arguments);
             this.summaryWidget = registry.byId(this.id + "_Summary");
+            this.deleteBtn = registry.byId(this.id + "Delete");
         },
 
         startup: function (args) {
@@ -229,6 +230,15 @@ define([
                 var tab = context.ensureLFPane(item.Name, item);
                 context.selectChild(tab, true);
             });
+            this.subfilesGrid.on("dgrid-select", function (evt) {
+                context.deleteBtn.set("disabled", true);
+            });
+            this.subfilesGrid.on("dgrid-deselect", function (evt) {
+                var selections = context.subfilesGrid.getSelected();
+                if (selections.length === 0) {
+                    context.deleteBtn.set("disabled", false);
+                }
+            });
             this.subfilesGrid.on(".dgrid-row:dblclick", function (evt) {
                 var item = context.subfilesGrid.row(evt).data;
                 var tab = context.ensureLFPane(item.Name, item);

esp/src/eclwatch/SourceFilesWidget.js

@@ -21,6 +21,7 @@ define([
     "dojo/_base/array",
     "dojo/on",
 
+    "dgrid/tree",
     "dgrid/selector",
 
     "hpcc/GridDetailsWidget",
@@ -29,7 +30,7 @@ define([
     "hpcc/ESPUtil"
 
 ], function (declare, lang, i18n, nlsHPCC, arrayUtil, on,
-                selector,
+                tree, selector,
                 GridDetailsWidget, ESPWorkunit, DelayLoadWidget, ESPUtil) {
     return declare("SourceFilesWidget", [GridDetailsWidget], {
         i18n: nlsHPCC,
@@ -57,6 +58,12 @@ define([
         },
 
         createGrid: function (domID) {
+            this.store.mayHaveChildren = function (item) {
+                return item.IsSuperFile;
+            };
+            this.store.getChildren = function (parent, options) {
+                return context.store.query({__hpcc_parentName: parent.Name});
+            };
             var retVal = new declare([ESPUtil.Grid(false, true)])({
                 store: this.store,
                 columns: {
@@ -64,18 +71,18 @@ define([
                         width: 27,
                         selectorType: 'checkbox'
                     }),
-                    Name: {
+                    Name: tree({
                         label: "Name", sortable: true,
                         formatter: function (Name, row) {
                             return dojoConfig.getImageHTML(row.IsSuperFile ? "folder_table.png" : "file.png") + "&nbsp;<a href='#' class='dgrid-row-url'>" + Name + "</a>";
                         }
-                    },
+                    }),
                     Count: { label: "Usage", width: 72, sortable: true }
                 }
             }, domID);
 
             var context = this;
-            retVal.on("." + this.id + "dgrid-row-url:click", function (evt) {
+            retVal.on(".dgrid-row-url:click", function (evt) {
                 if (context._onRowDblClick) {
                     var row = context.grid.row(evt).data;
                     context._onRowDblClick(row);
@@ -127,7 +134,7 @@ define([
                         row.sequence = idx;
                     });
                     context.store.setData(sourceFiles);
-                    context.grid.refresh();
+                    context.grid.set("query", { __hpcc_parentName: "" });
                 }
             });
         }

esp/src/eclwatch/WUQueryWidget.js

@@ -208,7 +208,6 @@ define([
         //  Implementation  ---
         getFilter: function () {
             var retVal = this.filter.toObject();
-            retval.Wuid =  retVal.Wuid.toUpperCase().trim();
             if (retVal.StartDate && retVal.FromTime) {
                 lang.mixin(retVal, {
                     StartDate: this.getISOString("FromDate", "FromTime")

esp/src/eclwatch/templates/WUQueryWidget.html

@@ -20,7 +20,7 @@
                     <div id="${id}Filter" data-dojo-type="FilterDropDownWidget">
                         <p id="${id}ArchivedWarning" style="display:none">${i18n.ArchivedWarning}</p>
                         <input id="${id}Type" title="${i18n.ArchivedOnly}" name="Type" colspan="2" data-dojo-attach-event="onClick:_onFilterType" data-dojo-props="value:'archived workunits'" data-dojo-type="dijit.form.CheckBox" />
-                        <input id="${id}Wuid" title="${i18n.WUID}:" name="Wuid" colspan="2" style="width:100%" data-dojo-props="trim: true, placeHolder:'W20130222-171723'" data-dojo-type="dijit.form.TextBox" />
+                        <input id="${id}Wuid" title="${i18n.WUID}:" name="Wuid" colspan="2" style="width:100%" data-dojo-props="trim: true, uppercase: true, placeHolder:'W20130222-171723'" data-dojo-type="dijit.form.TextBox" />
                         <input id="${id}Owner" title="${i18n.Owner}:" name="Owner" colspan="2" data-dojo-props="trim: true, placeHolder:'${i18n.jsmi}'" data-dojo-type="dijit.form.TextBox" />
                         <input id="${id}Jobname" title="${i18n.JobName}:" name="Jobname" colspan="2" data-dojo-props="trim: true, placeHolder:'${i18n.log_analysis_1}'" data-dojo-type="dijit.form.TextBox" />
                         <input id="${id}ClusterTargetSelect" title="${i18n.Cluster}:" name="Cluster" colspan="2" data-dojo-props="trim: true, placeHolder:'${i18n.Owner}'" data-dojo-type="TargetSelectWidget" />

esp/xslt/wsecl3_form.xsl

@@ -92,7 +92,7 @@
                 <link rel="stylesheet" type="text/css" href="/esp/files/gen_form.css"/>
                 <script type="text/javascript" src="/esp/files/req_array.js"/>
                 <script type="text/javascript" src="/esp/files/hashtable.js"/>
-                <script type="text/javascript" src="/esp/files/gen_form.js"/>
+                <script type="text/javascript" src="/esp/files/gen_form_wsecl.js"/>
                 <script type="text/javascript"><xsl:text disable-output-escaping="yes">
                 <![CDATA[
   var isIE = (navigator.appName == "Microsoft Internet Explorer");
@@ -114,7 +114,7 @@
                         <xsl:text disable-output-escaping="yes"><![CDATA[ + "<table id='"+newId+"'> </table></hr>"]]></xsl:text>
                     </xsl:if>
                     <xsl:text disable-output-escaping="yes"><![CDATA[
-       + "<input type='hidden' id='"+newId+"_ItemCt' name='"+newId+".itemcount' value='0' />"
+       + "<input type='hidden' id='"+newId+"_ItemCt' name='"+newId+".itemcount!' value='0' />"
           + "&nbsp;<input type='button' id='"+newId+"_AddBtn' onclick='appendRow(\""+newId+"\",\""+itemName+"\",get_"+typeName+"_Item)' value='Add' /> "
           + "<input type='button' id='"+newId+"_RvBtn' onclick='removeRow(\""+newId+"\",-1)' value='Delete' disabled='true' />" ]]></xsl:text>
                     <xsl:if test="not($useTableBorder)">

initfiles/bin/init_thor

@@ -28,7 +28,7 @@ rm -f ${SENTINEL}
 
 killed() {
         echo "Stopping"
-        $deploydir/stop_thor $deploydir
+        kill_process ${SENTINEL} ${PID_NAME} 3
         exit 255
 }
 

initfiles/componentfiles/thor/run_thor

@@ -120,6 +120,7 @@ while [[ 1 ]]; do
         esac
     else
         echo failed to start thormaster$LCR, pausing for 30 seconds
+        $deploydir/stop_thor $deploydir
         sleep 30
     fi
     if [[ ! -e $SENTINEL ]]; then

thorlcr/activities/funnel/thfunnelslave.cpp

@@ -76,10 +76,6 @@ class CParallelFunnel : public CSimpleInterface, implements IRowStream
             bool started = false;
             try
             {
-                { 
-                    CriticalBlock b(stopCrit);
-                    if (stopping) return;
-                }
                 if (funnel.startInputs)
                 {
                     IThorDataLink *_input = QUERYINTERFACE(input.get(), IThorDataLink);

thorlcr/activities/lookupjoin/thlookupjoinslave.cpp

@@ -97,6 +97,7 @@ class CBroadcaster : public CSimpleInterface
     InterruptableSemaphore allDoneSem;
     CriticalSection allDoneLock, bcastOtherCrit;
     bool allDone, allDoneWaiting, allRequestStop, stopping, stopRecv;
+    broadcast_flags stopFlag;
     Owned<IBitSet> slavesDone, slavesStopping;
 
     class CRecv : implements IThreaded
@@ -367,6 +368,7 @@ public:
         slavesStopping.setown(createThreadSafeBitSet());
         mpTag = TAG_NULL;
         recvInterface = NULL;
+        stopFlag = bcastflag_null;
     }
     void start(IBCastReceive *_recvInterface, mptag_t _mpTag, bool _stopping)
     {
@@ -382,6 +384,7 @@ public:
     void reset()
     {
         allDone = allDoneWaiting = allRequestStop = stopping = false;
+        stopFlag = bcastflag_null;
         slavesDone->reset();
         slavesStopping->reset();
     }
@@ -435,10 +438,22 @@ public:
     {
         return slavesStopping->test(myNode-1);
     }
+    broadcast_flags queryStopFlag() { return stopFlag; }
+    bool stopRequested()
+    {
+        if (bcastflag_null != queryStopFlag()) // if this node has requested to stop immediately
+            return true;
+        return allRequestStop; // if not, if all have request to stop
+    }
     void setStopping()
     {
         slavesStopping->set(myNode-1, true);
     }
+    void stop(broadcast_flags flag)
+    {
+        setStopping();
+        stopFlag = flag;
+    }
 };
 
 /* CMarker processes a sorted set of rows, comparing every adjacent row.
@@ -923,17 +938,17 @@ protected:
                         throw MakeActivityException(this, 0, "Out of memory: Unable to add any more rows to RHS");
 
                     rightSerializer->serialize(mbser, (const byte *)row.get());
-                    if (mb.length() >= MAX_SEND_SIZE || broadcaster.isStopping())
+                    if (mb.length() >= MAX_SEND_SIZE || broadcaster.stopRequested())
                         break;
                 }
                 if (0 == mb.length())
                     break;
-                if (broadcaster.isStopping())
-                    sendItem->setFlag(bcastflag_spilt);
+                if (broadcaster.stopRequested())
+                    sendItem->setFlag(broadcaster.queryStopFlag());
                 ThorCompress(mb, sendItem->queryMsg());
                 if (!broadcaster.send(sendItem))
                     break;
-                if (broadcaster.isStopping())
+                if (broadcaster.stopRequested())
                     break;
                 mb.clear();
                 broadcaster.resetSendItem(sendItem);
@@ -946,8 +961,8 @@ protected:
         }
 
         sendItem.setown(broadcaster.newSendItem(bcast_stop));
-        if (broadcaster.isStopping())
-            sendItem->setFlag(bcastflag_spilt);
+        if (broadcaster.stopRequested())
+            sendItem->setFlag(broadcaster.queryStopFlag());
         ActPrintLog("Sending final RHS broadcast packet");
         broadcaster.send(sendItem); // signals stop to others
     }
@@ -1445,7 +1460,7 @@ protected:
         setLocalLookup(true);
         ActPrintLog("Clearing non-local rows - cause: %s", msg);
 
-        broadcaster.setStopping(); // signals to broadcast to start stopping
+        broadcaster.stop(bcastflag_spilt); // signals to broadcast to start stopping immediately and to signal spilt to others
 
         rowidx_t clearedRows = 0;
         if (rhsCollated)