Merge remote-tracking branch 'origin/candidate-3.10.x'

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

docs/UsingConfigManager/UsingConfigManager.xml

@@ -1483,9 +1483,9 @@ sudo -u hpcc cp /etc/HPCCSystems/source/NewEnvironment.xml /etc/HPCCSystems/envi
                     </row>
 
                     <row>
-                      <entry><emphasis>maxCompileThreads</emphasis></entry>
+                      <entry><emphasis>maxEclccProcesses</emphasis></entry>
 
-                      <entry>The maximum number of compile threads</entry>
+                      <entry>The maximum number of eclcc processes that will be launched in parallel</entry>
                     </row>
 
                     <row>

ecl/eclccserver/eclccserver.cpp

@@ -574,7 +574,10 @@ int main(int argc, const char *argv[])
         getEclCCServerQueueNames(queueNames, globals->queryProp("@name"));
         if (!queueNames.length())
             throw MakeStringException(0, "No clusters found to listen on");
-        EclccServer server(queueNames.str(), globals->getPropInt("@maxCompileThreads", 1));
+        // The option has been renamed to avoid confusion with the similarly-named eclcc option, but
+        // still accept the old name if the new one is not present.
+        unsigned maxThreads = globals->getPropInt("@maxEclccProcesses", globals->getPropInt("@maxCompileThreads", 4));
+        EclccServer server(queueNames.str(), maxThreads);
         // if we got here, eclserver is successfully started and all options are good, so create the "sentinel file" for re-runs from the script
         // put in its own "scope" to force the flush
         writeSentinelFile(sentinelFile);

ecl/hqlcpp/hqlcpp.cpp

@@ -8812,9 +8812,11 @@ void HqlCppTranslator::doBuildAssignHashElement(BuildCtx & ctx, HashCodeCreator
             //fallthrough
             if (creator.optimize() && hasOutOfLineRows(elem->queryType()))
             {
+                creator.beginCondition(ctx);
                 BuildCtx iterctx(ctx);
                 BoundRow * row = buildDatasetIterate(iterctx, elem, false);
                 doBuildAssignHashElement(iterctx, creator, elem->queryNormalizedSelector(), elem->queryRecord());
+                creator.endCondition(iterctx);
                 return;
             }
             else

ecl/regress/dedupall5.ecl

@@ -0,0 +1,33 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+idRecord := { unsigned id; };
+
+namesRecord :=
+            RECORD
+string        surname;
+string        forename;
+dataset(idRecord) ids;
+            END;
+
+namesTable := dataset([
+    {'Fred', 'Smith', [1,2,3]},
+    {'Fred', 'Smith', [1,2,4]},
+    {'John', 'Smith', [4,5,6]},
+    {'John', 'Smith', [4,5,6]}], namesRecord);
+
+output(dedup(NOFOLD(namesTable), ALL));

esp/eclwatch/ws_XSLT/wuidcommon.xslt

@@ -1244,6 +1244,19 @@
           </a>
         </td>
       </xsl:if>
+      <xsl:if test="(Type = 'xml') or (Type = 'hint')">
+        <td>
+          <a href="/esp/iframe?esp_iframe_title=ECL Workunit - {$wuid} - {Description}&amp;inner=
+                   /WsWorkunits/WUFile%3fWuid%3d{$wuid}%26Name%3d{Name}%26IPAddress%3d{IPAddress}%26Description%3d{Description}%26Type%3dXML" >
+            <xsl:value-of select="Description"/>
+          </a>
+        </td>
+        <td>
+          <a href="javascript:void(0)" onclick="getOptions('{Description}', '/WsWorkunits/WUFile/{Name}?Wuid={$wuid}&amp;Name={Name}&amp;IPAddress={IPAddress}&amp;Description={Description}&amp;Type=xml', false); return false;">
+            download
+          </a>
+        </td>
+      </xsl:if>
       <xsl:if test="Type = 'res'">
         <td>
           <a href="/WsWorkunits/WUFile/res.txt?Wuid={$wuid}&amp;Type=res">res.txt</a>

esp/services/ws_workunits/ws_workunitsHelpers.cpp

@@ -436,6 +436,30 @@ void WsWuInfo::getTimers(IEspECLWorkunit &info, unsigned flags)
     }
 }
 
+struct mapEnums { int val; const char *str; };
+
+mapEnums queryFileTypes[] = {
+   { FileTypeCpp, "cpp" },
+   { FileTypeDll, "dll" },
+   { FileTypeResText, "res" },
+   { FileTypeHintXml, "hint" },
+   { FileTypeXml, "xml" },
+   { FileTypeSize,  NULL },
+};
+
+const char *getEnumText(int value, mapEnums *map)
+{
+    const char *defval = map->str;
+    while (map->str)
+    {
+        if (value==map->val)
+            return map->str;
+        map++;
+    }
+    assertex(!"Unexpected value in setEnum");
+    return defval;
+}
+
 void WsWuInfo::getHelpers(IEspECLWorkunit &info, unsigned flags)
 {
     try
@@ -478,9 +502,8 @@ void WsWuInfo::getHelpers(IEspECLWorkunit &info, unsigned flags)
         }
 
         IArrayOf<IEspECLHelpFile> helpers;
-        getHelpFiles(query, FileTypeCpp, helpers);
-        getHelpFiles(query, FileTypeDll, helpers);
-        getHelpFiles(query, FileTypeResText, helpers);
+        for (unsigned i = 0; i < FileTypeSize; i++)
+            getHelpFiles(query, (WUFileType) i, helpers);
 
         getWorkunitThorLogInfo(helpers, info);
 
@@ -1455,19 +1478,7 @@ void WsWuInfo::getHelpFiles(IConstWUQuery* query, WUFileType type, IArrayOf<IEsp
         cur.getName(name);
         Owned<IEspECLHelpFile> h= createECLHelpFile("","");
         h->setName(name.str());
-
-        switch (type)
-        {
-            case FileTypeCpp:
-                h->setType("cpp");
-                break;
-            case FileTypeDll:
-                h->setType("dll");
-                break;
-            default:
-                h->setType("res");
-                break;
-        }
+        h->setType(getEnumText(type, queryFileTypes));
 
         if (version > 1.31)
         {
@@ -1890,6 +1901,41 @@ void WsWuInfo::getWorkunitCpp(const char *cppname, const char* description, cons
     appendIOStreamContent(buf, ios.get(), forDownload);
 }
 
+void WsWuInfo::getWorkunitAssociatedXml(const char* name, const char* ipAddress, const char* plainText,
+                                        const char* description, bool forDownload, MemoryBuffer& buf)
+{
+    if (isEmpty(description)) //'File Name' as shown in WU Details page
+        throw MakeStringException(ECLWATCH_INVALID_INPUT, "File not specified.");
+    if (isEmpty(ipAddress))
+        throw MakeStringException(ECLWATCH_INVALID_INPUT, "File location not specified.");
+    if (isEmpty(name)) //file name with full path
+        throw MakeStringException(ECLWATCH_INVALID_FILE_NAME, "File path not specified.");
+
+    RemoteFilename rfn;
+    rfn.setRemotePath(name);
+    SocketEndpoint ep(ipAddress);
+    rfn.setIp(ep);
+
+    Owned<IFile> rFile = createIFile(rfn);
+    if (!rFile)
+        throw MakeStringException(ECLWATCH_CANNOT_OPEN_FILE, "Cannot open %s.", description);
+    OwnedIFileIO rIO = rFile->openShared(IFOread,IFSHfull);
+    if (!rIO)
+        throw MakeStringException(ECLWATCH_CANNOT_READ_FILE,"Cannot read %s.", description);
+    OwnedIFileIOStream ios = createBufferedIOStream(rIO);
+    if (!ios)
+        throw MakeStringException(ECLWATCH_CANNOT_READ_FILE,"Cannot read %s.", description);
+
+    const char* header;
+    if (plainText && (!stricmp(plainText, "yes")))
+        header = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>";
+    else
+        header = "<?xml version=\"1.0\" encoding=\"UTF-8\"?><?xml-stylesheet href=\"../esp/xslt/xmlformatter.xsl\" type=\"text/xsl\"?>";
+
+    buf.append(strlen(header), header);
+    appendIOStreamContent(buf, ios.get(), forDownload);
+}
+
 
 
 

esp/services/ws_workunits/ws_workunitsHelpers.hpp

@@ -173,6 +173,7 @@ public:
     void getWorkunitArchiveQuery(MemoryBuffer& buf);
     void getWorkunitDll(StringBuffer &name, MemoryBuffer& buf);
     void getWorkunitXml(const char* plainText, MemoryBuffer& buf);
+    void getWorkunitAssociatedXml(const char* name, const char* IPAddress, const char* plainText, const char* description, bool forDownload, MemoryBuffer& buf);
     void getWorkunitCpp(const char* cppname, const char* description, const char* ipAddress, MemoryBuffer& buf, bool forDownload);
     void getEventScheduleFlag(IEspECLWorkunit &info);
     unsigned getWorkunitThorLogInfo(IArrayOf<IEspECLHelpFile>& helpers, IEspECLWorkunit &info);

esp/services/ws_workunits/ws_workunitsService.cpp

@@ -2057,7 +2057,7 @@ void doWUQueryWithSort(IEspContext &context, IEspWUQueryRequest & req, IEspWUQue
         if (results.length() > (aindex_t)pagesize)
             results.pop();
 
-        if(begin + pagesize < numWUs)
+        if(unsigned (begin + pagesize) < numWUs)
         {
             resp.setNextPage(begin + pagesize);
             resp.setPageEndAt(begin + pagesize);
@@ -2649,6 +2649,18 @@ bool CWsWorkunitsEx::onWUFile(IEspContext &context,IEspWULogFileRequest &req, IE
                 winfo.getWorkunitEclAgentLog(req.getName(), mb);
                 openSaveFile(context, opt, "eclagent.log", HTTP_TYPE_TEXT_PLAIN, mb, resp);
             }
+            else if (strieq(File_XML,req.getType()) && notEmpty(req.getName()))
+            {
+                const char* name  = req.getName();
+                const char* ptr = strrchr(name, '/');
+                if (ptr)
+                    ptr++;
+                else
+                    ptr = name;
+
+                winfo.getWorkunitAssociatedXml(name, req.getIPAddress(), req.getPlainText(), req.getDescription(), opt > 0, mb);
+                openSaveFile(context, opt, ptr, HTTP_TYPE_APPLICATION_XML, mb, resp);
+            }
             else if (strieq(File_XML,req.getType()))
             {
                 winfo.getWorkunitXml(req.getPlainText(), mb);

initfiles/componentfiles/configxml/eclccserver.xsd

@@ -192,10 +192,10 @@
                 </xs:annotation>
             </xs:attribute>
             <xs:attribute name="traceLevel" type="xs:nonNegativeInteger" use="optional" default="1"/>
-            <xs:attribute name="maxCompileThreads" type="xs:nonNegativeInteger" use="optional" default="4">
+            <xs:attribute name="maxEclccProcesses" type="xs:nonNegativeInteger" use="optional" default="4">
                 <xs:annotation>
                     <xs:appinfo>
-                        <tooltip>Number of compile threads.</tooltip>
+                        <tooltip>Maximum number of instances of eclcc that will be launched in parallel.</tooltip>
                     </xs:appinfo>
                 </xs:annotation>
             </xs:attribute>

initfiles/etc/DIR_NAME/environment.xml.in

@@ -387,7 +387,7 @@
                     daliServers="mydali"
                     description="EclCCServer process"
                     enableSysLog="true"
-                    maxCompileThreads="4"
+                    maxEclccProcesses="4"
                     name="myeclccserver"
                     traceLevel="1">
    <Instance computer="localhost"