Merge branch 'candidate-6.2.0' into candidate-6.4.0

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

.gitmodules

@@ -43,3 +43,6 @@
 [submodule "libcouchbase"]
 	path = plugins/couchbase/libcouchbase
 	url = https://github.com/hpcc-systems/libcouchbase.git
+[submodule "plugins/redis/hiredis"]
+	path = plugins/redis/hiredis
+	url = https://github.com/hpcc-systems/hiredis.git

common/deftype/deftype.cpp

@@ -3679,7 +3679,7 @@ void XmlSchemaBuilder::getXmlTypeName(StringBuffer & xmlType, ITypeInfo & type)
             type.getECLType(typesXml);
             typesXml.append("\"><xs:restriction base=\"xs:decimal\">");
             typesXml.appendf("<xs:totalDigits value=\"%d\"/>", type.getDigits());
-            typesXml.appendf("<xs:fractionalDigits value=\"%d\" fixed=\"true\"/>", type.getPrecision());
+            typesXml.appendf("<xs:fractionDigits value=\"%d\" fixed=\"true\"/>", type.getPrecision());
             typesXml.append("</xs:restriction></xs:simpleType>").newline();
         }
         break;

docs/ECLLanguageReference/ECLR_mods/Value-Unicode.xml

@@ -6,7 +6,7 @@
 
   <para><emphasis role="bold">UNICODE<indexterm>
       <primary>UNICODE</primary>
-    </indexterm>[</emphasis><emphasis>locale</emphasis><emphasis
+    </indexterm>[</emphasis><emphasis>_locale</emphasis><emphasis
   role="bold">][</emphasis><emphasis>n</emphasis><emphasis
   role="bold">]<indexterm>
       <primary>UNICODE value type</primary>
@@ -30,8 +30,8 @@
         // utf-16-encoded string
 UNICODE4 MyUnicodeString := U'abcd';
         // same as: (UNICODE)'abcd'
-UNICODEde5 MyUnicodeString := U'abcd\353';
+UNICODE_de5 MyUnicodeString := U'abcd\353';
         // becomes 'abcdë' with a German locale
-UNICODEde5 MyUnicodeString := U'abcdë';
+UNICODE_de5 MyUnicodeString := U'abcdë';
         // same as previous example</programlisting>
 </sect1>

docs/HPCCClientTools/CT_Mods/CT_ECL_IDE.xml

@@ -210,22 +210,20 @@
             <listitem>
               <para>Specify your Working Folder.</para>
 
-              <para>When you are running your locally, your queries are
+              <para>When you submit to a Local target, your queries are
               compiled and stored in this location.</para>
 
-              <para>To compile a query locally rather than on the thor or
-              hthor of your environment, select <emphasis
-              role="bold">Local</emphasis> as the <emphasis
-              role="bold">Target</emphasis> before pressing <emphasis
-              role="bold">Submit</emphasis> on the <emphasis
+              <para>To compile a query locally rather than on a cluster in
+              your environment, select <emphasis role="bold">Local</emphasis>
+              as the <emphasis role="bold">Target</emphasis> before pressing
+              <emphasis role="bold">Submit</emphasis> on the <emphasis
               role="bold">Builder</emphasis> window.</para>
 
               <!-- -->
 
               <para>If you are running under Windows and want to compile
               locally, install the Microsoft VS 2008 C++ compiler (either
-              Express or Professional edition) and Linux users need
-              GCC.</para>
+              Express or Professional edition). Linux users need GCC.</para>
             </listitem>
 
             <listitem>

docs/HPCCMonitoring/HPCCMonitoringAndReporting.xml

@@ -148,7 +148,6 @@
           <para>System Load</para>
         </listitem>
       </itemizedlist></para>
-
   </chapter>
 
   <chapter id="Ganglya_Overview">
@@ -1191,6 +1190,18 @@
                   </row>
 
                   <row>
+                    <entry>-set_esp_username_pw</entry>
+
+                    <entry>Set specific login credentials for ESP checks. All
+                    fields are required (esp name, user name, password). Can
+                    be specified more than once to support multiple ESP
+                    servers.</entry>
+
+                    <entry><para>&lt;esp name&gt;</para><para>&lt;user
+                    name&gt;</para><para>&lt;password&gt;</para></entry>
+                  </row>
+
+                  <row>
                     <entry>-override_check_all_disks</entry>
 
                     <entry>check_all_disk plugin name</entry>
@@ -1421,11 +1432,8 @@
       </sect2>
     </sect1>
 
-        
-            <xi:include href="HPCCMonitoring/MonRep-Mods/NagiosInECLWa.xml"
-                        xpointer="Nagios_in_ECLWatch"
-                        xmlns:xi="http://www.w3.org/2001/XInclude" />
-        
-
+    <xi:include href="HPCCMonitoring/MonRep-Mods/NagiosInECLWa.xml"
+                xpointer="Nagios_in_ECLWatch"
+                xmlns:xi="http://www.w3.org/2001/XInclude" />
   </chapter>
 </book>

docs/HPCCSystemAdmin/HPCCSystemAdministratorsGuide.xml

@@ -1109,6 +1109,32 @@ lock=/var/lock/HPCCSystems</programlisting>
             </varlistentry>
           </variablelist></para>
       </sect2>
+
+      <sect2 id="ConfiguringRemoteAccessOverTLS">
+        <title>Remote Access over TLS</title>
+
+        <para>Configuring your system for remote file access over Transport
+        Layer Security (TLS) requires modifying the <emphasis
+        role="bold">dafilesrv</emphasis> setting in the
+        <emphasis>environment.conf</emphasis> file. </para>
+
+        <para>To do this either uncomment (if they are already there), or add
+        the following lines to the <emphasis>environment.conf</emphasis> file.
+        Then set the values as appropriate for your system. </para>
+
+        <para><programlisting>#enable SSL for dafilesrv remote file access
+dfsUseSSL=true
+dfsSSLCertFile=/certfilepath/certfile
+dfsSSLPrivateKeyFile=/keyfilepath/keyfile</programlisting>Set the <emphasis
+        role="blue">dfsUseSSL=true</emphasis> and set the value for the paths
+        to point to the certificate and key file paths on your system. Then
+        deploy the <emphasis>environment.conf</emphasis> file (and cert/key
+        files) to all nodes as appropriate. </para>
+
+        <para>When dafilesrv is enabled for TLS (port 7600), it can still
+        connect over a non-TLS connection (port 7100) to allow legacy clients
+        to work.</para>
+      </sect2>
     </sect1>
 
     <!--Inclusions-As-Sect1-->
@@ -1199,6 +1225,7 @@ lock=/var/lock/HPCCSystems</programlisting>
           </variablelist></para>
       </sect2>
     </sect1>
+
     <xi:include href="HPCCSystemAdmin/SA-Mods/CassandraWUServer.xml"
                 xpointer="CassandraWUStorage"
                 xmlns:xi="http://www.w3.org/2001/XInclude" />
@@ -1761,12 +1788,11 @@ heapUseTransparentHugePages</programlisting>
       </sect2>
     </sect1>
 
-     <xi:include href="RoxieReference/RoxieRefMods/RoxieCapacityPlanning.xml"
+    <xi:include href="RoxieReference/RoxieRefMods/RoxieCapacityPlanning.xml"
                 xpointer="Capacity_Planning"
                 xmlns:xi="http://www.w3.org/2001/XInclude" />
- 
 
-     <xi:include href="HPCCSystemAdmin/SA-Mods/SysAdminConfigMod.xml"
+    <xi:include href="HPCCSystemAdmin/SA-Mods/SysAdminConfigMod.xml"
                 xpointer="Sample_Sizings"
                 xmlns:xi="http://www.w3.org/2001/XInclude" />
   </chapter>

docs/Installing_and_RunningTheHPCCPlatform/Inst-Mods/hpcc_ldap.xml

@@ -297,6 +297,13 @@
     role="bold">initldap</emphasis> utility to create the required default
     HPCC Admin user on your LDAP server.</para>
 
+    <para>If you choose to use LDAP authentication you must enable LDAP
+    security in your HPCC System configuration. With LDAP security enabled on
+    your system you can then choose to enable file scope security. You can
+    choose to use LDAP authentication without enabling file scope security.
+    The following sections describe how to enable LDAP authentication and file
+    scope security for your HPCC system.</para>
+
     <!--***Note: (9/2014) Adding Documentation for initLdap.*** -->
 
     <sect3 id="UsingLDAP_ConnectToConfigMgr">
@@ -606,9 +613,8 @@
       <listitem>
         <?dbfo keep-together="always"?>
 
-        <para>To enable the file scope permissions, configure security for the
-        Dali Server. If you are not interested in file scope permissions you
-        can skip this step.</para>
+        <para>To enable the file scope permissions, configure the file scope
+        security for the Dali Server.</para>
 
         <para>In the Navigator pane, click on the <emphasis role="bold">Dali
         Server – mydali </emphasis><graphic
@@ -628,8 +634,35 @@
             </listitem>
 
             <listitem>
+              <para>Set the <emphasis role="bold">checkScopeScans</emphasis>
+              value to <emphasis>true</emphasis>.</para>
+
+              <para>Only set this value to true when you want file scope
+              security enabled. Security settings can have three
+              states.</para>
+
+              <itemizedlist>
+                <listitem>
+                  <para>None, no authentication and no file scope
+                  security.</para>
+                </listitem>
+
+                <listitem>
+                  <para>LDAP security for authentication only, without
+                  enabling file scope security.</para>
+                </listitem>
+
+                <listitem>
+                  <para>LDAP authentication and file scope security
+                  enabled.</para>
+                </listitem>
+              </itemizedlist>
+            </listitem>
+
+            <listitem>
               <para>Change the LDAP values as appropriate to match the
-              settings in your LDAP server.</para>
+              settings in your LDAP server component in configuration
+              manager.</para>
 
               <para>For example, change the <emphasis
               role="bold">ldapServer</emphasis> to the value you gave your
@@ -643,8 +676,8 @@
               are supplied. This is similar to a guest account, so it should
               be an account with <emphasis role="bold">very</emphasis> limited
               access, if used at all. To disable access without credentials,
-              leave <emphasis role="bold">filesDefaultUser</emphasis> blank.
-              </para>
+              leave <emphasis role="bold">filesDefaultUser</emphasis>
+              blank.</para>
 
               <para>The <emphasis role="bold">filesDefaultPassword</emphasis>
               is the password for that account.</para>
@@ -672,14 +705,17 @@
 
             <listitem>
               <para>Locate the <emphasis role="bold">ldapUser</emphasis> field
-              and verify that there is a "<emphasis>roxie</emphasis>"
+              and verify that there is a valid HPCC user who is a member of
+              the Authenticated Users group on your LDAP server. For example,
+              the "<emphasis>roxie</emphasis>" user assumes that the
+              "<emphasis>roxie</emphasis>" user is a valid HPCC authenticated
               user.</para>
             </listitem>
 
             <listitem>
-              <para>You can add password security for Roxie by adding it to
-              the <emphasis role="bold">ldapPassword</emphasis> field on the
-              same tab.</para>
+              <para>Add the password security for Roxie by adding it to the
+              <emphasis role="bold">ldapPassword</emphasis> field on the same
+              tab.</para>
             </listitem>
           </orderedlist></para>
       </listitem>
@@ -699,12 +735,12 @@
               fileref="../../images/caution.png" /></entry>
 
               <entry><para>In order to run Roxie queries with File Scope
-              security, ensure that the roxie user is created in the list of
+              security, ensure that a Roxie user is created in the list of
               authenticated users.</para>In the following section, <link
               linkend="Adding_Users"><emphasis>Adding and editing
-              users</emphasis></link>, add "<emphasis>roxie</emphasis>" as a
-              user and make sure the password is the same as the one entered
-              in Configuration Manager.</entry>
+              users</emphasis></link>, add the <emphasis>roxie</emphasis> user
+              and make sure that password is the same as the one entered in
+              Configuration Manager.</entry>
             </row>
           </tbody>
         </tgroup>

ecl/hqlcpp/hqlcppds.cpp

@@ -2431,12 +2431,11 @@ void HqlCppTranslator::doBuildDataset(BuildCtx & ctx, IHqlExpression * expr, CHq
             if (format == FormatLinkedDataset || format == FormatArrayDataset)
             {
                 IHqlExpression * choosenLimit = NULL;
-                if ((op == no_choosen) && !isChooseNAllLimit(expr->queryChild(1)) && !queryRealChild(expr, 2))
+                if ((op == no_choosen) && !isChooseNAllLimit(expr->queryChild(1)) && !queryRealChild(expr, 2) && (canEvaluateInline(&ctx, expr->queryChild(0)) || !canIterateInline(&ctx, expr->queryChild(0))))
                 {
                     choosenLimit = expr->queryChild(1);
                     expr = expr->queryChild(0);
                 }
-
                 //MORE: Extract limit and choosen and pass as parameters
                 builder.setown(createLinkedDatasetBuilder(record, choosenLimit));
             }
@@ -2808,7 +2807,7 @@ void HqlCppTranslator::buildDatasetAssign(BuildCtx & ctx, const CHqlBoundTarget
         else
         {
             IHqlExpression * choosenLimit = NULL;
-            if ((op == no_choosen) && !isChooseNAllLimit(expr->queryChild(1)) && !queryRealChild(expr, 2))
+            if ((op == no_choosen) && !isChooseNAllLimit(expr->queryChild(1)) && !queryRealChild(expr, 2) && (canEvaluateInline(&ctx, expr->queryChild(0)) || !canIterateInline(&ctx, expr->queryChild(0))))
             {
                 choosenLimit = expr->queryChild(1);
                 expr = expr->queryChild(0);
@@ -3824,20 +3823,25 @@ BoundRow * HqlCppTranslator::buildDatasetIterateChoosen(BuildCtx & ctx, IHqlExpr
             boundHigh.expr.setown(createValue(no_add, LINK(boundHigh.queryType()), LINK(boundHigh.expr), LINK(boundLow.expr)));
     }
 
-    BoundRow * cursor = buildDatasetIterate(ctx, expr->queryChild(0), needToBreak);
+    BoundRow * cursor = buildDatasetIterate(ctx, expr->queryChild(0), boundHigh.expr != nullptr);
 
     if (cursor)
     {
         OwnedHqlExpr inc = createValue(no_postinc, counter->getType(), LINK(counter));
         ctx.addExpr(inc);
 
-        OwnedHqlExpr cond;
         if (boundLow.expr)
-            extendConditionOwn(cond, no_and, createBoolExpr(no_gt, LINK(counter), LINK(boundLow.expr)));
-        if (boundHigh.expr)
-            extendConditionOwn(cond, no_and, createBoolExpr(no_le, LINK(counter), LINK(boundHigh.expr)));
-        if (cond)
+        {
+            OwnedHqlExpr cond = createBoolExpr(no_gt, LINK(counter), LINK(boundLow.expr));
             ctx.addFilter(cond);
+        }
+        if (boundHigh.expr)
+        {
+            BuildCtx breakctx(ctx);
+            OwnedHqlExpr cond = createBoolExpr(no_gt, LINK(counter), LINK(boundHigh.expr));
+            breakctx.addFilter(cond);
+            breakctx.addBreak();
+        }
     }
     return cursor;
 }

ecl/hqlcpp/hqlhtcpp.cpp

@@ -9598,7 +9598,7 @@ static IHqlExpression * forceToCorrectEncoding(IHqlExpression * expr, IAtom * en
             }
             return expr->clone(args);
         }
-        else
+        else if (!isNumericType(expr->queryType()))
         {
             OwnedHqlExpr cast = ensureExprType(expr, unknownUtf8Type);
             OwnedHqlExpr transfer = createValue(no_typetransfer, LINK(unknownStringType), LINK(cast));

ecl/hqlcpp/hqlttcpp.cpp

@@ -9415,9 +9415,12 @@ IHqlExpression * HqlLinkedChildRowTransformer::createTransformedBody(IHqlExpress
             }
         }
         break;
-    case no_attr:
     case no_embedbody:
-        //Don't change the type of an embed body - otherwise result it will become link counted when not expected.
+        if (expr->queryAttribute(languageAtom))
+            break;
+        //Don't change the type of an embedded C++ body - otherwise result it will become link counted when not expected.
+        // Fall into...
+    case no_attr:
         return LINK(expr);
     }
     return QuickHqlTransformer::createTransformedBody(expr);

ecl/regress/implicitlink.ecl

@@ -0,0 +1,33 @@
+IMPORT Python;
+
+nested := RECORD
+  integer value;
+END;
+
+parent := RECORD
+  DATASET(nested) nest;
+END;
+
+DATASET(parent) getP() := EMBED(Python)
+  return [
+          [
+           1,2,3,4
+          ],
+          [
+           5,6,7,8
+          ]
+         ]
+ENDEMBED;
+
+pcode(DATASET(parent) p) := EMBED(Python)
+  for child in p:
+   for c2 in child.nest:
+    print c2.value,
+ENDEMBED;
+
+ds := getp(); 
+//ds := DATASET([{[{1},{3}]}], parent);
+//ds;
+pcode(ds);
+
+

esp/logging/logginglib/datafieldmap.cpp

@@ -50,8 +50,8 @@ void readLogSourceCfg(IPropertyTree* cfg, unsigned& logSourceCount, StringAttr&
     ForEach(*iter)
     {
         ensureInputString(iter->query().queryProp("@name"), false, name, -1, "LogSource @name required");
-        ensureInputString(iter->query().queryProp("@maptologgroup"), true, groupName, -1, "LogSource @maptologgroup required");
-        ensureInputString(iter->query().queryProp("@maptodb"), true, dbName, -1, "LogSource @maptodb required");
+        ensureInputString(iter->query().queryProp("@mapToLogGroup"), true, groupName, -1, "LogSource @mapToLogGroup required");
+        ensureInputString(iter->query().queryProp("@mapToDB"), true, dbName, -1, "LogSource @mapToDB required");
         Owned<CLogSource> logSource = new CLogSource(name.str(), groupName.str(), dbName.str());
         logSources.setValue(name.str(), logSource);
         logSourceCount++;
@@ -70,7 +70,7 @@ void CLogTable::loadMappings(IPropertyTree& fieldList)
         IPropertyTree &map = itr->query();
 
         ensureInputString(map.queryProp("@name"), false, name, -1, "Field @name required");
-        ensureInputString(map.queryProp("@mapto"), true, mapTo, -1, "Field @mapto required");
+        ensureInputString(map.queryProp("@mapTo"), true, mapTo, -1, "Field @mapTo required");
         ensureInputString(map.queryProp("@type"), true, fieldType, -1, "Field @type required");
         defaultValue = map.queryProp("@default");
         defaultValue.trim();

esp/src/eclwatch/TopologyWidget.js

@@ -264,7 +264,7 @@ define([
                         disabled: function (item) {
                             if (item.__hpcc_treeItem) {
                                 if (context.viewModeTargets.checked) {
-                                    if (item.__hpcc_treeItem.Type === "HoleCluster" || item.__hpcc_treeItem.Type === "ThorCluster" || item.__hpcc_treeItem.Type === "RoxieCluster") {
+                                    if (item.__hpcc_treeItem.Type === "HoleCluster" || item.__hpcc_treeItem.Type === "ThorCluster" || item.__hpcc_treeItem.Type === "RoxieCluster" || item.hasLogs()) {
                                         return false;
                                     }
                                 }
@@ -274,7 +274,7 @@ define([
                                     }
                                 }
                                 if (context.viewModeMachines.checked) {
-                                    if (item.__hpcc_children.length > 0) {
+                                    if (item.__hpcc_children.length > 0 || item.hasLogs()) {
                                         return false;
                                     }
                                 }
@@ -306,7 +306,7 @@ define([
                 var selection = context.grid.getSelected();
                 if (selection.length) {
                     context.detailsWidget.init(selection[0]);
-                    if (context.viewModeTargets.checked === true && selection[0].__hpcc_parentNode) {
+                    if (context.viewModeTargets.checked === true && selection[0].__hpcc_parentNode && selection[0].hasLogs().length === 0) {
                         context.filter.disable(false);
                     } else if (context.viewModeServices.checked === true && selection[0].__hpcc_parentNode) {
                         context.filter.disable(false);

initfiles/componentfiles/configxml/@temp/CMakeLists.txt

@@ -24,6 +24,7 @@ FOREACH ( iFILES
     ${CMAKE_CURRENT_SOURCE_DIR}/roxiePlugins.xsl
     ${CMAKE_CURRENT_SOURCE_DIR}/esp_service_DynamicESDL.xsl
     ${CMAKE_CURRENT_SOURCE_DIR}/esp_service_wslogging.xsl
+    ${CMAKE_CURRENT_SOURCE_DIR}/logging_agent.xsl
 )
     Install ( FILES ${iFILES} DESTINATION componentfiles/configxml/@temp COMPONENT Runtime )
 ENDFOREACH ( iFILES )

initfiles/componentfiles/configxml/@temp/esp_service_DynamicESDL.xsl

@@ -19,6 +19,7 @@
 
 <xsl:stylesheet version="1.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform" xml:space="default" xmlns:seisint="http://seisint.com"  xmlns:set="http://exslt.org/sets" exclude-result-prefixes="seisint set">
     <xsl:import href="esp_service.xsl"/>
+    <xsl:import href="logging_agent.xsl"/>
 
     <xsl:template match="EspService">
         <xsl:param name="bindingNode"/>
@@ -100,7 +101,7 @@
                         <xsl:message terminate="yes">ESP NetAddress is undefined!</xsl:message>
                     </xsl:if>
                     <xsl:variable name="wsloggingUrl"><xsl:text>http://</xsl:text><xsl:value-of select="$espNetAddress"/><xsl:text>:</xsl:text><xsl:value-of select="$espPort"/></xsl:variable>
-                        <LogAgent name="{$agentName}" type="LogAgent" services="GetTransactionSeed,UpdateLog" plugin="espserverloggingagent">
+                        <LogAgent name="{$agentName}" type="LogAgent" services="GetTransactionSeed,UpdateLog,GetTransactionID" plugin="espserverloggingagent">
                             <ESPServer url="{$wsloggingUrl}" user="{$agentNode/@User}" password="{$agentNode/@Password}"/>
                             <xsl:if test="string($agentNode/@MaxServerWaitingSeconds) != ''">
                                 <MaxServerWaitingSeconds><xsl:value-of select="$agentNode/@MaxServerWaitingSeconds"/></MaxServerWaitingSeconds>
@@ -123,6 +124,11 @@
                             <xsl:if test="string($agentNode/@QueueSizeSignal) != ''">
                                 <QueueSizeSignal><xsl:value-of select="$agentNode/@QueueSizeSignal"/></QueueSizeSignal>
                             </xsl:if>
+
+                            <xsl:call-template name="LogSourceMap">
+                                <xsl:with-param name="agentNode" select="$agentNode"/>
+                            </xsl:call-template>
+
                             <Filters>
                                 <xsl:for-each select="$agentNode/Filter">
                                     <Filter value="{current()/@filter}" type="{current()/@type}"/>

initfiles/componentfiles/configxml/@temp/esp_service_wslogging.xsl

@@ -1,7 +1,27 @@
 <?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+################################################################################
+#    HPCC SYSTEMS software Copyright (C) 2016 HPCC Systems®.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+################################################################################
+-->
+
 <xsl:stylesheet version="1.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform" xml:space="default"
 xmlns:seisint="http://seisint.com"  xmlns:set="http://exslt.org/sets" exclude-result-prefixes="seisint set">
     <xsl:import href="esp_service.xsl"/>
+    <xsl:import href="logging_agent.xsl"/>
 
     <xsl:template match="EspService">
         <xsl:param name="authNode"/>
@@ -19,85 +39,10 @@ xmlns:seisint="http://seisint.com"  xmlns:set="http://exslt.org/sets" exclude-re
         <xsl:variable name="bindName" select="concat('WsLoggingService', '_', $bindingNode/@name, '_', $process)"/>
 
         <EspService name="{$serviceName}" type="{$serviceType}" plugin="{$servicePlugin}">
-            <xsl:if test="string(@LoggingManager) != ''">
-                <xsl:variable name="managerName" select="@LoggingManager"/>
-                <xsl:variable name="managerNode" select="/Environment/Software/LoggingManager[@name=$managerName]"/>
-
-                <xsl:if test="not($managerNode)">
-                    <xsl:message terminate="yes">Logging Manager is undefined!</xsl:message>
-                </xsl:if>
-
-                <xsl:if test="not($managerNode/ESPLoggingAgent[1]/@ESPLoggingAgent)">
-                     <xsl:message terminate="yes">ESP Logging Agent <xsl:value-of select="$managerNode/@ESPLoggingAgent[1]"/>is undefined!</xsl:message>
-                </xsl:if>
-
-                <LoggingManager name="{$managerNode/@name}">
-
-                <xsl:for-each select="$managerNode/ESPLoggingAgent">
-                    <xsl:variable name="agentName" select="@ESPLoggingAgent"/>
-                    <xsl:variable name="agentNode" select="/Environment/Software/ESPLoggingAgent[@name=$agentName]"/>
-
-                    <xsl:if test="not($agentNode)">
-                        <xsl:message terminate="yes">An ESP Logging Agent <xsl:value-of select="$agentName"/>  for <xsl:value-of select="$managerNode/@name"/> is undefined!</xsl:message>
-                    </xsl:if>
-
-                    <xsl:if test="string($agentNode/@ESPServer) = ''">
-                        <xsl:message terminate="yes">ESP server is undefined for <xsl:value-of select="$agentName"/> </xsl:message>
-                    </xsl:if>
-
-                    <xsl:variable name="espServer" select="$agentNode/@ESPServer"/>
-                    <xsl:variable name="espNode" select="/Environment/Software/EspProcess[@name=$espServer]"/>
-
-                    <xsl:if test="not($espNode)">
-                        <xsl:message terminate="yes">ESP process is undefined!</xsl:message>
-                    </xsl:if>
-
-                    <xsl:variable name="espPort" select="$espNode/EspBinding[@service='wslogging']/@port"/>
-
-                    <xsl:if test="string($espPort) = ''">
-                        <xsl:message terminate="yes">ESP server port is undefined for <xsl:value-of select="$espServer"/>!</xsl:message>
-                    </xsl:if>
-
-                    <xsl:variable name="espNetAddress" select="$espNode/Instance/@netAddress"/>
-
-                    <xsl:if test="string($espNetAddress) = ''">
-                        <xsl:message terminate="yes">ESP NetAddress is undefined!</xsl:message>
-                    </xsl:if>
-                    <xsl:variable name="wsloggingUrl"><xsl:text>http://</xsl:text><xsl:value-of select="$espNetAddress"/><xsl:text>:</xsl:text><xsl:value-of select="$espPort"/></xsl:variable>
-                        <LogAgent name="{$agentName}" type="LogAgent" services="GetTransactionSeed,UpdateLog" plugin="espserverloggingagent">
-                            <ESPServer url="{$wsloggingUrl}" user="{$agentNode/@User}" password="{$agentNode/@Password}"/>
-                            <xsl:if test="string($agentNode/@MaxServerWaitingSeconds) != ''">
-                                <MaxServerWaitingSeconds><xsl:value-of select="$agentNode/@MaxServerWaitingSeconds"/></MaxServerWaitingSeconds>
-                            </xsl:if>
-                            <xsl:if test="string($agentNode/@FailSafe) != ''">
-                                <FailSafe><xsl:value-of select="$agentNode/@FailSafe"/></FailSafe>
-                            </xsl:if>
-                            <xsl:if test="string($agentNode/@FailSafeLogsDir) != ''">
-                                <FailSafeLogsDir><xsl:value-of select="$agentNode/@FailSafeLogsDir"/></FailSafeLogsDir>
-                            </xsl:if>
-                            <xsl:if test="string($agentNode/@MaxLogQueueLength) != ''">
-                                <MaxLogQueueLength><xsl:value-of select="$agentNode/@MaxLogQueueLength"/></MaxLogQueueLength>
-                            </xsl:if>
-                            <xsl:if test="string($agentNode/@MaxTriesGTS) != ''">
-                                <MaxTriesGTS><xsl:value-of select="$agentNode/@MaxTriesGTS"/></MaxTriesGTS>
-                            </xsl:if>
-                            <xsl:if test="string($agentNode/@MaxTriesRS) != ''">
-                                <MaxTriesRS><xsl:value-of select="$agentNode/@MaxTriesRS"/></MaxTriesRS>
-                            </xsl:if>
-                            <xsl:if test="string($agentNode/@QueueSizeSignal) != ''">
-                                <QueueSizeSignal><xsl:value-of select="$agentNode/@QueueSizeSignal"/></QueueSizeSignal>
-                            </xsl:if>
-                            <Filters>
-                                <xsl:for-each select="$agentNode/Filter">
-                                    <Filter value="{current()/@filter}" type="{current()/@type}"/>
-                                </xsl:for-each>
-                            </Filters>
-                        </LogAgent>
-                </xsl:for-each>
-
-                </LoggingManager>
-
+            <xsl:if test="string(CassandraLoggingAgents[1]/@CassandraLoggingAgent) = ''">
+                <xsl:message terminate="yes">Logging Agent is undefined!</xsl:message>
             </xsl:if>
+            <xsl:call-template name="CassandraLoggingAgent"/>
         </EspService>
 
         <EspBinding name="{$bindName}" service="{$serviceName}" protocol="{$bindingNode/@protocol}" type="{$bindType}" plugin="{$servicePlugin}" netAddress="0.0.0.0" port="{$bindingNode/@port}">

initfiles/componentfiles/configxml/@temp/logging_agent.xsl

@@ -0,0 +1,129 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+################################################################################
+#    HPCC SYSTEMS software Copyright (C) 2016 HPCC Systems®.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+################################################################################
+-->
+
+<xsl:stylesheet version="1.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform" xml:space="default"
+xmlns:set="http://exslt.org/sets">
+    <xsl:template name="LogSourceMap">
+        <xsl:param name="agentNode"/>
+        <LogSourceMap>
+            <xsl:for-each select="$agentNode/LogSourceMap/LogSource">
+                <xsl:if test="string(current()/@name) = ''">
+                    <xsl:message terminate="yes">LogSource name is undefined!</xsl:message>
+                </xsl:if>
+                <xsl:if test="string(current()/@mapToDB) = ''">
+                    <xsl:message terminate="yes">LogSource mapToDB is undefined for <xsl:value-of select="current()/@name"/>!</xsl:message>
+                </xsl:if>
+                <xsl:if test="string(current()/@mapToLogGroup) = ''">
+                    <xsl:message terminate="yes">LogSource mapToLogGroup is undefined for <xsl:value-of select="current()/@name"/>!</xsl:message>
+                </xsl:if>
+                <LogSource name="{current()/@name}" maptodb="{current()/@mapToDB}" maptologgroup="{current()/@mapToLogGroup}"/>
+            </xsl:for-each>
+        </LogSourceMap>
+    </xsl:template>
+
+    <xsl:template name="LogGroup">
+        <xsl:param name="agentNode"/>
+        <xsl:for-each select="$agentNode/LogGroup">
+            <LogGroup name="{current()/@name}">
+                <xsl:for-each select="current()/Fieldmap">
+                    <Fieldmap table="{current()/@table}">
+                        <xsl:for-each select="current()/Field">
+                            <xsl:if test="string(current()/@name) = ''">
+                                <xsl:message terminate="yes">Field name is undefined!</xsl:message>
+                            </xsl:if>
+                            <xsl:if test="string(current()/@mapTo) = ''">
+                                <xsl:message terminate="yes">Field mapTo is undefined for <xsl:value-of select="current()/@name"/>!</xsl:message>
+                            </xsl:if>
+                            <xsl:if test="string(current()/@type) = ''">
+                                <xsl:message terminate="yes">Field type is undefined for <xsl:value-of select="current()/@name"/>!</xsl:message>
+                            </xsl:if>
+                            <xsl:choose>
+                                <xsl:when test="string(current()/@default) = ''">
+                                    <Field name="{current()/@name}" mapto="{current()/@mapTo}" type="{current()/@type}"/>
+                                </xsl:when>
+                                <xsl:otherwise>
+                                    <Field name="{current()/@name}" mapto="{current()/@mapTo}" type="{current()/@type}" default="{current()/@default}"/>
+                                </xsl:otherwise>
+                            </xsl:choose>
+                        </xsl:for-each>
+                    </Fieldmap>
+                </xsl:for-each>
+            </LogGroup>
+        </xsl:for-each>
+    </xsl:template>
+
+    <xsl:template name="CassandraLoggingAgent">
+        <xsl:for-each select="CassandraLoggingAgents">
+            <xsl:choose>
+                <xsl:when test="string(@CassandraLoggingAgent) = ''">
+                    <xsl:message terminate="yes">Cassandra Logging Agent name is undefined!</xsl:message>
+                </xsl:when>
+                <xsl:otherwise>
+                    <xsl:variable name="agentName" select="@CassandraLoggingAgent"/>
+                    <xsl:variable name="agentNode" select="/Environment/Software/CassandraLoggingAgent[@name=$agentName]"/>
+                    <xsl:if test="not($agentNode)">
+                        <xsl:message terminate="yes"><xsl:value-of select="$agentName"/> does not match with any Cassandra Logging Agent inside this environment!</xsl:message>
+                    </xsl:if>
+                    <xsl:if test="string($agentNode/@serverIP) = ''">
+                        <xsl:message terminate="yes">Cassandra server network address is undefined for <xsl:value-of select="$agentName"/>!</xsl:message>
+                    </xsl:if>
+                    <LogAgent name="{$agentName}" type="LogAgent" services="GetTransactionSeed,UpdateLog,GetTransactionID" plugin="cassandralogagent">
+                        <Cassandra server="{$agentNode/@serverIP}" dbUser="{$agentNode/@userName}" dbPassWord="{$agentNode/@userPassword}" dbName="{$agentNode/@ksName}"/>
+                        <xsl:if test="string($agentNode/@FailSafe) != ''">
+                            <FailSafe><xsl:value-of select="$agentNode/@FailSafe"/></FailSafe>
+                        </xsl:if>
+                        <xsl:if test="string($agentNode/@FailSafeLogsDir) != ''">
+                            <FailSafeLogsDir><xsl:value-of select="$agentNode/@FailSafeLogsDir"/></FailSafeLogsDir>
+                        </xsl:if>
+                        <xsl:if test="string($agentNode/@MaxLogQueueLength) != ''">
+                            <MaxLogQueueLength><xsl:value-of select="$agentNode/@MaxLogQueueLength"/></MaxLogQueueLength>
+                        </xsl:if>
+                        <xsl:if test="string($agentNode/@MaxTriesGTS) != ''">
+                            <MaxTriesGTS><xsl:value-of select="$agentNode/@MaxTriesGTS"/></MaxTriesGTS>
+                        </xsl:if>
+                        <xsl:if test="string($agentNode/@MaxTriesRS) != ''">
+                            <MaxTriesRS><xsl:value-of select="$agentNode/@MaxTriesRS"/></MaxTriesRS>
+                        </xsl:if>
+                        <xsl:if test="string($agentNode/@QueueSizeSignal) != ''">
+                            <QueueSizeSignal><xsl:value-of select="$agentNode/@QueueSizeSignal"/></QueueSizeSignal>
+                        </xsl:if>
+                        <xsl:if test="string($agentNode/@defaultTransaction) != ''">
+                            <defaultTransaction><xsl:value-of select="$agentNode/@defaultTransaction"/></defaultTransaction>
+                        </xsl:if>
+                        <xsl:if test="string($agentNode/@loggingTransaction) != ''">
+                            <LoggingTransaction><xsl:value-of select="$agentNode/@loggingTransaction"/></LoggingTransaction>
+                        </xsl:if>
+                        <xsl:if test="string($agentNode/@logSourcePath) != ''">
+                            <LogSourcePath><xsl:value-of select="$agentNode/@logSourcePath"/></LogSourcePath>
+                        </xsl:if>
+
+                        <xsl:call-template name="LogSourceMap">
+                            <xsl:with-param name="agentNode" select="$agentNode"/>
+                        </xsl:call-template>
+                        <xsl:call-template name="LogGroup">
+                            <xsl:with-param name="agentNode" select="$agentNode"/>
+                        </xsl:call-template>
+                    </LogAgent>
+                </xsl:otherwise>
+            </xsl:choose>
+        </xsl:for-each>
+    </xsl:template>
+
+</xsl:stylesheet>

initfiles/componentfiles/configxml/CMakeLists.txt

@@ -88,6 +88,7 @@ FOREACH( iFILES
     ${CMAKE_CURRENT_SOURCE_DIR}/loggingmanager.xsd
     ${CMAKE_CURRENT_SOURCE_DIR}/esploggingagent.xsd
     ${CMAKE_CURRENT_SOURCE_DIR}/wslogging.xsd
+    ${CMAKE_CURRENT_SOURCE_DIR}/cassandraloggingagent.xsd
     ${CMAKE_CURRENT_SOURCE_DIR}/daliplugin.xsd
 )
     Install ( FILES ${iFILES} DESTINATION componentfiles/configxml COMPONENT Runtime)

initfiles/componentfiles/configxml/buildsetCC.xml.in

@@ -330,6 +330,12 @@
     </BuildSet>
     <BuildSet deployable="no"
              installSet="deploy_map.xml"
+             name="cassandraloggingagent"
+             path="componentfiles/cassandraloggingagent"
+             processName="CassandraLoggingAgent"
+             schema="cassandraloggingagent.xsd"/>
+    <BuildSet deployable="no"
+             installSet="deploy_map.xml"
              name="esploggingagent"
              path="componentfiles/esploggingagent"
              processName="ESPLoggingAgent"

initfiles/componentfiles/configxml/cassandraloggingagent.xsd

@@ -0,0 +1,259 @@
+<?xml version="1.0" encoding="utf-8"?>
+
+<!--
+################################################################################
+#    HPCC SYSTEMS software Copyright (C) 2016 HPCC Systems®.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+################################################################################
+-->
+
+<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema" elementFormDefault="qualified">
+  <xs:include schemaLocation="environment.xsd"/>
+  <xs:element name="CassandraLoggingAgent">
+    <xs:complexType>
+      <xs:annotation>
+        <xs:appinfo>
+            <title>Cassandra Logging Agent</title>
+        </xs:appinfo>
+      </xs:annotation>
+      <xs:attribute name="build" type="buildType" use="required">
+        <xs:annotation>
+          <xs:appinfo>
+            <viewType>hidden</viewType>
+          </xs:appinfo>
+        </xs:annotation>
+      </xs:attribute>
+      <xs:attribute name="buildSet" type="buildSetType" use="required">
+        <xs:annotation>
+          <xs:appinfo>
+            <viewType>hidden</viewType>
+          </xs:appinfo>
+        </xs:annotation>
+      </xs:attribute>
+      <xs:attribute name="name" type="xs:string" use="required">
+        <xs:annotation>
+          <xs:appinfo>
+            <tooltip>Name for this logging agent</tooltip>
+          </xs:appinfo>
+        </xs:annotation>
+      </xs:attribute>
+      <xs:attribute name="description" type="xs:string" use="optional" default="Cassandra Logging Agent">
+        <xs:annotation>
+          <xs:appinfo>
+            <tooltip>Description for this logging agent</tooltip>
+          </xs:appinfo>
+        </xs:annotation>
+      </xs:attribute>
+      <xs:attribute name="MaxTriesGTS" type="xs:nonNegativeInteger" use="optional" default="3">
+        <xs:annotation>
+          <xs:appinfo>
+            <tooltip>Maximum retries of sending GetTransactionSeed requests when failed to get a response.</tooltip>
+          </xs:appinfo>
+        </xs:annotation>
+      </xs:attribute>
+      <xs:attribute name="MaxTriesRS" type="xs:nonNegativeInteger" use="optional" default="3">
+        <xs:annotation>
+          <xs:appinfo>
+            <tooltip>Maximum retries of sending LogUpdate requests when failed to get a response.</tooltip>
+          </xs:appinfo>
+        </xs:annotation>
+      </xs:attribute>
+      <xs:attribute name="MaxLogQueueLength" type="xs:nonNegativeInteger" use="optional" default="500000">
+        <xs:annotation>
+          <xs:appinfo>
+            <tooltip>maximum queue length for log update requests.</tooltip>
+          </xs:appinfo>
+        </xs:annotation>
+      </xs:attribute>
+      <xs:attribute name="QueueSizeSignal" type="xs:nonNegativeInteger" use="optional" default="10000">
+        <xs:annotation>
+          <xs:appinfo>
+            <tooltip>Add warning to esp log when the queue length of log update requests reaches this value.</tooltip>
+          </xs:appinfo>
+        </xs:annotation>
+      </xs:attribute>
+      <xs:attribute name="FailSafe" type="xs:boolean" use="optional" default="true">
+        <xs:annotation>
+          <xs:appinfo>
+            <tooltip>Enable FailSafe functionality.</tooltip>
+          </xs:appinfo>
+        </xs:annotation>
+      </xs:attribute>
+      <xs:attribute name="FailSafeLogsDir" type="xs:string" use="optional" default="">
+        <xs:annotation>
+          <xs:appinfo>
+            <tooltip>Log directory where Failsafe files are stored.</tooltip>
+          </xs:appinfo>
+        </xs:annotation>
+      </xs:attribute>
+      <xs:attribute name="logSourcePath" type="xs:string" use="required">
+        <xs:annotation>
+          <xs:appinfo>
+            <tooltip>The xpath inside a log request for LogSource.</tooltip>
+          </xs:appinfo>
+        </xs:annotation>
+      </xs:attribute>
+      <xs:attribute name="defaultTransaction" type="xs:string" use="optional" default="">
+        <xs:annotation>
+          <xs:appinfo>
+            <tooltip>if no applicaqtion name is given, which applicaqtion name should be used to get a transaction seed?</tooltip>
+          </xs:appinfo>
+        </xs:annotation>
+      </xs:attribute>
+      <xs:attribute name="loggingTransaction" type="xs:string" use="optional" default="">
+        <xs:annotation>
+          <xs:appinfo>
+            <tooltip>The applicaqtion name used to get a transaction seed for this logging agent.</tooltip>
+          </xs:appinfo>
+        </xs:annotation>
+      </xs:attribute>
+      <xs:attributeGroup ref="Cassandra"/>
+      <xs:element name="LogSourceMap" maxOccurs="unbounded">
+        <xs:annotation>
+          <xs:appinfo>
+            <title>Log Source Map</title>
+          </xs:appinfo>
+        </xs:annotation>
+        <xs:complexType>
+          <xs:attribute name="name" type="xs:string" use="required">
+            <xs:annotation>
+              <xs:appinfo>
+                <colIndex>1</colIndex>
+              </xs:appinfo>
+            </xs:annotation>
+          </xs:attribute>
+          <xs:attribute name="mapToDB" type="xs:string" use="required">
+            <xs:annotation>
+              <xs:appinfo>
+                <colIndex>2</colIndex>
+              </xs:appinfo>
+            </xs:annotation>
+          </xs:attribute>
+          <xs:attribute name="mapToLogGroup" type="xs:string" use="required">
+            <xs:annotation>
+              <xs:appinfo>
+                <colIndex>3</colIndex>
+              </xs:appinfo>
+            </xs:annotation>
+          </xs:attribute>
+        </xs:complexType>
+      </xs:element>
+      <xs:element name="LogGroup" maxOccurs="unbounded">
+        <xs:annotation>
+          <xs:appinfo>
+            <title>Log Group</title>
+          </xs:appinfo>
+        </xs:annotation>
+        <xs:complexType>
+          <xs:sequence>
+            <xs:element name="FieldMap" minOccurs="0" maxOccurs="unbounded">
+              <xs:annotation>
+                <xs:appinfo>
+                  <viewType>list</viewType>
+                  <title>Field Map</title>
+                </xs:appinfo>
+              </xs:annotation>
+              <xs:complexType>
+                <xs:attribute name="name" type="xs:string" use="required">
+                  <xs:annotation>
+                    <xs:appinfo>
+                      <colIndex>1</colIndex>
+                    </xs:appinfo>
+                  </xs:annotation>
+                </xs:attribute>
+                <xs:attribute name="mapTo" type="xs:string" use="required">
+                  <xs:annotation>
+                    <xs:appinfo>
+                      <colIndex>2</colIndex>
+                    </xs:appinfo>
+                  </xs:annotation>
+                </xs:attribute>
+                <xs:attribute name="type" type="xs:string" use="required" default="string">
+                  <xs:annotation>
+                    <xs:appinfo>
+                      <colIndex>3</colIndex>
+                    </xs:appinfo>
+                  </xs:annotation>
+                  <xs:simpleType>
+                    <xs:restriction base="xs:string">
+                      <xs:enumeration value="string"/>
+                      <xs:enumeration value="numeric"/>
+                    </xs:restriction>
+                  </xs:simpleType>
+                </xs:attribute>
+                <xs:attribute name="default" type="xs:string" use="optional">
+                  <xs:annotation>
+                    <xs:appinfo>
+                      <colIndex>4</colIndex>
+                    </xs:appinfo>
+                  </xs:annotation>
+                </xs:attribute>
+              </xs:complexType>
+            </xs:element>
+          </xs:sequence>
+          <xs:attribute name="name" type="xs:string" use="required">
+            <xs:annotation>
+              <xs:appinfo>
+                <colIndex>1</colIndex>
+              </xs:appinfo>
+            </xs:annotation>
+          </xs:attribute>
+        </xs:complexType>
+      </xs:element>
+    </xs:complexType>
+  </xs:element>
+
+  <xs:attributeGroup name="Cassandra">
+    <xs:attribute name="Cassandra" type="cassandraType" use="required">
+      <xs:annotation>
+        <xs:appinfo>
+          <tooltip>Specifies the Cassandra process.</tooltip>
+        </xs:appinfo>
+      </xs:annotation>
+    </xs:attribute>
+    <xs:attribute name="serverIP" type="xs:string" use="required">
+      <xs:annotation>
+        <xs:appinfo>
+          <tooltip>Specifies the network address of the data store server.</tooltip>
+          <required>true</required>
+        </xs:appinfo>
+      </xs:annotation>
+    </xs:attribute>
+    <xs:attribute name="ksName" type="xs:string" use="optional" >
+      <xs:annotation>
+         <xs:appinfo>
+           <tooltip>Specifies the name of the keyspace in the data store server.</tooltip>
+           <required>true</required>
+         </xs:appinfo>
+      </xs:annotation>
+    </xs:attribute>
+    <xs:attribute name="userName" type="xs:string" use="optional" >
+      <xs:annotation>
+        <xs:appinfo>
+          <tooltip>Specifies the user name for accessing the data store server.</tooltip>
+          <required>true</required>
+        </xs:appinfo>
+      </xs:annotation>
+    </xs:attribute>
+    <xs:attribute name="userPassword" type="xs:string" use="optional" >
+      <xs:annotation>
+        <xs:appinfo>
+          <viewType>password</viewType>
+          <tooltip>Specifies the user's password for accessing the data store server.</tooltip>
+          <required>true</required>
+        </xs:appinfo>
+      </xs:annotation>
+    </xs:attribute>
+  </xs:attributeGroup>
+</xs:schema>

initfiles/componentfiles/configxml/cgencomplist_linux.xml

@@ -93,6 +93,7 @@
   <Component name="ws_account" processName='EspService' schema='esp_service_account.xsd' deployable='no'>
     <File name="@temp/esp_service.xsl" method="esp_service_module" destName="WsAccount.xml" destPath="@temp"/>
   </Component>
+  <Component name="CassandraLoggingAgent" processName='CassandraLoggingAgent' schema='cassandraloggingagent.xsd' deployable='no'/>
   <Component name="ESPLoggingAgent" processName='ESPLoggingAgent' schema='esploggingagent.xsd' deployable='no'/>
   <Component name="LoggingManager" processName='LoggingManager' schema='loggingmanager.xsd' deployable='no'/>
   <Component name="WsLogging" processName='EspService' schema='wslogging.xsd' deployable='no'>

initfiles/componentfiles/configxml/cgencomplist_win.xml

@@ -96,6 +96,7 @@
   <Component name="ws_account" processName='EspService' schema='esp_service_account.xsd' deployable='no'>
     <File name="@temp\esp_service.xsl" method="esp_service_module" destName="WsAccount.xml" destPath="@temp"/>
   </Component>
+  <Component name="cassandraloggingagent" processName='CassandraLoggingAgent' schema='cassandraloggingagent.xsd' deployable='no'/>
   <Component name="ESPLoggingAgent" processName='ESPLoggingAgent' schema='esploggingagent.xsd' deployable='no'/>
   <Component name="LoggingManager" processName='LoggingManager' schema='loggingmanager.xsd' deployable='no'/>
   <Component name="WsLogging" processName='EspService' schema='wslogging.xsd' deployable='no'>

initfiles/componentfiles/configxml/esploggingagent.xsd

@@ -92,6 +92,36 @@
           </xs:appinfo>
         </xs:annotation>
       </xs:attribute>
+      <xs:element name="LogSourceMap" maxOccurs="unbounded">
+        <xs:annotation>
+          <xs:appinfo>
+            <title>Log Source Map</title>
+          </xs:appinfo>
+        </xs:annotation>
+        <xs:complexType>
+          <xs:attribute name="name" type="xs:string" use="required">
+            <xs:annotation>
+              <xs:appinfo>
+                <colIndex>1</colIndex>
+              </xs:appinfo>
+            </xs:annotation>
+          </xs:attribute>
+          <xs:attribute name="mapToDB" type="xs:string" use="required">
+            <xs:annotation>
+              <xs:appinfo>
+                <colIndex>2</colIndex>
+              </xs:appinfo>
+            </xs:annotation>
+          </xs:attribute>
+          <xs:attribute name="mapToLogGroup" type="xs:string" use="required">
+            <xs:annotation>
+              <xs:appinfo>
+                <colIndex>3</colIndex>
+              </xs:appinfo>
+            </xs:annotation>
+          </xs:attribute>
+        </xs:complexType>
+      </xs:element>
       <xs:sequence>
         <xs:element name="Filter" minOccurs="0" maxOccurs="unbounded">
           <xs:annotation>

initfiles/etc/DIR_NAME/genenvrules.conf

@@ -1,7 +1,7 @@
 
 [Algorithm]
 max_comps_per_node=4
-do_not_generate=SiteCertificate,dfuplus,soapplus,eclplus,ldapServer,ws_account,eclserver,ecldirect,DynamicESDL,esploggingagent,loggingmanager,wslogging,daliplugin
+do_not_generate=SiteCertificate,dfuplus,soapplus,eclplus,ldapServer,ws_account,eclserver,ecldirect,DynamicESDL,cassandraloggingagent,esploggingagent,loggingmanager,wslogging,daliplugin
 avoid_combo=dali-eclagent,dali-sasha
 comps_on_all_nodes=dafilesrv,ftslave
 exclude_from_comps_on_all_nodes=ldapServer

lib2/CMakeLists.txt

@@ -26,6 +26,7 @@ if (APPLE)
     list(APPEND DYLIBS ${OPENSSL_LIBRARIES})
     list(APPEND DYLIBS ${ZLIB_LIBRARIES})
     list(APPEND DYLIBS ${LIBXML2_LIBRARIES})
+    list(APPEND DYLIBS ${CBLAS_LIBRARIES})
 elseif (WIN32)
     #TODO:  Should find these dlls not assume them.
     if (NOT USE_NATIVE_LIBRARIES)
@@ -107,6 +108,12 @@ foreach(dylib ${DYLIBS})
             set(fixupCommand "${fixupCommand}\r\nexecute_process(COMMAND install_name_tool -change \"${dylib_1_path}\" \"@loader_path/../lib2/${dylib_name_ext}\" \${file})")
         endif ()
 
+        if ("${dylib_path}" MATCHES "^/usr/local/Cellar/.*$")
+            get_filename_component(dylib_dir ${dylib} DIRECTORY)
+            set(dylib_link_path "${dylib_dir}/${dylib_name_ext}")
+            set(fixupCommand "${fixupCommand}\r\nexecute_process(COMMAND install_name_tool -change \"${dylib_link_path}\" \"@loader_path/../lib2/${dylib_name_ext}\" \${file})")
+        endif ()
+
         install(CODE "
             file(GLOB files \"\$ENV{DESTDIR}\${CMAKE_INSTALL_PREFIX}/${EXEC_DIR}/*\" \"\$ENV{DESTDIR}\${CMAKE_INSTALL_PREFIX}/${LIB_DIR}/*.dylib\" \"\$ENV{DESTDIR}\${CMAKE_INSTALL_PREFIX}/plugins/*.dylib\" \"\$ENV{DESTDIR}\${CMAKE_INSTALL_PREFIX}/lib2/*.dylib\")
             foreach(file \${files})

plugins/javaembed/javaembed.cpp

@@ -33,6 +33,10 @@
 #include "thorxmlwrite.hpp"
 #include "esdl_def.hpp"
 
+#ifndef _WIN32
+ #include <sys/resource.h>
+#endif
+
 static const char * compatibleVersions[] = {
     "Java Embed Helper 1.0.0",
     NULL };
@@ -128,6 +132,21 @@ public:
 
         // Options we know we always want set
         optionStrings.append("-Xrs");
+#ifdef RLIMIT_STACK
+        // JVM has a habit of reducing the stack limit on main thread to 1M - probably dates back to when it was actually an increase...
+        StringBuffer stackOption("-Xss");
+        struct rlimit limit;
+        rlim_t slim = 0;
+        if (getrlimit (RLIMIT_STACK, &limit)==0)
+            slim = limit.rlim_cur;
+        if (!slim)
+            slim = 8*1024*1024;
+        if (slim >= 1*1024*1024)
+        {
+            stackOption.append((__uint64) slim);
+            optionStrings.append(stackOption);
+        }
+#endif
 
         // These may be useful for debugging
 #ifdef _DEBUG

plugins/redis/CMakeLists.txt

@@ -25,40 +25,69 @@
 project(redis)
 
 if(REDIS)
-    ADD_PLUGIN(redis PACKAGES HIREDIS MINVERSION 0.13.0) #NOTE: If MINVERSION is altered then min versions in redis.hpp must also be altered.
-    if(MAKE_REDIS)
-        set(
-            SRCS
-            redis.hpp
-            redis.cpp)
+    add_custom_command(OUTPUT ${PROJECT_BINARY_DIR}/libhiredis.so.0.13
+        COMMAND make dynamic
+        COMMAND mv libhiredis.so ${PROJECT_BINARY_DIR}/libhiredis.so.0.13
+        COMMAND make clean
+        WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/hiredis
+        COMMENT Building hiredis c library)
+    add_custom_command(OUTPUT ${PROJECT_BINARY_DIR}/libhiredis.so
+        COMMAND ln -sf libhiredis.so.0.13 libhiredis.so
+        DEPENDS ${PROJECT_BINARY_DIR}/libhiredis.so.0.13
+        WORKING_DIRECTORY ${PROJECT_BINARY_DIR})
+    add_custom_target(hiredis-build ALL
+        DEPENDS ${PROJECT_BINARY_DIR}/libhiredis.so)
 
-        include_directories(
-            ./../../system/include
-            ./../../rtl/eclrtl
-            ./../../rtl/include
-            ./../../common/deftype
-            ./../../system/jlib
-            ${HIREDIS_INCLUDE_DIR})
+    install(CODE "set(ENV{LD_LIBRARY_PATH} \"\$ENV{LD_LIBRARY_PATH}:${PROJECT_BINARY_DIR}\")")
 
-        add_definitions(-D_USRDLL -DECL_REDIS_EXPORTS)
+    add_library(hiredis SHARED IMPORTED)
+    set_property(TARGET hiredis PROPERTY IMPORTED_LOCATION ${PROJECT_BINARY_DIR}/libhiredis.so)
+    add_dependencies(hiredis hiredis-build)
 
-        HPCC_ADD_LIBRARY(redis SHARED ${SRCS})
-        if(${CMAKE_VERSION} VERSION_LESS "2.8.9")
-            message(WARNING "Cannot set NO_SONAME. shlibdeps will give warnings when package is installed")
-        elseif(NOT APPLE)
-            set_target_properties(redis PROPERTIES NO_SONAME 1)
-        endif()
+    install(FILES
+        ${PROJECT_BINARY_DIR}/libhiredis.so
+        ${PROJECT_BINARY_DIR}/libhiredis.so.0.13
+        DESTINATION ${LIB_DIR}
+        PERMISSIONS OWNER_WRITE OWNER_READ OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE
+        COMPONENT Runtime)
+
+    install(FILES
+        ${PROJECT_SOURCE_DIR}/hiredis/COPYING
+        DESTINATION ${LIB_DIR}
+        COMPONENT Runtime
+        RENAME libhiredis.copywrite)
+
+    set(
+        SRCS
+        redis.hpp
+        redis.cpp)
+
+    include_directories(
+        ./../../system/include
+        ./../../rtl/eclrtl
+        ./../../rtl/include
+        ./../../common/deftype
+        ./../../system/jlib
+        ./hiredis)
 
-        install(
-            TARGETS redis
-            DESTINATION plugins)
+    add_definitions(-D_USRDLL -DECL_REDIS_EXPORTS)
 
-        target_link_libraries(
-            redis
-            eclrtl
-            jlib
-            ${HIREDIS_LIBRARY})
+    HPCC_ADD_LIBRARY(redis SHARED ${SRCS})
+    if(${CMAKE_VERSION} VERSION_LESS "2.8.9")
+        message(WARNING "Cannot set NO_SONAME. shlibdeps will give warnings when package is installed")
+    elseif(NOT APPLE)
+        set_target_properties(redis PROPERTIES NO_SONAME 1)
     endif()
+
+    install(
+        TARGETS redis
+        DESTINATION plugins)
+
+    target_link_libraries(
+        redis
+        eclrtl
+        jlib
+        hiredis)
 endif()
 
 if(PLATFORM OR CLIENTTOOLS_ONLY)

plugins/redis/hiredis

@@ -0,0 +1 @@
+Subproject commit 010756025e8cefd1bc66c6d4ed3b1648ef6f1f95

roxie/ccd/ccdserver.cpp

@@ -2121,7 +2121,8 @@ public:
             DBGLOG("RecordPullerThread::stop");
         {
             CriticalBlock c(crit); // stop is called on our consumer's thread. We need to take care calling stop for our input to make sure it is not in mid-nextRow etc etc.
-            inputStream->stop();
+            if (inputStream)
+                inputStream->stop();
         }
         RestartableThread::join();
     }