Merge branch 'candidate-7.2.x'

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

dali/base/dasess.cpp

@@ -86,7 +86,7 @@ interface ISessionManagerServer: implements IConnectionMonitor
     virtual void stopSession(SessionId sessid,bool failed) = 0;
     virtual void setClientAuth(IDaliClientAuthConnection *authconn) = 0;
     virtual void setLDAPconnection(IDaliLdapConnection *_ldapconn) = 0;
-    virtual bool authorizeConnection(int role,bool revoke) = 0;
+    virtual bool authorizeConnection(const INode *client, DaliClientRole role) = 0;
     virtual void start() = 0;
     virtual void ready() = 0;
     virtual void stop() = 0;
@@ -372,13 +372,7 @@ public:
     bool remove(const CProcessSessionState *state, ISessionManagerServer *manager)
     {
         CHECKEDCRITICALBLOCK(mapprocesssect,60000);
-        if (SuperHashTableOf<CProcessSessionState,INode>::removeExact((CProcessSessionState *)state))
-        {
-            if (manager)
-                manager->authorizeConnection(state->queryRole(), true);
-            return true;
-        }
-        return false;
+        return SuperHashTableOf<CProcessSessionState,INode>::removeExact((CProcessSessionState *)state);
     }
 
     unsigned count()
@@ -525,9 +519,17 @@ public:
                 int role=0;
                 if (mb.length()-mb.getPos()>=sizeof(role)) { // a capability block present
                     mb.read(role);
-                    if (!manager.authorizeConnection(role,false)) {
+                    if (!manager.authorizeConnection(node, (DaliClientRole) role))
+                    {
+                        MilliSleep(2000); // Delay makes rapid probing of all possible roles slightly more painful.
                         SocketEndpoint sender = mb.getSender();
                         mb.clear();
+                        mb.append((SessionId) 0);
+                        INode *na = queryNullNode();
+                        Owned<IGroup> dummyCoven = createIGroup(1, &na);
+                        dummyCoven->serialize(mb);
+                        Owned<IException> e = makeStringException(666, "Access denied!");
+                        serializeException(e, mb);
                         coven.reply(mb);
                         MilliSleep(100+getRandom()%1000); // Causes client to 'work' for a short time.
                         Owned<INode> node = createINode(sender);
@@ -1094,12 +1096,6 @@ public:
         assertex(!"setLdapFlags called on client");
     }
 
-    bool authorizeConnection(DaliClientRole,bool)
-    {
-        return true;
-    }
-
-
     SessionId startSession(SecurityToken tok, SessionId parentid)
     {
         CMessageBuffer mb;
@@ -1315,7 +1311,6 @@ public:
         workthreadsem.signal(10);
         stopping = false;
         ldapsig.signal();
-
     }
     ~CCovenSessionManager()
     {
@@ -1627,7 +1622,7 @@ public:
     }
 
 
-    bool authorizeConnection(int role,bool revoke)
+    bool authorizeConnection(const INode *client, DaliClientRole role)
     {
         return true;
     }
@@ -2014,6 +2009,8 @@ bool registerClientProcess(ICommunicator *comm, IGroup *& retcoven,unsigned time
                 }
                 mb.read(mySessionId);
                 retcoven = deserializeIGroup(mb);
+                if (!mySessionId)
+                    throw deserializeException(mb);
                 return true;
             }
         }

docs/BuildTools/fo.xsl

@@ -1,4 +1,4 @@
-<?xml version='1.0'?>
+<?xml version='1.0'?>
 <xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
                 xmlns:fo="http://www.w3.org/1999/XSL/Format"
                 version='1.0'>
@@ -138,15 +138,16 @@
 
 <!--para-syntax-role-elim-hyph-17-->
 <xsl:template match="para[@role='syntax']">
-  <fo:block>
+  <fo:block space-after="6pt" space-before="12pt" space-before.precedence="1">
    <xsl:attribute name="hyphenate">false</xsl:attribute>
    <xsl:attribute name="text-align">left</xsl:attribute>
-    <xsl:apply-templates/>
+  <xsl:apply-templates/>
   </fo:block>
 </xsl:template>
 
 
 
+
 <xsl:attribute-set name="monospace.verbatim.properties">
  <xsl:attribute name="font-size">
    0.83em

docs/BuildTools/fo.xsl.in

@@ -139,7 +139,7 @@
 
 <!--para-syntax-role-elim-hyph-17-->
 <xsl:template match="para[@role='syntax']">
-  <fo:block>
+  <fo:block space-after="6pt" space-before="12pt" space-before.precedence="1">
    <xsl:attribute name="hyphenate">false</xsl:attribute>
    <xsl:attribute name="text-align">left</xsl:attribute>
     <xsl:apply-templates/>

docs/EN_US/ECLLanguageReference/ECLR_mods/BltInFunc-OUTPUT.xml

@@ -23,8 +23,8 @@
   </emphasis><emphasis role="bold">) ] ] [, ALGORITHM(</emphasis>
   <emphasis>name </emphasis><emphasis role="bold">) ] );</emphasis></para>
 
-  <para><emphasis role="bold">[</emphasis><emphasis>attr</emphasis>
-  :=<emphasis role="bold"> ]
+  <para role="syntax"><emphasis
+  role="bold">[</emphasis><emphasis>attr</emphasis> :=<emphasis role="bold"> ]
   OUTPUT(</emphasis><emphasis>recordset</emphasis><emphasis role="bold">,
   [</emphasis><emphasis> format </emphasis><emphasis
   role="bold">]</emphasis><emphasis> ,file
@@ -41,8 +41,8 @@
   role="bold">) ] ] [, ALGORITHM(</emphasis> <emphasis>name
   </emphasis><emphasis role="bold">) ] );</emphasis></para>
 
-  <para><emphasis role="bold">[</emphasis><emphasis>attr</emphasis>
-  :=<emphasis role="bold"> ]
+  <para role="syntax"><emphasis
+  role="bold">[</emphasis><emphasis>attr</emphasis> :=<emphasis role="bold"> ]
   OUTPUT(</emphasis><emphasis>recordset</emphasis><emphasis role="bold">,
   [</emphasis><emphasis> format </emphasis><emphasis
   role="bold">]</emphasis><emphasis> , file </emphasis><emphasis role="bold">
@@ -59,8 +59,8 @@
   role="bold">) ] ] [, ALGORITHM(</emphasis> <emphasis>name
   </emphasis><emphasis role="bold">) ] );</emphasis></para>
 
-  <para><emphasis role="bold">[</emphasis><emphasis>attr</emphasis>
-  :=<emphasis role="bold"> ]
+  <para role="syntax"><emphasis
+  role="bold">[</emphasis><emphasis>attr</emphasis> :=<emphasis role="bold"> ]
   OUTPUT(</emphasis><emphasis>recordset</emphasis><emphasis role="bold">,
   [</emphasis><emphasis> format </emphasis><emphasis
   role="bold">]</emphasis><emphasis> , file </emphasis><emphasis role="bold">
@@ -349,7 +349,7 @@
             <entry><emphasis role="bold">NAMED</emphasis></entry>
 
             <entry>Specifies the result name that appears in the workunit. Not
-            valid if the file parameter is present. </entry>
+            valid if the file parameter is present.</entry>
           </row>
 
           <row>
@@ -406,8 +406,8 @@
   <sect2 id="OUTPUT_Field_Names">
     <title>OUTPUT Field Names</title>
 
-    <para>Field names in an "on the fly" record format {...} must be unique or a
-    syntax error results. For example:</para>
+    <para>Field names in an "on the fly" record format {...} must be unique or
+    a syntax error results. For example:</para>
 
     <programlisting>          OUTPUT(person(), {module1.attr1, module2.attr1});</programlisting>
 

esp/clients/wsdfuaccess/wsdfuaccess.cpp

@@ -566,12 +566,12 @@ class DFUAccessTests : public CppUnit::TestFixture
         CPPUNIT_TEST(testDaFsStreamingGrouped);
         CPPUNIT_TEST(testDaFsStreamingCompressedAndGrouped);
         CPPUNIT_TEST(testFinish);
-   CPPUNIT_TEST_SUITE_END();
+    CPPUNIT_TEST_SUITE_END();
 
-   unsigned serverPort = DAFILESRV_PORT+1; // do not use standard port, which if in a URL will be converted to local path if IP is local
-   StringBuffer basePath;
-   Owned<CSimpleInterface> serverThread;
-   Owned<IFileDescriptor> fileDesc;
+    static unsigned serverPort;
+    StringBuffer basePath;
+    Owned<CSimpleInterface> serverThread;
+    Owned<IFileDescriptor> fileDesc;
 protected:
     void testStartServer()
     {
@@ -771,9 +771,15 @@ protected:
     }
 };
 
+/* MP_START_PORT -> MP_END_PORT is the MP reserved dynamic port range, and is used here for convenience.
+ * MP_START_PORT is used as starting point to find an available port for the temporary dafilesrv service in these unittests.
+ * All (MP) components using this range always check and find an unused port.
+ */
+unsigned DFUAccessTests::serverPort = MP_START_PORT;
+
+
 CPPUNIT_TEST_SUITE_REGISTRATION( DFUAccessTests );
 CPPUNIT_TEST_SUITE_NAMED_REGISTRATION( DFUAccessTests, "DFUAccessTests" );
 
 
 #endif // _USE_CPPUNIT
-

esp/src/eclwatch/HPCCPlatformOpsWidget.js

@@ -31,6 +31,7 @@ define([
             postCreate: function (args) {
                 this.inherited(arguments);
                 registry.byId(this.id + "_Permissions").set("disabled", true);
+                registry.byId(this.id + "_Monitoring").set("disabled", !dojoConfig.monitoringEnabled);
             },
 
             startup: function (args) {

esp/src/eclwatch/HPCCPlatformWidget.js

@@ -167,8 +167,11 @@ define([
                     request: {}
                 }).then(function (response) {
                     if (lang.exists("GetComponentStatusResponse.ComponentStatus", response)) {
+                        dojoConfig.monitoringEnabled = true;
                         var status = response.GetComponentStatusResponse.ComponentStatus
                         context.checkMonitoring(status);
+                    } else {
+                        dojoConfig.monitoringEnabled = false;
                     }
                 });
 

esp/xslt/esdl2cpp_cmake.xslt

@@ -18,6 +18,7 @@
 -->
 <xsl:stylesheet version="1.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform" xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns:soap="http://schemas.xmlsoap.org/wsdl/soap/" xmlns:http="http://schemas.xmlsoap.org/wsdl/http/" xmlns:mime="http://schemas.xmlsoap.org/wsdl/mime/" xmlns:wsdl="http://schemas.xmlsoap.org/wsdl/">
     <xsl:output method="text" omit-xml-declaration="yes" indent="no"/>
+    <xsl:param name="installdir"/>
 <xsl:template match="esxdl">
 <xsl:apply-templates select="EsdlService"/>
 </xsl:template>
@@ -26,7 +27,7 @@
     <xsl:variable name="servicename" select="@name"/>
 <xsl:text></xsl:text>cmake_minimum_required(VERSION 3.0)
 project (<xsl:value-of select="$servicename"/>ServicePlugin)
-
+set(CMAKE_INSTALL_PREFIX "<xsl:value-of select="$installdir"/>")
 if (("${HPCC_SOURCE_DIR}" STREQUAL "") OR ("${HPCC_BUILD_DIR}" STREQUAL "") OR ("${CMAKE_BUILD_TYPE}" STREQUAL
  ""))
     message (FATAL_ERROR "Please specify HPCC_SOURCE_DIR, HPCC_BUILD_DIR and CMAKE_BUILD_TYPE")
@@ -43,7 +44,8 @@ add_library (<xsl:value-of select="$servicename"/>Service SHARED <xsl:value-of s
                                        <xsl:value-of select="$servicename"/>Service.cpp
                                        <xsl:value-of select="$servicename"/>Service.hpp
                                        )
-target_link_libraries (<xsl:value-of select="$servicename"/>Service jlib)<xsl:text>
+target_link_libraries (<xsl:value-of select="$servicename"/>Service jlib)
+install(TARGETS <xsl:value-of select="$servicename"/>Service DESTINATION plugins)<xsl:text>
 </xsl:text>
 </xsl:template>
 </xsl:stylesheet>

fs/dafsserver/dafsserver.cpp

@@ -4776,7 +4776,11 @@ int setDaliServerTrace(byte flags)
 #include "unittests.hpp"
 #include "rmtfile.hpp"
 
-static unsigned serverPort = DAFILESRV_PORT+1; // do not use standard port, which if in a URL will be converted to local parth if IP is local
+/* MP_START_PORT -> MP_END_PORT is the MP reserved dynamic port range, and is used here for convenience.
+ * MP_START_PORT is used as starting point to find an available port for the temporary dafilesrv service in these unittests.
+ * All (MP) components using this range always check and find an unused port.
+ */
+static unsigned serverPort = MP_START_PORT;
 static StringBuffer basePath;
 static Owned<CSimpleInterface> serverThread;
 

initfiles/examples/EsdlExample/CppPlugin/EsdlExampleService.cpp

@@ -7,13 +7,13 @@ CppEchoPersonInfoResponse* EsdlExampleService::CppEchoPersonInfo(EsdlContext* co
 {
     Owned<CppEchoPersonInfoResponse> resp = new CppEchoPersonInfoResponse();
     //Fill in logic
-    resp->count.setown(new Integer(0));
-    if(request->Name)
+    resp->m_count.setown(new Integer(0));
+    if (request->m_Name)
     {
-        resp->count.setown(new Integer(1));
-        resp->Name.set(request->Name.get());
+        resp->m_count.setown(new Integer(1));
+        resp->m_Name.set(request->m_Name.get());
     }
-    appendArray(resp->Addresses, request->Addresses);
+    appendArray(resp->m_Addresses, request->m_Addresses);
     return resp.getClear();
 }
 

rtl/eclrtl/rtlnewkey.cpp

@@ -14,6 +14,8 @@
 ############################################################################## */
 
 #include <initializer_list>
+#include <algorithm>
+
 #include "jlib.hpp"
 #include "jdebug.hpp"
 #include "jsort.hpp"

rtl/eclrtl/rtlqstr.cpp

@@ -19,6 +19,8 @@
 #include "platform.h"
 #include <math.h>
 #include <stdio.h>
+#include <algorithm>
+
 #include "jexcept.hpp"
 #include "jmisc.hpp"
 #include "jutil.hpp"

testing/regress/ecl-test

@@ -85,27 +85,7 @@ class RegressMain:
             self.parser_query.print_help()
             exit()
         eclfiles=[]   # List for ECL filenames to be executed
-        for ecl in self.args.query:
-            if  ('*' in ecl) or ('?' in ecl):
-                # If there is any wildcard in ECL file name, resolve it
-                eclwild = os.path.join(self.regress.dir_ec, ecl)
-                eclfiles.extend( glob.glob(eclwild))
-            else:
-                # We have simple ECL file in parameter list, put it on the eclfile list
-                eclPath = os.path.join(self.regress.dir_ec, ecl)
-                if os.path.isfile(eclPath):
-                    eclfiles.append(eclPath)
-                else:
-                    logging.error("%s. ECL file '%s' doesn't exist!" % (1,  eclPath))
-                    raise Error("4001")
-
-        if len(eclfiles) > 1:
-            # Remove duplicates
-            tempList = list(set(eclfiles))
-            eclfiles = tempList
-
-            # Sort ECL filenames to ensure correct execution order
-            eclfiles.sort()
+        eclfiles = self.processEclList(self.args.query)
 
         # Go through the cluster list
         for engine in self.targetEngines:
@@ -152,6 +132,31 @@ class RegressMain:
             else:
                 self.regress.runSuiteP(engine, cluster, self.regress.suites[engine])
 
+    def processEclList(self, eclItems):
+        eclFiles = []
+        for ecl in eclItems:
+            if  ('*' in ecl) or ('?' in ecl):
+                # If there is any wildcard in ECL file name, resolve it
+                eclwild = os.path.join(self.regress.dir_ec, ecl)
+                eclFiles.extend( glob.glob(eclwild))
+            else:
+                # We have simple ECL file in parameter list, put it on the eclfile list
+                eclPath = os.path.join(self.regress.dir_ec, ecl)
+                if os.path.isfile(eclPath):
+                    eclFiles.append(eclPath)
+                else:
+                    logging.error("%s. ECL file '%s' doesn't exist!" % (1,  eclPath))
+                    raise Error("4001")
+
+        if len(eclFiles) > 1:
+            # Remove duplicates
+            tempList = list(set(eclFiles))
+            eclFiles = tempList
+
+            # Sort ECL filenames
+            eclFiles.sort()
+
+        return eclFiles
 
     def main(self):
         prog = "ecl-test"
@@ -217,6 +222,8 @@ class RegressMain:
                                 type=checkPqParam,  default = 1,   metavar="flushDiskCachePolicy")
         executionParser.add_argument('--runcount', help="Execute individual test case(s) in given times. Default value is 1",
                                 type=checkPqParam,  default = 1,   metavar="runcount")
+        executionParser.add_argument('--excludeFile', '--ef', help="Exclude files of suite. Wildcard is enabled. Default value is 'none'",
+                                nargs=1,  default = ['none'],   metavar="filespec[,filespec,...]")
 
 
         parser = argparse.ArgumentParser(prog=prog, description=description,  parents=[helperParser, commonParser,  executionParser])
@@ -384,6 +391,18 @@ class RegressMain:
             self.regress = Regression(self.args)
             logging.debug("Suite full path:%s",  regressionSuiteFullPath)
 
+            # process file exclusion
+            if self.args.excludeFile[0] != 'none':
+                excludeFiles = []
+                excludeItems = self.args.excludeFile[0].split(',')
+                excludeFiles = self.processEclList(excludeItems)
+
+                self.args.excludeFileSet = set(excludeFiles)
+                pass
+            else:
+                self.args.excludeFileSet = set()
+
+
             if self.args.func == 'list':
                 self.listClusters()
             elif self.args.func == 'query':

testing/regress/hpcc/regression/suite.py

@@ -115,6 +115,11 @@ class Suite:
                             exclusionReason=' ECL excluded'
 
                         if not exclude:
+                            if ecl in args.excludeFileSet:
+                                exclude = True
+                                exclusionReason=' ECL excluded by --excludeFile parameter'
+
+                        if not exclude:
                             self.addFileToSuite(eclfile)
                         else:
                             self.exclude.append(format(file, "30")+exclusionReason)

thorlcr/activities/indexread/thindexread.cpp

@@ -205,7 +205,7 @@ protected:
             superSubIndex++;
             f = &iter->query();
             if (width != f->numParts()-1)
-                throw MakeActivityException(this, 0, "Super key %s, with mixture of sub key width are not supported.", f->queryLogicalName());
+                throw MakeActivityException(this, 0, "Unsupported: Super key '%s' contains subfiles with differing numbers of file parts.", f->queryLogicalName());
         }
     }
 

tools/esdlcmd/esdlcmd_core.cpp

@@ -1013,7 +1013,9 @@ public:
             DBGLOG("ATTENTION: File %s already exists, won't generate again", filefullpath.str());
         else
         {
-            cmdHelper.defHelper->loadTransform( xsltpath, NULL, EsdlXslToCppCMake);
+            Owned<IProperties> params = createProperties();
+            params->setProp("installdir", INSTALL_DIR);
+            cmdHelper.defHelper->loadTransform( xsltpath, params.get(), EsdlXslToCppCMake);
             cmdHelper.defHelper->toMicroService( *structs, outputBuffer, EsdlXslToCppCMake, NULL, optFlags );
             saveAsFile(sourcedir.str(), "CMakeLists.txt", outputBuffer.str(), NULL);
         }