Merge branch 'candidate-6.4.8'

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

ecl/hqlcpp/hqlcpp.cpp

@@ -7002,6 +7002,7 @@ void HqlCppTranslator::doBuildExprCast(BuildCtx & ctx, ITypeInfo * to, CHqlBound
             return;
         case type_unicode:
         case type_varunicode:
+        case type_utf8:
             if ((from->getTypeCode() == to->getTypeCode()) && (to->getSize() == UNKNOWN_LENGTH))
             {
                 tgt.set(pure);

ecl/regress/issue18830.ecl

@@ -0,0 +1,42 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2017 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+namesRecord :=
+            RECORD, locale('us')
+unicode        surname;
+utf8           forename;
+varunicode     address;
+unicode20        surname20;
+varunicode20     address20;
+            END;
+
+namesRecord2 :=
+            RECORD
+unicode         surname;
+utf8            forename;
+varunicode      address;
+unicode20       surname20;
+varunicode20    address20;
+            END;
+
+namesTable := dataset('x',namesRecord,FLAT);
+
+output(project(nofold(namesTable),
+    transform(namesRecord2,
+        SELF:=LEFT;
+    )
+));

esp/services/esdl_svc_engine/esdl_binding.cpp

@@ -539,7 +539,8 @@ void EsdlServiceImpl::handleServiceRequest(IEspContext &context,
                 const char * key = (const char *)cur.getKey();
                 features.appendf("%s%s:%s", (index++ == 0 ? "" : ", "), key, getSecAccessFlagName(*methaccessmap.getValue(key)));
             }
-            throw MakeStringException(-1, "%s::%s access denied - Required features: %s.", srvdef.queryName(), mthName, features.str());
+            const char * user = context.queryUserId();
+            throw MakeStringException(-1, "%s::%s access denied for user '%s' - Method requires: '%s'.", srvdef.queryName(), mthName, (user && *user) ? user : "Anonymous", features.str());
         }
     }
 

esp/xslt/esdl2ecl.xslt

@@ -229,6 +229,7 @@ end;
 		<xsl:when test="$basic_type='int'"><xsl:text disable-output-escaping="yes">integer</xsl:text><xsl:if test="not(@ecl_max_len)"><xsl:value-of select="$size"/></xsl:if></xsl:when>
 		<xsl:when test="$basic_type='long'"><xsl:text disable-output-escaping="yes">integer4</xsl:text><xsl:if test="not(@ecl_max_len)"><xsl:value-of select="$size"/></xsl:if></xsl:when>
 		<xsl:when test="$basic_type='short'"><xsl:text disable-output-escaping="yes">integer2</xsl:text></xsl:when>
+		<xsl:when test="$basic_type='int64'"><xsl:text disable-output-escaping="yes">integer8</xsl:text></xsl:when>
 		<xsl:when test="$basic_type='bool'"><xsl:text disable-output-escaping="yes">boolean</xsl:text></xsl:when>
 		<xsl:when test="$basic_type='string'"><xsl:text disable-output-escaping="yes">string</xsl:text><xsl:if test="not(@ecl_max_len)"><xsl:value-of select="$size"/></xsl:if></xsl:when>
 		<xsl:when test="$basic_type='double'"><xsl:text disable-output-escaping="yes">real8</xsl:text></xsl:when>

plugins/py3embed/py3embed.cpp

@@ -253,6 +253,8 @@ static void releaseContext()
 
 // Use a global object to ensure that the Python interpreter is initialized on main thread
 
+static HINSTANCE keepLoadedHandle;
+
 static class Python3xGlobalState
 {
 public:
@@ -290,6 +292,9 @@ public:
     }
     ~Python3xGlobalState()
     {
+        if (keepLoadedHandle)
+            FreeSharedObject(keepLoadedHandle);  // Must be process termination - ok to free now (and helps stop lockups at closedown in some Python libraries eg Tensorflow).
+
         if (threadContext)
             delete threadContext;   // The one on the main thread won't get picked up by the thread hook mechanism
         threadContext = NULL;
@@ -509,7 +514,7 @@ MODULE_INIT(INIT_PRIORITY_STANDARD)
                     if (tail)
                     {
                         tail[strlen(SharedObjectExtension)] = 0;
-                        HINSTANCE h = LoadSharedObject(fullName, false, false);
+                        keepLoadedHandle = LoadSharedObject(fullName, false, false);
                         break;
                     }
                 }

plugins/pyembed/pyembed.cpp

@@ -246,6 +246,8 @@ static void releaseContext()
 
 // Use a global object to ensure that the Python interpreter is initialized on main thread
 
+static HINSTANCE keepLoadedHandle;
+
 static class Python27GlobalState
 {
 public:
@@ -283,6 +285,9 @@ public:
     }
     ~Python27GlobalState()
     {
+        if (keepLoadedHandle)
+            FreeSharedObject(keepLoadedHandle);  // Must be process termination - ok to free now (and helps stop lockups at closedown in some Python libraries eg Tensorflow).
+
         if (threadContext)
             delete threadContext;   // The one on the main thread won't get picked up by the thread hook mechanism
         threadContext = NULL;
@@ -474,7 +479,7 @@ protected:
 
 MODULE_INIT(INIT_PRIORITY_STANDARD)
 {
-    // Make sure we are never unloaded (as Python may crash if we are)
+    // Make sure we are never dynamically unloaded (as Python may crash if we are)
     // we do this by doing a dynamic load of the pyembed library
     // This also allows eclcc to be able to use the library for constant folding
 #ifdef _WIN32
@@ -493,8 +498,7 @@ MODULE_INIT(INIT_PRIORITY_STANDARD)
     StringBuffer modname;
     if (findLoadedModule(modname, "libpy2embed"))
     {
-        HINSTANCE h = LoadSharedObject(modname, false, false);
-        // Deliberately leak this handle
+        keepLoadedHandle = LoadSharedObject(modname, false, false);
     }
 #endif
     return true;

system/security/LdapSecurity/ldapconnection.cpp

@@ -1502,7 +1502,7 @@ public:
                 filter.append("uid=");
             filter.append(username);
 
-            char* attrs[] = {"cn", "userAccountControl", "pwdLastSet", "givenName", "sn", "employeeNumber", "distinguishedName",NULL};
+            char* attrs[] = {"cn", "userAccountControl", "pwdLastSet", "givenName", "sn", "employeeId", "distinguishedName",NULL};
 
             Owned<ILdapConnection> lconn = m_connections->getConnection();
             LDAP* sys_ld = ((CLdapConnection*)lconn.get())->getLd();
@@ -1612,7 +1612,7 @@ public:
                         user.setPasswordExpiration(expiry);
                     }
                 }
-                else if(stricmp(attribute, "employeeNumber") == 0)
+                else if(stricmp(attribute, "employeeId") == 0)
                 {
                     CLDAPGetValuesLenWrapper vals(sys_ld, entry, attribute);
                     if (vals.hasValues())
@@ -2017,7 +2017,7 @@ public:
             Owned<ILdapConnection> lconn = m_connections->getConnection();
             LDAP* ld = ((CLdapConnection*)lconn.get())->getLd();
 
-            char        *attrs[] = {"cn", "givenName", "sn", "gidnumber", "uidnumber", "homedirectory", "loginshell", "objectClass", "employeeNumber", NULL};
+            char        *attrs[] = {"cn", "givenName", "sn", "gidnumber", "uidnumber", "homedirectory", "loginshell", "objectClass", "employeeId", NULL};
             CLDAPMessage searchResult;
             int rc = ldap_search_ext_s(ld, (char*)basedn, LDAP_SCOPE_SUBTREE, (char*)filter.str(), attrs, 0, NULL, NULL, &timeOut, LDAP_NO_LIMIT,   &searchResult.msg );
 
@@ -2066,7 +2066,7 @@ public:
                                 valind++;
                             }
                         }
-                        else if(stricmp(attribute, "employeeNumber") == 0)
+                        else if(stricmp(attribute, "employeeId") == 0)
                             user.setEmployeeID(vals.queryCharValue(0));
                     }
                 }
@@ -2436,7 +2436,7 @@ public:
             filter.appendf(")(|(%s=*%s*)(%s=*%s*)(%s=*%s*)))", act_fieldname, searchstr, "givenName", searchstr, "sn", searchstr);
         }
 
-        char *attrs[] = {act_fieldname, sid_fieldname, "cn", "userAccountControl", "pwdLastSet", "employeeNumber", NULL};
+        char *attrs[] = {act_fieldname, sid_fieldname, "cn", "userAccountControl", "pwdLastSet", "employeeId", NULL};
 
         CPagedLDAPSearch pagedSrch(ld, m_ldapconfig->getLdapTimeout(), (char*)m_ldapconfig->getUserBasedn(), LDAP_SCOPE_SUBTREE, (char*)filter.str(), attrs);
         for (message = pagedSrch.getFirstEntry(); message; message = pagedSrch.getNextEntry())
@@ -2515,7 +2515,7 @@ public:
                             ((CLdapSecUser*)user.get())->setUserID(atoi(vals.queryCharValue(0)));
                     }
                 }
-                else if(stricmp(attribute, "employeeNumber") == 0)
+                else if(stricmp(attribute, "employeeId") == 0)
                 {
                     CLDAPGetValuesLenWrapper vals(ld, message, attribute);
                     if (vals.hasValues())
@@ -2707,7 +2707,7 @@ public:
             LDAPMod employeeID_attr =
             {
                 LDAP_MOD_REPLACE,
-                "employeeNumber",
+                "employeeId",
                 employeeID_values
             };
 
@@ -6007,7 +6007,7 @@ private:
         LDAPMod employeeID_attr =
         {
             LDAP_MOD_ADD,
-            "EmployeeNumber",
+            "employeeId",
             employeeID_values
         };
 

thorlcr/activities/firstn/thfirstnslave.cpp

@@ -223,7 +223,7 @@ class CFirstNSlaveGlobal : public CFirstNSlaveBase, implements ILookAheadStopNot
         {
             if (RCUNBOUND == maxres)
             {
-                maxres = getDataLinkCount();
+                maxres = getDataLinkCount() + skipped;
                 sendCount();
             }
         }

thorlcr/activities/join/thjoinslave.cpp

@@ -204,7 +204,15 @@ public:
         {
             secondaryInputIndex = index;
             secondaryInputStream = queryInputStream(secondaryInputIndex);
-            if (!isFastThrough(_input.itdl))
+            if (isFastThrough(_input.itdl))
+            {
+                if (queryInput(index)->isGrouped())
+                {
+                    secondaryInputStream = createUngroupStream(secondaryInputStream);
+                    Owned<IEngineRowStream> old = replaceInputStream(secondaryInputIndex, secondaryInputStream);
+                }
+            }
+            else
             {
                 IStartableEngineRowStream *lookAhead = createRowStreamLookAhead(this, secondaryInputStream, queryRowInterfaces(_input.itdl), JOIN_SMART_BUFFER_SIZE, isSmartBufferSpillNeeded(_input.itdl->queryFromActivity()),
                                                             false, RCUNBOUND, this, &container.queryJob().queryIDiskUsage());

thorlcr/graph/thgraphmaster.cpp

@@ -1647,12 +1647,14 @@ bool CJobMaster::go()
         ~CWorkunitPauseHandler() { stop(); }
         void stop()
         {
-            CriticalBlock b(crit);
-            if (watcher)
+            Owned<IWorkUnitWatcher> _watcher;
             {
-                watcher->unsubscribe();
-                watcher.clear();
+                CriticalBlock b(crit);
+                if (!watcher)
+                    return;
+                _watcher.setown(watcher.getClear());
             }
+            _watcher->unsubscribe();
         }
         void notify(WUSubscribeOptions flags)
         {

thorlcr/thorutil/thormisc.cpp

@@ -50,6 +50,7 @@
 #include "rtlds_imp.hpp"
 #include "rtlformat.hpp"
 #include "rmtfile.hpp"
+#include "roxiestream.hpp"
 
 
 #define SDS_LOCK_TIMEOUT 30000
@@ -1278,16 +1279,15 @@ IRowServer *createRowServer(CActivityBase *activity, IRowStream *seq, ICommunica
     return new CRowServer(activity, seq, comm, mpTag);
 }
 
-IRowStream *createUngroupStream(IRowStream *input)
+IEngineRowStream *createUngroupStream(IRowStream *input)
 {
-    class CUngroupStream : public CSimpleInterface, implements IRowStream
+    class CUngroupStream : public CSimpleInterfaceOf<IEngineRowStream>
     {
         IRowStream *input;
     public:
         CUngroupStream(IRowStream *_input) : input(_input) { input->Link(); }
         ~CUngroupStream() { input->Release(); }
-        IMPLEMENT_IINTERFACE_USING(CSimpleInterface);
-        virtual const void *nextRow() 
+        virtual const void *nextRow() override
         {
             const void *ret = input->nextRow(); 
             if (ret) 
@@ -1295,10 +1295,11 @@ IRowStream *createUngroupStream(IRowStream *input)
             else
                 return input->nextRow();
         }
-        virtual void stop()
+        virtual void stop() override
         {
             input->stop();
         }
+        virtual void resetEOF() override { throwUnexpected(); }
     };
     return new CUngroupStream(input);
 }

thorlcr/thorutil/thormisc.hpp

@@ -477,7 +477,8 @@ interface IRowServer : extends IInterface
 extern graph_decl IRowStream *createRowStreamFromNode(CActivityBase &activity, unsigned node, ICommunicator &comm, mptag_t mpTag, const bool &abortSoon);
 extern graph_decl IRowServer *createRowServer(CActivityBase *activity, IRowStream *seq, ICommunicator &comm, mptag_t mpTag);
 
-extern graph_decl IRowStream *createUngroupStream(IRowStream *input);
+interface IEngineRowStream;
+extern graph_decl IEngineRowStream *createUngroupStream(IRowStream *input);
 
 interface IThorRowInterfaces;
 extern graph_decl void sendInChunks(ICommunicator &comm, rank_t dst, mptag_t mpTag, IRowStream *input, IThorRowInterfaces *rowIf);