Merge branch 'candidate-7.2.0' into candidate-7.2.x

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

ecl/eclcc/eclcc.cpp

@@ -1589,7 +1589,10 @@ void EclCC::processXmlFile(EclCompileInstance & instance, const char *archiveXML
     //Items first in the list have priority -Dxxx=y overrides all
     processDefinitions(repositories);
     repositories.append(*LINK(pluginsRepository));
-    if (archiveTree->getPropBool("@useLocalSystemLibraries", false)) // Primarily for testing.
+
+    //Default to using the local system libraries so that updates are kept in sync with the plugins
+    bool useLocalSystemLibraries = archiveTree->getPropBool("@useLocalSystemLibraries", true);
+    if (useLocalSystemLibraries)
         repositories.append(*LINK(libraryRepository));
 
     Owned<IFileContents> contents;

ecl/hqlcpp/hqlcpp.cpp

@@ -146,7 +146,7 @@ WorkflowItem::WorkflowItem(IHqlExpression * _function) : wfid(0), function(_func
     assertex(body->getOperator() == no_outofline);
     IHqlExpression * ecl = body->queryChild(0);
     exprs.append(*createValue(no_return_stmt, makeVoidType(), LINK(ecl)));
-    label.append(function->queryId());
+    label.append(function->queryId()->queryStr());
 }
 
 IHqlExpression * WorkflowItem::getFunction() const

ecl/hqlcpp/hqliproj.cpp

@@ -916,7 +916,10 @@ bool isSensibleRecord(IHqlExpression * record)
                     IHqlExpression * limit = cur->queryAttribute(countAtom);
                     if (!limit)
                         limit = cur->queryAttribute(sizeAtom);
-                    if (limit && !limit->isConstant())
+                    //Previously constant counts were allowed, but that currently causes issues with the field translation code.
+                    //reinstate the following code once that is addressed
+                    //if (limit && !limit->isConstant())
+                    if (limit)
                         return false;
                     break;
                 }

ecl/hqlcpp/hqlttcpp.cpp

@@ -6482,7 +6482,7 @@ IHqlExpression * WorkflowTransformer::extractCommonWorkflow(IHqlExpression * exp
 
     s.appendf("AutoWorkflow: Spotted %s ", getOpString(expr->getOperator()));
     if (expr->queryId())
-        s.append("[").append(expr->queryId()).append("] ");
+        s.append("[").append(expr->queryId()->queryStr()).append("] ");
     s.append(" to common up between workflow items [").append(wfid).append("]");
     DBGLOG("%s", s.str());
     translator.addWorkunitException(SeverityInformation, 0, s.str(), location);

ecl/regress/ecllib.eclxml

@@ -0,0 +1,54 @@
+<Archive build="internal_7.2.1-closedown0"
+         eclVersion="7.2.1"
+         legacyImport="0"
+         legacyWhen="0">
+ <Query attributePath="_local_directory_.temp"/>
+ <Module key="_local_directory_" name="_local_directory_">
+  <Attribute key="temp"
+             name="temp"
+             sourcePath="/home/gavin/dev/hpcc/ecl/regress/temp.ecl"
+             ts="1553856593000000">
+   &#32;import Std.Str;
+
+output(Str.toUpperCase(&apos;zz&apos;));&#10;
+  </Attribute>
+ </Module>
+ <Module key="std" name="std">
+  <Attribute key="str"
+             name="Str"
+             sourcePath="/home/gavin/dev/hpcc/ecllibrary/std/Str.ecl"
+             ts="1545228834000000">
+EXPORT Str := MODULE
+
+
+/*
+  Since this is primarily a wrapper for a plugin, all the definitions for this standard library
+  module are included in a single file.  Generally I would expect them in individual files.
+  */
+
+IMPORT lib_stringlib;
+
+/**
+ * Return the argument string with all lower case characters converted to upper case.
+ *
+ * @param src           The string that is being converted.
+ */
+
+EXPORT STRING ToUpperCase(STRING src, unsigned newFromParam = 0, unsigned newToParam = -1) := lib_stringlib.StringLib.StringToUpperCase(src, newFromParam, unsigned newToParam);
+
+END;&#10;
+  </Attribute>
+ </Module>
+ <Module flags="5"
+         fullname="/home/gavin/buildr/RelWithDebInfo/libs/libstringlib.so"
+         key="lib_stringlib"
+         name="lib_stringlib"
+         plugin="libstringlib.so"
+         sourcePath="lib_stringlib"
+         ts="1553256631000000"
+         version="STRINGLIB 1.1.14">
+  <Text>export StringLib := SERVICE:fold
+  string StringToUpperCase(const string src) : c,pure,entrypoint=&apos;slStringToUpperCase&apos;;
+END;</Text>
+ </Module>
+</Archive>

plugins/javaembed/HpccUtils.java

@@ -36,8 +36,11 @@ public class HpccUtils  implements Iterator
     {
         return _hasNext(handle);
     }
-    public java.lang.Object next()
+    public java.lang.Object next() throws NoSuchElementException
     {
-        return _next(handle);
+        java.lang.Object ret = _next(handle);
+        if (ret == null)
+           throw new NoSuchElementException();
+        return ret;
     }
 }

plugins/javaembed/javaembed.cpp

@@ -4329,6 +4329,9 @@ protected:
         ret.append(classname).append('.');
         switch (persistMode)
         {
+        case persistThread:
+            ret.append(__uint64(GetCurrentThreadId()));
+            break;
         case persistGlobal:
             ret.append("global");
             break;

system/jlib/jstring.hpp

@@ -41,7 +41,7 @@ public:
     StringBuffer(StringBuffer && value);
     StringBuffer(size_t len, const char *value);
     StringBuffer(const StringBuffer & value);
-    StringBuffer(bool useInternal);
+    explicit StringBuffer(bool useInternal);
     StringBuffer(char value);
     ~StringBuffer();
 

system/security/LdapSecurity/ldapconnection.cpp

@@ -3457,14 +3457,20 @@ public:
 
     virtual bool getResources(LDAP* ld, SecResourceType rtype, const char * basedn, const char* prefix, IArrayOf<ISecResource>& resources)
     {
+        if(rtype == RT_FILE_SCOPE || rtype == RT_WORKUNIT_SCOPE)
+        {
+            assertex(isEmptyString(prefix));
+
+            getManagedScopeTree(rtype, basedn, resources);
+            return true;
+        }
+
         char        *attribute;
         LDAPMessage *message;
 
         StringBuffer basednbuf;
         LdapUtils::normalizeDn(basedn, m_ldapconfig->getBasedn(), basednbuf);
-        StringBuffer filter("objectClass=*");
 
-        TIMEVAL timeOut = {m_ldapconfig->getLdapTimeout(),0};
         const char* fldname;
         LdapServerType servertype = m_ldapconfig->getServerType();
         if(servertype == ACTIVE_DIRECTORY && (rtype == RT_DEFAULT || rtype == RT_MODULE || rtype == RT_SERVICE))
@@ -3473,7 +3479,7 @@ public:
             fldname = "ou";
         char        *attrs[] = {(char*)fldname, "description", NULL};
 
-        CPagedLDAPSearch pagedSrch(ld, m_ldapconfig->getLdapTimeout(), (char*)basednbuf.str(), LDAP_SCOPE_ONELEVEL, (char*)filter.str(), attrs);
+        CPagedLDAPSearch pagedSrch(ld, m_ldapconfig->getLdapTimeout(), (char*)basednbuf.str(), LDAP_SCOPE_ONELEVEL, "objectClass=*", attrs);
         for (message = pagedSrch.getFirstEntry(); message; message = pagedSrch.getNextEntry())
         {
             // Go through the search results by checking message types
@@ -3511,16 +3517,6 @@ public:
                 CLdapSecResource* resource = new CLdapSecResource(resourcename.str());
                 resource->setDescription(descbuf.str());
                 resources.append(*resource);
-                if(rtype == RT_FILE_SCOPE || rtype == RT_WORKUNIT_SCOPE)
-                {
-                    StringBuffer nextbasedn;
-                    nextbasedn.append("ou=").append(curname.str()).append(",").append(basedn);
-                    StringBuffer nextprefix;
-                    if(prefix != NULL && *prefix != '\0')
-                        nextprefix.append(prefix);
-                    nextprefix.append(curname.str()).append("::");
-                    getResources(ld, rtype, nextbasedn.str(), nextprefix.str(), resources);
-                }
             }
         }
 
@@ -3529,20 +3525,28 @@ public:
 
     virtual bool getResourcesEx(SecResourceType rtype, const char * basedn, const char* prefix, const char* searchstr, IArrayOf<ISecResource>& resources)
     {
+        if(rtype == RT_FILE_SCOPE || rtype == RT_WORKUNIT_SCOPE)
+        {
+            assertex(isEmptyString(searchstr));
+            assertex(isEmptyString(prefix));
+
+            getManagedScopeTree(rtype, basedn, resources);
+            return true;
+        }
+
         char        *attribute;
         LDAPMessage *message;
 
         StringBuffer basednbuf;
         LdapUtils::normalizeDn(basedn, m_ldapconfig->getBasedn(), basednbuf);
         StringBuffer filter("objectClass=*");
+
         if(searchstr && *searchstr && strcmp(searchstr, "*") != 0)
         {
             filter.insert(0, "(&(");
             filter.appendf(")(|(%s=*%s*)))", "uNCName", searchstr);
         }
 
-        TIMEVAL timeOut = {m_ldapconfig->getLdapTimeout(),0};
-
         Owned<ILdapConnection> lconn = m_connections->getConnection();
         LDAP* ld = ((CLdapConnection*)lconn.get())->getLd();
 
@@ -3582,6 +3586,7 @@ public:
                     }
                 }
             }
+
             if(curname.length() == 0)
                 continue;
             StringBuffer resourcename;
@@ -3591,16 +3596,6 @@ public:
             CLdapSecResource* resource = new CLdapSecResource(resourcename.str());
             resource->setDescription(descbuf.str());
             resources.append(*resource);
-            if(rtype == RT_FILE_SCOPE || rtype == RT_WORKUNIT_SCOPE)
-            {
-                StringBuffer nextbasedn;
-                nextbasedn.append("ou=").append(curname.str()).append(",").append(basedn);
-                StringBuffer nextprefix;
-                if(prefix != NULL && *prefix != '\0')
-                    nextprefix.append(prefix);
-                nextprefix.append(curname.str()).append("::");
-                getResources(ld, rtype, nextbasedn.str(), nextprefix.str(), resources);
-            }
         }
 
         return true;
@@ -6173,20 +6168,20 @@ private:
         return addResource(RT_FILE_SCOPE, user, resource, PT_ADMINISTRATORS_AND_USER, m_ldapconfig->getResourceBasedn(RT_FILE_SCOPE));
     }
 
-    virtual aindex_t getManagedFileScopes(IArrayOf<ISecResource>& scopes)
+    virtual aindex_t getManagedScopeTree(SecResourceType rtype, const char * basedn, IArrayOf<ISecResource>& scopes)
     {
         //Get array of all file scopes listed in files baseDN
         StringBuffer basednbuf;
-        LdapUtils::normalizeDn(m_ldapconfig->getResourceBasedn(RT_FILE_SCOPE), m_ldapconfig->getBasedn(), basednbuf);
+        LdapUtils::normalizeDn(basedn ? basedn : m_ldapconfig->getResourceBasedn(rtype), m_ldapconfig->getBasedn(), basednbuf);
         basednbuf.toLowerCase();//Will look something like "ou=files,ou=dataland_ecl,dc=internal,dc=sds". Lowercase ensures proper strstr with StringArray elements below
 
-        TIMEVAL timeOut = {m_ldapconfig->getLdapTimeout(),0};
         Owned<ILdapConnection> lconn = m_connections->getConnection();
         LDAP* ld = ((CLdapConnection*)lconn.get())->getLd();
         char *attrs[] = {"canonicalName", NULL};
 
         //Call LDAP to get the complete OU tree underneath basdnbuf
         CPagedLDAPSearch pagedSrch(ld, m_ldapconfig->getLdapTimeout(), (char*)basednbuf.str(), LDAP_SCOPE_SUBTREE, "objectClass=*", attrs);
+        StringArray arrScopes;
         for (LDAPMessage *message = pagedSrch.getFirstEntry(); message; message = pagedSrch.getNextEntry())
         {
             CLDAPGetAttributesWrapper   atts(ld, message);
@@ -6224,14 +6219,26 @@ private:
 
                         if (!sb.isEmpty())
                         {
-                            CLdapSecResource* resource = new CLdapSecResource(sb);
-                            scopes.append(*resource);
+                            arrScopes.append(sb);
                         }
                     }
                 }
             }
         }
-        DBGLOG("getManagedFileScopes() found %d scopes under '%s'", scopes.length(), basednbuf.str());
+
+        //Build sorted IArrayOf<ISecResource> from arrScopes
+        if (arrScopes.length())
+        {
+            arrScopes.sortAscii(false);
+            ForEachItemIn(i, arrScopes)
+            {
+                const char * scope= arrScopes.item(i);
+                CLdapSecResource* resource = new CLdapSecResource(scope);
+                scopes.append(*resource);
+            }
+        }
+
+        DBGLOG("getManagedScopeTree() found %d scopes under '%s'", scopes.length(), basednbuf.str());
         return scopes.length();
     }
 

system/security/LdapSecurity/ldapconnection.hpp

@@ -312,7 +312,7 @@ interface ILdapClient : extends IInterface
     virtual ILdapConfig* queryConfig() = 0;
     virtual const char* getPasswordStorageScheme() = 0;
     virtual bool createUserScope(ISecUser& user) = 0;
-    virtual aindex_t getManagedFileScopes(IArrayOf<ISecResource>& scopes) = 0;
+    virtual aindex_t getManagedScopeTree(SecResourceType rtype, const char * basedn, IArrayOf<ISecResource>& scopes) = 0;
     virtual SecAccessFlags queryDefaultPermission(ISecUser& user) = 0;
 
     //Data View related interfaces

system/security/LdapSecurity/ldapsecurity.cpp

@@ -1468,9 +1468,9 @@ bool CLdapSecManager::createUserScopes()
 }
 
 
-aindex_t CLdapSecManager::getManagedFileScopes(IArrayOf<ISecResource>& scopes)
+aindex_t CLdapSecManager::getManagedScopeTree(SecResourceType rtype, const char * basedn, IArrayOf<ISecResource>& scopes)
 {
-    return m_ldap_client->getManagedFileScopes(scopes);
+    return m_ldap_client->getManagedScopeTree(rtype, basedn, scopes);
 }
 
 SecAccessFlags CLdapSecManager::queryDefaultPermission(ISecUser& user)

system/security/LdapSecurity/ldapsecurity.ipp

@@ -469,7 +469,7 @@ public:
     }
 
     virtual bool createUserScopes();
-    virtual aindex_t getManagedFileScopes(IArrayOf<ISecResource>& scopes);
+    virtual aindex_t getManagedScopeTree(SecResourceType rtype, const char * basedn, IArrayOf<ISecResource>& scopes);
     virtual SecAccessFlags queryDefaultPermission(ISecUser& user);
     virtual bool clearPermissionsCache(ISecUser &user);
     virtual bool authenticateUser(ISecUser & user, bool * superUser);

system/security/shared/basesecurity.hpp

@@ -240,7 +240,7 @@ public:
         throwUnexpected();
     }
 
-    aindex_t getManagedFileScopes(IArrayOf<ISecResource>& scopes)
+    aindex_t getManagedScopeTree(SecResourceType rtype, const char * basedn, IArrayOf<ISecResource>& scopes)
     {
         throwUnexpected();
     }

system/security/shared/caching.cpp

@@ -567,7 +567,7 @@ bool CPermissionsCache::queryPermsManagedFileScope(ISecUser& sec_user, const cha
         {
             removeAllManagedFileScopes();
             IArrayOf<ISecResource> scopes;
-            aindex_t count = m_secMgr->getManagedFileScopes(scopes);
+            aindex_t count = m_secMgr->getManagedScopeTree(RT_FILE_SCOPE, nullptr, scopes);
             if (count)
                 addManagedFileScopes(scopes);
             m_defaultPermission = SecAccess_Unknown;//trigger refresh

system/security/shared/seclib.hpp

@@ -339,7 +339,7 @@ interface ISecManager : extends IInterface
     virtual const char * getDescription() = 0;
     virtual unsigned getPasswordExpirationWarningDays() = 0;
     virtual bool createUserScopes() = 0;
-    virtual aindex_t getManagedFileScopes(IArrayOf<ISecResource>& scopes) = 0;
+    virtual aindex_t getManagedScopeTree(SecResourceType rtype, const char * basedn, IArrayOf<ISecResource>& scopes) = 0;
     virtual SecAccessFlags queryDefaultPermission(ISecUser& user) = 0;
     virtual bool clearPermissionsCache(ISecUser & user) = 0;
     virtual bool authenticateUser(ISecUser & user, bool * superUser) = 0;

testing/regress/ecl/dsarray.ecl

@@ -0,0 +1,49 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2019 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+
+pointRec := { real8 x, real8 y };
+
+shapeRecord :=
+            RECORD
+unsigned        id;
+pointRec        corners[4];
+unsigned        extra := 10;
+            END;
+
+
+mkSquare(real8 x, real8 y, real8 w) := dataset([{x,y},{x+w,y},{x+w,y+w},{x,y+w}], pointRec);
+mkRectangle(real8 x, real8 y, real8 w, real8 h) := dataset([{x,y},{x+w,y},{x+w,y+h},{x,y+h}], pointRec);
+
+shapesDs := DATASET([
+                {1, mkSquare(-1,-1,2)},
+                {2, mkSquare(0,4,2)},
+                {3, mkSquare(1,3,12)},
+                {4, mkRectangle(0,0,1000,0.001)},
+                {5, mkRectangle(1,2,3,4)},
+                {0, mkSquare(0,0,0)}
+                ], shapeRecord)(id != 0);
+
+spilledDs := global(nofold(shapesDs));
+
+projDs := PROJECT(spilledDs, shapeRecord - [extra]);
+
+sequential(
+    output(shapesDs,          { name := 'Area of ' + (string)id + ' = ', area := (corners[3].x-corners[1].x) * (corners[3].y-corners[1].y) });
+    output(nofold(spilledDs), { name := 'Area of ' + (string)id + ' = ', area := (corners[3].x-corners[1].x) * (corners[3].y-corners[1].y) });
+    output(nofold(projDs),    { name := 'Area of ' + (string)id + ' = ', area := (corners[3].x-corners[1].x) * (corners[3].y-corners[1].y) });
+);

testing/regress/ecl/dsarray2.ecl

@@ -0,0 +1,50 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2019 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+
+pointRec := { real8 x, real8 y };
+
+shapeRecord :=
+            RECORD
+unsigned        id;
+unsigned        numCorners;
+dataset(pointRec, count(SELF.numCorners)) corners;
+unsigned        extra := 10;
+            END;
+
+
+mkSquare(real8 x, real8 y, real8 w) := dataset([{x,y},{x+w,y},{x+w,y+w},{x,y+w}], pointRec);
+mkRectangle(real8 x, real8 y, real8 w, real8 h) := dataset([{x,y},{x+w,y},{x+w,y+h},{x,y+h}], pointRec);
+
+shapesDs := DATASET([
+                {1, 4, mkSquare(-1,-1,2)},
+                {2, 4, mkSquare(0,4,2)},
+                {3, 4, mkSquare(1,3,12)},
+                {4, 4, mkRectangle(0,0,1000,0.001)},
+                {5, 4, mkRectangle(1,2,3,4)},
+                {0, 4, mkSquare(0,0,0)}
+                ], shapeRecord)(id != 0);
+
+spilledDs := global(nofold(shapesDs));
+
+projDs := PROJECT(spilledDs, shapeRecord - [extra]);
+
+sequential(
+    output(shapesDs,          { name := 'Area of ' + (string)id + ' = ', area := (corners[3].x-corners[1].x) * (corners[3].y-corners[1].y) });
+    output(nofold(spilledDs), { name := 'Area of ' + (string)id + ' = ', area := (corners[3].x-corners[1].x) * (corners[3].y-corners[1].y) });
+    output(nofold(projDs),    { name := 'Area of ' + (string)id + ' = ', area := (corners[3].x-corners[1].x) * (corners[3].y-corners[1].y) });
+);

testing/regress/ecl/key/dsarray.xml

@@ -0,0 +1,21 @@
+<Dataset name='Result 1'>
+ <Row><name>Area of 1 = </name><area>4.0</area></Row>
+ <Row><name>Area of 2 = </name><area>4.0</area></Row>
+ <Row><name>Area of 3 = </name><area>144.0</area></Row>
+ <Row><name>Area of 4 = </name><area>1.0</area></Row>
+ <Row><name>Area of 5 = </name><area>12.0</area></Row>
+</Dataset>
+<Dataset name='Result 2'>
+ <Row><name>Area of 1 = </name><area>4.0</area></Row>
+ <Row><name>Area of 2 = </name><area>4.0</area></Row>
+ <Row><name>Area of 3 = </name><area>144.0</area></Row>
+ <Row><name>Area of 4 = </name><area>1.0</area></Row>
+ <Row><name>Area of 5 = </name><area>12.0</area></Row>
+</Dataset>
+<Dataset name='Result 3'>
+ <Row><name>Area of 1 = </name><area>4.0</area></Row>
+ <Row><name>Area of 2 = </name><area>4.0</area></Row>
+ <Row><name>Area of 3 = </name><area>144.0</area></Row>
+ <Row><name>Area of 4 = </name><area>1.0</area></Row>
+ <Row><name>Area of 5 = </name><area>12.0</area></Row>
+</Dataset>

testing/regress/ecl/key/dsarray2.xml

@@ -0,0 +1,21 @@
+<Dataset name='Result 1'>
+ <Row><name>Area of 1 = </name><area>4.0</area></Row>
+ <Row><name>Area of 2 = </name><area>4.0</area></Row>
+ <Row><name>Area of 3 = </name><area>144.0</area></Row>
+ <Row><name>Area of 4 = </name><area>1.0</area></Row>
+ <Row><name>Area of 5 = </name><area>12.0</area></Row>
+</Dataset>
+<Dataset name='Result 2'>
+ <Row><name>Area of 1 = </name><area>4.0</area></Row>
+ <Row><name>Area of 2 = </name><area>4.0</area></Row>
+ <Row><name>Area of 3 = </name><area>144.0</area></Row>
+ <Row><name>Area of 4 = </name><area>1.0</area></Row>
+ <Row><name>Area of 5 = </name><area>12.0</area></Row>
+</Dataset>
+<Dataset name='Result 3'>
+ <Row><name>Area of 1 = </name><area>4.0</area></Row>
+ <Row><name>Area of 2 = </name><area>4.0</area></Row>
+ <Row><name>Area of 3 = </name><area>144.0</area></Row>
+ <Row><name>Area of 4 = </name><area>1.0</area></Row>
+ <Row><name>Area of 5 = </name><area>12.0</area></Row>
+</Dataset>