Merge branch 'candidate-6.0.0' into candidate-6.0.2

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

cmake_modules/FindJNI.cmake

@@ -156,16 +156,13 @@ JAVA_APPEND_LIBRARY_DIRECTORIES(JAVA_AWT_LIBRARY_DIRECTORIES
   /usr/lib/jvm/jre/lib/{libarch}
   /usr/local/lib/java/jre/lib/{libarch}
   /usr/local/share/java/jre/lib/{libarch}
-  /usr/lib/j2sdk1.4-sun/jre/lib/{libarch}
-  /usr/lib/j2sdk1.5-sun/jre/lib/{libarch}
-  /opt/sun-jdk-1.5.0.04/jre/lib/{libarch}
-  /usr/lib/jvm/java-6-sun/jre/lib/{libarch}
-  /usr/lib/jvm/java-1.5.0-sun/jre/lib/{libarch}
-  /usr/lib/jvm/java-6-sun-1.6.0.00/jre/lib/{libarch}       # can this one be removed according to #8821 ? Alex
+  /usr/lib/j2sdk1.7-sun/jre/lib/{libarch}
+  /usr/lib/jvm/java-7-sun/jre/lib/{libarch}
+  /usr/lib/jvm/java-1.7.0-sun/jre/lib/{libarch}
+  /usr/lib/jvm/java-7-sun-1.7.0.00/jre/lib/{libarch}       # can this one be removed according to #8821 ? Alex
   /usr/lib/jvm/java-7-openjdk-{libarch}/jre/lib/{libarch}  # Ubuntu 13.04 location
-  /usr/lib/jvm/java-6-openjdk-{libarch}/jre/lib/{libarch}  # Ubuntu 12.10 location
-  /usr/lib/jvm/java-6-openjdk/jre/lib/{libarch}
-  /usr/lib/jvm/java-1.6.0-openjdk-1.6.0.0/jre/lib/{libarch}        # fedora
+  /usr/lib/jvm/java-8-openjdk-{libarch}/jre/lib/{libarch}  # Ubuntu 16.04 location
+  /usr/lib/jvm/java-9-openjdk-{libarch}/jre/lib/{libarch}
   # Debian specific paths for default JVM
   /usr/lib/jvm/default-java/jre/lib/{libarch}
   /usr/lib/jvm/default-java/jre/lib
@@ -190,17 +187,16 @@ JAVA_APPEND_LIBRARY_DIRECTORIES(JAVA_AWT_EXPANDED_INCLUDE_DIRECTORIES
   /usr/lib/java/include
   /usr/local/lib/java/include
   /usr/lib/jvm/java/include
-  /usr/lib/jvm/java-6-sun/include
-  /usr/lib/jvm/java-1.5.0-sun/include
-  /usr/lib/jvm/java-6-sun-1.6.0.00/include       # can this one be removed according to #8821 ? Alex
+  /usr/lib/jvm/java-7-sun/include
+  /usr/lib/jvm/java-1.7.0-sun/include
+  /usr/lib/jvm/java-7-sun-1.7.0.00/include       # can this one be removed according to #8821 ? Alex
   /usr/lib/jvm/java-7-openjdk-{libarch}/include  # Ubuntu 13.04 location
-  /usr/lib/jvm/java-6-openjdk-{libarch}/jre/lib/{libarch}  # Ubuntu 12.10 location
-  /usr/lib/jvm/java-6-openjdk-{libarch}/include
-  /usr/lib/jvm/java-6-openjdk/include
+  /usr/lib/jvm/java-8-openjdk-{libarch}/include  # Ubuntu 16.04 location
+  /usr/lib/jvm/java-9-openjdk-{libarch}/include  #
   /usr/local/share/java/include
-  /usr/lib/j2sdk1.4-sun/include
-  /usr/lib/j2sdk1.5-sun/include
-  /opt/sun-jdk-1.5.0.04/include
+  /usr/lib/j2sdk1.7-sun/include
+  /usr/lib/j2sdk1.8-sun/include
+  /usr/lib/j2sdk1.9-sun/include
   # Debian specific path for default JVM
   /usr/lib/jvm/default-java/include
   )

dali/ft/filecopy.cpp

@@ -78,6 +78,7 @@
 #define ANtransferBufferSize "@transferBufferSize"
 #define ANencryptKey        "@encryptKey"
 #define ANdecryptKey        "@decryptKey"
+#define ANumask             "@umask"
 
 #define PNpartition         "partition"
 #define PNprogress          "progress"
@@ -369,6 +370,9 @@ bool FileTransferThread::performTransfer()
             progress.item(i2).serializeExtra(msg, 1);
 
         //NB: Any extra data must be appended at the end...
+
+        msg.append(sprayer.fileUmask);
+
         if (!catchWriteBuffer(socket, msg))
             throwError1(RFSERR_TimeoutWaitConnect, url.str());
 
@@ -592,6 +596,24 @@ FileSprayer::FileSprayer(IPropertyTree * _options, IPropertyTree * _progress, IR
     encryptKey.set(options->queryProp(ANencryptKey));
     decryptKey.set(options->queryProp(ANdecryptKey));
 
+    fileUmask = -1;
+    const char *umaskStr = options->queryProp(ANumask);
+    if (umaskStr)
+    {
+        char *eptr = NULL;
+        errno = 0;
+        fileUmask = (int)strtol(umaskStr, &eptr, 8);
+        if (errno || *eptr != '\0')
+        {
+            LOG(MCdebugInfo, job, "Invalid umask value <%s> ignored", umaskStr);
+            fileUmask = -1;
+        }
+        else
+        {
+            // never strip off owner
+            fileUmask &= 077;
+        }
+    }
 }
 
 
@@ -816,25 +838,6 @@ void FileSprayer::beforeTransfer()
         checker.For(targets.ordinality(), 25, true, true);
     }
 
-    int umask = -1;
-    if (options->hasProp("@umask"))
-    {
-        StringBuffer umaskStr;
-        options->getProp("@umask", umaskStr);
-        errno = 0;
-        umask = (int)strtol(umaskStr.str(), NULL, 8);
-        if (errno)
-        {
-            LOG(MCdebugInfo, job, "Invalid umask value <%s> ignored", umaskStr.str());
-            umask = -1;
-        }
-        else
-        {
-            // never strip off owner
-            umask &= 077;
-        }
-    }
-
     if (!isRecovering && !usePullOperation())
     {
         try {
@@ -852,8 +855,8 @@ void FileSprayer::beforeTransfer()
                     if (!dir->exists())
                     {
                         dir->createDirectory();
-                        if (umask != -1)
-                            dir->setFilePermissions(~umask&0777);
+                        if (fileUmask != -1)
+                            dir->setFilePermissions(~fileUmask&0777);
                     }
                 }
             }
@@ -892,8 +895,8 @@ void FileSprayer::beforeTransfer()
                     remote.getPath(name);
                     throwError1(DFTERR_CouldNotCreateOutput, name.str());
                 }
-                if (umask != -1)
-                    file->setFilePermissions(~umask&0666);
+                if (fileUmask != -1)
+                    file->setFilePermissions(~fileUmask&0666);
                 //Create the headers on the utf files.
                 unsigned headerSize = getHeaderSize(tgtFormat.type);
                 if (headerSize)

dali/ft/filecopy.ipp

@@ -322,6 +322,7 @@ protected:
     bool                    preserveCompression;
     offset_t                headerSize;
     offset_t                footerSize;
+    int                     fileUmask;
 };
 
 

dali/ft/fttransform.cpp

@@ -498,6 +498,7 @@ TransferServer::TransferServer(ISocket * _masterSocket)
     compressedInput = false;
     compressOutput = false;
     transferBufferSize = DEFAULT_STD_BUFFER_SIZE;
+    fileUmask = -1;
 }
 
 void TransferServer::sendProgress(OutputProgress & curProgress)
@@ -642,9 +643,16 @@ void TransferServer::deserializeAction(MemoryBuffer & msg, unsigned action)
     ForEachItemIn(i1, progress)
         progress.item(i1).deserializeExtra(msg, 1);
 
+    if (msg.remaining())
+        msg.read(fileUmask);
+
     LOG(MCdebugProgress, unknownJob, "throttle(%d), transferBufferSize(%d)", throttleNicSpeed, transferBufferSize);
     PROGLOG("compressedInput(%d), compressedOutput(%d), copyCompressed(%d)", compressedInput?1:0, compressOutput?1:0, copyCompressed?1:0);
     PROGLOG("encrypt(%d), decrypt(%d)", encryptKey.isEmpty()?0:1, decryptKey.isEmpty()?0:1);
+    if (fileUmask != -1)
+        PROGLOG("umask(0%o)", fileUmask);
+    else
+        PROGLOG("umask(default)");
 
     //---Finished deserializing ---
     displayProgress(progress);
@@ -893,6 +901,10 @@ processedProgress:
                     renameDfuTempToFinal(curPartition.outputName);
 
                     OwnedIFile output = createIFile(curPartition.outputName);
+
+                    if (fileUmask != -1)
+                        output->setFilePermissions(~fileUmask&0666);
+
                     if (mirror || replicate)
                     {
                         OwnedIFile input = createIFile(curPartition.inputName);

dali/ft/fttransform.ipp

@@ -243,6 +243,7 @@ protected:
     size32_t                transferBufferSize;
     StringAttr              encryptKey;
     StringAttr              decryptKey;
+    int                     fileUmask;
 };
 
 

docs/DynamicESDL/DESDL-Mods/ESDLecl_null.xml

@@ -14,10 +14,10 @@
   <para><emphasis role="bold">Example:</emphasis></para>
 
   <programlisting>[ecl_null(0)] int Age;
-[ecl_null('false')] bool IsMatch;   </programlisting>
+[ecl_null("false")] bool IsMatch;   </programlisting>
 
   <para>Age 0 means there is no Age data for this person. So, if Age is 0, the
-  &lt;Age&gt; tag is not returned. </para>
+  &lt;Age&gt; tag is not returned.</para>
 
   <para>Without this attribute, <emphasis
   role="bold">&lt;Age&gt;0&lt;/Age&gt;</emphasis> would be returned.</para>

docs/DynamicESDL/DynamicESDL_Includer.xml

@@ -502,14 +502,14 @@ OUTPUT(ds_out, NAMED('AddThisResponse')); </programlisting>
           </listitem>
 
           <listitem>
-            <para>Save the file as MathServiceConfig.xml</para>
+            <para>Save the file as MathSvcCfg.xml</para>
           </listitem>
 
           <listitem>
             <para>Bind the service methods to the queries using a the
             configuration XML.</para>
 
-            <programlisting>esdl bind-service myesp 8003 MathService.1 MathService --config MathServiceConfig.xml -s nnn.nnn.nnn.nnn -p 8010
+            <programlisting>esdl bind-service myesp 8003 MathService.1 MathService --config MathSvcCfg.xml -s nnn.nnn.nnn.nnn -p 8010
 </programlisting>
 
             <para>Where myesp is the name of your ESP process, 8003 is the

docs/ECLLanguageReference/ECLR_mods/BltInFunc-BUILD.xml

@@ -4,8 +4,8 @@
 <sect1 id="BUILD">
   <title>BUILD</title>
 
-  <para><emphasis role="bold">[</emphasis><emphasis>attrname</emphasis> :=<emphasis
-  role="bold"> ] BUILD<indexterm>
+  <para><emphasis role="bold">[</emphasis><emphasis>attrname</emphasis>
+  :=<emphasis role="bold"> ] BUILD<indexterm>
       <primary>BUILD</primary>
     </indexterm>(</emphasis><emphasis>baserecset</emphasis><emphasis
   role="bold">, [</emphasis><emphasis> indexrec </emphasis><emphasis
@@ -14,8 +14,8 @@
   role="bold">] );</emphasis></para>
 
   <para><emphasis role="bold">[</emphasis><emphasis>attrname</emphasis>
-  :=<emphasis role="bold">
-  ] BUILD(</emphasis><emphasis>baserecset</emphasis><emphasis
+  :=<emphasis role="bold"> ]
+  BUILD(</emphasis><emphasis>baserecset</emphasis><emphasis
   role="bold">,</emphasis><emphasis> keys</emphasis><emphasis
   role="bold">,</emphasis><emphasis> payload</emphasis><emphasis role="bold">,
   </emphasis><emphasis>indexfile </emphasis><emphasis role="bold">[,
@@ -27,6 +27,11 @@
   </emphasis><emphasis role="bold">[, </emphasis><emphasis>options
   </emphasis><emphasis role="bold">] );</emphasis></para>
 
+  <para><emphasis role="bold">[</emphasis><emphasis>attrname</emphasis>
+  :=<emphasis role="bold"> ] BUILD(</emphasis><emphasis> indexdef, dataset,
+  </emphasis><emphasis role="bold">[, </emphasis><emphasis>options
+  </emphasis><emphasis role="bold">] );</emphasis></para>
+
   <para><emphasis role="bold">BUILD(</emphasis><emphasis> library
   </emphasis><emphasis role="bold">);</emphasis></para>
 
@@ -126,12 +131,12 @@
       </tgroup>
     </informaltable></para>
 
-  <para>The first three forms of the <emphasis role="bold">BUILD
+  <para>The first four forms of the <emphasis role="bold">BUILD
   </emphasis>action create index files. Indexes are automatically compressed,
   minimizing overhead associated with using indexed record access. The keyword
-  BUILDINDEX may be used in place of BUILD in these forms.</para>
+  BUILDINDEX may be used in place of BUILD in these forms. </para>
 
-  <para>The fourth form creates an external query library<indexterm>
+  <para>The fifth form creates an external query library<indexterm>
       <primary>query library</primary>
     </indexterm>—a workunit that implements the specified
   <emphasis>library</emphasis>. This is similar to creating a .DLL in Windows
@@ -563,6 +568,28 @@ BUILD(Vehicles,{st,city},{lname},'vkey::st.city');
     <programlisting>nameKey := INDEX(mainTable,{surname,forename,filepos},'name.idx');
 BUILD(nameKey); //gets all info from the INDEX definition
 </programlisting>
+
+    <para><emphasis role="bold">[</emphasis><emphasis>attrname</emphasis>
+    :=<emphasis role="bold"> ] BUILD(</emphasis><emphasis> indexdef, dataset
+    </emphasis><emphasis role="bold">[, </emphasis><emphasis>options
+    </emphasis><emphasis role="bold">] );</emphasis></para>
+
+    <para>Form 4 creates an index file on a dataset using a previously defined
+    INDEX definition.</para>
+
+    <para>This is used to build an index where the dataset definition is
+    complex. This allows the index to be logically separated from the dataset
+    from which it is created. This is especially useful when the dataset
+    definition is very complicated (Mb of source) because when the index is
+    subsequently used in a query, all the code to create it is also
+    parsed.</para>
+
+    <para>Example:</para>
+
+    <programlisting>ds = DATASET(100, TRANSFORM({ unsigned id }, SELF.id := COUNTER));
+i := INDEX({ unsigned id }, 'myIndex');
+BUILD(i, ds);
+</programlisting>
   </sect2>
 
   <sect2 id="BUILD_a_Query_Library">
@@ -571,7 +598,7 @@ BUILD(nameKey); //gets all info from the INDEX definition
     <para><emphasis role="bold">BUILD(</emphasis><emphasis> library
     </emphasis><emphasis role="bold">);</emphasis></para>
 
-    <para>Form 4 creates an external query library <emphasis role="bold">for
+    <para>Form 5 creates an external query library <emphasis role="bold">for
     use in hthor or Roxie, only</emphasis>.</para>
 
     <para>A query library allows a set of related attributes to be packaged as

esp/esdllib/esdl_def.cpp

@@ -1826,7 +1826,7 @@ public:
         return enumDef;
     }
 
-    bool loadXMLDefinitionFrombuffer(const StringBuffer & xmlDef, const char * path="", int indent = 0)
+    bool loadXMLDefinitionFrombuffer(const StringBuffer & xmlDef, const char * path="", int indent = 0, bool loadImports = true)
     {
         bool esxdlFound = false;
 
@@ -1857,7 +1857,14 @@ public:
                                 xpp->readStartTag(stag);
                                 //we're loading from buffer, no need to load includes from file
                                 if(!stricmp(stag.getLocalName(), "EsdlInclude"))
+                                {
+                                    if (loadImports)
+                                    {
+                                        const char *incname = stag.getValue("file");
+                                        loadfile(path, incname, indent+3);
+                                    }
                                     xpp->skipSubTree();
+                                }
                                 else if(!stricmp(stag.getLocalName(), "EsdlStruct"))
                                     loadStruct(xpp.get(), stag, EsdlTypeStruct);
                                 else if(!stricmp(stag.getLocalName(), "EsdlRequest"))
@@ -1959,7 +1966,7 @@ void EsdlDefinition::addDefinitionFromXML(const StringBuffer & xmlDef, const cha
         TimeSection ts("adding XML ESDL definition");
 
         EsdlDefLoader loader(this);
-        if (loader.loadXMLDefinitionFrombuffer(xmlDef))
+        if (loader.loadXMLDefinitionFrombuffer(xmlDef, "", 0, false))
             added.setValue(esdlDefId, true);
         else
             throw MakeStringException(-1, "Could not load XML ESDL def: %s", xmlDef.str());

esp/src/eclwatch/css/hpcc.css

@@ -1147,4 +1147,10 @@ margin-left:-20px;
     text-align: left;
     overflow: hidden;
     width: 80px;
+}
+
+.dojoxUploaderFileListHeader th {
+    background-color:#eee;
+    padding:3px;
+    text-align: left;
 }

initfiles/componentfiles/configxml/roxie.xsd.in

@@ -441,6 +441,13 @@
     </xs:attribute>
  </xs:attributeGroup>
  <xs:attributeGroup name="Options">
+    <xs:attribute name="affinity" type="xs:nonNegativeInteger" use="optional" default="0">
+      <xs:annotation>
+        <xs:appinfo>
+          <tooltip>If non-zero, binds the roxie process to use the specified cores only (bitmask)</tooltip>
+        </xs:appinfo>
+      </xs:annotation>
+    </xs:attribute>
     <xs:attribute name="allFilesDynamic" type="xs:boolean" use="optional" default="false">
       <xs:annotation>
         <xs:appinfo>
@@ -479,6 +486,13 @@
         </xs:appinfo>
       </xs:annotation>
     </xs:attribute>
+    <xs:attribute name="coresPerQuery" type="xs:nonNegativeInteger" use="optional" default="0">
+      <xs:annotation>
+        <xs:appinfo>
+          <tooltip>If non-zero, binds each incoming query to use the specified number of cores only</tooltip>
+        </xs:appinfo>
+      </xs:annotation>
+    </xs:attribute>
     <xs:attribute name="crcResources" type="xs:boolean" use="optional" default="false">
       <xs:annotation>
         <xs:appinfo>

plugins/kafka/CMakeLists.txt

@@ -103,7 +103,13 @@ if(KAFKA)
             DESTINATION plugins)
 
         install(
-            FILES ${LIBRDKAFKA_LIB} ${LIBRDKAFKA_LIB_REAL} ${LIBRDKAFKACPP_LIB} ${LIBRDKAFKACPP_LIB_REAL}
+            FILES ${LIBRDKAFKA_LIB_REAL} ${LIBRDKAFKACPP_LIB_REAL}
+            DESTINATION ${LIB_DIR}
+            PERMISSIONS OWNER_WRITE OWNER_READ OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE
+            COMPONENT Runtime)
+
+        install(
+            FILES ${LIBRDKAFKA_LIB} ${LIBRDKAFKACPP_LIB}
             DESTINATION ${LIB_DIR}
             COMPONENT Runtime)
 

roxie/ccd/CMakeLists.txt

@@ -121,4 +121,8 @@ IF (USE_OPENSSL)
     )
 ENDIF()
 
+IF (USE_TBBMALLOC)
+   target_link_libraries ( ccd ${TBBMALLOC_LIBRARIES})
+ENDIF()
+
 

system/jlib/CMakeLists.txt

@@ -198,10 +198,6 @@ target_link_libraries ( jlib
         lz4
        )
 
-if (USE_TBBMALLOC)
-   target_link_libraries ( jlib ${TBBMALLOC_LIBRARIES})
-endif()
-
 if ( ${HAVE_LIBDL} )
 target_link_libraries ( jlib dl)
 endif ( ${HAVE_LIBDL} )

system/security/LdapSecurity/ldapsecurity.cpp

@@ -536,9 +536,9 @@ void CLdapSecManager::init(const char *serviceName, IPropertyTree* cfg)
     int cachetimeout = cfg->getPropInt("@cacheTimeout", 5);
 
     if (cfg->getPropBool("@sharedCache", true))
-        m_permissionsCache = CPermissionsCache::queryInstance();
+        m_permissionsCache.setown(CPermissionsCache::getInstance(cfg->queryProp("@name")));
     else
-        m_permissionsCache = new CPermissionsCache();
+        m_permissionsCache.setown(new CPermissionsCache());
 
     m_permissionsCache->setCacheTimeout( 60 * cachetimeout);
     m_permissionsCache->setTransactionalEnabled(true);
@@ -554,8 +554,6 @@ CLdapSecManager::CLdapSecManager(const char *serviceName, IPropertyTree &config)
 
 CLdapSecManager::~CLdapSecManager()
 {
-    if (!m_cfg->getPropBool("@sharedCache", true))
-        delete m_permissionsCache;
 }
 
 //interface ISecManager : extends IInterface

system/security/LdapSecurity/ldapsecurity.ipp

@@ -335,7 +335,7 @@ private:
     IUserArray m_user_array;
     Monitor m_monitor;
     Owned<IProperties> m_extraparams;
-    CPermissionsCache * m_permissionsCache;
+    Owned<CPermissionsCache> m_permissionsCache;
     bool m_cache_off[RT_SCOPE_MAX];
     bool m_usercache_off;
     bool authenticate(ISecUser* user);

system/security/shared/caching.cpp

@@ -18,6 +18,11 @@
 #include "caching.hpp"
 #include "jtime.hpp"
 
+//define a container for multiple instances of a security manager cache
+typedef map<string, CPermissionsCache*> MapCache;
+static CriticalSection mapCacheCS;//guards modifications to the cache map
+static MapCache g_mapCache;
+
 /**********************************************************
  *     CResPermissionsCache                               *
  *     (used by CPermissionsCache defined below)          *
@@ -210,6 +215,11 @@ void CResPermissionsCache::remove(SecResourceType rtype, const char* resourcenam
 
 CPermissionsCache::~CPermissionsCache()
 {
+    if (!m_secMgrClass.isEmpty())
+    {
+        CriticalBlock block(mapCacheCS);
+        g_mapCache.erase(m_secMgrClass.str());
+    }
     flush();
 }
 
@@ -625,3 +635,22 @@ void CPermissionsCache::flush()
     m_lastManagedFileScopesRefresh = 0;
     m_defaultPermission = SecAccess_Unknown;//trigger refresh
 }
+
+CPermissionsCache* CPermissionsCache::getInstance(const char * _secMgrClass)
+{
+    const char * secMgrClass = (_secMgrClass != nullptr  &&  *_secMgrClass) ? _secMgrClass : "genericSecMgrClass";
+
+    CriticalBlock block(mapCacheCS);
+    MapCache::iterator it = g_mapCache.find(secMgrClass);
+    if (it != g_mapCache.end())//exists in cache
+    {
+        LINK((*it).second);
+        return (*it).second;
+    }
+    else
+    {
+        CPermissionsCache * instance = new CPermissionsCache(_secMgrClass);
+        g_mapCache.insert(pair<string, CPermissionsCache*>(secMgrClass, instance));
+        return instance;
+    }
+}

system/security/shared/caching.hpp

@@ -18,7 +18,7 @@
 #ifndef _CACHING_HPP__
 #define _CACHING_HPP__
 #pragma warning(disable:4786)
- 
+
 #include "jliball.hpp"
 #include "seclib.hpp"
 #undef new
@@ -39,10 +39,10 @@ using std::string;
 
 typedef pair<time_t, ISecResource*> ResPermCacheEntry;
 typedef pair<string, SecResourceType> SecCacheKeyEntry;
-//this a cache for a given user that stores permissions for individual resources 
-//along with their timestamps when they were fetched.  The cache is periodically 
+//this a cache for a given user that stores permissions for individual resources
+//along with their timestamps when they were fetched.  The cache is periodically
 //cleaned up to remove stale (older than 5 minutes) entries - triggered by a lookup
-//itself.  Note that each user has an instance of this cache, which is stored in 
+//itself.  Note that each user has an instance of this cache, which is stored in
 //another map (CPermissionsCache) as defined below.
 //
 //
@@ -134,34 +134,27 @@ public:
 
 // main cache that stores all user-specific caches (defined by CResPermissionsCache above)
 //
-static CriticalSection PCCritSect;//guards instance factory
-static CPermissionsCache* instance = nullptr;//accessed via CPermissionsCache::queryInstance()
-
-class CPermissionsCache
+class CPermissionsCache : public CInterface, implements IInterface
 {
 public:
-    CPermissionsCache()
+    IMPLEMENT_IINTERFACE
+
+    CPermissionsCache(const char * _secMgrClass = nullptr)
     {
         m_cacheTimeout = 300;
         m_transactionalEnabled = false;
         m_secMgr = NULL;
         m_lastManagedFileScopesRefresh = 0;
         m_defaultPermission = SecAccess_Unknown;
+        m_secMgrClass.set(_secMgrClass);
     }
 
     virtual ~CPermissionsCache();
 
-    static CPermissionsCache* queryInstance()
-    {
-        {
-            CriticalBlock block(PCCritSect);
-            if (instance == nullptr)
-            {
-                instance = new CPermissionsCache();
-            }
-        }
-        return instance;
-    }
+    //Returns an owned reference to a shared cache of a given Sec Mgr class type.
+    //Call this method with a unique class string ("LDAP", "MyOtherSecMgr")
+    //to create a cache shared amongst security managers of the same class
+    static CPermissionsCache* getInstance(const char * _secMgrClass);
 
     //finds cached permissions for a number of resources and sets them in
     //and also returns status in the boolean array passed in
@@ -206,6 +199,8 @@ private:
     MapUserCache m_userCache;
     mutable ReadWriteLock m_userCacheRWLock;    //guards m_userCache
 
+    StringAttr                  m_secMgrClass;
+
     //Managed File Scope support
     int                         m_defaultPermission;
     map<string, ISecResource*>  m_managedFileScopesMap;
@@ -216,5 +211,4 @@ private:
 
 time_t getThreadCreateTime();
 
-
 #endif

thorlcr/activities/diskread/thdiskreadslave.cpp

@@ -526,6 +526,8 @@ class CDiskNormalizeSlave : public CDiskReadSlaveActivityRecord
 
     class CNormalizePartHandler : public CDiskRecordPartHandler
     {
+        typedef CDiskRecordPartHandler PARENT;
+
         RtlDynamicRowBuilder outBuilder;
         CDiskNormalizeSlave &activity;
         const void * nextrow;
@@ -537,34 +539,39 @@ class CDiskNormalizeSlave : public CDiskReadSlaveActivityRecord
         {
             nextrow = NULL;
         }
-
-        ~CNormalizePartHandler()
+        virtual void close(CRC32 &fileCRC) override
         {
             if (nextrow)
-                CDiskRecordPartHandler::prefetchDone();
+                prefetchDone();
+            PARENT::close(fileCRC);
         }
-
-        const void *nextRow()
+        const void *nextRow() override
         {
             // logic here is a bit obscure
-            if (eoi || activity.queryAbortSoon()) {
+            if (eoi || activity.queryAbortSoon())
+            {
                 eoi = true;
                 return NULL;
             }
-            loop {
-                if (nextrow) {
-                    while (activity.helper->next()) {
+            loop
+            {
+                if (nextrow)
+                {
+                    while (activity.helper->next())
+                    {
                         size32_t sz = activity.helper->transform(outBuilder.ensureRow());
                         if (sz) 
                             return outBuilder.finalizeRowClear(sz);
                     }
-                    CDiskRecordPartHandler::prefetchDone();
+                    prefetchDone();
                 }
-                nextrow = CDiskRecordPartHandler::prefetchRow();
+                nextrow = prefetchRow();
                 if (!nextrow)
                     break;
-                if (activity.segMonitorsMatch(nextrow)) {
-                    if (activity.helper->first(nextrow)) {
+                if (activity.segMonitorsMatch(nextrow))
+                {
+                    if (activity.helper->first(nextrow))
+                    {
                         size32_t sz = activity.helper->transform(outBuilder.ensureRow());
                         if (sz) 
                             return outBuilder.finalizeRowClear(sz);
@@ -572,7 +579,7 @@ class CDiskNormalizeSlave : public CDiskReadSlaveActivityRecord
                     }
                 }
                 nextrow = NULL;
-                CDiskRecordPartHandler::prefetchDone();
+                prefetchDone();
             }
             eoi = true;
             return NULL;

thorlcr/activities/enth/thenthslave.cpp

@@ -28,7 +28,7 @@ protected:
     Semaphore finishedSem;
     rowcount_t counter = 0, localRecCount = 0;
     rowcount_t denominator = 0, numerator = 0;
-    IEngineRowStream *originalInputStream = nullptr;
+    Owned<IEngineRowStream> originalInputStream;
 
     bool haveLocalCount() { return RCUNBOUND != localRecCount; }
     inline bool wanted()
@@ -82,8 +82,8 @@ protected:
         else
         {
             localRecCount = RCUNBOUND;
-            IStartableEngineRowStream *lookAhead = createRowStreamLookAhead(this, originalInputStream, queryRowInterfaces(input), ENTH_SMART_BUFFER_SIZE, true, false, RCUNBOUND, this, &container.queryJob().queryIDiskUsage());
-            setLookAhead(0, lookAhead); // NB: this is post base start()
+            IStartableEngineRowStream *lookAhead = createRowStreamLookAhead(this, inputStream, queryRowInterfaces(input), ENTH_SMART_BUFFER_SIZE, true, false, RCUNBOUND, this, &container.queryJob().queryIDiskUsage());
+            originalInputStream.setown(replaceInputStream(0, lookAhead)); // NB: this is post base start()
             lookAhead->start();
         }
     }
@@ -94,11 +94,6 @@ public:
     {
         appendOutputLinked(this);
     }
-    virtual void setInputStream(unsigned index, CThorInput &_input, bool consumerOrdered) override
-    {
-        PARENT::setInputStream(index, _input, consumerOrdered);
-        originalInputStream = inputStream;
-    }
     virtual void start() override
     {
         ActivityTimer s(totalCycles, timeActivities);
@@ -108,6 +103,16 @@ public:
         denominator = validRC(helper->getProportionDenominator());
         numerator = validRC(helper->getProportionNumerator());
     }
+    virtual void stop() override
+    {
+        PARENT::stop();
+
+        // restore original inputStream if lookAhead was installed, to avoid base start spuriously starting previously installed lookahead
+        if (originalInputStream)
+        {
+            Owned<IEngineRowStream> lookAhead = replaceInputStream(0, originalInputStream.getClear());
+        }
+    }
     virtual bool isGrouped() const override { return false; }
     void getMetaInfo(ThorDataLinkMetaInfo &info)
     {

thorlcr/activities/firstn/thfirstnslave.cpp

@@ -213,7 +213,7 @@ class CFirstNSlaveGlobal : public CFirstNSlaveBase, implements ILookAheadStopNot
     rowcount_t maxres, skipped, totallimit;
     bool firstget;
     ThorDataLinkMetaInfo inputMeta;
-    IEngineRowStream *originalInputStream = nullptr;
+    Owned<IEngineRowStream> originalInputStream;
 
 protected:
     virtual void doStop()
@@ -232,11 +232,6 @@ public:
         PARENT::init(data, slaveData);
         mpTag = container.queryJobChannel().deserializeMPTag(data);
     }
-    virtual void setInputStream(unsigned index, CThorInput &_input, bool consumerOrdered) override
-    {
-        PARENT::setInputStream(index, _input, consumerOrdered);
-        originalInputStream = inputStream;
-    }
     virtual void start() override
     {
         ActivityTimer s(totalCycles, timeActivities);
@@ -249,11 +244,19 @@ public:
         totallimit = (rowcount_t)helper->getLimit();
         rowcount_t _skipCount = validRC(helper->numToSkip()); // max
         rowcount_t maxRead = (totallimit>(RCUNBOUND-_skipCount))?RCUNBOUND:totallimit+_skipCount;
-        IStartableEngineRowStream *lookAhead = createRowStreamLookAhead(this, originalInputStream, queryRowInterfaces(input), FIRSTN_SMART_BUFFER_SIZE, isSmartBufferSpillNeeded(this), false,
+        IStartableEngineRowStream *lookAhead = createRowStreamLookAhead(this, inputStream, queryRowInterfaces(input), FIRSTN_SMART_BUFFER_SIZE, isSmartBufferSpillNeeded(this), false,
                                                                               maxRead, this, &container.queryJob().queryIDiskUsage()); // if a very large limit don't bother truncating
-        setLookAhead(0, lookAhead);
+        originalInputStream.setown(replaceInputStream(0, lookAhead));
         lookAhead->start();
     }
+    virtual void stop() override
+    {
+        PARENT::stop();
+        if (originalInputStream)
+        {
+            Owned<IEngineRowStream> lookAhead = replaceInputStream(0, originalInputStream.getClear());
+        }
+    }
     virtual void abort()
     {
         PARENT::abort();

thorlcr/activities/hashdistrib/thhashdistribslave.cpp

@@ -1990,7 +1990,7 @@ public:
         CriticalBlock block(stopsect);  // can be called async by distribute
         if (!inputstopped)
         {
-            PARENT::stop();
+            PARENT::stopInput(0);
             inputstopped = true;
         }
     }
@@ -2024,6 +2024,7 @@ public:
         }
         stopInput();
         instrm.clear();
+        dataLinkStop();
     }
     virtual void kill() override
     {

thorlcr/graph/graph_lcr.cmake

@@ -75,4 +75,8 @@ target_link_libraries ( graph_lcr
          roxiemem
     )
 
+if (USE_TBBMALLOC)
+   target_link_libraries ( graph_lcr ${TBBMALLOC_LIBRARIES})
+endif()
+
 

thorlcr/graph/thgraphslave.cpp

@@ -190,6 +190,16 @@ void CSlaveActivity::setInputStream(unsigned index, CThorInput &_input, bool con
     }
 }
 
+IEngineRowStream *CSlaveActivity::replaceInputStream(unsigned index, IEngineRowStream *_inputStream)
+{
+    CThorInput &_input = inputs.item(index);
+    IEngineRowStream *prevInputStream = _input.stream.getClear();
+    _input.stream.setown(_inputStream);
+    if (0 == index)
+        inputStream = _inputStream;
+    return prevInputStream;
+}
+
 void CSlaveActivity::setLookAhead(unsigned index, IStartableEngineRowStream *lookAhead)
 {
     CThorInput &_input = inputs.item(index);

thorlcr/graph/thgraphslave.hpp

@@ -167,6 +167,7 @@ public:
     virtual void connectInputStreams(bool consumerOrdered);
 
     void setLookAhead(unsigned index, IStartableEngineRowStream *lookAhead);
+    IEngineRowStream *replaceInputStream(unsigned index, IEngineRowStream *_inputStream);
     IThorDataLink *queryOutput(unsigned index) const;
     IThorDataLink *queryInput(unsigned index) const;
     IEngineRowStream *queryInputStream(unsigned index) const;

tools/esdlcmd/esdl-publish.cpp

@@ -75,10 +75,10 @@ public:
     virtual void usage()
     {
          printf(
-                "   -s, --server <ip>            IP of server running ESDL services\n"
-                "   --port <port>                ESDL services port\n"
-                "   -u, --username <name>        Username for accessing ESDL services\n"
-                "   -pw, --password <pw>         Password for accessing ESDL services\n"
+                "   -s, --server <ip>            IP of server running WsESDLConfig service\n"
+                "   --port <port>                WsESDLConfig service port\n"
+                "   -u, --username <name>        Username for accessing WsESDLConfig service\n"
+                "   -pw, --password <pw>         Password for accessing WsESDLConfig service\n"
                 "   --version <ver>              ESDL service version\n"
                 );
         EsdlCmdCommon::usage();
@@ -291,7 +291,7 @@ public:
                 "   ESDLServiceName                                  The Name of the ESDL Service (as defined in the ESDL Definition)\n"
 
                 "\nOptions (use option flag followed by appropriate value):\n"
-                "   --config <file|xml>                              Configuration XML for all methods associated with the target Service\n"
+                "   --config <file|\"xml\">                              Configuration XML for all methods associated with the target Service\n"
                 "   --overwrite                                      Overwrite binding if it already exists\n");
 
                 EsdlPublishCmdCommon::usage();
@@ -449,21 +449,22 @@ public:
     void usage()
     {
         printf( "\nUsage:\n\n"
-                "esdl unbind-service <TargetESPProcessName> <TargetESPBindingPort | TargetESPServiceName> [command options]\n\n"
-                "   TargetESPProcessName                             The target ESP Process name\n"
-                "   TargetESPBindingPort | TargetESPServiceName      Either target ESP binding port or target ESP service name\n");
+                "esdl unbind-service <TargetESPProcessName> <TargetESPServiceName> [command options]\n\n"
+                "   TargetESPProcessName      The target ESP Process name\n"
+                "   TargetESPServiceName      The target ESP Service name\n"
+                "\n\nOptions:\n"
+              );
 
         EsdlPublishCmdCommon::usage();
 
-        printf( "\n   Use this command to unpublish ESDL Service based bindings.\n"
-                "   To unbind an ESDL Service, provide the target ESP process name\n"
+        printf( "\n   Use this command to un-bind ESDL Service based bindings.\n"
+                "   To un-bind an ESDL Service, provide the target ESP process name\n"
                 "   (ESP Process which will host the ESP Service as defined in the ESDL Definition.) \n"
-                "   It is also necessary to provide the Port on which this service is configured to run (ESP Binding),\n"
-                "   and the name of the service you are unbinding.\n"
+                "   It is also necessary to provide the name of the service.\n"
                 );
 
         printf("\nExample:"
-                ">esdl unbind-service myesp 8088 \n"
+                ">esdl unbind-service myesp wsmyservice \n"
                 );
     }
 

tools/esdlcmd/esdlcmd_core.cpp

@@ -248,27 +248,27 @@ public:
     void printOptions()
     {
         puts("Options:");
-        puts("  --version <version number> : Constrain to interface version\n");
-        puts("  --method <method name>[;<method name>]* : Constrain to list of specific method(s)\n" );
-        puts("  --xslt <xslt file path> : Path to '/xslt/esxdl2xsd.xslt' file to transform EsdlDef to XSD\n" );
-        puts("  --preprocess-output <raw output directory> : Output pre-processed xml file to specified directory before applying XSLT transform\n" );
+        puts("  --version <version number> : Constrain to interface version");
+        puts("  --method <method name>[;<method name>]* : Constrain to list of specific method(s)" );
+        puts("  --xslt <xslt file path> : Path to '/xslt/esxdl2xsd.xslt' file to transform EsdlDef to XSD" );
+        puts("  --preprocess-output <raw output directory> : Output pre-processed xml file to specified directory before applying XSLT transform" );
         puts("  --annotate <all | none> : Flag turning on either all annotations or none. By default annotations are generated " );
         puts("                    for Enumerations. Setting the flag to 'none' will disable even those. Setting it" );
-        puts("                    to 'all' will enable additional annotations such as collapsed, cols, form_ui, html_head and rows.\n");
+        puts("                    to 'all' will enable additional annotations such as collapsed, cols, form_ui, html_head and rows.");
         puts("  --noopt : Turns off the enforcement of 'optional' attributes on elements. If no -noopt is specified then all elements with an 'optional'" );
-        puts("       will be included in the output. By default 'optional' filtering is enforced.\n");
+        puts("       will be included in the output. By default 'optional' filtering is enforced.");
         puts("  -opt,--optional <param value> : Value to use for optional tag filter when gathering dependencies" );
         puts("                       An example: passing 'internal' when some Esdl definition objects have the attribute");
-        puts("                       optional(\"internal\") will ensure they appear in the XSD, otherwise they'd be filtered out\n");
-        puts("  -tns,--target-namespace <target namespace> : The target namespace, passed to the transform via the parameter 'tnsParam'\n" );
+        puts("                       optional(\"internal\") will ensure they appear in the XSD, otherwise they'd be filtered out");
+        puts("  -tns,--target-namespace <target namespace> : The target namespace, passed to the transform via the parameter 'tnsParam'" );
         puts("                            used for the final output of the XSD. If not supplied will default to " );
-        puts("                            http://webservices.seisint.com/<service name>\n" );
-        puts("  -n <int>: Number of times to run transform after loading XSLT. Defaults to 1.\n" );
-        puts("  --show-inheritance : Turns off the collapse feature. Collapsing optimizes the XML output to strip out structures\n" );
-        puts("                        only used for inheritance, and collapses their elements into their child. That simplifies the\n" );
+        puts("                            http://webservices.seisint.com/<service name>" );
+        puts("  -n <int>: Number of times to run transform after loading XSLT. Defaults to 1." );
+        puts("  --show-inheritance : Turns off the collapse feature. Collapsing optimizes the XML output to strip out structures" );
+        puts("                        only used for inheritance, and collapses their elements into their child. That simplifies the" );
         puts("                        stylesheet. By default this option is on.");
-        puts("  --no-arrayof : Supresses the use of the arrrayof element. arrayof optimizes the XML output to include 'ArrayOf...'\n" );
-        puts("                        structure definitions for those EsdlArray elements with no item_tag attribute. Works in conjunction\n" );
+        puts("  --no-arrayof : Supresses the use of the arrrayof element. arrayof optimizes the XML output to include 'ArrayOf...'" );
+        puts("                        structure definitions for those EsdlArray elements with no item_tag attribute. Works in conjunction" );
         puts("                        with an optimized stylesheet that doesn't generate these itself. This defaults to on.");
     }
 
@@ -276,9 +276,9 @@ public:
     {
         puts("Usage:");
         puts("esdl xsd sourcePath serviceName [options]\n" );
-        puts("\nsourcePath must be absolute path to the ESDL Definition file containing the" );
-        puts("EsdlService definition for the service you want to work with.\n" );
-        puts("serviceName EsdlService definition for the service you want to work with.\n" );
+        puts("\nsourcePath - Absolute path to the EXSDL Definition file ( XML generated from ECM )" );
+        puts("               which contains ESDL Service definition.\n" );
+        puts("serviceName  - Name of ESDL Service defined in the given EXSDL file.\n" );
 
         printOptions();
         EsdlConvertCmd::usage();
@@ -574,9 +574,9 @@ public:
     {
         puts("Usage:");
         puts("esdl wsdl sourcePath serviceName [options]\n" );
-        puts("\nsourcePath must be absolute path to the ESDL Definition file containing the" );
-        puts("EsdlService definition for the service you want to work with.\n" );
-        puts("serviceName EsdlService definition for the service you want to work with.\n" );
+        puts("\nsourcePath - Absolute path to the EXSDL Definition file ( XML generated from ECM )" );
+        puts("               which contains ESDL Service definition.\n" );
+        puts("serviceName  - Name of ESDL Service defined in the given EXSDL file.\n" );
 
         printOptions();
         puts("  --wsdladdress  Defines the output WSDL's location address\n");
@@ -766,22 +766,23 @@ public:
     void printOptions()
     {
         puts("Options:");
-        puts("  --version <version number> : Constrain to interface version\n");
-        puts("  --method <method name>[;<method name>]* : Constrain to list of specific method(s)\n" );
-        puts("  --xslt <xslt file path> : Path to xslt files used to transform EsdlDef to Java code\n" );
-        puts("  --preprocess-output <raw output directory> : Output pre-processed xml file to specified directory before applying XSLT transform\n" );
-        puts("  --show-inheritance : Turns off the collapse feature. Collapsing optimizes the XML output to strip out structures\n" );
-        puts("                        only used for inheritance, and collapses their elements into their child. That simplifies the\n" );
+        puts("  --version <version number> : Constrain to interface version");
+        puts("  --method <method name>[;<method name>]* : Constrain to list of specific method(s)" );
+        puts("  --xslt <xslt file path> : Path to xslt files used to transform EsdlDef to Java code" );
+        puts("  --preprocess-output <raw output directory> : Output pre-processed xml file to specified directory before applying XSLT transform" );
+        puts("  --show-inheritance : Turns off the collapse feature. Collapsing optimizes the XML output to strip out structures" );
+        puts("                        only used for inheritance, and collapses their elements into their child. That simplifies the" );
         puts("                        stylesheet. By default this option is on.");
     }
 
     virtual void usage()
     {
         puts("Usage:");
+
         puts("esdl java sourcePath serviceName [options]\n" );
-        puts("\nsourcePath must be absolute path to the ESDL Definition file containing the" );
-        puts("EsdlService definition for the service you want to work with.\n" );
-        puts("serviceName EsdlService definition for the service you want to work with.\n" );
+        puts("\nsourcePath - Absolute path to the EXSDL Definition file ( XML generated from ECM )" );
+        puts("               which contains ESDL Service definition.\n" );
+        puts("serviceName  - Name of ESDL Service defined in the given EXSDL file.\n" );
 
         printOptions();
         EsdlConvertCmd::usage();