Merge branch 'candidate-6.4.24'

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

docs/EN_US/ConfiguringHPCC/ConfiguringHPCC.xml

@@ -2556,6 +2556,28 @@ sudo -u hpcc cp /etc/HPCCSystems/source/NewEnvironment.xml /etc/HPCCSystems/envi
             </varlistentry>
           </variablelist>
         </sect3>
+        <sect3>
+          <title>Redundancy</title>
+          <para>Roxie can be configured to utilize a few different redundancy
+          models. <itemizedlist>
+              <listitem>
+                <para>Simple Redundancy - One channel per slave. Most commonly
+                used for a single node Roxie.</para>
+              </listitem>
+              <listitem>
+                <para>Full Redundancy - More slaves than the number of
+                channels. Multiple slaves host each channel.</para>
+              </listitem>
+              <listitem>
+                <para>Overloaded Redundancy - There are multiple channels per
+                slave.</para>
+              </listitem>
+              <listitem>
+                <para>Cyclic Redundancy - Each node hosts multiple channels in
+                rotation. The most commonly used configuration.</para>
+              </listitem>
+            </itemizedlist></para>
+        </sect3>
       </sect2>
 
       <sect2 id="Topology" role="brk">

docs/EN_US/ECLProgrammersGuide/PRG_Mods/PrG_Creating_example_data.xml

@@ -10,9 +10,8 @@
     <para>All the example code for the <emphasis>Programmer's Guide</emphasis>
     is available for download from the HPCC Systems website from the same page
     that the PDF is available (<ulink
-    url="http://hpccsystems.com/download/docs/learning-ecl">click
-    here</ulink>). To make this code available for use in the ECL IDE, you
-    simply:</para>
+    url="https://hpccsystems.com/training/documentation/learning-ecl">https://hpccsystems.com/training/documentation/learning-ecl</ulink>).
+    To make this code available for use in the ECL IDE, you simply:</para>
 
     <orderedlist>
       <listitem>

docs/EN_US/HPCCDataTutorial/DataTutorial.xml

@@ -116,7 +116,7 @@
           <para>Download a raw data file</para>
 
           <para>There are links to data file available at <ulink
-          url="http://hpccsystems.com/community/docs/data-tutorial-guide">http://hpccsystems.com/community/docs/data-tutorial-guide</ulink></para>
+          url="https://hpccsystems.com/training/documentation/learning-ecl">https://hpccsystems.com/training/documentation/learning-ecl</ulink></para>
 
           <para>The download is approximately 30 MB (compressed) and is
           available in either ZIP or .tar.gz format. Choose the appropriate

esp/logging/logginglib/datafieldmap.cpp

@@ -85,12 +85,14 @@ void CLogTable::loadMappings(IPropertyTree& fieldList)
 void CLogGroup::loadMappings(IPropertyTree& fieldList)
 {
     StringBuffer tableName;
+    bool enableLogID;
     Owned<IPropertyTreeIterator> itr = fieldList.getElements("Fieldmap");
     ForEach(*itr)
     {
         ensureInputString(itr->query().queryProp("@table"), true, tableName, -1, "Fieldmap @table required");
+        enableLogID = itr->query().getPropBool("@enableLogID", true);
 
-        Owned<CLogTable> table = new CLogTable(tableName.str());
+        Owned<CLogTable> table = new CLogTable(tableName.str(), enableLogID);
         table->loadMappings(itr->query());
         CIArrayOf<CLogField>& logFields = table->getLogFields();
         if (logFields.length() < 1)

esp/logging/logginglib/datafieldmap.hpp

@@ -47,12 +47,14 @@ public:
 class CLogTable : public CInterface
 {
     StringAttr tableName;
+    bool enableLogID;
     CIArrayOf<CLogField> logFields;
 public:
-    CLogTable(const char* _tableName) : tableName(_tableName) {};
+    CLogTable(const char* _tableName, bool _enableLogID) : tableName(_tableName), enableLogID(_enableLogID) {};
     virtual ~CLogTable() {};
 
     const char* getTableName() { return tableName.get(); };
+    bool getEnableLogID() { return enableLogID; }
     void setTableName(const char* _tableName) { return tableName.set(_tableName); };
     void loadMappings(IPropertyTree& cfg);
     CIArrayOf<CLogField>& getLogFields() { return logFields; };

esp/logging/logginglib/loggingagentbase.cpp

@@ -232,7 +232,9 @@ bool CDBLogAgentBase::buildUpdateLogStatement(IPropertyTree* logRequest, const c
 
     //add any default fields that may be required but not in request.
     addMissingFields(logFields, handledFields, fields, values);
-    appendFieldInfo("log_id", logID, fields, values, true);
+    if (table.getEnableLogID()) {
+        appendFieldInfo("log_id", logID, fields, values, true);
+    }
 
     setUpdateLogStatement(logDB, table.getTableName(), fields.str(), values.str(), updateDBStatement);
     return true;

system/security/shared/caching.cpp

@@ -79,14 +79,14 @@ int CResPermissionsCache::lookup( IArrayOf<ISecResource>& resources, bool* pFoun
         if (it != m_resAccessMap.end())//exists in cache
         {
             ResPermCacheEntry& resParamCacheEntry = (*it).second;
-            const time_t timestamp = resParamCacheEntry.first;
+            const time_t timeExpiry = resParamCacheEntry.first + m_pParentCache->getCacheTimeout();
 
-            if (timestamp < tstamp)//entry was not stale during last cleanup but is stale now
+            if (timeExpiry < tstamp)//entry was not stale during last cleanup but is stale now
                 *pFound++ = false;
             else if(!m_pParentCache->isCacheEnabled() && m_pParentCache->isTransactionalEnabled())//m_pParentCache->getOriginalTimeout() == 0)
             {
                 time_t tctime = getThreadCreateTime();
-                if(tctime <= 0 || timestamp < tctime)
+                if(tctime <= 0 || timeExpiry < tctime)
                 {
                     *pFound++ = false;
                 }
@@ -532,25 +532,31 @@ inline void CPermissionsCache::removeAllManagedFileScopes()
 
     etc. Until full scope path checked, or no read permissions hit on ancestor scope.
 */
+static CriticalSection msCacheSyncCS;//for managed scopes cache syncronization
 bool CPermissionsCache::queryPermsManagedFileScope(ISecUser& sec_user, const char * fullScope, StringBuffer& managedScope, SecAccessFlags * accessFlags)
 {
+    unsigned start = msTick();
     if (!fullScope || !*fullScope)
     {
         *accessFlags = queryDefaultPermission(sec_user);
         return true;
     }
 
-    time_t now;
-    time(&now);
-    if (m_secMgr && (0 == m_lastManagedFileScopesRefresh || m_lastManagedFileScopesRefresh < (now - m_cacheTimeout)))
+    if (m_secMgr)
     {
-        removeAllManagedFileScopes();
-        IArrayOf<ISecResource> scopes;
-        aindex_t count = m_secMgr->getManagedFileScopes(scopes);
-        if (count)
-            addManagedFileScopes(scopes);
-        m_defaultPermission = SecAccess_Unknown;//trigger refresh
-        m_lastManagedFileScopesRefresh = now;
+        CriticalBlock block(msCacheSyncCS);
+        time_t now;
+        time(&now);
+        if (0 == m_lastManagedFileScopesRefresh || ((now - m_lastManagedFileScopesRefresh) > m_cacheTimeout))
+        {
+            removeAllManagedFileScopes();
+            IArrayOf<ISecResource> scopes;
+            aindex_t count = m_secMgr->getManagedFileScopes(scopes);
+            if (count)
+                addManagedFileScopes(scopes);
+            m_defaultPermission = SecAccess_Unknown;//trigger refresh
+            time(&m_lastManagedFileScopesRefresh);
+        }
     }
 
     if (m_managedFileScopesMap.empty())
@@ -601,9 +607,7 @@ bool CPermissionsCache::queryPermsManagedFileScope(ISecUser& sec_user, const cha
                 {
                     *accessFlags = res->getAccessFlags();
                     managedScope.append(const_cast<char *>(res->getName()));
-#ifdef _DEBUG
-                    DBGLOG("FileScope %s for %s(%s) access denied %d",fullScope, sec_user.getName(), res->getName(), *accessFlags);
-#endif
+                    DBGLOG("FileScope %s for %s(%s) access denied %d at scope %s, took %dms",fullScope, sec_user.getName(), res->getName(), *accessFlags, scope, msTick()-start);
                     return true;
                 }
                 else
@@ -619,7 +623,7 @@ bool CPermissionsCache::queryPermsManagedFileScope(ISecUser& sec_user, const cha
             *accessFlags = matchedRes->getAccessFlags();
             managedScope.append(const_cast<char *>(matchedRes->getName()));
 #ifdef _DEBUG
-            DBGLOG("FileScope %s for %s(%s) access granted %d", fullScope, sec_user.getName(), matchedRes->getName(), *accessFlags);
+            DBGLOG("FileScope %s for %s(%s) access granted %d, took %dms", fullScope, sec_user.getName(), matchedRes->getName(), *accessFlags, msTick()-start);
 #endif
             rc = true;
         }
@@ -628,7 +632,7 @@ bool CPermissionsCache::queryPermsManagedFileScope(ISecUser& sec_user, const cha
             managedScope.append(const_cast<char *>(res->getName()));
 
 #ifdef _DEBUG
-            DBGLOG("FileScope %s for %s(%s) managed but not cached", fullScope, sec_user.getName(), res->getName());
+            DBGLOG("FileScope %s for %s(%s) managed but not cached, took %dms", fullScope, sec_user.getName(), res->getName(), msTick()-start);
 #endif
             rc = false;//need to go to LDAP to check
         }
@@ -637,7 +641,7 @@ bool CPermissionsCache::queryPermsManagedFileScope(ISecUser& sec_user, const cha
     {
         *accessFlags = queryDefaultPermission(sec_user);
 #ifdef _DEBUG
-        DBGLOG("FileScope %s for %s not managed, using default %d", fullScope, sec_user.getName(),*accessFlags);
+        DBGLOG("FileScope %s for %s not managed, using default %d, took %dms", fullScope, sec_user.getName(),*accessFlags, msTick()-start);
 #endif
         rc = true;
     }