Merge branch 'candidate-6.0.4'

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

common/workunit/workunit.cpp

@@ -485,7 +485,10 @@ public:
             return false;
         ensureUniqueStatistic();
         if (!firstSubGraph())
-            return false;
+        {
+            if (!nextGraph())
+                return false;
+        }
 
         valid = true;
         return true;

dali/base/dacoven.cpp

@@ -37,7 +37,7 @@ extern void closedownDFS();
 // base is saved in store whenever block exhausted, so replacement coven servers can restart 
 
 // server side versioning.
-#define ServerVersion    "3.12"
+#define ServerVersion    "3.13"
 #define MinClientVersion "1.5"
 
 

dali/base/dadfs.cpp

@@ -78,6 +78,7 @@ enum MDFSRequestKind
     MDFS_ITERATE_RELATIONSHIPS,
     MDFS_SET_FILE_PROTECT,
     MDFS_ITERATE_FILTEREDFILES,
+    MDFS_ITERATE_FILTEREDFILES2,
     MDFS_MAX
 };
 
@@ -9837,10 +9838,8 @@ public:
         mb.writeDirect(0,sizeof(count),&count);
     }
 
-    void iterateFilteredFiles(CMessageBuffer &mb,StringBuffer &trc)
+    void iterateFilteredFiles(TransactionLog &transactionLog, CMessageBuffer &mb,StringBuffer &trc, bool returnAllFilesFlag)
     {
-        TransactionLog transactionLog(*this, MDFS_ITERATE_FILTEREDFILES, mb.getSender());
-
         Owned<IUserDescriptor> udesc;
         StringAttr filters;
         bool recursive;
@@ -9886,7 +9885,8 @@ public:
             e->Release();
             returnAllMatchingFiles = false;
         }
-        mb.append(returnAllMatchingFiles);
+        if (returnAllFilesFlag)
+            mb.append(returnAllMatchingFiles);
 
         tookMs = msTick()-start;
         if (tookMs>100)
@@ -9906,6 +9906,18 @@ public:
         mb.writeDirect(0,sizeof(count),&count);
     }
 
+    void iterateFilteredFiles(CMessageBuffer &mb,StringBuffer &trc)
+    {
+        TransactionLog transactionLog(*this, MDFS_ITERATE_FILTEREDFILES, mb.getSender());
+        iterateFilteredFiles(transactionLog, mb, trc, false);
+    }
+
+    void iterateFilteredFiles2(CMessageBuffer &mb,StringBuffer &trc)
+    {
+        TransactionLog transactionLog(*this, MDFS_ITERATE_FILTEREDFILES2, mb.getSender());
+        iterateFilteredFiles(transactionLog, mb, trc, true);
+    }
+
     void iterateRelationships(CMessageBuffer &mb,StringBuffer &trc)
     {
         TransactionLog transactionLog(*this, MDFS_ITERATE_RELATIONSHIPS, mb.getSender());
@@ -10124,54 +10136,71 @@ public:
         int fn;
         mb.read(fn);
 
-        try {
-            switch (fn) {
-            case MDFS_ITERATE_FILES: {
-                    iterateFiles(mb,trc);                    
+        try
+        {
+            switch (fn)
+            {
+                case MDFS_ITERATE_FILES:
+                {
+                    iterateFiles(mb, trc);
+                    break;
                 }
-                break;
-            case MDFS_ITERATE_FILTEREDFILES: {
-                    iterateFilteredFiles(mb,trc);
+                case MDFS_ITERATE_FILTEREDFILES: // legacy, newer clients will send MDFS_ITERATE_FILTEREDFILES2
+                {
+                    iterateFilteredFiles(mb, trc);
+                    break;
                 }
-                break;
-            case MDFS_ITERATE_RELATIONSHIPS: {
-                    iterateRelationships(mb,trc);                    
+                case MDFS_ITERATE_FILTEREDFILES2:
+                {
+                    iterateFilteredFiles2(mb, trc);
+                    break;
                 }
-                break;
-            case MDFS_GET_FILE_TREE: {
-                    getFileTree(mb,trc);
+                case MDFS_ITERATE_RELATIONSHIPS:
+                {
+                    iterateRelationships(mb, trc);
+                    break;
                 }
-                break;
-            case MDFS_GET_GROUP_TREE: {
-                    getGroupTree(mb,trc);
+                case MDFS_GET_FILE_TREE:
+                {
+                    getFileTree(mb, trc);
+                    break;
                 }
-                break;
-            case MDFS_SET_FILE_ACCESSED: {
-                    setFileAccessed(mb,trc);
+                case MDFS_GET_GROUP_TREE:
+                {
+                    getGroupTree(mb, trc);
+                    break;
                 }
-                break;
-            case MDFS_SET_FILE_PROTECT: {
-                    setFileProtect(mb,trc);
+                case MDFS_SET_FILE_ACCESSED:
+                {
+                    setFileAccessed(mb, trc);
+                    break;
                 }
-                break;
-            default: {
+                case MDFS_SET_FILE_PROTECT:
+                {
+                    setFileProtect(mb, trc);
+                    break;
+                }
+                default:
+                {
                     mb.clear();
+                    break;
                 }
-                break;
             }
         }
-        catch (IException *e) {
+        catch (IException *e)
+        {
             int err=-1; // exception marker
             mb.clear().append(err); 
             serializeException(e, mb); 
             e->Release();
         }
-        coven.reply(mb);    
-        if (block0.slow()) {
+        coven.reply(mb);
+        if (block0.slow())
+        {
             SocketEndpoint ep = mb.getSender();
             ep.getUrlStr(block0.appendMsg(trc).append(" from "));
         }
-    }   
+    }
 
     void nodeDown(rank_t rank)
     {
@@ -10187,6 +10216,8 @@ public:
             return ret.append("MDFS_ITERATE_FILES");
         case MDFS_ITERATE_FILTEREDFILES:
             return ret.append("MDFS_ITERATE_FILTEREDFILES");
+        case MDFS_ITERATE_FILTEREDFILES2:
+            return ret.append("MDFS_ITERATE_FILTEREDFILES2");
         case MDFS_ITERATE_RELATIONSHIPS:
             return ret.append("MDFS_ITERATE_RELATIONSHIPS");
         case MDFS_GET_FILE_TREE:
@@ -12072,12 +12103,13 @@ extern da_decl const char* getDFUQResultFieldName(DFUQResultField feild)
     return DFUQResultFieldNames[feild];
 }
 
-IPropertyTreeIterator *deserializeFileAttrIterator(MemoryBuffer& mb, DFUQResultField* localFilters, const char* localFilterBuf)
+IPropertyTreeIterator *deserializeFileAttrIterator(MemoryBuffer& mb, unsigned numFiles, DFUQResultField* localFilters, const char* localFilterBuf)
 {
     class CFileAttrIterator: public CInterface, implements IPropertyTreeIterator
     {
         Owned<IPropertyTree> cur;
         StringArray fileNodeGroups;
+        size32_t fileDataStart;
 
         void setFileNodeGroup(IPropertyTree *attr, const char* group, StringArray& nodeGroupFilter)
         {
@@ -12182,15 +12214,19 @@ IPropertyTreeIterator *deserializeFileAttrIterator(MemoryBuffer& mb, DFUQResultF
         IMPLEMENT_IINTERFACE;
         MemoryBuffer mb;
         unsigned numfiles;
-        bool allMatchingFilesReceived;
         StringArray nodeGroupFilter;
 
+        CFileAttrIterator(MemoryBuffer &_mb, unsigned _numfiles) : numfiles(_numfiles)
+        {
+            /* not particuarly nice, but buffer contains extra meta info ahead of serialized file info
+             * record position to rewind to, if iterator reused.
+             */
+            fileDataStart = _mb.getPos();
+            mb.swapWith(_mb);
+        }
         bool first()
         {
-            mb.reset();
-            mb.read(numfiles);
-            mb.read(allMatchingFilesReceived);
-
+            mb.reset(fileDataStart);
             return next();
         }
 
@@ -12237,8 +12273,7 @@ IPropertyTreeIterator *deserializeFileAttrIterator(MemoryBuffer& mb, DFUQResultF
             }
         }
 
-    } *fai = new CFileAttrIterator;
-    mb.swapWith(fai->mb);
+    } *fai = new CFileAttrIterator(mb, numFiles);
     fai->setLocalFilters(localFilters, localFilterBuf);
     return fai;
 }
@@ -12247,7 +12282,13 @@ IPropertyTreeIterator *CDistributedFileDirectory::getDFAttributesTreeIterator(co
     const char* localFilterBuf, IUserDescriptor* user, bool& allMatchingFilesReceived, INode* foreigndali, unsigned foreigndalitimeout)
 {
     CMessageBuffer mb;
-    mb.append((int)MDFS_ITERATE_FILTEREDFILES).append(filters).append(true);
+    CDaliVersion serverVersionNeeded("3.13");
+    bool legacy = (queryDaliServerVersion().compare(serverVersionNeeded) < 0);
+    if (legacy)
+        mb.append((int)MDFS_ITERATE_FILTEREDFILES);
+    else
+        mb.append((int)MDFS_ITERATE_FILTEREDFILES2);
+    mb.append(filters).append(true);
     if (user)
         user->serialize(mb);
 
@@ -12259,8 +12300,11 @@ IPropertyTreeIterator *CDistributedFileDirectory::getDFAttributesTreeIterator(co
 
     unsigned numfiles;
     mb.read(numfiles);
-    mb.read(allMatchingFilesReceived);
-    return deserializeFileAttrIterator(mb, localFilters, localFilterBuf);
+    if (legacy)
+        allMatchingFilesReceived = true; // don't know any better
+    else
+        mb.read(allMatchingFilesReceived);
+    return deserializeFileAttrIterator(mb, numfiles, localFilters, localFilterBuf);
 }
 
 IDFAttributesIterator* CDistributedFileDirectory::getLogicalFilesSorted(

ecllibrary/teststd/Date/TestFormat.ecl

@@ -41,14 +41,6 @@ EXPORT TestFormat := MODULE
     ASSERT(Date.DateToString(20110302, '%d %b %Y') = '02 Mar 2011');
     ASSERT(Date.DateToString(20111202, '%d %B %Y') = '02 December 2011');
 
-    ASSERT(Date.MatchDateString('1dec2011',DateFormats) = 20111201);
-    ASSERT(Date.MatchDateString('2011dec1',DateFormats) = 20111201);
-    ASSERT(Date.MatchDateString('1 december 2011',DateFormats) = 20111201);
-    ASSERT(Date.MatchDateString('2011\tdecem\t01',DateFormats) = 20111201);
-    ASSERT(Date.MatchDateString('20111201',DateFormats) = 20111201);
-    ASSERT(Date.MatchDateString('2011-12-01',DateFormats) = 20111201);
-    ASSERT(Date.MatchDateString('1/12/2011',DateFormats) = 20111201);
-
     ASSERT(Date.MatchTimeString('123456',TimeFormats) = 123456);
     ASSERT(Date.MatchTimeString('12:34:56',TimeFormats) = 123456);
     ASSERT(Date.MatchTimeString('12:34',TimeFormats) = 123400);

initfiles/sbin/prerm.in

@@ -28,9 +28,7 @@ fi
 # Stop all services before cleaning up
 ######################################
 /etc/init.d/hpcc-init stop
-if [ -f "${PID_DIR}/mydafilesrv.pid" ]; then
-    /etc/init.d/dafilesrv stop
-fi
+/etc/init.d/dafilesrv stop
 
 # Remove edits from /etc files
 ##############################

system/security/shared/caching.cpp

@@ -228,7 +228,7 @@ int CPermissionsCache::lookup( ISecUser& sec_user, IArrayOf<ISecResource>& resou
 {
     const char* userId = sec_user.getName();
     int nFound;
-    ReadLockBlock readLock(m_resPermCacheRWLock);
+    CriticalBlock block(m_resPermCacheLock);
     MapResPermissionsCache::const_iterator i = m_resPermissionsMap.find( userId ); 
     if (i != m_resPermissionsMap.end())
     {
@@ -252,7 +252,7 @@ int CPermissionsCache::lookup( ISecUser& sec_user, IArrayOf<ISecResource>& resou
 void CPermissionsCache::add( ISecUser& sec_user, IArrayOf<ISecResource>& resources )
 {
     const char* user = sec_user.getName();
-    WriteLockBlock writeLock(m_resPermCacheRWLock);
+    CriticalBlock block(m_resPermCacheLock);
     MapResPermissionsCache::const_iterator i = m_resPermissionsMap.find( user ); 
     CResPermissionsCache* pResPermissionsCache;
 
@@ -282,7 +282,7 @@ void CPermissionsCache::removePermissions( ISecUser& sec_user)
 #ifdef _DEBUG
         DBGLOG("CACHE: CPermissionsCache Removing permissions for user %s", user);
 #endif
-        WriteLockBlock writeLock(m_resPermCacheRWLock);
+        CriticalBlock block(m_resPermCacheLock);
         m_resPermissionsMap.erase(user); 
     }
 }
@@ -290,7 +290,7 @@ void CPermissionsCache::removePermissions( ISecUser& sec_user)
 void CPermissionsCache::remove(SecResourceType rtype, const char* resourcename)
 {
     MapResPermissionsCache::const_iterator i;
-    WriteLockBlock writeLock(m_resPermCacheRWLock);
+    CriticalBlock block(m_resPermCacheLock);
     MapResPermissionsCache::const_iterator iEnd = m_resPermissionsMap.end(); 
 
     for (i = m_resPermissionsMap.begin(); i != iEnd; i++)
@@ -310,7 +310,7 @@ bool CPermissionsCache::lookup(ISecUser& sec_user)
         return false;
 
     string key(username);
-    ReadLockBlock readLock(m_userCacheRWLock );
+    CriticalBlock block(m_userCacheLock);
 
     MapUserCache::iterator it = m_userCache.find(key);
     if (it == m_userCache.end())
@@ -364,7 +364,7 @@ ISecUser* CPermissionsCache::getCachedUser( ISecUser& sec_user)
         return NULL;
 
     string key(username);
-    ReadLockBlock readLock(m_userCacheRWLock );
+    CriticalBlock block(m_userCacheLock);
     MapUserCache::iterator it = m_userCache.find(key);
     if (it == m_userCache.end())
         return NULL;
@@ -382,7 +382,7 @@ void CPermissionsCache::add(ISecUser& sec_user)
         return;
     
     string key(username);
-    WriteLockBlock writeLock(m_userCacheRWLock );
+    CriticalBlock block(m_userCacheLock);
     MapUserCache::iterator it = m_userCache.find(key);
     CachedUser* user = NULL;
     if (it != m_userCache.end())
@@ -403,7 +403,7 @@ void CPermissionsCache::removeFromUserCache(ISecUser& sec_user)
     if(username && *username)
     {
         string key(username);
-        WriteLockBlock writeLock(m_userCacheRWLock );
+        CriticalBlock block(m_userCacheLock);
         MapUserCache::iterator it = m_userCache.find(key);
         if (it != m_userCache.end())
         {
@@ -419,7 +419,7 @@ void CPermissionsCache::removeFromUserCache(ISecUser& sec_user)
 
 bool CPermissionsCache::addManagedFileScopes(IArrayOf<ISecResource>& scopes)
 {
-    WriteLockBlock writeLock(m_scopesRWLock);
+    CriticalBlock block(m_scopesLock);
     ForEachItemIn(x, scopes)
     {
         ISecResource* scope = &scopes.item(x);
@@ -445,7 +445,7 @@ bool CPermissionsCache::addManagedFileScopes(IArrayOf<ISecResource>& scopes)
 
 inline void CPermissionsCache::removeManagedFileScopes(IArrayOf<ISecResource>& scopes)
 {
-    WriteLockBlock writeLock(m_scopesRWLock);
+    CriticalBlock block(m_scopesLock);
     ForEachItemIn(x, scopes)
     {
         ISecResource* scope = &scopes.item(x);
@@ -466,7 +466,7 @@ inline void CPermissionsCache::removeManagedFileScopes(IArrayOf<ISecResource>& s
 
 inline void CPermissionsCache::removeAllManagedFileScopes()
 {
-    WriteLockBlock writeLock(m_scopesRWLock);
+    CriticalBlock block(m_scopesLock);
     map<string, ISecResource*>::const_iterator cit;
     map<string, ISecResource*>::const_iterator iEnd = m_managedFileScopesMap.end();
 
@@ -538,7 +538,7 @@ bool CPermissionsCache::queryPermsManagedFileScope(ISecUser& sec_user, const cha
     ISecResource *matchedRes = NULL;
     ISecResource *res = NULL;
     bool isManaged = false;
-    ReadLockBlock readLock(m_scopesRWLock);
+    CriticalBlock block(m_scopesLock);
     for(unsigned i = 0; i < scopes.length(); i++)
     {
         const char* scope = scopes.item(i);
@@ -616,8 +616,10 @@ int CPermissionsCache::queryDefaultPermission(ISecUser& user)
 }
 void CPermissionsCache::flush()
 {
+    // MORE - is this safe? m_defaultPermossion and m_lastManagedFileScopesRefresh are unprotected,
+    // and entries could be added to the first cache while the second is being cleared - does that matter?
     {
-        WriteLockBlock writeLock(m_resPermCacheRWLock);
+        CriticalBlock block(m_resPermCacheLock);
         MapResPermissionsCache::const_iterator i;
         MapResPermissionsCache::const_iterator iEnd = m_resPermissionsMap.end();
         for (i = m_resPermissionsMap.begin(); i != iEnd; i++)
@@ -625,7 +627,7 @@ void CPermissionsCache::flush()
         m_resPermissionsMap.clear();
     }
     {
-        WriteLockBlock writeLock(m_userCacheRWLock );
+        CriticalBlock block(m_userCacheLock);
         MapUserCache::const_iterator ui;
         MapUserCache::const_iterator uiEnd = m_userCache.end();
         for (ui = m_userCache.begin(); ui != uiEnd; ui++)

system/security/shared/caching.hpp

@@ -191,20 +191,20 @@ private:
     typedef std::map<string, CachedUser*> MapUserCache;
 
     MapResPermissionsCache m_resPermissionsMap;  //user specific resource permissions cache
-    mutable ReadWriteLock m_resPermCacheRWLock; //guards m_resPermissionsMap
+    mutable CriticalSection m_resPermCacheLock; //guards m_resPermissionsMap - DO NOT use RW lock as the lookup modifies the cache to remove expired entries
 
     int m_cacheTimeout; //cleanup cycle period
     bool m_transactionalEnabled;
 
     MapUserCache m_userCache;
-    mutable ReadWriteLock m_userCacheRWLock;    //guards m_userCache
+    mutable CriticalSection m_userCacheLock;    //guards m_userCache - DO NOT use RW lock as the lookup modifies the cache to remove expired entries
 
     StringAttr                  m_secMgrClass;
 
     //Managed File Scope support
     int                         m_defaultPermission;
     map<string, ISecResource*>  m_managedFileScopesMap;
-    mutable ReadWriteLock       m_scopesRWLock;//guards m_managedFileScopesMap
+    mutable CriticalSection     m_scopesLock;  //guards m_managedFileScopesMap
     ISecManager *               m_secMgr;
     time_t                      m_lastManagedFileScopesRefresh;
 };

testing/regress/hpcc/util/util.py

@@ -140,8 +140,8 @@ def abortWorkunit(wuid):
 
 def createZAP(wuid,  taskId):
     retVal = 'Error in create ZAP'
-    # http://localhost:8010/WsWorkunits/WUCreateZAPInfo?Wuid=<wuid>&ProblemDescription=<problem_description>
-    host = "http://"+gConfig.espIp+gConfig.espSocket+"/WsWorkunits/WUCreateZAPInfo?Wuid="+wuid+"&ProblemDescription=\"Failed+in+OBT\""
+    # http://localhost:8010/WsWorkunits/WUCreateZAPInfo?Wuid=<wuid>&ProblemDescription=<problem_description>&IncludeThorSlaveLog="on"
+    host = "http://"+gConfig.espIp+gConfig.espSocket+"/WsWorkunits/WUCreateZAPInfo?Wuid="+wuid+"&ProblemDescription=\"Failed+in+OBT\"&IncludeThorSlaveLog=on"
     logging.debug("%3d. createZAP(%s, host :'%s')",  taskId,  wuid, host)
 
     state = 'OK'

thorlcr/activities/join/thjoin.cpp

@@ -263,7 +263,7 @@ public:
                             ActPrintLog("JOIN barrier.2 raised");
                             imaster->SortDone();
                             // NB on the cosort should use same serializer as sort (but in fact it only gets used when 0 rows on primary side)
-                            imaster->SortSetup(secondaryRowIf, secondaryCompare, primaryKeySerializer, true, false, NULL, NULL); //serializers OK
+                            imaster->SortSetup(secondaryRowIf, secondaryCompare, primaryKeySerializer, true, false, NULL, primaryRowIf); //serializers OK
                             ActPrintLog("JOIN waiting for barrier.3");
                             if (barrier->wait(false)) // local sort complete
                             {