Merge branch 'candidate-5.4.0'

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

common/workunit/pkgimpl.hpp

@@ -520,6 +520,8 @@ public:
                 }
 
                 filelist->addFilesFromQuery(cw, this, queryid);
+
+                unsigned unmatchedCount=0;
                 Owned<IReferencedFileIterator> refFiles = filelist->getFiles();
                 ForEach(*refFiles)
                 {
@@ -549,6 +551,7 @@ public:
                     }
                     VStringBuffer fullname("%s/%s", queryid, rf.getLogicalName());
                     unmatchedFiles.append(fullname);
+                    unmatchedCount++;
                 }
 
                 const IHpccPackage *matched = matchPackage(queryid);
@@ -557,6 +560,12 @@ public:
                     const char *matchId = matched->queryTree()->queryProp("@id");
                     if (!referencedPackages.getValue(matchId))
                         referencedPackages.setValue(matchId, true);
+                    if (unmatchedCount && matched->isCompulsory())
+                    {
+                        VStringBuffer msg("Compulsory query %s has query files not defined in package %s", queryid, matchId);
+                        err.append(msg.str());
+                        isValid=false;
+                    }
                 }
                 else
                     unmatchedQueries.append(queryid);

dali/base/dadfs.cpp

@@ -1419,6 +1419,8 @@ class CDistributedFileTransaction: public CInterface, implements IDistributedFil
         }
         bool find(const char *subFile, bool sub)
         {
+            StringBuffer tmp;
+            subFile = normalizeLFN(subFile, tmp);
             HTMapping *match = subFilesByName.find(subFile);
             if (match)
                 return true;

ecl/ecl-package/ecl-package.cpp

@@ -737,6 +737,7 @@ public:
         if (errors.ordinality()>0)
         {
             validateMessages = true;
+            fputs("   Validation Failed!\n", stderr);
             fputs("   Error(s):\n", stderr);
             ForEachItemIn(i, errors)
                 fprintf(stderr, "      %s\n", errors.item(i));

rtl/eclrtl/eclrtl.cpp

@@ -307,23 +307,26 @@ private:
 };
 
 typedef MapStringTo<RTLUnicodeConverter, char const *> MapStrToUnicodeConverter;
-MapStrToUnicodeConverter *unicodeConverterMap;
+static __thread MapStrToUnicodeConverter *unicodeConverterMap = NULL;
+static __thread ThreadTermFunc prevThreadTerminator = NULL;
 CriticalSection ucmCrit;
 
-MODULE_INIT(INIT_PRIORITY_STANDARD)
-{
-    unicodeConverterMap = new MapStrToUnicodeConverter;
-    return true;
-}
-MODULE_EXIT()
+static void clearUnicodeConverterMap()
 {
     delete unicodeConverterMap;
+    if (prevThreadTerminator)
+        (*prevThreadTerminator)();
 }
 
-
 RTLUnicodeConverter * queryRTLUnicodeConverter(char const * codepage)
 {
-    CriticalBlock b(ucmCrit);
+    if (!unicodeConverterMap) // NB: one per thread, so no contention
+    {
+        unicodeConverterMap = new MapStrToUnicodeConverter(GetCurrentThreadId());
+        // Use thread terminator hook to clear them up on thread exit.
+        // NB: May need to revisit if not on a jlib Thread.
+        prevThreadTerminator = addThreadTermFunc(clearUnicodeConverterMap);
+    }
     RTLUnicodeConverter * conv = unicodeConverterMap->getValue(codepage);
     if(!conv)
     {

system/mp/mpcomm.cpp

@@ -460,7 +460,7 @@ public:
     void stop();
     unsigned short getPort() { return port; }
     void setPort(unsigned short _port) { port = _port; }
-    CMPChannel &lookup(const SocketEndpoint &remoteep);
+    CMPChannel *lookup(const SocketEndpoint &remoteep);
     ISocketSelectHandler &querySelectHandler() { return *selecthandler; };
     CBufferQueue &getReceiveQ() { return receiveq; }
     bool recv(CMessageBuffer &mbuf, const SocketEndpoint *ep, mptag_t tag, CTimeMon &tm);
@@ -1932,7 +1932,8 @@ int CMPConnectThread::run()
                 PROGLOG("MP: Connect Thread: after read %s",tmp1.str());
 #endif
                 checkSelfDestruct(&id[0],sizeof(id));
-                if (!parent->lookup(_remoteep).attachSocket(sock,_remoteep,hostep,false,&rd,addrval)) {
+                Linked<CMPChannel> channel = parent->lookup(_remoteep);
+                if (!channel->attachSocket(sock,_remoteep,hostep,false,&rd,addrval)) {
 #ifdef _FULLTRACE       
                     PROGLOG("MP Connect Thread: lookup failed");
 #endif
@@ -2014,7 +2015,7 @@ public:
 
 //-----------------------------------------------------------------------------------
 
-CMPChannel &CMPServer::lookup(const SocketEndpoint &endpoint)
+CMPChannel *CMPServer::lookup(const SocketEndpoint &endpoint)
 {
     // there is an assumption here that no removes will be done within this loop
     CriticalBlock block(serversect);
@@ -2042,7 +2043,7 @@ CMPChannel &CMPServer::lookup(const SocketEndpoint &endpoint)
         e = new CMPChannel(this,ep);
         add(*e);
     }
-    return *e;
+    return LINK(e);
 }
 
 
@@ -2338,11 +2339,11 @@ public:
         }
 
         CTimeMon tm(timeout);
-        CMPChannel &channel = parent->lookup(dst->endpoint());
+        Linked<CMPChannel> channel = parent->lookup(dst->endpoint());
         unsigned remaining;
         if (tm.timedout(&remaining))
             return false;
-        if (!channel.send(mbuf,tag,mbuf.getReplyTag(),tm,false))
+        if (!channel->send(mbuf,tag,mbuf.getReplyTag(),tm,false))
             return false;
         mbuf.clear(); // for consistent semantics
         return true;
@@ -2352,11 +2353,11 @@ public:
     {
         CriticalBlock block(verifysect);
         CTimeMon tm(timeout);
-        CMPChannel &channel = parent->lookup(node->endpoint());
+        Linked<CMPChannel> channel = parent->lookup(node->endpoint());
         unsigned remaining;
         if (tm.timedout(&remaining))
             return false;
-        return channel.verifyConnection(tm,true);
+        return channel->verifyConnection(tm,true);
     }
 
     void verifyAll(StringBuffer &log)
@@ -2395,12 +2396,12 @@ public:
                 else
                     doverify = (myrank<rank);
                 if (doverify) {
-                    CMPChannel &channel = parent->lookup(group->queryNode(rank).endpoint());
+                    Linked<CMPChannel> channel = parent->lookup(group->queryNode(rank).endpoint());
                     unsigned remaining;
                     if (tm.timedout(&remaining)) {
                         return false;
                     }
-                    if (!channel.verifyConnection(tm,true)) {
+                    if (!channel->verifyConnection(tm,true)) {
                         return false;
                     }
                 }
@@ -2410,8 +2411,8 @@ public:
             ForEachNodeInGroup(rank,*group) {
                 bool doverify = ((rank&1)==(myrank&1))?(myrank<rank):(myrank>rank);
                 if (doverify) {
-                    CMPChannel &channel = parent->lookup(group->queryNode(rank).endpoint());
-                    while (!channel.verifyConnection(tm,false)) {
+                    Linked<CMPChannel> channel = parent->lookup(group->queryNode(rank).endpoint());
+                    while (!channel->verifyConnection(tm,false)) {
                         unsigned remaining;
                         if (tm.timedout(&remaining))
                             return false;
@@ -2508,9 +2509,9 @@ public:
     void disconnect(INode *node)
     {
         CriticalBlock block(verifysect);
-        CMPChannel &channel = parent->lookup(node->endpoint());
-        channel.closeSocket();
-        parent->removeChannel(&channel);
+        Linked<CMPChannel> channel = parent->lookup(node->endpoint());
+        channel->closeSocket();
+        parent->removeChannel(channel);
     }
 
     CInterCommunicator(CMPServer *_parent)
@@ -2537,7 +2538,7 @@ class CCommunicator: public CInterface, public ICommunicator
         return group->queryNode(rank).endpoint();
     }
 
-    CMPChannel &queryChannel(rank_t rank)
+    CMPChannel *getChannel(rank_t rank)
     {
         return parent->lookup(queryEndpoint(rank));
     }
@@ -2585,11 +2586,11 @@ public:
                 endrank = dstrank;
             for (;dstrank<=endrank;dstrank++) {
                 if (dstrank!=myrank) {
-                    CMPChannel &channel = queryChannel(dstrank);
+                    Linked<CMPChannel> channel = getChannel(dstrank);
                     unsigned remaining;
                     if (tm.timedout(&remaining))
                         return false;
-                    if (!channel.send(mbuf,tag,mbuf.getReplyTag(),tm,false))
+                    if (!channel->send(mbuf,tag,mbuf.getReplyTag(),tm,false))
                         return false;
                 }
             }
@@ -2654,11 +2655,11 @@ public:
         assertex(rank!=RANK_RANDOM);
         assertex(rank!=RANK_ALL);
         CTimeMon tm(timeout);
-        CMPChannel &channel = queryChannel(rank);
+        Linked<CMPChannel> channel = getChannel(rank);
         unsigned remaining;
         if (tm.timedout(&remaining))
             return false;
-        return channel.verifyConnection(tm,true);
+        return channel->verifyConnection(tm,true);
     }
 
     bool verifyAll(bool duplex, unsigned timeout)
@@ -2676,12 +2677,12 @@ public:
                 else
                     doverify = (myrank<rank);
                 if (doverify) {
-                    CMPChannel &channel = queryChannel(rank);
+                    Linked<CMPChannel> channel = getChannel(rank);
                     unsigned remaining;
                     if (tm.timedout(&remaining)) {
                         return false;
                     }
-                    if (!channel.verifyConnection(tm,true)) 
+                    if (!channel->verifyConnection(tm,true))
                         return false;
                 }
             }
@@ -2690,8 +2691,8 @@ public:
             ForEachNodeInGroup(rank,*group) {
                 bool doverify = ((rank&1)==(myrank&1))?(myrank<rank):(myrank>rank);
                 if (doverify) {
-                    CMPChannel &channel = queryChannel(rank);
-                    while (!channel.verifyConnection(tm,false)) {
+                    Linked<CMPChannel> channel = getChannel(rank);
+                    while (!channel->verifyConnection(tm,false)) {
                         unsigned remaining;
                         if (tm.timedout(&remaining))
                             return false;
@@ -2806,12 +2807,12 @@ public:
         }
             
         CTimeMon tm(timeout);
-        CMPChannel &channel = parent->lookup(mbuf.getSender());
+        Linked<CMPChannel> channel = parent->lookup(mbuf.getSender());
         unsigned remaining;
         if (tm.timedout(&remaining)) {
             return false;
         }
-        if (channel.send(mbuf,replytag,TAG_NULL,tm, true)) {
+        if (channel->send(mbuf,replytag,TAG_NULL,tm, true)) {
             mbuf.setReplyTag(TAG_NULL);
             return true;
         }
@@ -2827,9 +2828,9 @@ public:
     void disconnect(INode *node)
     {
         CriticalBlock block(verifysect);
-        CMPChannel &channel = parent->lookup(node->endpoint());
-        channel.closeSocket();
-        parent->removeChannel(&channel);
+        Linked<CMPChannel> channel = parent->lookup(node->endpoint());
+        channel->closeSocket();
+        parent->removeChannel(channel);
     }
 
     CCommunicator(CMPServer *_parent,IGroup *_group, bool _outer)

testing/regress/ecl/key/loopall.xml

@@ -51,50 +51,56 @@
  <Row><surname>X                   </surname><forename>Z         </forename><age>400</age></Row>
 </Dataset>
 <Dataset name='Result 5'>
+ <Row><surname>Halliday            </surname><forename>Gavin     </forename><age>31</age></Row>
+ <Row><surname>Halliday            </surname><forename>Liz       </forename><age>30</age></Row>
+ <Row><surname>Salter              </surname><forename>Abi       </forename><age>10</age></Row>
+ <Row><surname>X                   </surname><forename>Z         </forename><age>25</age></Row>
+</Dataset>
+<Dataset name='Result 6'>
  <Row><surname>Halliday            </surname><forename>Gavin     </forename><age>124</age></Row>
  <Row><surname>Halliday            </surname><forename>Liz       </forename><age>120</age></Row>
  <Row><surname>X                   </surname><forename>Z         </forename><age>100</age></Row>
  <Row><surname>Salter              </surname><forename>Abi       </forename><age>160</age></Row>
 </Dataset>
-<Dataset name='Result 6'>
+<Dataset name='Result 7'>
  <Row><surname>Halliday            </surname><forename>Gavin     </forename><age>124</age></Row>
  <Row><surname>Halliday            </surname><forename>Liz       </forename><age>120</age></Row>
  <Row><surname>X                   </surname><forename>Z         </forename><age>100</age></Row>
  <Row><surname>Salter              </surname><forename>Abi       </forename><age>160</age></Row>
 </Dataset>
-<Dataset name='Result 7'>
+<Dataset name='Result 8'>
  <Row><surname>Halliday            </surname><forename>Gavin     </forename><age>31744</age></Row>
  <Row><surname>Halliday            </surname><forename>Liz       </forename><age>30720</age></Row>
  <Row><surname>Salter              </surname><forename>Abi       </forename><age>10240</age></Row>
  <Row><surname>X                   </surname><forename>Z         </forename><age>25600</age></Row>
 </Dataset>
-<Dataset name='Result 8'>
+<Dataset name='Result 9'>
  <Row><surname>Halliday            </surname><forename>Gavin     </forename><age>124</age></Row>
  <Row><surname>Halliday            </surname><forename>Liz       </forename><age>120</age></Row>
  <Row><surname>X                   </surname><forename>Z         </forename><age>100</age></Row>
  <Row><surname>Salter              </surname><forename>Abi       </forename><age>80</age></Row>
 </Dataset>
-<Dataset name='Result 9'>
+<Dataset name='Result 10'>
  <Row><surname>Halliday            </surname><forename>Gavin     </forename><age>3968</age></Row>
  <Row><surname>Halliday            </surname><forename>Liz       </forename><age>3840</age></Row>
  <Row><surname>Salter              </surname><forename>Abi       </forename><age>1280</age></Row>
  <Row><surname>X                   </surname><forename>Z         </forename><age>3200</age></Row>
 </Dataset>
-<Dataset name='Result 10'>
+<Dataset name='Result 11'>
  <Row><surname>Halliday            </surname><forename>Gavin     </forename><age>186</age></Row>
  <Row><surname>Halliday            </surname><forename>Liz       </forename><age>180</age></Row>
  <Row><surname>X                   </surname><forename>Z         </forename><age>150</age></Row>
  <Row><surname>Salter              </surname><forename>Abi       </forename><age>240</age></Row>
 </Dataset>
-<Dataset name='Result 11'>
+<Dataset name='Result 12'>
  <Row><surname>Halliday            </surname><forename>Gavin     </forename><age>186</age></Row>
  <Row><surname>Halliday            </surname><forename>Liz       </forename><age>180</age></Row>
  <Row><surname>X                   </surname><forename>Z         </forename><age>150</age></Row>
  <Row><surname>Salter              </surname><forename>Abi       </forename><age>240</age></Row>
 </Dataset>
-<Dataset name='Result 12'>
+<Dataset name='Result 13'>
  <Row><parentfield>parent1changed</parentfield><children><Row><surname>Halliday            </surname><forename>Gavin     </forename><age>51</age></Row><Row><surname>Halliday            </surname><forename>Liz       </forename><age>50</age></Row><Row><surname>Salter              </surname><forename>Abi       </forename><age>30</age></Row><Row><surname>X                   </surname><forename>Z         </forename><age>45</age></Row></children></Row>
 </Dataset>
-<Dataset name='Result 13'>
+<Dataset name='Result 14'>
  <Row><parentfield>parent1changed</parentfield><children><Row><surname>Halliday            </surname><forename>Gavin     </forename><age>62</age></Row><Row><surname>Halliday            </surname><forename>Liz       </forename><age>120</age></Row><Row><surname>X                   </surname><forename>Z         </forename><age>100</age></Row><Row><surname>Salter              </surname><forename>Abi       </forename><age>80</age></Row></children></Row>
 </Dataset>

testing/regress/ecl/loopall.ecl

@@ -43,6 +43,9 @@ output(loop(namesTable2, 10, left.age <= 60, project(rows(left), transform(names
 //Case 3: global loop test
 output(loop(namesTable2, sum(rows(left), age) < 1000, project(rows(left), transform(namesRecord, self.age := left.age*two; self := left))));
 
+// Test loopFirstTime(), should not execute body of loop
+output(loop(namesTable2, sum(rows(left), age) < 1000 AND false, project(rows(left), transform(namesRecord, self.age := left.age*two; self := left))));
+
 //Case 4: global loop test with row filter
 output(loop(namesTable2, left.age < 100, exists(rows(left)) and sum(rows(left), age) < 1000, project(rows(left), transform(namesRecord, self.age := left.age*two; self := left))));
 

thorlcr/activities/loop/thloopslave.cpp

@@ -267,6 +267,8 @@ public:
         loopPending.setown(createOverflowableBuffer(*this, this, false, true));
         loopPendingCount = 0;
         finishedLooping = ((container.getKind() == TAKloopcount) && (maxIterations == 0));
+        if ((flags & IHThorLoopArg::LFnewloopagain) && !helper->loopFirstTime())
+            finishedLooping = true;
         curInput.set(input);
         lastMs = msTick();