HPCC-9112 Large sets of files sprayed using wildcard may be truncated

This change modifies CRemoteDirectoryIterator::appendBuf() to properly
handle end of stream and end of block conditions. Before this fix if the
directory contents spanned more than one transfer buffer between nodes, only
the first buffer's contents would be iterated by next(), because the
block_end_flag was left in the buffer, and next() interpreted it to mean
'no_more_valid_items'.

This fix removes the block_end_flags.

It also changes a variable name from "b" to "isValidEntry" in the
CRemoteDirectoryIterator::next() method to improve code self-documentation.

Finally it changes some information text message in dfuplus.cpp because
the same log was generated in two different function (fixedSpray and
variableSpray).

Signed-off-by: Attila Vamos <attila.vamos@gmail.com>

common/remote/sockfile.cpp

@@ -1347,8 +1347,22 @@ public:
         size32_t rest = _buf.length()-_buf.getPos();
         const byte *rb = (const byte *)_buf.readDirect(rest);
         bool ret = true;
+        // At the last byte of the rb (rb[rest-1]) is the stream live flag
+        // 		True if the stream has more data
+        //		False at the end of stream
+        // the previous byte (rb[rest-2]) is the flag to signals there are more
+        // valid entry in this block
+        // 		True if there are valid directory entry follows this flag
+        //		False if there are no more valid entry in this block aka end of block
+        // If there are more data in the stream, the end of block flag should be remove
         if (rest&&(rb[rest-1]!=0)) {
-            rest--;
+            rest--; // remove stream live flag
+
+            if(rest && (0 == rb[rest-1]))
+            {
+            	rest--; //Remove end of block flag
+            }
+
             ret = false;  // continuation
         }
         buf.append(rest,rb);
@@ -1379,9 +1393,9 @@ public:
             curisdir = false;
             if (buf.getPos()>=buf.length())
                 return false;
-            byte b;
-            buf.read(b);
-            curvalid = b!=0;
+            byte isValidEntry;
+            buf.read(isValidEntry);
+            curvalid = isValidEntry!=0;
             if (!curvalid) 
                 return false;
             buf.read(curisdir);

dali/dfuplus/dfuplus.cpp

@@ -318,7 +318,7 @@ bool CDfuPlusHelper::fixedSpray(const char* srcxml,const char* srcip,const char*
         req->setTransferBufferSize(globals->getPropInt("transferbuffersize"));
 
     if(srcxml == NULL)
-        info("\nSpraying from %s on %s to %s\n", srcfile, srcip, dstname);
+        info("\nFixed spraying from %s on %s to %s\n", srcfile, srcip, dstname);
     else
         info("\nSpraying to %s\n", dstname);
 
@@ -424,7 +424,7 @@ bool CDfuPlusHelper::variableSpray(const char* srcxml,const char* srcip,const ch
         req->setTransferBufferSize(globals->getPropInt("transferbuffersize"));
 
     if(srcxml == NULL)
-        info("\nSpraying from %s on %s to %s\n", srcfile, srcip, dstname);
+        info("\nVariable spraying from %s on %s to %s\n", srcfile, srcip, dstname);
     else
         info("\nSpraying to %s\n", dstname);
     Owned<IClientSprayResponse> result = sprayclient->SprayVariable(req);