HPCC-13477 FLZ compressor fixes (2) for when input buffer > internal buffer

Signed-off-by: Mark Kelly <mark.kelly@lexisnexis.com>

system/jlib/jfile.cpp

@@ -6025,7 +6025,7 @@ public:
     virtual void get(size32_t len, void * ptr)
     {
         if (len>buffer.remaining()) {
-            ERRLOG("CFileSerialStream::get read past end of stream.4(%u,%u)",(unsigned)len,(unsigned)buffer.remaining());
+            ERRLOG("CMemoryBufferSerialStream::get read past end of stream.4(%u,%u)",(unsigned)len,(unsigned)buffer.remaining());
             throw MakeStringException(-1,"CMemoryBufferSerialStream::get read past end of stream (%u,%u)",(unsigned)len,(unsigned)buffer.remaining());
         }
         const void * data = buffer.readDirect(len);

system/jlib/jflz.cpp

@@ -616,6 +616,7 @@ class jlib_decl CFastLZCompressor : public CInterface, public ICompressor
     bool trailing;
     byte *outbuf;
     size32_t outlen;
+    size32_t wrmax;
 
     inline void setinmax()
     {
@@ -637,11 +638,13 @@ class jlib_decl CFastLZCompressor : public CInterface, public ICompressor
         if (trailing)
             return;
         size32_t toflush = (inlenblk==COMMITTED)?inlen:inlenblk;
+        if (toflush == 0)
+            return;
         assertex(outlen+sizeof(size32_t)*2+toflush+fastlzSlack(toflush)<=blksz);
         size32_t *cmpsize = (size32_t *)(outbuf+outlen);
         byte *out = (byte *)(cmpsize+1);
         *cmpsize = (size32_t)fastlz_compress(inbuf, (int)toflush, out, ht);
-        if (*cmpsize<=toflush) {
+        if (*cmpsize<toflush) {
             *(size32_t *)outbuf += toflush;
             outlen += *cmpsize+sizeof(size32_t);
             if (inlenblk==COMMITTED)
@@ -665,6 +668,7 @@ public:
         outlen = 0;
         outbuf = NULL;      // only set on close
         bufalloc = 0;
+        wrmax = 0;          // set at open
     }
 
     virtual ~CFastLZCompressor()
@@ -676,6 +680,7 @@ public:
 
     virtual void open(void *buf,size32_t max)
     {
+        wrmax = max;
         if (buf) {
             if (bufalloc) {
                 free(outbuf);
@@ -723,16 +728,30 @@ public:
 
     size32_t write(const void *buf,size32_t len)
     {
-        if (len+inlen>inmax) {
-            if (trailing)
-                return 0;
-            flushcommitted();
-            if (len+inlen>inmax) 
-                len = inmax-inlen;
+        // no more than wrmax per write
+        size32_t lenb = wrmax;
+        byte *b = (byte *)buf;
+        size32_t written = 0;
+        while (len)
+        {
+            if (len < lenb)
+                lenb = len;
+            if (lenb+inlen>inmax) {
+                if (trailing)
+                    return written;
+                flushcommitted();
+                if (lenb+inlen>inmax)
+                    lenb = inmax-inlen;
+            }
+            if (lenb == 0)
+                return written;
+            memcpy(inbuf+inlen,b,lenb);
+            b += lenb;
+            inlen += lenb;
+            len -= lenb;
+            written += lenb;
         }
-        memcpy(inbuf+inlen,buf,len);
-        inlen += len;
-        return len;
+        return written;
     }
 
     void *  bufptr() 
@@ -791,7 +810,7 @@ public:
     }
 
     virtual void expand(void *buf)
-{
+    {
         if (!outlen)
             return;
         if (buf) {

tools/copyexp/copyexp.cpp

@@ -70,7 +70,7 @@ static const char *formatTime(unsigned t,StringBuffer &str)
     str.clear();
     if (t>100000)
         str.appendf("%ds",t/1000);
-    else if (t>100000)
+    else
         str.appendf("%dms",t);
     return str.str();
 
@@ -99,7 +99,7 @@ static void printStats(offset_t filesize,unsigned start,unsigned startu)
     if (elapsed<1000)
         printf("%" I64F "d bytes copied, at %.2f MB/s in %s\n",filesize,((((double)filesize)/(1024*1024))/elapsedu)*1000000,formatTimeU(elapsedu,tmp));
     else
-        printf("%" I64F "d bytes copied, at %.2f MB/s in %s\n",filesize,((((double)filesize)/(1024*1024))/elapsed)*1000,formatTime(elapsed*1000,tmp));
+        printf("%" I64F "d bytes copied, at %.2f MB/s in %s\n",filesize,((((double)filesize)/(1024*1024))/elapsed)*1000,formatTime(elapsed,tmp));
 }
 
 int copyExpanded(const char *from, const char *to, bool stats)