HPCC-24047 Change default file comp from LZW to LZ4 for Thor pub files

Signed-off-by: Mark Kelly <mark.kelly@lexisnexisrisk.com>

common/thorhelper/thorcommon.hpp

@@ -110,10 +110,10 @@ inline void setCompFlag(const char *compStr, unsigned &flags)
     {
         if (0 == stricmp("FLZ", compStr))
             flags |= rw_fastlz;
-        else if (0 == stricmp("LZ4", compStr))
-            flags |= rw_lz4;
-        else // not specifically FLZ or LZ4 so set to LZW (or rowdif)
+        else if (0 == stricmp("LZW", compStr))
             flags |= rw_lzw;
+        else // not specifically FLZ or LZW so set to default LZ4
+            flags |= rw_lz4;
     }
     else // default is LZ4
         flags |= rw_lz4;
@@ -121,28 +121,24 @@ inline void setCompFlag(const char *compStr, unsigned &flags)
 
 inline unsigned getCompMethod(unsigned flags)
 {
-    unsigned compMethod = COMPRESS_METHOD_LZW;
+    unsigned compMethod = COMPRESS_METHOD_LZ4;
     if (TestRwFlag(flags, rw_lzw))
         compMethod = COMPRESS_METHOD_LZW;
     else if (TestRwFlag(flags, rw_fastlz))
         compMethod = COMPRESS_METHOD_FASTLZ;
-    else if (TestRwFlag(flags, rw_lz4))
-        compMethod = COMPRESS_METHOD_LZ4;
     return compMethod;
 }
 
 inline unsigned getCompMethod(const char *compStr)
 {
-    unsigned compMethod = COMPRESS_METHOD_LZW;
+    unsigned compMethod = COMPRESS_METHOD_LZ4;
     if (!isEmptyString(compStr))
     {
         if (0 == stricmp("FLZ", compStr))
             compMethod = COMPRESS_METHOD_FASTLZ;
-        else if (0 == stricmp("LZ4", compStr))
-            compMethod = COMPRESS_METHOD_LZ4;
+        else if (0 == stricmp("LZW", compStr))
+            compMethod = COMPRESS_METHOD_LZW;
     }
-    else // default is LZ4
-        compMethod = COMPRESS_METHOD_LZ4;
     return compMethod;
 }
 

dali/ft/fttransform.cpp

@@ -846,7 +846,7 @@ processedProgress:
                     decrypt(key,encryptKey);
                     compressor.setown(createAESCompressor256(key.length(),key.str()));
                 }
-                outio.setown(createCompressedFileWriter(outio, 0, true, compressor));
+                outio.setown(createCompressedFileWriter(outio, 0, true, compressor, COMPRESS_METHOD_LZW));
             }
 
             LOG(MCdebugProgress, unknownJob, "Start pulling to file: %s", localFilename.str());
@@ -972,7 +972,7 @@ bool TransferServer::push()
                     decrypt(key,encryptKey);
                     compressor.setown(createAESCompressor256(key.length(),key.str()));
                 }
-                outio.setown(createCompressedFileWriter(outio, 0, true, compressor));
+                outio.setown(createCompressedFileWriter(outio, 0, true, compressor, COMPRESS_METHOD_LZW));
             }
             out.setown(createIOStream(outio));
             if (!compressOutput)

ecl/hthor/hthor.cpp

@@ -560,7 +560,7 @@ void CHThorDiskWriteActivity::open()
     }
     Owned<IFileIO> io;
     if(blockcompressed)
-        io.setown(createCompressedFileWriter(file, groupedMeta->getFixedSize(), extend, true, ecomp));
+        io.setown(createCompressedFileWriter(file, groupedMeta->getFixedSize(), extend, true, ecomp, COMPRESS_METHOD_LZW));
     else
         io.setown(file->open(extend ? IFOwrite : IFOcreate));
     if(!io)

roxie/ccd/ccdserver.cpp

@@ -11799,7 +11799,7 @@ public:
             blockcompressed = true;
         }
         if (blockcompressed)
-            io.setown(createCompressedFileWriter(writer->queryFile(), (diskmeta->isFixedSize() ? diskmeta->getFixedSize() : 0), extend, true, ecomp));
+            io.setown(createCompressedFileWriter(writer->queryFile(), (diskmeta->isFixedSize() ? diskmeta->getFixedSize() : 0), extend, true, ecomp, COMPRESS_METHOD_LZW));
         else
             io.setown(writer->queryFile()->open(extend ? IFOwrite : IFOcreate));
         if (!io)

system/jlib/jlzw.cpp

@@ -2808,13 +2808,13 @@ MODULE_INIT(INIT_PRIORITY_STANDARD)
         virtual ICompressor *getCompressor(const char *options) { return createLZWCompressor(true); }
         virtual IExpander *getExpander(const char *options) { return createLZWExpander(true); }
     };
-    ICompressHandler *flzCompressor = new CFLZCompressHandler();
-    addCompressorHandler(flzCompressor);
     addCompressorHandler(new CAESCompressHandler());
     addCompressorHandler(new CDiffCompressHandler());
     addCompressorHandler(new CLZWCompressHandler());
-    addCompressorHandler(new CLZ4CompressHandler());
-    defaultCompressor.set(flzCompressor);
+    addCompressorHandler(new CFLZCompressHandler());
+    ICompressHandler *lz4Compressor = new CLZ4CompressHandler();
+    addCompressorHandler(lz4Compressor);
+    defaultCompressor.set(lz4Compressor);
     return true;
 }
 

system/jlib/jlzw.hpp

@@ -114,8 +114,8 @@ extern jlib_decl bool isCompressedFile(const char *filename);
 extern jlib_decl bool isCompressedFile(IFile *file);
 extern jlib_decl ICompressedFileIO *createCompressedFileReader(IFile *file,IExpander *expander=NULL, bool memorymapped=false, IFEflags extraFlags=IFEnone);
 extern jlib_decl ICompressedFileIO *createCompressedFileReader(IFileIO *fileio,IExpander *expander=NULL);
-extern jlib_decl ICompressedFileIO *createCompressedFileWriter(IFile *file,size32_t recordsize,bool append=false,bool setcrc=true,ICompressor *compressor=NULL, unsigned compMethod=COMPRESS_METHOD_LZW, IFEflags extraFlags=IFEnone);
-extern jlib_decl ICompressedFileIO *createCompressedFileWriter(IFileIO *fileio,size32_t recordsize,bool setcrc=true,ICompressor *compressor=NULL, unsigned compMethod=COMPRESS_METHOD_LZW);
+extern jlib_decl ICompressedFileIO *createCompressedFileWriter(IFile *file,size32_t recordsize,bool append=false,bool setcrc=true,ICompressor *compressor=NULL, unsigned compMethod=COMPRESS_METHOD_LZ4, IFEflags extraFlags=IFEnone);
+extern jlib_decl ICompressedFileIO *createCompressedFileWriter(IFileIO *fileio,size32_t recordsize,bool setcrc=true,ICompressor *compressor=NULL, unsigned compMethod=COMPRESS_METHOD_LZ4);
 
 #define COMPRESSEDFILECRC (~0U)
 

thorlcr/activities/thactivityutil.cpp

@@ -736,7 +736,7 @@ IFileIO *createMultipleWrite(CActivityBase *activity, IPartDescriptor &partDesc,
     Owned<IFileIO> fileio;
     if (compress)
     {
-        unsigned compMethod = COMPRESS_METHOD_LZW;
+        unsigned compMethod = COMPRESS_METHOD_LZ4;
         // rowdif used if recordSize > 0, else fallback to compMethod
         if (!ecomp)
         {

thorlcr/activities/thdiskbase.cpp

@@ -231,7 +231,7 @@ void CWriteMasterBase::publish()
                 if (0 != (diskHelperBase->getFlags() & TDXgrouped))
                     recordSize += 1;
             }
-            unsigned compMethod = COMPRESS_METHOD_LZW;
+            unsigned compMethod = COMPRESS_METHOD_LZ4;
             // rowdiff used if recordSize > 0, else fallback to compMethod
             if (getOptBool(THOROPT_COMP_FORCELZW, false))
             {

thorlcr/thorutil/thormisc.hpp

@@ -51,7 +51,7 @@
 #define THOROPT_HDIST_PULLBUFFER_SIZE "hdPullBufferSize"        // Distribute pull buffer size (receiver side limit, before spilling)
 #define THOROPT_HDIST_CANDIDATELIMIT  "hdCandidateLimit"        // Limits # of buckets to push to the writers when send buffer is full           (default = is 50% largest)
 #define THOROPT_HDIST_TARGETWRITELIMIT "hdTargetLimit"          // Limit # of writer threads working on a single target                          (default = unbound, but picks round-robin)
-#define THOROPT_HDIST_COMP            "hdCompressorType"        // Distribute compressor to use                                                  (default = "FLZ")
+#define THOROPT_HDIST_COMP            "hdCompressorType"        // Distribute compressor to use                                                  (default = "LZ4")
 #define THOROPT_HDIST_COMPOPTIONS     "hdCompressorOptions"     // Distribute compressor options, e.g. AES key                                   (default = "")
 #define THOROPT_SPLITTER_SPILL        "splitterSpill"           // Force splitters to spill or not, default is to adhere to helper setting       (default = -1)
 #define THOROPT_LOOP_MAX_EMPTY        "loopMaxEmpty"            // Max # of iterations that LOOP can cycle through with 0 results before errors  (default = 1000)