HPCC-16858 Detect compressed files when reattaching XREF files.

Check physical files, test to see if compressed so as to mark
the meta data correctly, also add size or compressedSize
dependent on whether compressed or not.

Signed-off-by: Jake Smith <jake.smith@lexisnexisrisk.com>

dali/dfuXRefLib/XRefFilesNode.cpp

@@ -21,6 +21,7 @@
 
 #include "XRefFilesNode.hpp"
 
+#include "jlzw.hpp"
 #include "dautils.hpp"
 
 //////////////////////////////////////////////////////////////////////
@@ -335,6 +336,9 @@ bool CXRefFilesNode::AttachPhysical(const char *Partmask,IUserDescriptor* udesc,
 
     unsigned numparts = subBranch->getPropInt("Numparts");
 
+    bool isCompressed = false;
+    bool first = true;
+    offset_t totalSize = 0;
     Owned<IPropertyTreeIterator> partItr =  subBranch->getElements("Part");
     for (partItr->first(); partItr->isValid(); partItr->next())
     {
@@ -348,7 +352,7 @@ bool CXRefFilesNode::AttachPhysical(const char *Partmask,IUserDescriptor* udesc,
         splitFilename(remoteFilePath.str(), &_drive, &_path, &_tail, &_ext);
         _filename.append(_tail.str());
         _filename.append(_ext.str());
-                
+
         const char* _node = part.queryProp("Node[1]");
         if (!_node||!*_node)
             _node = part.queryProp("RNode[1]");
@@ -360,8 +364,42 @@ bool CXRefFilesNode::AttachPhysical(const char *Partmask,IUserDescriptor* udesc,
         Owned<INode> node = createINode(_node);
         DBGLOG("Setting number %d for Node %s and name %s",part.getPropInt("Num")-1,_node,_filename.str());
         //Num is 0 based...
-        fileDesc->setPart(part.getPropInt("Num")-1,node,_filename.str());
+        unsigned partNo = part.getPropInt("Num")-1;
+
+        RemoteFilename rfn;
+        rfn.setPath(node->endpoint(), remoteFilePath);
+        Owned<IFile> iFile = createIFile(rfn);
+        offset_t physicalSize = iFile->size();
+        bool partCompressed = isCompressedFile(iFile);
+        if (first)
+        {
+            first = false;
+            isCompressed = partCompressed;
+        }
+        else if (isCompressed != partCompressed)
+        {
+            VStringBuffer err("%s - could not attach (mixed compressed/non-compressed physical parts detected)", Partmask);
+            ERRLOG("%s", err.str());
+            errstr.append(err.str());
+            return false;
+        }
+        Owned<IPropertyTree> partProps = createPTree("Part");
+        if (isCompressed)
+            partProps->setPropInt64("@compressedSize", physicalSize);
+        else
+            partProps->setPropInt64("@size", physicalSize);
+        totalSize += physicalSize;
+
+        fileDesc->setPart(partNo, node,_filename.str(), partProps);
+    }
+    IPropertyTree &props = fileDesc->queryProperties();
+    if (isCompressed)
+    {
+        props.setPropBool("@blockCompressed", true);
+        props.setPropInt64("@compressedSize", totalSize);
     }
+    else
+        props.setPropInt64("@size", totalSize);
 
     Owned<IDistributedFile> dFile = queryDistributedFileDirectory().createNew(fileDesc);
     dFile->attach(logicalName.str(),udesc);

system/jlib/jlzw.cpp

@@ -2326,66 +2326,94 @@ public:
 };
 
 
+static unsigned getCompressedMethod(__int64 compressedType)
+{
+    if (compressedType == COMPRESSEDFILEFLAG)
+        return COMPRESS_METHOD_LZW;
+    else if (compressedType == FASTCOMPRESSEDFILEFLAG)
+        return COMPRESS_METHOD_FASTLZ;
+    else if (compressedType == LZ4COMPRESSEDFILEFLAG)
+        return COMPRESS_METHOD_LZ4;
+    return 0;
+}
 
-        
+static bool isCompressedType(__int64 compressedType)
+{
+    return 0 != getCompressedMethod(compressedType);
+}
 
-ICompressedFileIO *createCompressedFileReader(IFileIO *fileio,IExpander *expander)
+bool isCompressedFile(IFileIO *iFileIO, CompressedFileTrailer *trailer=nullptr)
 {
-    if (fileio) {
-        offset_t fsize = fileio->size();
-        if (fsize>=sizeof(WinCompressedFileTrailer)) {  // thats 8 bytes bigger but I think doesn't matter
+    if (iFileIO)
+    {
+        offset_t fsize = iFileIO->size();
+        if (fsize>=sizeof(WinCompressedFileTrailer))  // thats 8 bytes bigger but I think doesn't matter
+        {
             WinCompressedFileTrailer wintrailer;
-            CompressedFileTrailer trailer;
-            if (fileio->read(fsize-sizeof(WinCompressedFileTrailer),sizeof(WinCompressedFileTrailer),&wintrailer)==sizeof(WinCompressedFileTrailer)) {
-                wintrailer.translate(trailer);
-                if ( (trailer.compressedType==COMPRESSEDFILEFLAG) ||
-                     (trailer.compressedType==FASTCOMPRESSEDFILEFLAG) ||
-                     (trailer.compressedType==LZ4COMPRESSEDFILEFLAG) )
-                {
-                    if (expander&&(trailer.recordSize!=0)) {
-                        throw MakeStringException(-1, "Compressed file format error(%d), Encrypted?",trailer.recordSize);
-                    }
-                    unsigned compMethod1 = COMPRESS_METHOD_LZW;
-                    if (trailer.compressedType == FASTCOMPRESSEDFILEFLAG)
-                        compMethod1 = COMPRESS_METHOD_FASTLZ;
-                    else if (trailer.compressedType == LZ4COMPRESSEDFILEFLAG)
-                        compMethod1 = COMPRESS_METHOD_LZ4;
-                    CCompressedFile *cfile = new CCompressedFile(fileio,NULL,trailer,ICFread,false,NULL,expander,compMethod1);
-                    return cfile;
-                }
+            CompressedFileTrailer _trailer;
+            if (!trailer)
+                trailer = &_trailer;
+            if (iFileIO->read(fsize-sizeof(WinCompressedFileTrailer),sizeof(WinCompressedFileTrailer),&wintrailer)==sizeof(WinCompressedFileTrailer))
+            {
+                wintrailer.translate(*trailer);
+                if (isCompressedType(trailer->compressedType))
+                    return true;
             }
         }
     }
-    return NULL;
+    return false;
+}
+
+
+
+bool isCompressedFile(const char *filename)
+{
+    Owned<IFile> iFile = createIFile(filename);
+    return isCompressedFile(iFile);
+}
+
+bool isCompressedFile(IFile *iFile)
+{
+    Owned<IFileIO> iFileIO = iFile->open(IFOread);
+    return isCompressedFile(iFileIO);
+}
+
+ICompressedFileIO *createCompressedFileReader(IFileIO *fileio,IExpander *expander)
+{
+    CompressedFileTrailer trailer;
+    if (isCompressedFile(fileio, &trailer))
+    {
+        if (expander&&(trailer.recordSize!=0))
+            throw MakeStringException(-1, "Compressed file format error(%d), Encrypted?",trailer.recordSize);
+        unsigned compMethod = getCompressedMethod(trailer.compressedType);
+        return new CCompressedFile(fileio,NULL,trailer,ICFread,false,NULL,expander,compMethod);
+    }
+    return nullptr;
 }
 
 
 ICompressedFileIO *createCompressedFileReader(IFile *file,IExpander *expander, bool memorymapped, IFEflags extraFlags)
 {
-    if (file) {
-        if (memorymapped) {
+    if (file)
+    {
+        if (memorymapped)
+        {
             Owned<IMemoryMappedFile> mmfile = file->openMemoryMapped();
-            if (mmfile) {
+            if (mmfile)
+            {
                 offset_t fsize = mmfile->fileSize();
-                if (fsize>=sizeof(WinCompressedFileTrailer)) {  // thats 8 bytes bigger but I think doesn't matter
+                if (fsize>=sizeof(WinCompressedFileTrailer))  // thats 8 bytes bigger but I think doesn't matter
+                {
                     WinCompressedFileTrailer wintrailer;
                     CompressedFileTrailer trailer;
                     memcpy(&wintrailer,mmfile->base()+fsize-sizeof(WinCompressedFileTrailer),sizeof(WinCompressedFileTrailer));
                     wintrailer.translate(trailer);
-                    if ( (trailer.compressedType==COMPRESSEDFILEFLAG) ||
-                         (trailer.compressedType==FASTCOMPRESSEDFILEFLAG) ||
-                         (trailer.compressedType==LZ4COMPRESSEDFILEFLAG) )
+                    unsigned compMethod = getCompressedMethod(trailer.compressedType);
+                    if (compMethod)
                     {
-                        if (expander&&(trailer.recordSize!=0)) {
+                        if (expander&&(trailer.recordSize!=0))
                             throw MakeStringException(-1, "Compressed file format error(%d), Encrypted?",trailer.recordSize);
-                        }
-                        unsigned compMethod1 = COMPRESS_METHOD_LZW;
-                        if (trailer.compressedType == FASTCOMPRESSEDFILEFLAG)
-                            compMethod1 = COMPRESS_METHOD_FASTLZ;
-                        else if (trailer.compressedType == LZ4COMPRESSEDFILEFLAG)
-                            compMethod1 = COMPRESS_METHOD_LZ4;
-                        CCompressedFile *cfile = new CCompressedFile(NULL,mmfile,trailer,ICFread,false,NULL,expander,compMethod1);
-                        return cfile;
+                        return new CCompressedFile(NULL,mmfile,trailer,ICFread,false,NULL,expander,compMethod);
                     }
                 }
             }
@@ -2404,26 +2432,21 @@ ICompressedFileIO *createCompressedFileWriter(IFileIO *fileio,size32_t recordsiz
 {
     CompressedFileTrailer trailer;
     offset_t fsize = fileio->size();
-    if (fsize) {
-        loop {
-            if (fsize>=sizeof(WinCompressedFileTrailer)) {  // thats 8 bytes bigger but I think doesn't matter
+    if (fsize)
+    {
+        loop
+        {
+            if (fsize>=sizeof(WinCompressedFileTrailer))  // thats 8 bytes bigger but I think doesn't matter
+            {
                 WinCompressedFileTrailer wintrailer;
                 CompressedFileTrailer trailer;
                 if (fileio->read(fsize-sizeof(WinCompressedFileTrailer),sizeof(WinCompressedFileTrailer),&wintrailer)==sizeof(WinCompressedFileTrailer)) {
                     wintrailer.translate(trailer);
-                    if ( (trailer.compressedType==COMPRESSEDFILEFLAG) ||
-                         (trailer.compressedType==FASTCOMPRESSEDFILEFLAG) ||
-                         (trailer.compressedType==LZ4COMPRESSEDFILEFLAG) )
+                    unsigned compMethod = getCompressedMethod(trailer.compressedType);
+                    if (compMethod)
                     {
                         // check trailer.compressedType against _compMethod
-                        unsigned compMethod1 = 0;
-                        if (trailer.compressedType == COMPRESSEDFILEFLAG)
-                            compMethod1 = COMPRESS_METHOD_LZW;
-                        else if (trailer.compressedType == FASTCOMPRESSEDFILEFLAG)
-                            compMethod1 = COMPRESS_METHOD_FASTLZ;
-                        else if (trailer.compressedType == LZ4COMPRESSEDFILEFLAG)
-                            compMethod1 = COMPRESS_METHOD_LZ4;
-                        if (_compMethod != compMethod1)
+                        if (_compMethod != compMethod)
                             throw MakeStringException(-1,"Appending to file with different compression method");
                         if ((recordsize==trailer.recordSize)||!trailer.recordSize)
                             break;
@@ -2434,7 +2457,8 @@ ICompressedFileIO *createCompressedFileWriter(IFileIO *fileio,size32_t recordsiz
             throw MakeStringException(-1,"Appending to file that is not compressed");
         }
     }
-    else {
+    else
+    {
         memset(&trailer,0,sizeof(trailer));
         trailer.crc = ~0U;
         if (_compMethod == COMPRESS_METHOD_FASTLZ)
@@ -2670,9 +2694,7 @@ IPropertyTree *getBlockedFileDetails(IFile *file)
             CompressedFileTrailer trailer;
             if (fileio->read(fsize-sizeof(WinCompressedFileTrailer),sizeof(WinCompressedFileTrailer),&wintrailer)==sizeof(WinCompressedFileTrailer)) {
                 wintrailer.translate(trailer);
-                if ( (trailer.compressedType==COMPRESSEDFILEFLAG) ||
-                     (trailer.compressedType==FASTCOMPRESSEDFILEFLAG) ||
-                     (trailer.compressedType==LZ4COMPRESSEDFILEFLAG) )
+                if (isCompressedType(trailer.compressedType))
                 {
                     trailer.setDetails(*tree);
                     unsigned nb = trailer.numBlocks();

system/jlib/jlzw.hpp

@@ -110,6 +110,8 @@ interface ICompressedFileIO: extends IFileIO
     virtual unsigned method()=0;
 };
 
+extern jlib_decl bool isCompressedFile(const char *filename);
+extern jlib_decl bool isCompressedFile(IFile *file);
 extern jlib_decl ICompressedFileIO *createCompressedFileReader(IFile *file,IExpander *expander=NULL, bool memorymapped=false, IFEflags extraFlags=IFEnone);
 extern jlib_decl ICompressedFileIO *createCompressedFileReader(IFileIO *fileio,IExpander *expander=NULL);
 extern jlib_decl ICompressedFileIO *createCompressedFileWriter(IFile *file,size32_t recordsize,bool append=false,bool setcrc=true,ICompressor *compressor=NULL, unsigned compMethod=COMPRESS_METHOD_LZW, IFEflags extraFlags=IFEnone);