Merge pull request #9491 from jakesmith/hpcc-16858

HPCC-16858 Detect compressed files when reattaching XREF files.

Reviewed-By: Mark Kelly <mark.kelly@lexisnexis.com>
Reviewed-By: Richard Chapman <rchapman@hpccsystems.com>

dali/dfuXRefLib/XRefFilesNode.cpp

@@ -21,6 +21,7 @@
 
 #include "XRefFilesNode.hpp"
 
+#include "jlzw.hpp"
 #include "dautils.hpp"
 
 //////////////////////////////////////////////////////////////////////
@@ -335,6 +336,9 @@ bool CXRefFilesNode::AttachPhysical(const char *Partmask,IUserDescriptor* udesc,
 
     unsigned numparts = subBranch->getPropInt("Numparts");
 
+    bool isCompressed = false;
+    bool first = true;
+    offset_t totalSize = 0;
     Owned<IPropertyTreeIterator> partItr =  subBranch->getElements("Part");
     for (partItr->first(); partItr->isValid(); partItr->next())
     {
@@ -348,7 +352,7 @@ bool CXRefFilesNode::AttachPhysical(const char *Partmask,IUserDescriptor* udesc,
         splitFilename(remoteFilePath.str(), &_drive, &_path, &_tail, &_ext);
         _filename.append(_tail.str());
         _filename.append(_ext.str());
-                
+
         const char* _node = part.queryProp("Node[1]");
         if (!_node||!*_node)
             _node = part.queryProp("RNode[1]");
@@ -360,8 +364,42 @@ bool CXRefFilesNode::AttachPhysical(const char *Partmask,IUserDescriptor* udesc,
         Owned<INode> node = createINode(_node);
         DBGLOG("Setting number %d for Node %s and name %s",part.getPropInt("Num")-1,_node,_filename.str());
         //Num is 0 based...
-        fileDesc->setPart(part.getPropInt("Num")-1,node,_filename.str());
+        unsigned partNo = part.getPropInt("Num")-1;
+
+        RemoteFilename rfn;
+        rfn.setPath(node->endpoint(), remoteFilePath);
+        Owned<IFile> iFile = createIFile(rfn);
+        offset_t physicalSize = iFile->size();
+        bool partCompressed = isCompressedFile(iFile);
+        if (first)
+        {
+            first = false;
+            isCompressed = partCompressed;
+        }
+        else if (isCompressed != partCompressed)
+        {
+            VStringBuffer err("%s - could not attach (mixed compressed/non-compressed physical parts detected)", Partmask);
+            ERRLOG("%s", err.str());
+            errstr.append(err.str());
+            return false;
+        }
+        Owned<IPropertyTree> partProps = createPTree("Part");
+        if (isCompressed)
+            partProps->setPropInt64("@compressedSize", physicalSize);
+        else
+            partProps->setPropInt64("@size", physicalSize);
+        totalSize += physicalSize;
+
+        fileDesc->setPart(partNo, node,_filename.str(), partProps);
+    }
+    IPropertyTree &props = fileDesc->queryProperties();
+    if (isCompressed)
+    {
+        props.setPropBool("@blockCompressed", true);
+        props.setPropInt64("@compressedSize", totalSize);
     }
+    else
+        props.setPropInt64("@size", totalSize);
 
     Owned<IDistributedFile> dFile = queryDistributedFileDirectory().createNew(fileDesc);
     dFile->attach(logicalName.str(),udesc);

system/jlib/jlzw.cpp

@@ -2326,66 +2326,94 @@ public:
 };
 
 
+static unsigned getCompressedMethod(__int64 compressedType)
+{
+    if (compressedType == COMPRESSEDFILEFLAG)
+        return COMPRESS_METHOD_LZW;
+    else if (compressedType == FASTCOMPRESSEDFILEFLAG)
+        return COMPRESS_METHOD_FASTLZ;
+    else if (compressedType == LZ4COMPRESSEDFILEFLAG)
+        return COMPRESS_METHOD_LZ4;
+    return 0;
+}
 
-        
+static bool isCompressedType(__int64 compressedType)
+{
+    return 0 != getCompressedMethod(compressedType);
+}
 
-ICompressedFileIO *createCompressedFileReader(IFileIO *fileio,IExpander *expander)
+bool isCompressedFile(IFileIO *iFileIO, CompressedFileTrailer *trailer=nullptr)
 {
-    if (fileio) {
-        offset_t fsize = fileio->size();
-        if (fsize>=sizeof(WinCompressedFileTrailer)) {  // thats 8 bytes bigger but I think doesn't matter
+    if (iFileIO)
+    {
+        offset_t fsize = iFileIO->size();
+        if (fsize>=sizeof(WinCompressedFileTrailer))  // thats 8 bytes bigger but I think doesn't matter
+        {
             WinCompressedFileTrailer wintrailer;
-            CompressedFileTrailer trailer;
-            if (fileio->read(fsize-sizeof(WinCompressedFileTrailer),sizeof(WinCompressedFileTrailer),&wintrailer)==sizeof(WinCompressedFileTrailer)) {
-                wintrailer.translate(trailer);
-                if ( (trailer.compressedType==COMPRESSEDFILEFLAG) ||
-                     (trailer.compressedType==FASTCOMPRESSEDFILEFLAG) ||
-                     (trailer.compressedType==LZ4COMPRESSEDFILEFLAG) )
-                {
-                    if (expander&&(trailer.recordSize!=0)) {
-                        throw MakeStringException(-1, "Compressed file format error(%d), Encrypted?",trailer.recordSize);
-                    }
-                    unsigned compMethod1 = COMPRESS_METHOD_LZW;
-                    if (trailer.compressedType == FASTCOMPRESSEDFILEFLAG)
-                        compMethod1 = COMPRESS_METHOD_FASTLZ;
-                    else if (trailer.compressedType == LZ4COMPRESSEDFILEFLAG)
-                        compMethod1 = COMPRESS_METHOD_LZ4;
-                    CCompressedFile *cfile = new CCompressedFile(fileio,NULL,trailer,ICFread,false,NULL,expander,compMethod1);
-                    return cfile;
-                }
+            CompressedFileTrailer _trailer;
+            if (!trailer)
+                trailer = &_trailer;
+            if (iFileIO->read(fsize-sizeof(WinCompressedFileTrailer),sizeof(WinCompressedFileTrailer),&wintrailer)==sizeof(WinCompressedFileTrailer))
+            {
+                wintrailer.translate(*trailer);
+                if (isCompressedType(trailer->compressedType))
+                    return true;
             }
         }
     }
-    return NULL;
+    return false;
+}
+
+
+
+bool isCompressedFile(const char *filename)
+{
+    Owned<IFile> iFile = createIFile(filename);
+    return isCompressedFile(iFile);
+}
+
+bool isCompressedFile(IFile *iFile)
+{
+    Owned<IFileIO> iFileIO = iFile->open(IFOread);
+    return isCompressedFile(iFileIO);
+}
+
+ICompressedFileIO *createCompressedFileReader(IFileIO *fileio,IExpander *expander)
+{
+    CompressedFileTrailer trailer;
+    if (isCompressedFile(fileio, &trailer))
+    {
+        if (expander&&(trailer.recordSize!=0))
+            throw MakeStringException(-1, "Compressed file format error(%d), Encrypted?",trailer.recordSize);
+        unsigned compMethod = getCompressedMethod(trailer.compressedType);
+        return new CCompressedFile(fileio,NULL,trailer,ICFread,false,NULL,expander,compMethod);
+    }
+    return nullptr;
 }
 
 
 ICompressedFileIO *createCompressedFileReader(IFile *file,IExpander *expander, bool memorymapped, IFEflags extraFlags)
 {
-    if (file) {
-        if (memorymapped) {
+    if (file)
+    {
+        if (memorymapped)
+        {
             Owned<IMemoryMappedFile> mmfile = file->openMemoryMapped();
-            if (mmfile) {
+            if (mmfile)
+            {
                 offset_t fsize = mmfile->fileSize();
-                if (fsize>=sizeof(WinCompressedFileTrailer)) {  // thats 8 bytes bigger but I think doesn't matter
+                if (fsize>=sizeof(WinCompressedFileTrailer))  // thats 8 bytes bigger but I think doesn't matter
+                {
                     WinCompressedFileTrailer wintrailer;
                     CompressedFileTrailer trailer;
                     memcpy(&wintrailer,mmfile->base()+fsize-sizeof(WinCompressedFileTrailer),sizeof(WinCompressedFileTrailer));
                     wintrailer.translate(trailer);
-                    if ( (trailer.compressedType==COMPRESSEDFILEFLAG) ||
-                         (trailer.compressedType==FASTCOMPRESSEDFILEFLAG) ||
-                         (trailer.compressedType==LZ4COMPRESSEDFILEFLAG) )
+                    unsigned compMethod = getCompressedMethod(trailer.compressedType);
+                    if (compMethod)
                     {
-                        if (expander&&(trailer.recordSize!=0)) {
+                        if (expander&&(trailer.recordSize!=0))
                             throw MakeStringException(-1, "Compressed file format error(%d), Encrypted?",trailer.recordSize);
-                        }
-                        unsigned compMethod1 = COMPRESS_METHOD_LZW;
-                        if (trailer.compressedType == FASTCOMPRESSEDFILEFLAG)
-                            compMethod1 = COMPRESS_METHOD_FASTLZ;
-                        else if (trailer.compressedType == LZ4COMPRESSEDFILEFLAG)
-                            compMethod1 = COMPRESS_METHOD_LZ4;
-                        CCompressedFile *cfile = new CCompressedFile(NULL,mmfile,trailer,ICFread,false,NULL,expander,compMethod1);
-                        return cfile;
+                        return new CCompressedFile(NULL,mmfile,trailer,ICFread,false,NULL,expander,compMethod);
                     }
                 }
             }
@@ -2404,26 +2432,21 @@ ICompressedFileIO *createCompressedFileWriter(IFileIO *fileio,size32_t recordsiz
 {
     CompressedFileTrailer trailer;
     offset_t fsize = fileio->size();
-    if (fsize) {
-        loop {
-            if (fsize>=sizeof(WinCompressedFileTrailer)) {  // thats 8 bytes bigger but I think doesn't matter
+    if (fsize)
+    {
+        loop
+        {
+            if (fsize>=sizeof(WinCompressedFileTrailer))  // thats 8 bytes bigger but I think doesn't matter
+            {
                 WinCompressedFileTrailer wintrailer;
                 CompressedFileTrailer trailer;
                 if (fileio->read(fsize-sizeof(WinCompressedFileTrailer),sizeof(WinCompressedFileTrailer),&wintrailer)==sizeof(WinCompressedFileTrailer)) {
                     wintrailer.translate(trailer);
-                    if ( (trailer.compressedType==COMPRESSEDFILEFLAG) ||
-                         (trailer.compressedType==FASTCOMPRESSEDFILEFLAG) ||
-                         (trailer.compressedType==LZ4COMPRESSEDFILEFLAG) )
+                    unsigned compMethod = getCompressedMethod(trailer.compressedType);
+                    if (compMethod)
                     {
                         // check trailer.compressedType against _compMethod
-                        unsigned compMethod1 = 0;
-                        if (trailer.compressedType == COMPRESSEDFILEFLAG)
-                            compMethod1 = COMPRESS_METHOD_LZW;
-                        else if (trailer.compressedType == FASTCOMPRESSEDFILEFLAG)
-                            compMethod1 = COMPRESS_METHOD_FASTLZ;
-                        else if (trailer.compressedType == LZ4COMPRESSEDFILEFLAG)
-                            compMethod1 = COMPRESS_METHOD_LZ4;
-                        if (_compMethod != compMethod1)
+                        if (_compMethod != compMethod)
                             throw MakeStringException(-1,"Appending to file with different compression method");
                         if ((recordsize==trailer.recordSize)||!trailer.recordSize)
                             break;
@@ -2434,7 +2457,8 @@ ICompressedFileIO *createCompressedFileWriter(IFileIO *fileio,size32_t recordsiz
             throw MakeStringException(-1,"Appending to file that is not compressed");
         }
     }
-    else {
+    else
+    {
         memset(&trailer,0,sizeof(trailer));
         trailer.crc = ~0U;
         if (_compMethod == COMPRESS_METHOD_FASTLZ)
@@ -2670,9 +2694,7 @@ IPropertyTree *getBlockedFileDetails(IFile *file)
             CompressedFileTrailer trailer;
             if (fileio->read(fsize-sizeof(WinCompressedFileTrailer),sizeof(WinCompressedFileTrailer),&wintrailer)==sizeof(WinCompressedFileTrailer)) {
                 wintrailer.translate(trailer);
-                if ( (trailer.compressedType==COMPRESSEDFILEFLAG) ||
-                     (trailer.compressedType==FASTCOMPRESSEDFILEFLAG) ||
-                     (trailer.compressedType==LZ4COMPRESSEDFILEFLAG) )
+                if (isCompressedType(trailer.compressedType))
                 {
                     trailer.setDetails(*tree);
                     unsigned nb = trailer.numBlocks();

system/jlib/jlzw.hpp

@@ -110,6 +110,8 @@ interface ICompressedFileIO: extends IFileIO
     virtual unsigned method()=0;
 };
 
+extern jlib_decl bool isCompressedFile(const char *filename);
+extern jlib_decl bool isCompressedFile(IFile *file);
 extern jlib_decl ICompressedFileIO *createCompressedFileReader(IFile *file,IExpander *expander=NULL, bool memorymapped=false, IFEflags extraFlags=IFEnone);
 extern jlib_decl ICompressedFileIO *createCompressedFileReader(IFileIO *fileio,IExpander *expander=NULL);
 extern jlib_decl ICompressedFileIO *createCompressedFileWriter(IFile *file,size32_t recordsize,bool append=false,bool setcrc=true,ICompressor *compressor=NULL, unsigned compMethod=COMPRESS_METHOD_LZW, IFEflags extraFlags=IFEnone);