Big-Data-HPC-Platform/system/jlib/jlzw.hpp

/*##############################################################################

    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems®.

    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.
    You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

    Unless required by applicable law or agreed to in writing, software
    distributed under the License is distributed on an "AS IS" BASIS,
    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    See the License for the specific language governing permissions and
    limitations under the License.
############################################################################## */



#ifndef JLZW_INCL
#define JLZW_INCL

#include "jiface.hpp"
#include "jfile.hpp"
#include <stdio.h>

interface jlib_decl ICompressor : public IInterface
{
    virtual void   open(MemoryBuffer &mb, size32_t initialSize=0)=0; // variable internally sized buffer
    virtual void   open(void *blk, size32_t blksize)=0;              // fixed size output
    virtual void   close()=0;
    virtual size32_t write(const void *buf,size32_t len)=0;             
                                                                            
    virtual void * bufptr()=0;
    virtual size32_t buflen()=0;
    virtual void   startblock()=0;                      // row based must call startblock/commitblock
    virtual void   commitblock()=0;
};

interface jlib_decl IExpander : public IInterface
{
    virtual size32_t init(const void *blk)=0; // returns size required
    virtual void   expand(void *target)=0;
    virtual void * bufptr()=0;
    virtual size32_t buflen()=0;
};


interface jlib_decl IRandRowExpander : public IInterface
{
    static inline bool isRand(const void *blk) { return *((const unsigned short *)blk+1)==0xffff; }
    virtual bool init(const void *blk,bool copy=true)=0;
    virtual size32_t rowSize() const = 0;       
    virtual unsigned numRows() const = 0;
    virtual bool expandRow(void *target,unsigned index) const =0;
    virtual size32_t expandRow(void *target,unsigned index,size32_t ofs,size32_t sz) const =0;
    virtual const byte *firstRow() const = 0;
    virtual int cmpRow(const void *target,unsigned index,size32_t ofs=0,size32_t sz=(size32_t)-1) const =0;
};




extern jlib_decl ICompressor *createLZWCompressor(bool supportbigendian=false); // bigendiansupport required for cross platform with solaris
extern jlib_decl IExpander *createLZWExpander(bool supportbigendian=false);

#define RLEMAXOVERHEAD 2
extern jlib_decl size32_t RLECompress(void *dst,const void *src,size32_t size);  // maximum will write is 2+size
extern jlib_decl size32_t RLEExpand(void *dst,const void *src,size32_t expsize); // returns amount read, expsize must be provided


extern jlib_decl size32_t DiffCompressFirst(const void *src,void *dst,void *buff,size32_t rs);  // compress first row (actually make bigger, but in same format as compression)
                                                                                            // buf need not be initialized
extern jlib_decl size32_t DiffCompress(const void *src,void *dst,void *buff,size32_t rs);       // compress subsequent rows (bufs set by previous DiffFirstCompress or DiffCompress
extern jlib_decl size32_t DiffCompress2(const void *src,void *dst,const void *prev,size32_t rs);// compress row (prev not updated)
extern jlib_decl size32_t DiffExpand(const void *src,void *dst,const void *prev,size32_t rs);   // expand row, prev must be passed previous expanded row 
extern jlib_decl size32_t DiffCompressedSize(const void *cmpressedsrc,size32_t rs);             // calculate compressed row size - rs is expanded size
inline size32_t MaxDiffCompressedRowSize (size32_t rowsize) { return rowsize+((rowsize+254)/255)*2; }


extern jlib_decl ICompressor *createRDiffCompressor(); // NB only supports transaction mode one row per transaction and fixed row size
extern jlib_decl IExpander *createRDiffExpander(); // NB only supports transaction mode one row per transaction and fixed row size

extern jlib_decl ICompressor *createRandRDiffCompressor(); // similar to RDiffCompressor except rows can be expanded individually
extern jlib_decl IRandRowExpander *createRandRDiffExpander(); // NB only supports fixed row size


//Some helper functions to make it easy to compress/decompress to memorybuffers.
extern jlib_decl void compressToBuffer(MemoryBuffer & out, size32_t len, const void * src);
extern jlib_decl void decompressToBuffer(MemoryBuffer & out, const void * src);
extern jlib_decl void decompressToBuffer(MemoryBuffer & out, MemoryBuffer & in);
extern jlib_decl void decompressToAttr(MemoryAttr & out, const void * src);
extern jlib_decl void decompressToBuffer(MemoryAttr & out, MemoryBuffer & in);
extern jlib_decl void appendToBuffer(MemoryBuffer & out, size32_t len, const void * src); //format as failed compression


#define COMPRESS_METHOD_ROWDIF 1
#define COMPRESS_METHOD_LZW    2
#define COMPRESS_METHOD_FASTLZ 3
#define COMPRESS_METHOD_LZMA   4
#define COMPRESS_METHOD_LZ4    5

interface ICompressedFileIO: extends IFileIO
{
    virtual unsigned dataCRC()=0;                   // CRC for data area (note total file CRC equals COMPRESSEDFILECRC)
    virtual size32_t recordSize()=0;                // 0 for lzw/fastlz, otherwise record length for row difference compression
    virtual size32_t blockSize()=0;                 // block size used
    virtual void setBlockSize(size32_t size)=0;     // only callable before any writes
    virtual bool readMode()=0;                      // true if created using createCompressedFileReader
    virtual unsigned method()=0;
};

extern jlib_decl bool isCompressedFile(const char *filename);
extern jlib_decl bool isCompressedFile(IFile *file);
extern jlib_decl ICompressedFileIO *createCompressedFileReader(IFile *file,IExpander *expander=NULL, bool memorymapped=false, IFEflags extraFlags=IFEnone);
extern jlib_decl ICompressedFileIO *createCompressedFileReader(IFileIO *fileio,IExpander *expander=NULL);
extern jlib_decl ICompressedFileIO *createCompressedFileWriter(IFile *file,size32_t recordsize,bool append=false,bool setcrc=true,ICompressor *compressor=NULL, unsigned compMethod=COMPRESS_METHOD_LZW, IFEflags extraFlags=IFEnone);
extern jlib_decl ICompressedFileIO *createCompressedFileWriter(IFileIO *fileio,size32_t recordsize,bool setcrc=true,ICompressor *compressor=NULL, unsigned compMethod=COMPRESS_METHOD_LZW);

#define COMPRESSEDFILECRC (~0U)

extern jlib_decl ICompressor *createAESCompressor(const void *key, unsigned keylen=0); // keylen>0 must be 16, 24, or 32 Bytes, if keylen=0 key is padded to 32bytes
extern jlib_decl IExpander *createAESExpander(const void *key, unsigned keylen=0);   // keylen>0 must be 16, 24, or 32 Bytes, if keylen=0 key is padded to 32bytes

extern jlib_decl ICompressor *createAESCompressor256(size32_t len, const void *key); // 256bit key
extern jlib_decl IExpander *createAESExpander256(size32_t len, const void *key);     // key is ascii and is padded

interface IPropertyTree;
extern jlib_decl IPropertyTree *getBlockedFileDetails(IFile *file);


interface ICompressHandler : extends IInterface
{
    virtual const char *queryType() const = 0;
    virtual ICompressor *getCompressor(const char *options=NULL) = 0;
    virtual IExpander *getExpander(const char *options=NULL) = 0;
};
extern jlib_decl void setDefaultCompressor(const char *type);
extern jlib_decl ICompressHandler *queryCompressHandler(const char *type);
extern jlib_decl ICompressHandler *queryDefaultCompressHandler();
extern jlib_decl bool addCompressorHandler(ICompressHandler *handler); // returns true if added, false if already registered
extern jlib_decl bool removeCompressorHandler(ICompressHandler *handler); // returns true if present and removed

extern jlib_decl ICompressor *getCompressor(const char *type, const char *options=NULL);
extern jlib_decl IExpander *getExpander(const char *type, const char *options=NULL);



#define MIN_ROWCOMPRESS_RECSIZE 8
#endif