Merge pull request #8778 from jamienoss/issue15210-directly-compile-gzipped-archives

HPCC-15210 Compile direct from .gz files

Reviewed-by: Gavin Halliday <ghalliday@hpccsystems.com>

ecl/eclcc/CMakeLists.txt

@@ -41,6 +41,7 @@ include_directories (
          ./../../rtl/include 
          ./../../system/jlib 
          ./../../common/remote
+         ./../../system/security/zcrypt
     )
 
 ADD_DEFINITIONS( -D_CONSOLE )
@@ -65,3 +66,9 @@ target_link_libraries ( eclcc
          hqlcpp
     )
 
+IF (USE_ZLIB)
+    target_link_libraries ( eclcc
+        ${ZLIB_LIBRARIES}
+        zcrypt
+    )
+ENDIF()

ecl/eclcc/eclcc.cpp

@@ -57,6 +57,10 @@
 #include <cppunit/ui/text/TestRunner.h>
 #endif
 
+#ifdef _USE_ZLIB
+#include "zcrypt.hpp"
+#endif
+
 //#define TEST_LEGACY_DEPENDENCY_CODE
 
 #define INIFILE "eclcc.ini"
@@ -876,7 +880,14 @@ void EclCC::getComplexity(IWorkUnit *wu, IHqlExpression * query, IErrorReceiver
 
 static bool convertPathToModule(StringBuffer & out, const char * filename)
 {
-    const char * dot = strrchr(filename, '.');
+    StringBuffer temp;
+#ifdef _USE_ZLIB
+    removeZipExtension(temp, filename);
+#else
+    temp.append(filename);
+#endif
+
+    const char * dot = strrchr(temp.str(), '.');
     if (dot)
     {
         if (!strieq(dot, ".ecl") && !strieq(dot, ".hql") && !strieq(dot, ".eclmod") && !strieq(dot, ".eclattr"))
@@ -885,7 +896,7 @@ static bool convertPathToModule(StringBuffer & out, const char * filename)
     else
         return false;
 
-    const unsigned copyLen = dot-filename;
+    const unsigned copyLen = dot-temp.str();
     if (copyLen == 0)
         return false;
 

ecl/hql/CMakeLists.txt

@@ -105,6 +105,7 @@ include_directories (
          ./../../system/jlib 
          ./../../ecl/hql 
          ./../../testing/unittests
+         ./../../system/security/zcrypt
     )
 
 if (WIN32)
@@ -144,4 +145,9 @@ target_link_libraries ( hql
          ${CPPUNIT_LIBRARIES}
     )
 
-
+IF (USE_ZLIB)
+    target_link_libraries ( hql
+        ${ZLIB_LIBRARIES}
+        zcrypt
+    )
+ENDIF()

ecl/hql/hqlcollect.cpp

@@ -25,6 +25,10 @@
 #include "hqlexpr.hpp"
 #include "hqlerrors.hpp"
 
+#ifdef _USE_ZLIB
+#include "zcrypt.hpp"
+#endif
+
 inline bool isNullOrBlank(const char * s) { return !s || !*s; }
 
 //DLLs should call CPluginCtx to manage memory
@@ -209,7 +213,14 @@ public:
 
 EclSourceType getEclSourceType(const char * tailname)
 {
-    const char *ext = strrchr(tailname, '.');
+    StringBuffer temp;
+#ifdef _USE_ZLIB
+    removeZipExtension(temp, tailname);
+#else
+    temp.append(tailname);
+#endif
+
+    const char *ext = strrchr(temp.str(), '.');
     if (!ext)
         return ESTnone;
     if (stricmp(ext, ".eclmod")==0 || stricmp(ext, ".hql")==0)
@@ -226,18 +237,24 @@ EclSourceType getEclSourceType(const char * tailname)
 static IIdAtom * deriveEclName(const char * filename)
 {
     if (!filename)
-        return NULL;
+        return nullptr;
+
+    StringBuffer tailname;
+#ifdef _USE_ZLIB
+    removeZipExtension(tailname, pathTail(filename));
+#else
+    tailname.append(pathTail(filename));
+#endif
 
-    const char * tailname = pathTail(filename);
-    const char *ext = strrchr(tailname, '.');
+    const char * ext = strrchr(tailname.str(), '.');
     IIdAtom * id;
     if (ext)
-        id = createIdAtom(tailname, ext-tailname);
+        id = createIdAtom(tailname.str(), ext-tailname.str());
     else
-        id = createIdAtom(tailname);
+        id = createIdAtom(tailname.str());
 
     if (!isCIdentifier(str(id)))
-        return NULL;
+        return nullptr;
     return id;
 }
 

ecl/hql/hqlexpr.cpp

@@ -50,6 +50,10 @@
 #include "hqldesc.hpp"
 #include "hqlir.hpp"
 
+#ifdef _USE_ZLIB
+#include "zcrypt.hpp"
+#endif
+
 //This nearly works - but there are still some examples which have problems - primarily libraries, old parameter syntax, enums and other issues.
 
 //#define ANNOTATE_EXPR_POSITION
@@ -7592,6 +7596,18 @@ bool CFileContents::preloadFromFile()
         sizeRead += rd;
         mb.setLength(sizeRead);
     } while (rd);
+
+#ifdef _USE_ZLIB
+    byte * contents = static_cast<byte *>(mb.bufferBase());
+    if (isgzipped(contents, sizeRead))
+    {
+        StringBuffer decoded;
+        gunzip(contents, sizeRead, decoded);
+        size_t decodedLength = (size_t)decoded.length();
+        mb.setBuffer(decodedLength, (void*)decoded.detach(), true);
+    }
+#endif
+
     ensureUtf8(mb);
     setContentsOwn(mb);
     return true;
@@ -7632,11 +7648,21 @@ void CFileContents::ensureLoaded()
     buffer.ensureCapacity(sizeToRead+1);
     byte * contents = static_cast<byte *>(buffer.reserve(sizeToRead));
     size32_t sizeRead = io->read(0, sizeToRead, contents);
-    ensureUtf8(buffer);
-    setContentsOwn(buffer);
-
     if (sizeRead != sizeToRead)
         throw MakeStringException(1, "File %s only read %u of %u bytes", file->queryFilename(), sizeRead, sizeToRead);
+
+#ifdef _USE_ZLIB
+    if (isgzipped(contents, sizeToRead))
+    {
+        StringBuffer decoded;
+        gunzip(contents, sizeToRead, decoded);
+        size_t decodedLength = (size_t)decoded.length();
+        buffer.setBuffer(decodedLength, (void*)decoded.detach(), true);
+    }
+#endif
+
+    ensureUtf8(buffer);
+    setContentsOwn(buffer);
 }
 
 CFileContents::CFileContents(const char *query, ISourcePath * _sourcePath, bool _isSigned)

ecl/regress/regress.sh

@@ -128,8 +128,10 @@ if [[ $eclcc != '' ]]; then
     echo "ECLCC=$eclcc" >> Makefile
     echo "TARGET=$target_dir" >> Makefile
     echo "FILES=\$(shell echo *.ecl*)" >> Makefile
-    echo "LOGS_=\$(FILES:%.ecl=\$(TARGET)/%.ecl.log)" >> Makefile
-    echo "LOGS=\$(LOGS_:%.eclxml=\$(TARGET)/%.eclxml.log)" >> Makefile
+    echo "LOGS0=\$(FILES:%.ecl=\$(TARGET)/%.ecl.log)" >> Makefile
+    echo "LOGS1=\$(LOGS0:%.ecl.gz=\$(TARGET)/%.ecl.log)" >> Makefile
+    echo "LOGS2=\$(LOGS1:%.eclxml.gz=\$(TARGET)/%.eclxml.log)" >> Makefile
+    echo "LOGS=\$(LOGS2:%.eclxml=\$(TARGET)/%.eclxml.log)" >> Makefile
     echo >> Makefile
     echo "all: \$(LOGS)" >> Makefile
     echo >> Makefile
@@ -139,8 +141,14 @@ if [[ $eclcc != '' ]]; then
     echo "\$(TARGET)/%.ecl.log: %.ecl" >> Makefile
     echo -e "\t\$(ECLCC) \$(FLAGS) $^" >> Makefile
     echo >> Makefile
+    echo "\$(TARGET)/%.ecl.log: %.ecl.gz" >> Makefile
+    echo -e "\t\$(ECLCC) \$(FLAGS) $^" >> Makefile
+    echo >> Makefile
     echo "\$(TARGET)/%.eclxml.log: %.eclxml" >> Makefile
     echo -e "\t\$(ECLCC) \$(FLAGS) $^" >> Makefile
+    echo >> Makefile
+    echo "\$(TARGET)/%.eclxml.log: %.eclxml.gz" >> Makefile
+    echo -e "\t\$(ECLCC) \$(FLAGS) $^" >> Makefile
 
     if [[ $query != '' ]]; then
         ## Compile all regressions

system/security/zcrypt/zcrypt.cpp

@@ -1166,3 +1166,18 @@ void gunzip(const byte* compressed, unsigned int comprLen, StringBuffer& sOutput
         throwGZipException("decompression", ret);
     }
 }
+
+bool isgzipped(const byte * content, size_t length)
+{
+    if (length < 2)
+        return false;
+    return (content[0] == 0x1f) && (content[1] == 0x8b);
+}
+
+void removeZipExtension(StringBuffer & target, const char * source)
+{
+    target.set(source);
+    const char * extension = strrchr(target.str(), '.');
+    if (extension && streq(extension, ".gz"))
+        target.remove(extension-target.str(), 3);
+}

system/security/zcrypt/zcrypt.hpp

@@ -100,5 +100,7 @@ typedef unsigned char byte;
 ZCRYPT_API char* gzip( const char* inputBuffer, unsigned int inputSize,
     unsigned int* outlen, int compressionLevel=GZ_DEFAULT_COMPRESSION);
 ZCRYPT_API void gunzip(const byte* compressed, unsigned int comprLen, StringBuffer& sOutput);
+ZCRYPT_API bool isgzipped(const byte* content, size_t length);
+ZCRYPT_API void removeZipExtension(StringBuffer & target, const char * source);
 
 #endif