FEATURE - support for reading directly from zip and other archives

Refactored zip and git file hook functionality to use dyaically loaded plugin
library structure.
Default USE_LIBARCHIVE of fon Windows, to avoid breaking everyone's builds
Update lastPos properly for more accurate detection of unsupported seek
requests.
Removed some unnecessary tracing.

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

cmake_modules/commonSetup.cmake

@@ -41,7 +41,7 @@ IF ("${COMMONSETUP_DONE}" STREQUAL "")
   cmake_policy ( SET CMP0011 NEW )
 
 
-  option(USE_BINUTILS "Enable use of binutils to embed workunit info into dhard objects" ON)
+  option(USE_BINUTILS "Enable use of binutils to embed workunit info into shared objects" ON)
   option(USE_CPPUNIT "Enable unit tests (requires cppunit)" OFF)
   option(USE_OPENLDAP "Enable OpenLDAP support (requires OpenLDAP)" ON)
   option(USE_ICU "Enable unicode support (requires ICU)" ON)
@@ -49,7 +49,11 @@ IF ("${COMMONSETUP_DONE}" STREQUAL "")
   option(Boost_USE_STATIC_LIBS "Use boost_regex static library for RPM BUILD" OFF)
   option(USE_OPENSSL "Configure use of OpenSSL" ON)
   option(USE_ZLIB "Configure use of zlib" ON)
-  option(USE_LIBARCHIVE "Configure use of libarchive" ON)
+  if (WIN32)
+    option(USE_LIBARCHIVE "Configure use of libarchive" OFF)   # libarchive rather less standard on windows systems
+  else()
+    option(USE_LIBARCHIVE "Configure use of libarchive" ON)
+  endif()
   option(USE_NATIVE_LIBRARIES "Search standard OS locations for thirdparty libraries" ON)
   option(USE_GIT_DESCRIBE "Use git describe to generate build tag" ON)
   option(CHECK_GIT_TAG "Require git tag to match the generated build tag" OFF)

common/remote/CMakeLists.txt

@@ -25,6 +25,8 @@
 
 project( remote ) 
 
+add_subdirectory(hooks)
+
 set (    SRCS 
          rmtfile.cpp 
          rmtpass.cpp 
@@ -32,8 +34,6 @@ set (    SRCS
          rmtssh.cpp 
          rmtsmtp.cpp 
          sockfile.cpp
-         gitfile.cpp
-         archive.cpp
          
          remoteerr.hpp
          rmtfile.hpp
@@ -42,8 +42,6 @@ set (    SRCS
          rmtspawn.hpp
          rmtssh.hpp
          sockfile.hpp
-         gitfile.hpp
-         archive.hpp
     )
 
 include_directories (
@@ -53,10 +51,6 @@ include_directories (
          ./../../system/jlib 
     )
 
-IF (USE_LIBARCHIVE)
-include_directories ( ${LIBARCHIVE_INCLUDE_DIR} )
-ENDIF()
-
 ADD_DEFINITIONS( -D_USRDLL -DREMOTE_EXPORTS )
 
 HPCC_ADD_LIBRARY( remote SHARED ${SRCS}  )
@@ -66,9 +60,3 @@ target_link_libraries ( remote
     jlib 
     mp
     )
-
-IF (USE_LIBARCHIVE)
-target_link_libraries ( remote
-    archive
-    )
-ENDIF()

common/remote/hooks/CMakeLists.txt

@@ -0,0 +1,31 @@
+################################################################################
+#    Copyright (C) 2011 HPCC Systems.
+#
+#    All rights reserved. This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU Affero General Public License as
+#    published by the Free Software Foundation, either version 3 of the
+#    License, or (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU Affero General Public License for more details.
+#
+#    You should have received a copy of the GNU Affero General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+################################################################################
+
+
+# Component: remote/hooks
+#####################################################
+# Description:
+# ------------
+#    Cmake Input File for remote/hooks
+#####################################################
+
+project( hooks )
+
+add_subdirectory(git)
+IF (USE_LIBARCHIVE)
+  add_subdirectory(libarchive)
+ENDIF()

common/remote/hooks/git/CMakeLists.txt

@@ -0,0 +1,45 @@
+################################################################################
+#    Copyright (C) 2011 HPCC Systems.
+#
+#    All rights reserved. This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU Affero General Public License as
+#    published by the Free Software Foundation, either version 3 of the
+#    License, or (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU Affero General Public License for more details.
+#
+#    You should have received a copy of the GNU Affero General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+################################################################################
+
+
+# Component: gitfile
+#####################################################
+# Description:
+# ------------
+#    Cmake Input File for git repository direct reading hook
+#####################################################
+
+project( gitfile )
+
+set (    SRCS
+         gitfile.cpp
+         gitfile.hpp
+    )
+
+include_directories (
+         ${HPCC_SOURCE_DIR}/system/include
+         ${HPCC_SOURCE_DIR}/system/jlib
+    )
+
+ADD_DEFINITIONS( -D_USRDLL -DGITFILE_EXPORTS )
+
+HPCC_ADD_LIBRARY( gitfile SHARED ${SRCS}  )
+install ( TARGETS gitfile DESTINATION ${OSSDIR}/filehooks )
+
+target_link_libraries ( gitfile
+    jlib
+    )

common/remote/gitfile.cpp

@@ -263,7 +263,7 @@ protected:
     bool isExisting;
 };
 
-extern REMOTE_API IFile *createGitFile(const char *gitFileName)
+static IFile *createGitFile(const char *gitFileName)
 {
     StringBuffer fname(gitFileName);
     assertex(fname.length());
@@ -436,7 +436,7 @@ protected:
 
 static CriticalSection *cs;
 
-extern REMOTE_API void installGitFileHook()
+extern GITFILE_API void installFileHook()
 {
     CriticalBlock b(*cs); // Probably overkill!
     if (!gitRepositoryFileHook)
@@ -446,7 +446,7 @@ extern REMOTE_API void installGitFileHook()
     }
 }
 
-extern REMOTE_API void removeGitFileHook()
+extern GITFILE_API void removeFileHook()
 {
     CriticalBlock b(*cs); // Probably overkill!
     if (gitRepositoryFileHook)
@@ -456,7 +456,7 @@ extern REMOTE_API void removeGitFileHook()
     }
 }
 
-MODULE_INIT(INIT_PRIORITY_REMOTE_RMTFILE)
+MODULE_INIT(INIT_PRIORITY_STANDARD)
 {
     cs = new CriticalSection;
     gitRepositoryFileHook = NULL;  // Not really needed, but you have to have a modinit to match a modexit
@@ -465,7 +465,11 @@ MODULE_INIT(INIT_PRIORITY_REMOTE_RMTFILE)
 
 MODULE_EXIT()
 {
-    removeGitFileHook();
+    if (gitRepositoryFileHook)
+    {
+        removeContainedFileHook(gitRepositoryFileHook);
+        gitRepositoryFileHook = NULL;
+    }
     ::Release(gitRepositoryFileHook);
     delete cs;
 }

common/remote/gitfile.hpp

@@ -21,10 +21,10 @@
 
 #include "jfile.hpp"
 
-#ifdef REMOTE_EXPORTS
-#define REMOTE_API __declspec(dllexport)
+#ifdef GITFILE_EXPORTS
+#define GITFILE_API __declspec(dllexport)
 #else
-#define REMOTE_API __declspec(dllimport)
+#define GITFILE_API __declspec(dllimport)
 #endif
 
 
@@ -33,8 +33,9 @@
  * Installs hooks into createIFile, spotting filenames of the form /my/directory/.git/{revision}/path/within/git
  */
 
-extern REMOTE_API IFile * createGitFile(const char * _filename); // create directly without installing createIFile hook
-extern REMOTE_API void installGitFileHook();
-extern REMOTE_API void removeGitFileHook();
+extern "C" {
+  extern GITFILE_API void installFileHook();
+  extern GITFILE_API void removeFileHook();
+};
 
 #endif

common/remote/hooks/libarchive/CMakeLists.txt

@@ -0,0 +1,48 @@
+################################################################################
+#    Copyright (C) 2011 HPCC Systems.
+#
+#    All rights reserved. This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU Affero General Public License as
+#    published by the Free Software Foundation, either version 3 of the
+#    License, or (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU Affero General Public License for more details.
+#
+#    You should have received a copy of the GNU Affero General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+################################################################################
+
+
+# Component: archivefile
+#####################################################
+# Description:
+# ------------
+#    Cmake Input File for archivefile
+#####################################################
+
+project( archivefile )
+
+set (    SRCS
+         archive.cpp
+         archive.hpp
+    )
+
+include_directories (
+         ${HPCC_SOURCE_DIR}/system/include
+         ${HPCC_SOURCE_DIR}/system/jlib
+    )
+
+include_directories ( ${LIBARCHIVE_INCLUDE_DIR} )
+
+ADD_DEFINITIONS( -D_USRDLL -DARCHIVEFILE_EXPORTS )
+
+HPCC_ADD_LIBRARY( archivefile SHARED ${SRCS}  )
+install ( TARGETS archivefile DESTINATION ${OSSDIR}/filehooks )
+
+target_link_libraries ( archivefile
+    jlib
+    archive
+    )

common/remote/archive.cpp

@@ -25,11 +25,8 @@
 #include "jfile.hpp"
 #include "jlog.hpp"
 #include "jregexp.hpp"
-#include "gitfile.hpp"
 #include "archive.hpp"
 
-#ifdef _USE_LIBARCHIVE
-
 #include <sys/stat.h>
 #include <archive.h>
 #include <archive_entry.h>
@@ -182,7 +179,6 @@ public:
         if (pos < lastPos)
             throw MakeStringException(0, "Only sequential access to contained file %s supported", fullName.get());
         byte *data = (byte *) _data;
-        lastPos = pos;
         size32_t lenRequested = len;
         while (len > 0 & pos < fileSize)
         {
@@ -211,6 +207,7 @@ public:
                 pos += copyLen;
             }
         }
+        lastPos = pos;
         return lenRequested - len;
     }
     virtual offset_t size()
@@ -367,7 +364,7 @@ protected:
     Linked<ArchiveEntry> entry;
 };
 
-extern REMOTE_API IFile *createIFileInArchive(const char *containedFileName)
+static IFile *createIFileInArchive(const char *containedFileName)
 {
     StringBuffer fname(containedFileName);
     assertex(fname.length());
@@ -450,7 +447,6 @@ public:
                         // Strip off a trailing /, then check that there is no / in the tail
                         if (strchr(tail, PATHSEPCHAR) == NULL &&  (!mask.length() || WildMatch(tail, mask, false)))
                         {
-                            DBGLOG("found file %s %s %s", container.get(), relDir.get(), tail.str());
                             entries.append(*new ArchiveEntry(entry));
                         }
                     }
@@ -523,7 +519,7 @@ protected:
     }
 } *archiveFileHook;
 
-extern REMOTE_API void installArchiveFileHook()
+extern ARCHIVEFILE_API void installFileHook()
 {
     SpinBlock b(*lock); // Probably overkill!
     if (!archiveFileHook)
@@ -533,7 +529,7 @@ extern REMOTE_API void installArchiveFileHook()
     }
 }
 
-extern REMOTE_API void removeArchiveFileHook()
+extern ARCHIVEFILE_API void removeFileHook()
 {
     SpinBlock b(*lock); // Probably overkill!
     if (archiveFileHook)
@@ -543,7 +539,7 @@ extern REMOTE_API void removeArchiveFileHook()
     }
 }
 
-MODULE_INIT(INIT_PRIORITY_REMOTE_RMTFILE)
+MODULE_INIT(INIT_PRIORITY_STANDARD)
 {
     lock = new SpinLock;
     signature = new RegExpr(ARCHIVE_SIGNATURE);
@@ -553,21 +549,12 @@ MODULE_INIT(INIT_PRIORITY_REMOTE_RMTFILE)
 
 MODULE_EXIT()
 {
-    removeArchiveFileHook();
+    if (archiveFileHook)
+    {
+        removeContainedFileHook(archiveFileHook);
+        archiveFileHook = NULL;
+    }
     delete signature;
     delete lock;
     ::Release(archiveFileHook);
 }
-#else
-extern REMOTE_API void installArchiveFileHook()
-{
-}
-
-extern REMOTE_API void removeArchiveFileHook()
-{
-}
-extern REMOTE_API IFile *createIFileInArchive(const char *containedFileName)
-{
-    throw MakeStringException(0, "System was built without archive file support");
-}
-#endif

common/remote/archive.hpp

@@ -21,10 +21,10 @@
 
 #include "jfile.hpp"
 
-#ifdef REMOTE_EXPORTS
-#define REMOTE_API __declspec(dllexport)
+#ifdef ARCHIVEFILE_EXPORTS
+#define ARCHIVEFILE_API __declspec(dllexport)
 #else
-#define REMOTE_API __declspec(dllimport)
+#define ARCHIVEFILE_API __declspec(dllimport)
 #endif
 
 
@@ -33,8 +33,8 @@
  * Installs hooks into createIFile, spotting filenames of the form /my/directory/myfile.zip/{password}/path/within/archive
  */
 
-extern REMOTE_API IFile * createIFileInArchive(const char * _filename); // create directly without installing createIFile hook
-extern REMOTE_API void installArchiveFileHook();
-extern REMOTE_API void removeArchiveFileHook();
-
+extern "C" {
+  extern ARCHIVEFILE_API void installFileHook();
+  extern ARCHIVEFILE_API void removeFileHook();
+};
 #endif

common/remote/rmtfile.cpp

@@ -607,6 +607,94 @@ public:
     return failures.ordinality();
 }
 
+static PointerArrayOf<SharedObject> *hookDlls;
+
+static void installFileHook(const char *hookFileSpec);
+
+extern REMOTE_API void installFileHooks(const char *hookFileSpec)
+{
+    if (!hookDlls)
+        hookDlls = new PointerArrayOf<SharedObject>;
+    const char * cursor = hookFileSpec;
+    for (;*cursor;)
+    {
+        StringBuffer file;
+        while (*cursor && *cursor != ENVSEPCHAR)
+            file.append(*cursor++);
+        if(*cursor)
+            cursor++;
+        if(!file.length())
+            continue;
+        installFileHook(file);
+    }
+}
+
+typedef void *(HookInstallFunction)();
+
+static void installFileHook(const char *hookFile)
+{
+    StringBuffer dirPath, dirTail, absolutePath;
+    splitFilename(hookFile, &dirPath, &dirPath, &dirTail, &dirTail);
+    makeAbsolutePath(dirPath.str(), absolutePath);
+    if (!containsFileWildcard(dirTail))
+    {
+        addPathSepChar(absolutePath).append(dirTail);
+        Owned<IFile> file = createIFile(absolutePath);
+        if (file->isDirectory() == foundYes)
+        {
+            installFileHooks(addPathSepChar(absolutePath).append('*'));
+        }
+        else if (file->isFile() == foundYes)
+        {
+            SharedObject *so = new SharedObject(); // MORE - this leaks! Kind-of deliberate right now...
+            if (so->load(file->queryFilename(), false))
+            {
+                HookInstallFunction *hookInstall = (HookInstallFunction *) GetSharedProcedure(so->getInstanceHandle(), "installFileHook");
+                hookInstall();
+                hookDlls->append(so);
+            }
+            else
+            {
+                so->unload();
+                delete so;
+                DBGLOG("File hook library %s could not be loaded", hookFile);
+            }
+        }
+        else
+        {
+            DBGLOG("File hook library %s not found", hookFile);
+        }
+    }
+    else
+    {
+        Owned<IDirectoryIterator> dir = createDirectoryIterator(absolutePath, dirTail);
+        ForEach(*dir)
+        {
+            const char *name = dir->query().queryFilename();
+            if (name && *name && *name != '.')
+                installFileHook(name);
+        }
+    }
+}
+
+// Should be called before closedown, ideally. MODEXIT tries to mop up but may be too late to do so cleanly
+
+extern REMOTE_API void removeFileHooks()
+{
+    if (hookDlls)
+    {
+        ForEachItemIn(idx, *hookDlls)
+        {
+            SharedObject *so = hookDlls->item(idx);
+            HookInstallFunction *hookInstall = (HookInstallFunction *) GetSharedProcedure(so->getInstanceHandle(), "removeFileHook");
+            if (hookInstall)
+                hookInstall();
+            delete so;
+        }
+        delete hookDlls;
+        hookDlls = NULL;
+    }
+}
 
 MODULE_INIT(INIT_PRIORITY_REMOTE_RMTFILE)
 {
@@ -627,5 +715,6 @@ MODULE_EXIT()
         ::Release(DaliServixIntercept);
         DaliServixIntercept = NULL;
     }
+    removeFileHooks();
 }
 

common/remote/rmtfile.hpp

@@ -94,5 +94,7 @@ extern REMOTE_API void setRemoteFileTimeouts(unsigned maxconnecttime,unsigned ma
                                 
 extern REMOTE_API unsigned validateNodes(const SocketEndpointArray &eps,const char *dataDir, const char *mirrorDir, bool chkver, const char *script, unsigned scripttimeout, SocketEndpointArray &failures, UnsignedArray &failedcodes, StringArray &failedmessages, const char *filename=NULL);
 
+extern REMOTE_API void installFileHooks(const char *filespec);
+extern REMOTE_API void removeFileHooks(); // Should be called before closedown
 
 #endif

ecl/eclcc/eclcc.cpp

@@ -35,13 +35,12 @@
 #include "hqlcollect.hpp"
 #include "hqlrepository.hpp"
 #include "hqlerror.hpp"
-#include "gitfile.hpp"
-#include "archive.hpp"
 
 #include "hqlgram.hpp"
 #include "hqltrans.ipp"
 
 #include "build-config.h"
+#include "rmtfile.hpp"
 
 #define INIFILE "eclcc.ini"
 #define SYSTEMCONFDIR CONFIG_DIR
@@ -231,6 +230,7 @@ protected:
     const char * programName;
 
     StringBuffer pluginsPath;
+    StringBuffer hooksPath;
     StringBuffer templatePath;
     StringBuffer eclLibraryPath;
     StringBuffer stdIncludeLibraryPath;
@@ -245,6 +245,7 @@ protected:
     FILE * batchLog;
 
     IFileArray inputFiles;
+    StringArray inputFileNames;
     StringArray debugOptions;
     StringArray compileOptions;
     StringArray linkOptions;
@@ -275,8 +276,6 @@ protected:
 
 static int doMain(int argc, const char *argv[])
 {
-    installGitFileHook();
-    installArchiveFileHook();
     EclCC processor(argv[0]);
     if (!processor.parseCommandLineOptions(argc, argv))
         return 1;
@@ -314,6 +313,7 @@ int main(int argc, const char *argv[])
 
     unsigned exitCode = doMain(argc, argv);
     releaseAtoms();
+    removeFileHooks();
     return exitCode;
 }
 
@@ -397,6 +397,7 @@ void EclCC::loadOptions()
     extractOption(libraryPath, globals, "ECLCC_LIBRARY_PATH", "libraryPath", syspath, ".\\cl\\lib");
     extractOption(includePath, globals, "ECLCC_INCLUDE_PATH", "includePath", syspath, ".\\cl\\include");
     extractOption(pluginsPath, globals, "ECLCC_PLUGIN_PATH", "plugins", syspath, ".\\plugins");
+    extractOption(hooksPath, globals, "HPCC_FILEHOOKS_PATH", "filehooks", syspath, ".\\filehooks");
     extractOption(templatePath, globals, "ECLCC_TPL_PATH", "templatePath", syspath, ".");
     extractOption(eclLibraryPath, globals, "ECLCC_ECLLIBRARY_PATH", "eclLibrariesPath", syspath, ".\\ecllibrary");
 #else
@@ -412,6 +413,7 @@ void EclCC::loadOptions()
     extractOption(libraryPath, globals, "ECLCC_LIBRARY_PATH", "libraryPath", syspath, "lib");
     extractOption(includePath, globals, "ECLCC_INCLUDE_PATH", "includePath", syspath, "componentfiles/cl/include");
     extractOption(pluginsPath, globals, "ECLCC_PLUGIN_PATH", "plugins", syspath, "plugins");
+    extractOption(hooksPath, globals, "HPCC_FILEHOOKS_PATH", "filehooks", syspath, "filehooks");
     extractOption(templatePath, globals, "ECLCC_TPL_PATH", "templatePath", syspath, "componentfiles");
     extractOption(eclLibraryPath, globals, "ECLCC_ECLLIBRARY_PATH", "eclLibrariesPath", syspath, "share/ecllibrary/");
 
@@ -432,6 +434,8 @@ void EclCC::loadOptions()
                 fprintf(stdout, "Logging to '%s'\n",lf.str());
         }
     }
+    if (hooksPath.length())
+        installFileHooks(hooksPath.str());
 
     if (!optNoCompile)
         setCompilerPath(compilerPath.str(), includePath.str(), libraryPath.str(), NULL, optTargetCompiler, logVerbose);
@@ -1181,6 +1185,18 @@ void EclCC::processReference(EclCompileInstance & instance, const char * queryAt
 bool EclCC::processFiles()
 {
     loadOptions();
+    ForEachItemIn(idx, inputFileNames)
+    {
+        processArgvFilename(inputFiles, inputFileNames.item(idx));
+    }
+    if (inputFiles.ordinality() == 0)
+    {
+        if (!optBatchMode && optQueryRepositoryReference)
+            return true;
+        ERRLOG("No input files could be opened");
+        return false;
+    }
+
 
     StringBuffer searchPath;
     searchPath.append(stdIncludeLibraryPath).append(ENVSEPCHAR).append(includeLibraryPath);
@@ -1421,7 +1437,7 @@ bool EclCC::parseCommandLineOptions(int argc, const char* argv[])
         }
         else if (strcmp(arg, "-")==0) 
         {
-            processArgvFilename(inputFiles, "stdin:");
+            inputFileNames.append("stdin:");
         }
         else if (arg[0] == '-')
         {
@@ -1430,7 +1446,7 @@ bool EclCC::parseCommandLineOptions(int argc, const char* argv[])
             return false;
         }
         else
-            processArgvFilename(inputFiles, arg);
+            inputFileNames.append(arg);
     }
     if (showHelp)
     {
@@ -1444,11 +1460,11 @@ bool EclCC::parseCommandLineOptions(int argc, const char* argv[])
 
     loadManifestOptions();
 
-    if (inputFiles.ordinality() == 0)
+    if (inputFileNames.ordinality() == 0)
     {
         if (!optBatchMode && optQueryRepositoryReference)
             return true;
-        ERRLOG("No input files supplied");
+        ERRLOG("No input filenames supplied");
         return false;
     }