Merge branch 'candidate-5.2.0'

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

cmake_modules/commonSetup.cmake

@@ -38,13 +38,19 @@ IF ("${COMMONSETUP_DONE}" STREQUAL "")
     then create a separate build directory and run 'cmake path_to_source [options]' there.")
 
   cmake_policy ( SET CMP0011 NEW )
-
+  if (NOT (CMAKE_MAJOR_VERSION LESS 3))
+    cmake_policy ( SET CMP0026 OLD )
+  endif()
   option(CLIENTTOOLS "Enable the building/inclusion of a Client Tools component." ON)
   option(PLATFORM "Enable the building/inclusion of a Platform component." ON)
   option(DEVEL "Enable the building/inclusion of a Development component." OFF)
   option(CLIENTTOOLS_ONLY "Enable the building of Client Tools only." OFF)
 
-  option(USE_BINUTILS "Enable use of binutils to embed workunit info into shared objects" ON)
+  if (APPLE OR WIN32)
+    option(USE_BINUTILS "Enable use of binutils to embed workunit info into shared objects" OFF)
+  else()
+    option(USE_BINUTILS "Enable use of binutils to embed workunit info into shared objects" ON)
+  endif()
   option(USE_CPPUNIT "Enable unit tests (requires cppunit)" OFF)
   option(USE_OPENLDAP "Enable OpenLDAP support (requires OpenLDAP)" ON)
   option(USE_ICU "Enable unicode support (requires ICU)" ON)
@@ -490,7 +496,7 @@ IF ("${COMMONSETUP_DONE}" STREQUAL "")
   ENDIF(MAKE_DOCS)
 
   IF ( NOT MAKE_DOCS_ONLY )
-      IF (USE_BINUTILS AND NOT WIN32)
+      IF (USE_BINUTILS AND NOT WIN32 AND NOT APPLE)
         find_package(BINUTILS)
         IF (BINUTILS_FOUND)
           add_definitions (-D_USE_BINUTILS)

common/dllserver/thorplugin.cpp

@@ -25,6 +25,10 @@
 #include "eclrtl.hpp"
 #ifdef _USE_BINUTILS
 #include "bfd.h"
+#elif defined(__APPLE__)
+#include <mach-o/getsect.h>
+#include <sys/mman.h>
+#include <sys/stat.h>
 #endif
 
 #include "thorplugin.hpp"
@@ -195,7 +199,10 @@ bool HelperDll::getResource(size32_t & len, const void * & data, const char * ty
     symName.append(type).append("_").append(id).append("_txt_start");
     data = (const void *) getEntry(symName.str());
     if (!data)
+    {
+        printf("Failed to locate symbol %s", symName.str());
         return false;
+    }
     byte bom;
     byte version;
     bool compressed;
@@ -269,8 +276,41 @@ extern bool getResourceFromFile(const char *filename, MemoryBuffer &data, const
         bfd_close (file);
    }
    return data.length() != 0;
+#elif defined(__APPLE__)
+    unsigned long len = 0;
+    struct stat stat_buf;
+    VStringBuffer sectname("%s_%u", type, id);
+    int fd = open(filename, O_RDONLY);
+    if (fd == -1 || fstat(fd, &stat_buf) == -1)
+    {
+        DBGLOG("Failed to load library %s: %d", filename, errno);
+        return false;
+    }
+    __uint64 size = stat_buf.st_size;
+    byte *start_addr = (byte *) mmap(0, size, PROT_READ, MAP_FILE | MAP_PRIVATE, fd, 0);
+    if (start_addr == MAP_FAILED)
+    {
+        DBGLOG("Failed to load library %s: %d", filename, errno);
+    }
+    else
+    {
+        // The first bytes are the Mach-O header
+        struct mach_header_64 *mh = (struct mach_header_64 *) start_addr;
+        if (mh->magic != MH_MAGIC_64)
+        {
+            DBGLOG("Failed to load library %s: Does not appear to be a Mach-O 64-bit binary", filename);
+        }
+        else
+        {
+            unsigned char *data2 = getsectiondata(mh, "__TEXT", sectname.str(), &len);
+            data.append(len, data2);
+        }
+        munmap(start_addr, size);
+    }
+    close(fd);
+    return len != 0;
 #else
-   UNIMPLEMENTED;
+    UNIMPLEMENTED;
 #endif
 }
 

ecl/eclcc/eclcc.cpp

@@ -772,6 +772,9 @@ void EclCC::instantECL(EclCompileInstance & instance, IWorkUnit *wu, const char
                 if (generateOk && !optNoCompile)
                 {
                     Owned<ICppCompiler> compiler = createCompiler(processName.str());
+                    StringBuffer mainName(processName);
+                    mainName.append(".cpp");
+                    compiler->addSourceFile(mainName);
                     compiler->setSaveTemps(optSaveTemps);
 
                     bool compileOk = true;
@@ -1704,8 +1707,9 @@ bool EclCC::generatePrecompiledHeader()
         ERRLOG("Cannot find eclinclude4.hpp");
         return false;
     }
-    Owned<ICppCompiler> compiler = createCompiler("eclinclude4.hpp", foundPath, NULL);
+    Owned<ICppCompiler> compiler = createCompiler("precompile", foundPath, NULL);
     compiler->setDebug(true);  // a precompiled header with debug can be used for no-debug, but not vice versa
+    compiler->addSourceFile("eclinclude4.hpp");
     compiler->setPrecompileHeader(true);
     if (compiler->compile())
     {

ecl/hqlcpp/hqlcpp.cpp

@@ -1135,6 +1135,20 @@ const char * HqlCppInstance::queryLibrary(unsigned idx)
     return NULL;
 }
 
+const char * HqlCppInstance::queryObjectFile(unsigned idx)
+{
+    if (objectFiles.isItem(idx))
+        return objectFiles.item(idx).text;
+    return NULL;
+}
+
+const char * HqlCppInstance::querySourceFile(unsigned idx)
+{
+    if (sourceFiles.isItem(idx))
+        return sourceFiles.item(idx).text;
+    return NULL;
+}
+
 HqlStmts * HqlCppInstance::querySection(IAtom * section)
 {
     ForEachItemIn(idx, sections)
@@ -1240,6 +1254,16 @@ void HqlCppInstance::useLibrary(const char * libname)
     insertUniqueString(modules, libname);
 }
 
+void HqlCppInstance::useObjectFile(const char * objname)
+{
+    insertUniqueString(objectFiles, objname);
+}
+
+void HqlCppInstance::useSourceFile(const char * srcname)
+{
+    insertUniqueString(sourceFiles, srcname);
+}
+
 void HqlCppInstance::addHint(const char * hintXml, ICodegenContextCallback * ctxCallback)
 {
     if (!hintFile)
@@ -1307,27 +1331,24 @@ void HqlCppInstance::flushResources(const char *filename, ICodegenContextCallbac
     {
         bool flushText = workunit->getDebugValueBool("flushResourceAsText", false);
 
-        StringBuffer path, trailing;
-        splitFilename(filename, &path, &path, &trailing, &trailing);
-
-        StringBuffer ln;
-        ln.append(path).append(SharedObjectPrefix).append(trailing).append(LibraryExtension);
 #ifdef __64BIT__
-        // ARMFIX: Map all the uses of this property and make sure
-        // they're not used to mean x86_64 (it shouldn't, though)
         bool target64bit = workunit->getDebugValueBool("target64bit", true);
 #else
         bool target64bit = workunit->getDebugValueBool("target64bit", false);
 #endif
-        resources.flush(ln.str(), flushText, target64bit);
+        StringBuffer resname;
+        bool isObjectFile = resources.flush(resname, filename, flushText, target64bit);
 
         StringBuffer resTextName;
-        if (flushText && resources.queryWriteText(resTextName, ln))
+        if (flushText && resources.queryWriteText(resTextName, resname))
         {
             Owned<IWUQuery> query = workunit->updateQuery();
             associateLocalFile(query, FileTypeHintXml, resTextName, "Workunit resource text", 0);
         }
-        useLibrary(filename);
+        if (isObjectFile)
+            useObjectFile(resname);
+        else
+            useSourceFile(resname);
     }
 }
 

ecl/hqlcpp/hqlcpp.hpp

@@ -113,6 +113,8 @@ interface HQLCPP_API IHqlCppInstance : public IInterface
 public:
     virtual HqlStmts * ensureSection(IAtom * section) = 0;
     virtual const char * queryLibrary(unsigned idx) = 0;
+    virtual const char * queryObjectFile(unsigned idx) = 0;
+    virtual const char * querySourceFile(unsigned idx) = 0;
     virtual HqlStmts * querySection(IAtom * section) = 0;
     virtual void addResource(const char * type, unsigned len, const void * data, IPropertyTree *manifestEntry, unsigned id=(unsigned)-1) = 0;
     virtual void addCompressResource(const char * type, unsigned len, const void * data, IPropertyTree *manifestEntry, unsigned id=(unsigned)-1) = 0;

ecl/hqlcpp/hqlcpp.ipp

@@ -129,6 +129,8 @@ public:
 
     virtual HqlStmts * ensureSection(IAtom * section);
     virtual const char * queryLibrary(unsigned idx);
+    virtual const char * queryObjectFile(unsigned idx);
+    virtual const char * querySourceFile(unsigned idx);
     virtual HqlStmts * querySection(IAtom * section);
     virtual void flushHints();
     virtual void flushResources(const char *filename, ICodegenContextCallback * ctxCallback);
@@ -141,6 +143,8 @@ public:
     bool useFunction(IHqlExpression * funcdef);
     void useInclude(const char * include);
     void useLibrary(const char * libname);
+    void useObjectFile(const char * objname);
+    void useSourceFile(const char * srcname);
     unsigned addStringResource(unsigned len, const char * body);
     void addHint(const char * hintXml, ICodegenContextCallback * ctxCallback);
 
@@ -155,6 +159,8 @@ public:
     CIArray             sections;
     IArray               helpers;
     StringAttrArray     modules;
+    StringAttrArray     objectFiles;
+    StringAttrArray     sourceFiles;
     StringAttrArray     includes;
     CIArray             extra;
     ResourceManager     resources;

ecl/hqlcpp/hqlecl.cpp

@@ -115,6 +115,7 @@ protected:
     unsigned defaultMaxCompileThreads;
     StringArray sourceFiles;
     StringArray libraries;
+    StringArray objects;
     offset_t totalGeneratedSize;
     bool checkForLocalFileUploads;
     bool noOutput;
@@ -151,11 +152,30 @@ void HqlDllGenerator::addLibrariesToCompiler()
         const char * lib = code->queryLibrary(idx);
         if (!lib)
             break;
-
         PrintLog("Adding library: %s", lib);
         addLibrary(lib);
         idx++;
     }
+    idx=0;
+    loop
+    {
+        const char * obj = code->queryObjectFile(idx);
+        if (!obj)
+            break;
+        PrintLog("Adding object file: %s", obj);
+        objects.append(obj);
+        idx++;
+    }
+    idx=0;
+    loop
+    {
+        const char * src = code->querySourceFile(idx);
+        if (!src)
+            break;
+        PrintLog("Adding source file: %s", src);
+        sourceFiles.append(src);
+        idx++;
+    }
 }
 
 
@@ -490,7 +510,7 @@ void HqlDllGenerator::doExpand(HqlCppTranslator & translator)
             expandCode(CHILD_MODULE_TEMPLATE, fullext, code, true, i+1, translator.queryOptions().targetCompiler);
 
             StringBuffer fullname;
-            fullname.append(wuname).append("_").append(i+1);
+            fullname.append(wuname).append("_").append(i+1).append(".cpp");
             sourceFiles.append(fullname);
         }
     }
@@ -535,6 +555,8 @@ bool HqlDllGenerator::doCompile(ICppCompiler * compiler)
 
     ForEachItemIn(idx, libraries)
         compiler->addLibrary(libraries.item(idx));
+    ForEachItemIn(idx2, objects)
+        compiler->addObjectFile(objects.item(idx2));
 
     StringBuffer options;
     StringBufferAdaptor linkOptionAdaptor(options);
@@ -577,8 +599,7 @@ bool HqlDllGenerator::doCompile(ICppCompiler * compiler)
         remove(temp.clear().append(wuname).append(".hpp").str());
         ForEachItemIn(i, sourceFiles)
         {
-            temp.clear().append(sourceFiles.item(i)).append(".cpp");
-            remove(temp.str());
+            remove(sourceFiles.item(i));
         }
     }
     return ok;
@@ -586,14 +607,8 @@ bool HqlDllGenerator::doCompile(ICppCompiler * compiler)
 
 void HqlDllGenerator::flushResources()
 {
-    StringBuffer resname(wuname);
-#ifdef _WIN32
-    resname.append(".res");
-#else
-    resname.append(".res.o");
-#endif
     if (code)
-        code->flushResources(resname.str(), ctxCallback);
+        code->flushResources(wuname, ctxCallback);
 }
 
 

ecl/hqlcpp/hqlres.cpp

@@ -407,7 +407,7 @@ void ResourceManager::flushAsText(const char *filename)
     fclose(f);
 }
 
-void ResourceManager::flush(const char *filename, bool flushText, bool target64bit)
+bool ResourceManager::flush(StringBuffer &filename, const char *basename, bool flushText, bool target64bit)
 {
     finalize();
 
@@ -415,12 +415,15 @@ void ResourceManager::flush(const char *filename, bool flushText, bool target64b
     // or that we want to access without having to run the dll/so
     // In linux there is no .res concept but we can achieve the same effect by generating an object file with a specially-named section 
     // bintils tools can be used to extract the data externally (internally we just have a named symbol for it)
+    // Alternatively we can generate an assembler file to create the equivalent object file, if binutils is not available
+    bool isObjectFile = true;
 #ifdef _WIN32
+    filename.append(basename).append(".res");
     int h = _open(filename, _O_WRONLY|_O_CREAT|_O_TRUNC|_O_BINARY|_O_SEQUENTIAL, _S_IREAD | _S_IWRITE | _S_IEXEC);
     
     //assertex(h != HFILE_ERROR);
     if (h == HFILE_ERROR) // error can not be ignored!
-        throwError1(HQLERR_ResourceCreateFailed, filename);
+        throwError1(HQLERR_ResourceCreateFailed, filename.str());
 
     totalbytes = 0;
     putbytes(h, "\x00\x00\x00\x00\x20\x00\x00\x00\xff\xff\x00\x00\xff\xff\x00\x00"
@@ -458,6 +461,7 @@ void ResourceManager::flush(const char *filename, bool flushText, bool target64b
     }
     _close(h);
 #elif defined(_USE_BINUTILS)
+    filename.append(basename).append(".res.o");
     asymbol **syms = NULL;
     bfd *file = NULL;
     StringArray names;  // need to make sure that the strings we use in symbol table have appropriate lifetime 
@@ -522,17 +526,42 @@ void ResourceManager::flush(const char *filename, bool flushText, bool target64b
         E->Release();
         //translate the assert exceptions into something else...
         StringBuffer msg;
-        msg.appendf("%s: %s", filename, bfd_errmsg(bfd_get_error()));
+        msg.appendf("%s: %s", filename.str(), bfd_errmsg(bfd_get_error()));
         delete syms;
         if (file)
             bfd_close_all_done(file); // allow bfd to clean up memory
         throwError1(HQLERR_ResourceCreateFailed, msg.str());
     }
 #else
-    UNIMPLEMENTED;
+    isObjectFile = false;
+    filename.append(basename).append(".res.s");
+    FILE *f = fopen(filename, "wt");
+    if (!f)
+        throwError1(HQLERR_ResourceCreateFailed, filename.str());
+    ForEachItemIn(idx, resources)
+    {
+        ResourceItem &s = (ResourceItem &) resources.item(idx);
+        const char *type = s.type.str();
+        unsigned id = s.id;
+        VStringBuffer binfile("%s_%s_%u.bin", filename.str(), type, id);
+        fprintf(f, " .section __TEXT,%s_%u\n", type, id);
+        fprintf(f, " .global _%s_%u_txt_start\n", type, id);  // For some reason apple needs a leading underbar and linux does not
+        fprintf(f, "_%s_%u_txt_start:\n", type, id);
+        fprintf(f, " .incbin \"%s\"\n", binfile.str());
+        FILE *bin = fopen(binfile, "wb");
+        if (!bin)
+        {
+            fclose(f);
+            throwError1(HQLERR_ResourceCreateFailed, binfile.str());
+        }
+        fwrite(s.data.get(), 1, s.data.length(), bin);
+        fclose(bin);
+    }
+    fclose(f);
 #endif
     if (flushText)
         flushAsText(filename);
+    return isObjectFile;
 }
 
 

ecl/hqlcpp/hqlres.hpp

@@ -39,7 +39,7 @@ public:
     void finalize();
 
     unsigned count();
-    void flush(const char *filename, bool flushText, bool target64bit);
+    bool flush(StringBuffer &filename, const char *basename, bool flushText, bool target64bit);
     void flushAsText(const char *filename);
     bool queryWriteText(StringBuffer & resTextName, const char * filename);
 private:

plugins/cassandra/CMakeLists.txt

@@ -35,14 +35,21 @@ if (USE_CASSANDRA)
     # until then, we build the required libraries from source
 
     # Build libuv, required by the cassandra driver but not available on all distros
-    add_custom_command(OUTPUT ${PROJECT_BINARY_DIR}/libuv.so
-                       COMMAND make builddir_name=${PROJECT_BINARY_DIR}
-                       COMMAND mv ${PROJECT_BINARY_DIR}/libuv.so ${PROJECT_BINARY_DIR}/libuv.so.0.10
-                       COMMAND ln -s ${PROJECT_BINARY_DIR}/libuv.so.0.10 ${PROJECT_BINARY_DIR}/libuv.so
-                       WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/libuv)
-    add_custom_target ( libuv ALL DEPENDS ${PROJECT_BINARY_DIR}/libuv.so )
-
-    set(LIBUV_LIBRARY ${PROJECT_BINARY_DIR}/libuv.so)
+    if (APPLE)
+      add_custom_command(OUTPUT ${PROJECT_BINARY_DIR}/libuv.dylib
+                         COMMAND make builddir_name=${PROJECT_BINARY_DIR}
+                         WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/libuv)
+      add_custom_target ( libuv ALL DEPENDS ${PROJECT_BINARY_DIR}/libuv.dylib )
+      set(LIBUV_LIBRARY ${PROJECT_BINARY_DIR}/libuv.dylib)
+    else()
+      add_custom_command(OUTPUT ${PROJECT_BINARY_DIR}/libuv.so
+                         COMMAND make builddir_name=${PROJECT_BINARY_DIR}
+                         COMMAND mv ${PROJECT_BINARY_DIR}/libuv.so ${PROJECT_BINARY_DIR}/libuv.so.0.10
+                         COMMAND ln -s ${PROJECT_BINARY_DIR}/libuv.so.0.10 ${PROJECT_BINARY_DIR}/libuv.so
+                         WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/libuv)
+      add_custom_target ( libuv ALL DEPENDS ${PROJECT_BINARY_DIR}/libuv.so )
+      set(LIBUV_LIBRARY ${PROJECT_BINARY_DIR}/libuv.so)
+    endif()
     set(LIBUV_INCLUDE_DIR ${PROJECT_SOURCE_DIR}/libuv/include/)
 
     # Build the Cassandra cpp driver, only presently available as source
@@ -57,12 +64,11 @@ if (USE_CASSANDRA)
     option(CASS_BUILD_STATIC "Build static library" OFF)
     option(CASS_BUILD_EXAMPLES "Build examples" OFF)
     set_property(GLOBAL PROPERTY FIND_LIBRARY_USE_LIB64_PATHS FALSE)
-    SET (_SAVE_Boost_FOUND "${Boost_FOUND}")  # Working around a bug in the cassandra cmake file
-    UNSET(Boost_FOUND)
+    SET (_SAVE_CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
+    SET (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-format-nonliteral") # Work around cassandra build error
     add_subdirectory (cpp-driver ${PROJECT_BINARY_DIR}/cassandra)
     add_dependencies( cassandra libuv )
-
-    SET (Boost_FOUND "${_SAVE_Boost_FOUND}")
+    SET (CMAKE_CXX_FLAGS "${_SAVE_CMAKE_CXX_FLAGS}")
 
     set(SRCS cassandraembed.cpp)
 
@@ -97,7 +103,11 @@ if (USE_CASSANDRA)
     # include them in our rpm
     # Note that the cassandra driver CMake file already includes the relevant install commands
 
-    install ( FILES ${PROJECT_BINARY_DIR}/libuv.so.0.10 DESTINATION ${LIB_DIR} COMPONENT Runtime )
+    if (APPLE)
+      install ( FILES ${PROJECT_BINARY_DIR}/libuv.dylib DESTINATION ${LIB_DIR} COMPONENT Runtime )
+    else()
+      install ( FILES ${PROJECT_BINARY_DIR}/libuv.so.0.10 DESTINATION ${LIB_DIR} COMPONENT Runtime )
+    endif()
 
     target_link_libraries ( cassandraembed
         cassandra

system/jlib/jcomp.cpp

@@ -82,17 +82,24 @@ static const char * COMPILE_ONLY[] = { "/c", "-c" };
 
 static const char * CC_OPTION_CORE[] = { "", "-fvisibility=hidden -DUSE_VISIBILITY=1" };
 static const char * LINK_OPTION_CORE[] = { "/DLL /libpath:." , "" };
-static const char * CC_OPTION_DEBUG[] = { "/Zm500 /EHsc /GR /Zi /nologo /bigobj", "-g -fPIC -pipe -O0" };
+static const char * CC_OPTION_DEBUG[] = { "/Zm500 /EHsc /GR /Zi /nologo /bigobj", "-g -fPIC  -O0" };
 
-static const char * DLL_LINK_OPTION_DEBUG[] = { "/BASE:" BASE_ADDRESS " /NOLOGO /LARGEADDRESSAWARE /INCREMENTAL:NO /DEBUG /DEBUGTYPE:CV", "-g -shared -L. -fPIC -pipe -O0" };
-static const char * EXE_LINK_OPTION_DEBUG[] = { "/BASE:" BASE_ADDRESS " /NOLOGO /LARGEADDRESSAWARE /INCREMENTAL:NO /DEBUG /DEBUGTYPE:CV", "-g -L. -Wl,-E -fPIC -pipe -O0" };
 
-static const char * CC_OPTION_RELEASE[] = { "/Zm500 /EHsc /GR /Oi /Ob1 /GF /nologo /bigobj", "-fPIC -pipe -O0" };
+static const char * CC_OPTION_RELEASE[] = { "/Zm500 /EHsc /GR /Oi /Ob1 /GF /nologo /bigobj", "-fPIC  -O0" };
 
 static const char * CC_OPTION_PRECOMPILEHEADER[] = { "", " -x c++-header" };
 
+#ifdef __APPLE__
+static const char * DLL_LINK_OPTION_DEBUG[] = { "/BASE:" BASE_ADDRESS " /NOLOGO /LARGEADDRESSAWARE /INCREMENTAL:NO /DEBUG /DEBUGTYPE:CV", "-g -shared -L. -fPIC -pipe -O0" };
+static const char * EXE_LINK_OPTION_DEBUG[] = { "/BASE:" BASE_ADDRESS " /NOLOGO /LARGEADDRESSAWARE /INCREMENTAL:NO /DEBUG /DEBUGTYPE:CV", "-g -L. -fPIC -pipe -O0 -Wl,-export_dynamic -v" };
+static const char * DLL_LINK_OPTION_RELEASE[] = { "/BASE:" BASE_ADDRESS " /NOLOGO /LARGEADDRESSAWARE /INCREMENTAL:NO", "-shared -L. -fPIC -pipe -O0" };
+static const char * EXE_LINK_OPTION_RELEASE[] = { "/BASE:" BASE_ADDRESS " /NOLOGO /LARGEADDRESSAWARE /INCREMENTAL:NO", "-L. -fPIC -pipe -O0 -Wl,-export_dynamic -v" };
+#else
+static const char * DLL_LINK_OPTION_DEBUG[] = { "/BASE:" BASE_ADDRESS " /NOLOGO /LARGEADDRESSAWARE /INCREMENTAL:NO /DEBUG /DEBUGTYPE:CV", "-g -shared -L. -fPIC -pipe -O0" };
+static const char * EXE_LINK_OPTION_DEBUG[] = { "/BASE:" BASE_ADDRESS " /NOLOGO /LARGEADDRESSAWARE /INCREMENTAL:NO /DEBUG /DEBUGTYPE:CV", "-g -L. -Wl,-E -fPIC -pipe -O0" };
 static const char * DLL_LINK_OPTION_RELEASE[] = { "/BASE:" BASE_ADDRESS " /NOLOGO /LARGEADDRESSAWARE /INCREMENTAL:NO", "-shared -L. -fPIC -pipe -O0" };
 static const char * EXE_LINK_OPTION_RELEASE[] = { "/BASE:" BASE_ADDRESS " /NOLOGO /LARGEADDRESSAWARE /INCREMENTAL:NO", "-L. -Wl,-E -fPIC -pipe -O0" };
+#endif
 
 static const char * LINK_TARGET[] = { " /out:", " -o " };
 static const char * DEFAULT_CC_LOCATION[] = { ".", "." };
@@ -239,9 +246,7 @@ static void setDirectoryPrefix(StringAttr & target, const char * source)
 
 CppCompiler::CppCompiler(const char * _coreName, const char * _sourceDir, const char * _targetDir, unsigned _targetCompiler, bool _verbose)
 {
-#define CORE_NAME allSources.item(0)
-    if (_coreName)
-        addSourceFile(_coreName);
+    coreName.set(_coreName);
     targetCompiler = _targetCompiler;
     createDLL = true;
 #ifdef _DEBUG
@@ -350,9 +355,15 @@ void CppCompiler::addLinkOption(const char * option)
 
 void CppCompiler::addSourceFile(const char * filename)
 {
+    DBGLOG("addSourceFile %s", filename);
     allSources.append(filename);
 }
 
+void CppCompiler::addObjectFile(const char * filename)
+{
+    linkerLibraries.append(" ").append(filename);
+}
+
 void CppCompiler::writeLogFile(const char* filepath, StringBuffer& log)
 {
     if(!filepath || !*filepath || !log.length())
@@ -467,8 +478,6 @@ bool CppCompiler::compileFile(IThreadPool * pool, const char * filename, Semapho
         addPathSepChar(cmdline);
     }
     cmdline.append(filename);
-    if (!precompileHeader)
-        cmdline.append(".cpp");
     cmdline.append("\" ");
     expandCompileOptions(cmdline);
 
@@ -655,7 +664,7 @@ bool CppCompiler::doLink()
     cmdline.append(linkerLibraries);
 
     StringBuffer outName;
-    outName.append(createDLL ? SharedObjectPrefix : NULL).append(CORE_NAME).append(createDLL ? SharedObjectExtension : ProcessExtension);
+    outName.append(createDLL ? SharedObjectPrefix : NULL).append(coreName).append(createDLL ? SharedObjectExtension : ProcessExtension);
     cmdline.append(LINK_TARGET[targetCompiler]).append("\"").append(targetDir).append(outName).append("\"");
 
     StringBuffer temp;
@@ -667,7 +676,7 @@ bool CppCompiler::doLink()
     DWORD runcode = 0;
     if (verbose)
         PrintLog("%s", expanded.str());
-    StringBuffer logFile = StringBuffer(CORE_NAME).append("_link.log.tmp");
+    StringBuffer logFile = StringBuffer(coreName).append("_link.log.tmp");
     logFiles.append(logFile);
     bool ret = invoke_program(expanded.str(), runcode, true, logFile) && (runcode == 0);
     linkFailed = !ret;
@@ -703,20 +712,28 @@ StringBuffer & CppCompiler::getObjectName(StringBuffer & out, const char * filen
 
 void CppCompiler::removeTemporaries()
 {
+    DBGLOG("Remove temporaries");
     switch (targetCompiler)
     {
     case Vs6CppCompiler:
     case GccCppCompiler:
         {
             StringBuffer temp;
-            remove(temp.clear().append(targetDir).append(CORE_NAME).append(".exp").str());
-            remove(getObjectName(temp.clear(), CORE_NAME).str());
-            remove(temp.clear().append(targetDir).append(CORE_NAME).append(".lib").str());
+            remove(temp.clear().append(targetDir).append(coreName).append(".exp").str());
+            remove(getObjectName(temp.clear(), coreName).str());
+            remove(temp.clear().append(targetDir).append(coreName).append(".lib").str());
 #ifdef _WIN32
-            remove(temp.clear().append(targetDir).append(CORE_NAME).append(".res.lib").str());
+            remove(temp.clear().append(targetDir).append(coreName).append(".res").str());
+#elif defined (_USE_BINUTILS)
+            remove(temp.clear().append(targetDir).append(coreName).append(".res.o").str());
 #else
-            remove(temp.clear().append(targetDir).append(CORE_NAME).append(".res.o.lib").str());
-            remove(temp.clear().append(targetDir).append("lib").append(CORE_NAME).append(".res.o.so").str());
+            temp.clear().append(coreName).append(".res.s*");
+            DBGLOG("Remove %s%s",targetDir.str(), temp.str());
+            Owned<IDirectoryIterator> resTemps = createDirectoryIterator(targetDir, temp.str());
+            ForEach(*resTemps)
+            {
+                remove(resTemps->getName(temp.clear().append(targetDir)).str());
+            }
 #endif
             break;
         }

system/jlib/jcomp.hpp

@@ -51,6 +51,7 @@ public:
     virtual void addInclude(const char * includePath) = 0;
     virtual void addLinkOption(const char * option) = 0;
     virtual void addSourceFile(const char * filename) = 0;
+    virtual void addObjectFile(const char * filename) = 0;
     virtual bool compile() = 0;
     virtual void extractErrors(IArrayOf<IError> & errors) = 0;
     virtual void setDebug(bool _debug) = 0;

system/jlib/jcomp.ipp

@@ -35,6 +35,7 @@ public:
     virtual void addLinkOption(const char * option);
     virtual void addInclude(const char * includePath);
     virtual void addSourceFile(const char * filename);
+    virtual void addObjectFile(const char * filename);
     virtual bool compile();
     virtual void extractErrors(IArrayOf<IError> & errors);
     virtual void setDebug(bool _debug);
@@ -71,6 +72,7 @@ protected:
     StringArray     allSources;
     StringArray     logFiles;
     StringAttr      ccLogPath;
+    StringAttr      coreName;
     unsigned        targetCompiler;
     unsigned        maxCompileThreads;
     bool            onlyCompile;

system/jlib/jdebug.cpp

@@ -47,7 +47,10 @@
 #include <sys/klog.h>
 #include <dirent.h>
 #endif
-
+#ifdef __APPLE__
+ #include <sys/param.h>
+ #include <sys/mount.h>
+#endif
 
 //===========================================================================
 #ifdef _DEBUG
@@ -1122,7 +1125,7 @@ void getMemStats(StringBuffer &out, unsigned &memused, unsigned &memtot)
 
 void getDiskUsage(char const * path, unsigned __int64 & total, unsigned __int64 & inUse)
 {
-#ifdef __linux__
+#if defined(__linux__) || defined(__APPLE__)
     struct statfs stfs;
     if(statfs(path, &stfs) < 0)
     {
@@ -1143,9 +1146,8 @@ void getDiskUsage(char const * path, unsigned __int64 & total, unsigned __int64
             inUse = total - (unsigned __int64)stfs.f_bfree * st.st_blksize;
         }
     }
-#endif
-#if defined (__FreeBSD__) || defined (__APPLE__)
-    UNIMPLEMENTED;
+#else
+    total = inUse = 0;
 #endif
 }
 

testing/regress/hpcc/common/error.py

@@ -49,4 +49,4 @@ class Error(Exception):
         return "Error (%s): %s " % (self.code, ERROR[self.code])
         
     def getErrorCode(self):
-        return self.code
+        return int(self.code)

testing/regress/hpcc/common/shell.py

@@ -60,6 +60,7 @@ class Shell:
             exception = CalledProcessError(
                 process.returncode, repr(args))
             exception.output = ''.join(filter(None, [stdout, stderr]))
+            logging.debug("exception.output:'%s'",  exception.output)
             raise Error('1001', err=repr(exception.output))
         return stdout
 

testing/regress/hpcc/util/ecl/cc.py

@@ -56,13 +56,15 @@ class ECLCC(Shell):
 
         if result.startswith( 'Error()'):
             retVal = False
-            ecl.diff += ecl.getEcl() + '\n  eclcc returns with:\n\t'
+            ecl.diff += ("%3d. Test: %s\n") % (ecl.getTaskId(), ecl.getBaseEclRealName())
+            ecl.diff += '  eclcc returns with:\n\t'
             try:
                 lines = repr(self.makeArchiveError).replace('\\n',  '\n\t').splitlines(True)
                 for line in lines:
-                    if  "Error" in line:
+                    lowerLine = line.lower()
+                    if  (": error " in lowerLine) or (": warning " in lowerLine):
                         ecl.diff += line.replace("'",  "")
-                    if "): error " in  line:
+                    if ("): error " in  line) or ("): warning " in lowerLine):
                         ecl.diff += line.replace("\\'", "'")
                 #ecl.diff += repr(self.makeArchiveError).replace('\\n',  '\n\t')
             except Exception as ex:

testing/regress/hpcc/util/ecl/command.py

@@ -131,7 +131,8 @@ class ECLcmd(Shell):
                     eclfile.diff = 'Error'
             else:
                 if queryWuid(eclfile.getJobname(), eclfile.getTaskId())['state'] == 'aborted':
-                    eclfile.diff = eclfile.ecl+'\n\t'+'Aborted ( reason: '+eclfile.getAbortReason()+' )'
+                    eclfile.diff = ("%3d. Test: %s\n") % (eclfile.taskId, eclfile.getBaseEclRealName())
+                    eclfile.diff += '\t'+'Aborted ( reason: '+eclfile.getAbortReason()+' )'
                     test = False
                 elif eclfile.getIgnoreResult():
                     logging.debug("%3d. Ignore result (ecl:'%s')", eclfile.getTaskId(),  eclfile.getBaseEcl())
@@ -142,7 +143,8 @@ class ECLcmd(Shell):
                         #output generation disabled with //nooutput tag
                         eclfile.diff = '-'
                     else:
-                        eclfile.diff = 'Output of '+eclfile.ecl +' test is:\n'+ data
+                        eclfile.diff = ("%3d. Test: %s\n") % (eclfile.taskId, eclfile.getBaseEclRealName())
+                        eclfile.diff += data
                     test = True
                 else:
                     test = eclfile.testResults()