Merge branch 'candidate-6.0.0'

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

.gitmodules

@@ -34,3 +34,6 @@
 [submodule "plugins/kafka/librdkafka"]
 	path = plugins/kafka/librdkafka
 	url = https://github.com/hpcc-systems/librdkafka.git
+[submodule "system/lz4_sm/lz4"]
+    path = system/lz4_sm/lz4
+    url = https://github.com/hpcc-systems/lz4.git

cmake_modules/FindMEMCACHED.cmake

@@ -41,10 +41,23 @@ IF (NOT LIBMEMCACHED_FOUND)
 
   SET (LIBMEMCACHED_LIBRARIES ${LIBMEMCACHEDCORE_LIBRARY} ${LIBMEMCACHEDUTIL_LIBRARY})
 
+  FILE (STRINGS "${LIBMEMCACHED_INCLUDE_DIR}/libmemcached-1.0/configure.h" version REGEX "#define LIBMEMCACHED_VERSION_STRING")
+  STRING(REGEX REPLACE "#define LIBMEMCACHED_VERSION_STRING " "" version "${version}")
+  STRING(REGEX REPLACE "\"" "" version "${version}")
+  SET (LIBMEMCACHED_VERSION_STRING ${version})
+  IF ("${LIBMEMCACHED_VERSION_STRING}" VERSION_EQUAL "${LIBMEMCACHED_FIND_VERSION}" OR "${LIBMEMCACHED_VERSION_STRING}" VERSION_GREATER "${LIBMEMCACHED_FIND_VERSION}")  
+    SET (LIBMEMCACHED_VERSION_OK 1)
+    SET (MSG "${DEFAULT_MSG}")
+  ELSE()
+    SET (LIBMEMCACHED_VERSION_OK 0)
+    SET(MSG "libmemcached version '${LIBMEMCACHED_VERSION_STRING}' incompatible with min version>=${LIBMEMCACHED_FIND_VERSION}")
+  ENDIF()
+      
   include(FindPackageHandleStandardArgs)
-  find_package_handle_standard_args(memcached DEFAULT_MSG
+  find_package_handle_standard_args(libmemcached ${MSG}
     LIBMEMCACHED_LIBRARIES
     LIBMEMCACHED_INCLUDE_DIR
+    LIBMEMCACHED_VERSION_OK
   )
 
   MARK_AS_ADVANCED(LIBMEMCACHED_INCLUDE_DIRS LIBMEMCACHED_LIBRARIES)

cmake_modules/commonSetup.cmake

@@ -40,13 +40,15 @@ IF ("${COMMONSETUP_DONE}" STREQUAL "")
   cmake_policy ( SET CMP0011 NEW )
   if (NOT (CMAKE_MAJOR_VERSION LESS 3))
     cmake_policy ( SET CMP0026 OLD )
-    cmake_policy ( SET CMP0054 NEW )
+    if (NOT (CMAKE_MINOR_VERSION LESS 1))
+      cmake_policy ( SET CMP0054 NEW )
+    endif()
   endif()
   option(CLIENTTOOLS "Enable the building/inclusion of a Client Tools component." ON)
   option(PLATFORM "Enable the building/inclusion of a Platform component." ON)
   option(DEVEL "Enable the building/inclusion of a Development component." OFF)
   option(CLIENTTOOLS_ONLY "Enable the building of Client Tools only." OFF)
-  option(TEST_PLUGINS "Enable the building of platform and all plugins for testing purposes" OFF)
+  option(INCLUDE_PLUGINS "Enable the building of platform and all plugins for testing purposes" OFF)
   option(PLUGIN "Enable building of a plugin" OFF)
   option(USE_SHLIBDEPS "Enable the use of dpkg-shlibdeps on ubuntu packagin" OFF)
 
@@ -59,6 +61,9 @@ IF ("${COMMONSETUP_DONE}" STREQUAL "")
   option(USE_OPENLDAP "Enable OpenLDAP support (requires OpenLDAP)" ON)
   option(USE_ICU "Enable unicode support (requires ICU)" ON)
   option(USE_BOOST_REGEX "Configure use of boost regex" ON)
+  # USE_C11_REGEX is only checked if USE_BOOST_REGEX is OFF
+  # to disable REGEX altogether turn both off
+  option(USE_C11_REGEX "Configure use of c++11 std::regex" ON)
   option(Boost_USE_STATIC_LIBS "Use boost_regex static library for RPM BUILD" OFF)
   option(USE_OPENSSL "Configure use of OpenSSL" ON)
   option(USE_ZLIB "Configure use of zlib" ON)
@@ -69,6 +74,11 @@ IF ("${COMMONSETUP_DONE}" STREQUAL "")
   endif()
   option(USE_LIBARCHIVE "Configure use of libarchive" ON)
   option(USE_URIPARSER "Configure use of uriparser" OFF)
+  if (APPLE OR WIN32)
+    option(USE_NUMA "Configure use of numa" OFF)
+  else()
+    option(USE_NUMA "Configure use of numa" ON)
+  endif()
   option(USE_NATIVE_LIBRARIES "Search standard OS locations for thirdparty libraries" ON)
   option(USE_GIT_DESCRIBE "Use git describe to generate build tag" ON)
   option(CHECK_GIT_TAG "Require git tag to match the generated build tag" OFF)
@@ -120,7 +130,7 @@ IF ("${COMMONSETUP_DONE}" STREQUAL "")
         set(PLUGIN ON)
         set(CLIENTTOOLS OFF)
         set(PLATFORM OFF)
-        set(TEST_PLUGINS OFF)
+        set(INCLUDE_PLUGINS OFF)
         set(USE_OPTIONAL OFF) # Force failure if we can't find the plugin dependencies
     endif()
 
@@ -222,7 +232,7 @@ IF ("${COMMONSETUP_DONE}" STREQUAL "")
   endif()
 
     # Leave REMBED OFF for compliance with licensing
-    if(TEST_PLUGINS)
+    if(INCLUDE_PLUGINS)
         set(V8EMBED ON)
         set(MEMCACHED ON)
         set(PYEMBED ON)
@@ -272,8 +282,9 @@ IF ("${COMMONSETUP_DONE}" STREQUAL "")
     message(FATAL_ERROR "No threading support found")
   ENDIF()
 
-  if (NOT APPLE AND NOT WIN32)
+  if (USE_NUMA)
     find_package(NUMA)
+    add_definitions (-D_USE_NUMA)
     if (NOT NUMA_FOUND)
       message(FATAL_ERROR "Support for numa not found")
     endif()
@@ -774,10 +785,22 @@ IF ("${COMMONSETUP_DONE}" STREQUAL "")
       if(USE_BOOST_REGEX)
         find_package(BOOST_REGEX)
         if (BOOST_REGEX_FOUND)
+          message(STATUS "BOOST_REGEX enabled")
           add_definitions (-D_USE_BOOST_REGEX)
         else()
           message(FATAL_ERROR "BOOST_REGEX requested but package not found")
         endif()
+      else(USE_BOOST_REGEX)
+        if (USE_C11_REGEX)
+          if ((NOT CMAKE_COMPILER_IS_GNUCC) OR (CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 4.9.0))
+            message(STATUS "C11_REGEX enabled")
+            add_definitions (-D_USE_C11_REGEX)
+          else()
+            message(STATUS "C11_REGEX requested but not supported on this platform")
+          endif()
+        else(USE_C11_REGEX)
+          message(STATUS "NO REGEX requested")
+        endif(USE_C11_REGEX)
       endif(USE_BOOST_REGEX)
 
       if(USE_OPENSSL)

cmake_modules/dependencies/jessie.cmake

@@ -0,0 +1 @@
+SET_DEPENDENCIES ( CPACK_DEBIAN_PACKAGE_DEPENDS g++ openssh-client openssh-server expect rsync libapr1 python )

cmake_modules/getpackagerevisionarch.sh

@@ -32,6 +32,10 @@ case "$ARCH" in
      ARCH="i386"
      ARCH2="i386"
      ;;
+   arm*)
+     ARCH="arm"
+     ARCH2="arm"
+     ;;
 esac
 
 
@@ -59,6 +63,13 @@ if [ -e /etc/debian_version ]; then
             OUTPUT="squeeze"
         fi
         ;;
+      8.*)
+        if [ ${NOARCH} -eq 0 ]; then
+            OUTPUT="jessie_${ARCH2}"
+        else
+            OUTPUT="jessie"
+        fi
+        ;;
       "sid")
         if [ ${NOARCH} -eq 0 ]; then
             OUTPUT="sid_${ARCH2}"

common/dllserver/thorplugin.cpp

@@ -249,6 +249,7 @@ static void secscan (bfd *file, sec_ptr sec, void *userParam)
         bfd_get_section_contents(file, sec, data, 0, size);
     }
 }
+static CriticalSection bfdCs;
 #endif
 
 static bool getResourceFromMappedFile(const char * filename, const byte * start_addr, MemoryBuffer &data, const char * type, unsigned id)
@@ -269,7 +270,7 @@ static bool getResourceFromMappedFile(const char * filename, const byte * start_
     unsigned char *data2 = getsectiondata(mh, "__TEXT", sectname.str(), &len);
     data.append(len, data2);
     return true;
-#else
+#elif defined (__64bit__)
     // The first bytes are the ELF header
     const Elf64_Ehdr * hdr = (const Elf64_Ehdr *) start_addr;
     if (memcmp(hdr->e_ident, ELFMAG, SELFMAG) != 0)
@@ -309,6 +310,46 @@ static bool getResourceFromMappedFile(const char * filename, const byte * start_
 
     DBGLOG("Failed to extract resource %s: Does not include a matching entry", filename);
     return false;
+#else
+    // The first bytes are the ELF header
+    const Elf32_Ehdr * hdr = (const Elf32_Ehdr *) start_addr;
+    if (memcmp(hdr->e_ident, ELFMAG, SELFMAG) != 0)
+    {
+        DBGLOG("Failed to extract resource %s: Does not appear to be a ELF binary", filename);
+        return false;
+    }
+    if (hdr->e_ident[EI_CLASS] != ELFCLASS32)
+    {
+        DBGLOG("Failed to extract resource %s: Does not appear to be a ELF 32-bit binary", filename);
+        return false;
+    }
+
+    //Check that there is a symbol table for the sections.
+    if (hdr->e_shstrndx == SHN_UNDEF)
+    {
+        DBGLOG("Failed to extract resource %s: Does not include a section symbol table", filename);
+        return false;
+    }
+
+    //Now walk the sections comparing the section names
+    Elf32_Half numSections = hdr->e_shnum;
+    const Elf32_Shdr * sectionHeaders = reinterpret_cast<const Elf32_Shdr *>(start_addr + hdr->e_shoff);
+    const Elf32_Shdr & symbolTableSection = sectionHeaders[hdr->e_shstrndx];
+    const char * symbolTable = (const char *)start_addr + symbolTableSection.sh_offset;
+    VStringBuffer sectname("%s_%u", type, id);
+    for (unsigned iSect= 0; iSect < numSections; iSect++)
+    {
+        const Elf32_Shdr & section = sectionHeaders[iSect];
+        const char * sectionName = symbolTable + section.sh_name;
+        if (streq(sectionName, sectname))
+        {
+            data.append(section.sh_size, start_addr + section.sh_offset);
+            return true;
+        }
+    }
+
+    DBGLOG("Failed to extract resource %s: Does not include a matching entry", filename);
+    return false;
 #endif
 }
 
@@ -332,6 +373,7 @@ extern bool getResourceFromFile(const char *filename, MemoryBuffer &data, const
     FreeLibrary(dllHandle);
     return true;
 #elif defined (_USE_BINUTILS)
+    CriticalBlock block(bfdCs);
     bfd_init ();
     bfd *file = bfd_openr(filename, NULL);
     if (file)

common/remote/sockfile.cpp

@@ -191,7 +191,7 @@ public:
 static CriticalSection              secureContextCrit;
 static Owned<ISecureSocketContext>  secureContext;
 
-#ifdef USE_OPENSSL
+#ifdef _USE_OPENSSL
 static ISecureSocket *createSecureSocket(ISocket *sock,SecureSocketType type)
 {
     {
@@ -894,7 +894,7 @@ class CRemoteBase: public CInterface
                     socket.setown(ISocket::connect(ep));
                 if (useSSL)
                 {
-#ifdef USE_OPENSSL
+#ifdef _USE_OPENSSL
                     Owned<ISecureSocket> ssock = createSecureSocket(socket.getClear(), ClientSocket);
                     int status = ssock->secure_connect();
                     if (status < 0)
@@ -5119,7 +5119,7 @@ public:
                     sock.setown(acceptsock->accept(true));
                     if (useSSL)
                     {
-#ifdef USE_OPENSSL
+#ifdef _USE_OPENSSL
                         Owned<ISecureSocket> ssock = createSecureSocket(sock.getClear(), ServerSocket);
                         int status = ssock->secure_accept();
                         if (status < 0)

common/thorhelper/roxiedebug.cpp

@@ -46,10 +46,9 @@ bool CDebugCommandHandler::checkCommand(IXmlWriter &out, const char *&supplied,
     }
 }
 
-void CDebugCommandHandler::doDebugCommand(IPropertyTree *query, IDebuggerContext *debugContext, FlushingStringBuffer &output)
+void CDebugCommandHandler::doDebugCommand(IPropertyTree *query, IDebuggerContext *debugContext, IXmlWriter &out)
 {
     const char *commandName = query->queryName();
-    CommonXmlWriter out(0, 1);
     if (strnicmp(commandName, "b", 1)==0 && checkCommand(out, commandName, "breakpoint"))
     {
         const char *mode = query->queryProp("@mode");
@@ -207,7 +206,6 @@ void CDebugCommandHandler::doDebugCommand(IPropertyTree *query, IDebuggerContext
     else
         throw MakeStringException(THORHELPER_DEBUG_ERROR, "Unknown command %s", commandName);
     out.outputEndNested(commandName);
-    output.append(out.str());
 }
 
 //=======================================================================================

common/thorhelper/roxiedebug.hpp

@@ -39,7 +39,13 @@ class THORHELPER_API CDebugCommandHandler : public CInterface, implements IInter
 public:
     IMPLEMENT_IINTERFACE;
     static bool checkCommand(IXmlWriter &out, const char *&supplied, const char *expected);
-    void doDebugCommand(IPropertyTree *query, IDebuggerContext *debugContext, FlushingStringBuffer &output);
+    void doDebugCommand(IPropertyTree *query, IDebuggerContext *debugContext, IXmlWriter &out);
+    void doDebugCommand(IPropertyTree *query, IDebuggerContext *debugContext, FlushingStringBuffer &output)
+    {
+        CommonXmlWriter out(0, 1);
+        doDebugCommand(query, debugContext, out);
+        output.append(out.str());
+    }
 };
 
 //=======================================================================================

common/thorhelper/roxiehelper.hpp

@@ -231,7 +231,6 @@ protected:
     UnsignedArray lengths;
 
     bool needsFlush(bool closing);
-    void startBlock();
 public:
     TextMarkupFormat mlFmt;      // controls whether xml/json elements are output
     bool isRaw;      // controls whether output as binary or ascii
@@ -260,11 +259,13 @@ public:
     virtual void flush(bool closing) ;
     virtual void addPayload(StringBuffer &s, unsigned int reserve=0);
     virtual void *getPayload(size32_t &length);
+    virtual void startBlock();
     virtual void startDataset(const char *elementName, const char *resultName, unsigned sequence, bool _extend = false, const IProperties *xmlns=NULL);
     virtual void startScalar(const char *resultName, unsigned sequence);
     virtual void setScalarInt(const char *resultName, unsigned sequence, __int64 value, unsigned size);
     virtual void setScalarUInt(const char *resultName, unsigned sequence, unsigned __int64 value, unsigned size);
     virtual void incrementRowCount();
+    void setTail(const char *value){tail.set(value);}
 };
 
 class THORHELPER_API FlushingJsonBuffer : public FlushingStringBuffer

common/thorhelper/thorcommon.cpp

@@ -1549,13 +1549,13 @@ IExtRowWriter *createRowWriter(IFile *iFile, IRowInterfaces *rowIf, unsigned fla
         size32_t fixedSize = rowIf->queryRowMetaData()->querySerializedDiskMeta()->getFixedSize();
         if (fixedSize && TestRwFlag(flags, rw_grouped))
             ++fixedSize; // row writer will include a grouping byte
-        iFileIO.setown(createCompressedFileWriter(iFile, fixedSize, TestRwFlag(flags, rw_extend), TestRwFlag(flags, rw_compressblkcrc), compressor, TestRwFlag(flags, rw_fastlz)));
+        iFileIO.setown(createCompressedFileWriter(iFile, fixedSize, TestRwFlag(flags, rw_extend), TestRwFlag(flags, rw_compressblkcrc), compressor, getCompMethod(flags)));
     }
     else
         iFileIO.setown(iFile->open((flags & rw_extend)?IFOwrite:IFOcreate));
     if (!iFileIO)
         return NULL;
-    flags &= ~((unsigned)(rw_compress|rw_fastlz|rw_compressblkcrc));
+    flags &= ~COMP_MASK;
     return createRowWriter(iFileIO, rowIf, flags);
 }
 
@@ -1576,7 +1576,7 @@ IExtRowWriter *createRowWriter(IFileIO *iFileIO, IRowInterfaces *rowIf, unsigned
 
 IExtRowWriter *createRowWriter(IFileIOStream *strm, IRowInterfaces *rowIf, unsigned flags)
 {
-    if (0 != (flags & (rw_compress|rw_fastlz|rw_extend|rw_buffered|rw_compressblkcrc)))
+    if (0 != (flags & (rw_extend|rw_buffered|COMP_MASK)))
         throw MakeStringException(0, "Unsupported createRowWriter flags");
     Owned<CRowStreamWriter> writer = new CRowStreamWriter(strm, rowIf->queryRowSerializer(), rowIf->queryRowAllocator(), TestRwFlag(flags, rw_grouped), TestRwFlag(flags, rw_crc), TestRwFlag(flags, rw_autoflush));
     return writer.getClear();

common/thorhelper/thorcommon.hpp

@@ -20,6 +20,7 @@
 
 #include "jiface.hpp"
 #include "jcrc.hpp"
+#include "jlzw.hpp"
 #include "jsort.hpp"
 #include "jdebug.hpp"
 #include "jfile.hpp"
@@ -90,11 +91,58 @@ enum RowReaderWriterFlags
     rw_compressblkcrc = 0x10, // block compression, this sets/checks crc's at block level
     rw_fastlz         = 0x20, // if rw_compress
     rw_autoflush      = 0x40,
-    rw_buffered       = 0x80
+    rw_buffered       = 0x80,
+    rw_lzw            = 0x100, // if rw_compress
+    rw_lz4            = 0x200  // if rw_compress
 };
 #define DEFAULT_RWFLAGS (rw_buffered|rw_autoflush|rw_compressblkcrc)
 inline bool TestRwFlag(unsigned flags, RowReaderWriterFlags flag) { return 0 != (flags & flag); }
 
+#define COMP_MASK (rw_compress|rw_compressblkcrc|rw_fastlz|rw_lzw|rw_lz4)
+#define COMP_TYPE_MASK (rw_fastlz|rw_lzw|rw_lz4)
+inline void setCompFlag(const StringBuffer compStr, unsigned &flags)
+{
+    flags &= ~COMP_TYPE_MASK;
+    if (compStr.length())
+    {
+        if (0 == stricmp("FLZ", compStr.str()))
+            flags |= rw_fastlz;
+        else if (0 == stricmp("LZ4", compStr.str()))
+            flags |= rw_lz4;
+        else // not specifically FLZ or LZ4 so set to LZW (or rowdif)
+            flags |= rw_lzw;
+    }
+    else // default is LZ4
+        flags |= rw_lz4;
+}
+
+inline unsigned getCompMethod(unsigned flags)
+{
+    unsigned compMethod = COMPRESS_METHOD_LZW;
+    if (TestRwFlag(flags, rw_lzw))
+        compMethod = COMPRESS_METHOD_LZW;
+    else if (TestRwFlag(flags, rw_fastlz))
+        compMethod = COMPRESS_METHOD_FASTLZ;
+    else if (TestRwFlag(flags, rw_lz4))
+        compMethod = COMPRESS_METHOD_LZ4;
+    return compMethod;
+}
+
+inline unsigned getCompMethod(const StringBuffer compStr)
+{
+    unsigned compMethod = COMPRESS_METHOD_LZW;
+    if (compStr.length())
+    {
+        if (0 == stricmp("FLZ", compStr.str()))
+            compMethod = COMPRESS_METHOD_FASTLZ;
+        else if (0 == stricmp("LZ4", compStr.str()))
+            compMethod = COMPRESS_METHOD_LZ4;
+    }
+    else // default is LZ4
+        compMethod = COMPRESS_METHOD_LZ4;
+    return compMethod;
+}
+
 interface IExtRowStream: extends IRowStream
 {
     virtual offset_t getOffset() = 0;

common/thorhelper/thorsoapcall.cpp

@@ -937,7 +937,7 @@ public:
         return error.getLink();
     }
     inline IEngineRowAllocator * queryOutputAllocator() const { return outputAllocator; }
-#ifdef USE_OPENSSL
+#ifdef _USE_OPENSSL
     ISecureSocket *createSecureSocket(ISocket *sock)
     {
         {
@@ -1778,7 +1778,7 @@ public:
                     socket.setown(blacklist->connect(connUrl.port, connUrl.host, master->logctx, (unsigned)master->maxRetries, master->timeoutMS, master->roxieAbortMonitor));
                     if (stricmp(url.method, "https") == 0)
                     {
-#ifdef USE_OPENSSL
+#ifdef _USE_OPENSSL
                         Owned<ISecureSocket> ssock = master->createSecureSocket(socket.getClear());
                         if (ssock) 
                         {

dali/base/dasess.cpp

@@ -1503,7 +1503,7 @@ public:
         bool ok = true;
 #else
         bool ok = true;
-        if (ldapconn->getLDAPflags() & DLF_ENABLED)
+        if (ldapconn && ldapconn->getLDAPflags() & DLF_ENABLED)
             ok = ldapconn->clearPermissionsCache(udesc);
 #endif
         return ok;

deployment/configgen/configengcallback.hpp

@@ -138,7 +138,7 @@ class CConfigEngCallback: public CInterface, implements IDeploymentCallback
         StringBuffer errMsg(szMessage);
         String str(errMsg.trim());
 
-#ifdef USE_XALAN
+#ifdef _USE_XALAN
         if (str.lastIndexOf('[') > 0)
         {
           errMsg.clear();

deployment/deploy/DeploymentEngine.cpp

@@ -1813,7 +1813,7 @@ void CDeploymentEngine::setXsl(IXslProcessor* processor, IXslTransform* transfor
    m_externalFunction.setown(m_transform->createExternalFunction("addDeploymentFile", addDeploymentFile));
    m_transform->setExternalFunction(SEISINT_NAMESPACE, m_externalFunction.get(), true);
 
-#ifdef USE_OPENSSL
+#ifdef _USE_OPENSSL
    m_externalFunction2.setown(m_transform->createExternalFunction("siteCertificate", siteCertificateFunction));
    m_transform->setExternalFunction(SEISINT_NAMESPACE, m_externalFunction2.get(), true);
 #endif
@@ -2357,7 +2357,7 @@ void CDeploymentEngine::addDeploymentFile(StringBuffer &ret, const char *in, IXs
 //  siteCertificate
 //---------------------------------------------------------------------------
 /*static*/
-#ifdef USE_OPENSSL
+#ifdef _USE_OPENSSL
 void CDeploymentEngine::siteCertificateFunction(StringBuffer &ret, const char *in, IXslTransform*)
 {
     //input is of the format <processType>+<process name>+<instance name>+<output path>
@@ -2402,7 +2402,7 @@ void CDeploymentEngine::processCustomMethod(const char* method, const char *sour
         throw MakeStringException(0, "Process '%s': invalid method '%s' specified for file '%s'",
         m_name.get(), method, fileName);
 
-#ifdef USE_OPENSSL
+#ifdef _USE_OPENSSL
     siteCertificate(m_process, instanceName, outputFile);
 #else
     throw MakeStringException(0, "Process '%s' file '%s' method '%s': requires OpenSSL (disabled in build)",
@@ -2410,7 +2410,7 @@ void CDeploymentEngine::processCustomMethod(const char* method, const char *sour
 #endif
 }
 
-#ifdef USE_OPENSSL
+#ifdef _USE_OPENSSL
 //---------------------------------------------------------------------------
 //  processCustomMethod
 //---------------------------------------------------------------------------

deployment/deploy/DeploymentEngine.hpp

@@ -244,7 +244,7 @@ protected:
         EnvMachineOS os=MachineOsUnknown, const char* processName=NULL,bool isEspModuleOrPlugin=false);
    virtual void processCustomMethod(const char *method, const char *source, const char *outputFile, 
                                     const char *instanceName, EnvMachineOS os);
-#ifdef USE_OPENSSL
+#ifdef _USE_OPENSSL
    virtual void siteCertificate(IPropertyTree& process, const char *instanceName, const char *outputFile);
 #endif
 
@@ -289,7 +289,7 @@ protected:
                                    EnvMachineOS os, bool bCacheable, 
                                    const char* params=NULL);
    static void addDeploymentFile(StringBuffer &ret, const char *in, IXslTransform*);
-#ifdef USE_OPENSSL
+#ifdef _USE_OPENSSL
    static void siteCertificateFunction(StringBuffer &ret, const char *in, IXslTransform*);
 #endif
    virtual IEnvDeploymentEngine& getEnvDepEngine() const { return m_envDepEngine; }

ecl/eclcc/CMakeLists.txt

@@ -26,8 +26,6 @@
 project( eclcc ) 
 
 set (    SRCS
-         reservedwords.hpp
-         reservedwords.cpp
          eclcc.hpp
          eclcc.cpp
     )

ecl/eclcc/eclcc.cpp

@@ -276,7 +276,7 @@ public:
     }
 
     bool printKeywordsToXml();
-    bool parseCommandLineOptions(int argc, const char* argv[]);
+    int parseCommandLineOptions(int argc, const char* argv[]);
     void loadOptions();
     void loadManifestOptions();
     bool processFiles();
@@ -435,8 +435,9 @@ static int doMain(int argc, const char *argv[])
         return doSelfTest(argc, argv);
 
     EclCC processor(argc, argv);
-    if (!processor.parseCommandLineOptions(argc, argv))
-        return 1;
+    int ret = processor.parseCommandLineOptions(argc, argv);
+    if (ret != 0)
+        return ret;
 
     try
     {
@@ -1893,12 +1894,12 @@ bool EclCC::allowAccess(const char * category, bool isSigned)
 }
 
 //=========================================================================================
-bool EclCC::parseCommandLineOptions(int argc, const char* argv[])
+int EclCC::parseCommandLineOptions(int argc, const char* argv[])
 {
     if (argc < 2)
     {
         usage();
-        return false;
+        return 1;
     }
 
     ArgvIterator iter(argc, argv);
@@ -1976,7 +1977,15 @@ bool EclCC::parseCommandLineOptions(int argc, const char* argv[])
         else if (strcmp(arg, "-internal")==0)
         {
             outputSizeStmts();
-            testHqlInternals();
+            int error = testHqlInternals() + testReservedWords();  //NOTE: testReservedWords() depends on testHqlInternals() so must be be called after.
+            // report test result
+            if (error)
+            {
+                printf("%d error%s found!\n", error, error<=1?"":"s");
+                return 300;
+            }
+            else
+                printf("No errors\n");
         }
         else if (iter.matchFlag(tempBool, "-save-cpps"))
         {
@@ -2076,7 +2085,7 @@ bool EclCC::parseCommandLineOptions(int argc, const char* argv[])
             if (!checkFileExists(optIniFilename))
             {
                 ERRLOG("Error: INI file '%s' does not exist",optIniFilename.get());
-                return false;
+                return 1;
             }
         }
         else if (iter.matchFlag(optShowPaths, "-showpaths"))
@@ -2085,7 +2094,7 @@ bool EclCC::parseCommandLineOptions(int argc, const char* argv[])
         else if (iter.matchOption(optManifestFilename, "-manifest"))
         {
             if (!isManifestFileValid(optManifestFilename))
-                return false;
+                return 1;
         }
         else if (iter.matchOption(tempArg, "-split"))
         {
@@ -2094,7 +2103,7 @@ bool EclCC::parseCommandLineOptions(int argc, const char* argv[])
             if (!split)
             {
                 ERRLOG("Error: syntax is -split=part:splits\n");
-                return false;
+                return 1;
             }
             batchSplit = atoi(split+1);
             if (batchSplit == 0)
@@ -2108,7 +2117,7 @@ bool EclCC::parseCommandLineOptions(int argc, const char* argv[])
         else if (iter.matchOption(tempArg, "-platform") || /*deprecated*/ iter.matchOption(tempArg, "-target"))
         {
             if (!setTargetPlatformOption(tempArg.get(), optTargetClusterType))
-                return false;
+                return 1;
         }
         else if (iter.matchFlag(logVerbose, "-v") || iter.matchFlag(logVerbose, "--verbose"))
         {
@@ -2118,7 +2127,7 @@ bool EclCC::parseCommandLineOptions(int argc, const char* argv[])
         else if (strcmp(arg, "--version")==0)
         {
             fprintf(stdout,"%s %s\n", LANGUAGE_VERSION, BUILD_TAG);
-            return false;
+            return 1;
         }
         else if (startsWith(arg, "-Wc,"))
         {
@@ -2150,7 +2159,7 @@ bool EclCC::parseCommandLineOptions(int argc, const char* argv[])
         {
             ERRLOG("Error: unrecognised option %s",arg);
             usage();
-            return false;
+            return 1;
         }
         else
             inputFileNames.append(arg);
@@ -2158,7 +2167,7 @@ bool EclCC::parseCommandLineOptions(int argc, const char* argv[])
     if (showHelp)
     {
         usage();
-        return false;
+        return 1;
     }
 
     if (optComponentName.length())
@@ -2175,9 +2184,9 @@ bool EclCC::parseCommandLineOptions(int argc, const char* argv[])
     if (inputFileNames.ordinality() == 0 && !optKeywords)
     {
         if (optGenerateHeader || optShowPaths || (!optBatchMode && optQueryRepositoryReference))
-            return true;
+            return 0;
         ERRLOG("No input filenames supplied");
-        return false;
+        return 1;
     }
 
     if (optDebugMemLeak)
@@ -2187,7 +2196,7 @@ bool EclCC::parseCommandLineOptions(int argc, const char* argv[])
         initLeakCheck(title);
     }
 
-    return true;
+    return 0;
 }
 
 //=========================================================================================

ecl/hql/CMakeLists.txt

@@ -27,34 +27,35 @@
 project( hql ) 
 
 set (   SRCS 
-        hqlatoms.cpp 
-        hqlattr.cpp 
+        hqlatoms.cpp
+        hqlattr.cpp
         hqlcollect.cpp
         hqldesc.cpp
-        hqldsparam.cpp 
-        hqlerror.cpp 
-        hqlesp.cpp 
-        hqlexpr.cpp 
-        hqlfold.cpp 
-        hqlgram2.cpp 
-        hqlmanifest.cpp 
-        hqlmeta.cpp 
-        hqlopt.cpp 
-        hqlparse.cpp 
-        hqlpmap.cpp 
+        hqldsparam.cpp
+        hqlerror.cpp
+        hqlesp.cpp
+        hqlexpr.cpp
+        hqlfold.cpp
+        hqlgram2.cpp
+        hqlmanifest.cpp
+        hqlmeta.cpp
+        hqlopt.cpp
+        hqlparse.cpp
+        hqlpmap.cpp
         hqlplugininfo.cpp
-        hqlpregex.cpp 
+        hqlpregex.cpp
         hqlrepository.cpp
-        hqlscope.cpp 
-        hqlstack.cpp 
-        hqlthql.cpp 
-        hqltrans.cpp 
+        hqlscope.cpp
+        hqlstack.cpp
+        hqlthql.cpp
+        hqltrans.cpp
         hqlusage.cpp
-        hqlutil.cpp 
+        hqlutil.cpp
         hqlir.cpp
-        hqlvalid.cpp 
-        hqlwuerr.cpp 
-        hqlxmldb.cpp 
+        hqlvalid.cpp
+        hqlwuerr.cpp
+        hqlxmldb.cpp
+        reservedwords.cpp
 
         hqlgram.y
         hqllex.l
@@ -87,6 +88,7 @@ set (   SRCS
         hqlvalid.hpp
         hqlwuerr.hpp
         hqlxmldb.hpp
+        reservedwords.hpp
     )
 
 include_directories ( 
@@ -134,11 +136,11 @@ ADD_DEFINITIONS( -D_USRDLL -DHQL_EXPORTS -DHQLFOLD_EXPORTS -DHQLTRANS_EXPORTS )
 
 HPCC_ADD_LIBRARY( hql SHARED ${SRCS} ${CMAKE_CURRENT_BINARY_DIR}/hqlgram.cpp ${CMAKE_CURRENT_BINARY_DIR}/hqllex.cpp  )
 install ( TARGETS hql RUNTIME DESTINATION ${EXEC_DIR} LIBRARY DESTINATION ${LIB_DIR} )
-target_link_libraries ( hql  
+target_link_libraries ( hql
          jlib
-         nbcd 
-         eclrtl 
-         deftype 
+         nbcd
+         eclrtl
+         deftype
          ${CPPUNIT_LIBRARIES}
     )
 

ecl/hql/hqlattr.cpp

@@ -3007,6 +3007,7 @@ IHqlExpression * calcRowInformation(IHqlExpression * expr)
     case no_xmlproject:
     case no_call:
     case no_externalcall:
+    case no_embedbody:
         info.setUnknown(RCMfew);
         break;
     case no_colon:

ecl/hql/hqlexpr.cpp

@@ -12184,6 +12184,10 @@ IHqlExpression * createExternalFuncdefFromInternal(IHqlExpression * funcdef)
     if (functionBodyUsesContext(body))
         attrs.append(*LINK(cachedContextAttribute));
 
+    IHqlExpression *child = body->queryChild(0);
+    if (child && child->getOperator()==no_embedbody)
+        unwindAttribute(attrs, child, inlineAtom);
+
     ITypeInfo * returnType = funcdef->queryType()->queryChildType();
     OwnedHqlExpr externalExpr = createExternalReference(funcdef->queryId(), LINK(returnType), attrs);
     return replaceChild(funcdef, 0, externalExpr);

ecl/hql/hqlexpr.ipp

@@ -40,9 +40,6 @@
 #include "defvalue.hpp"
 #include "hqlexpr.hpp"
 
-#ifdef USE_TBB
-#include "tbb/scalable_allocator.h"
-#endif
 
 typedef byte transformdepth_t;
 #define TRANSFORM_DEPTH_MASK    0x7f
@@ -163,11 +160,6 @@ public:
     friend class CHqlExprMeta;
     typedef LinkedBaseIHqlExpression Parent;
 
-#ifdef USE_TBB
-    void *operator new(size32_t size) { return scalable_malloc(size); }
-    void operator delete(void *ptr) { return scalable_free(ptr); }
-#endif
-
 protected:
     unsigned hashcode;          // CInterface is 4 byte aligned in 64bits, so use this to pad
                                 // Worth storing because it significantly speeds up equality checking

ecl/hql/hqlgram.hpp

@@ -1234,6 +1234,7 @@ private:
 
 IHqlExpression *reparseTemplateFunction(IHqlExpression * funcdef, IHqlScope *scope, HqlLookupContext & ctx, bool hasFieldMap);
 extern HQL_API void resetLexerUniqueNames();        // to make regression suite consistent
-extern HQL_API void testHqlInternals();
+extern HQL_API int testHqlInternals();
+extern HQL_API int testReservedWords();
 
 #endif

ecl/hql/hqlgram2.cpp

@@ -48,6 +48,7 @@
 #include "hqlvalid.hpp"
 #include "hqlrepository.hpp"
 #include "hqlir.hpp"
+#include "reservedwords.hpp"
 
 #define ADD_IMPLICIT_FILEPOS_FIELD_TO_INDEX         TRUE
 #define FAST_FIND_FIELD
@@ -10551,7 +10552,7 @@ static void getTokenText(StringBuffer & msg, int token)
     case ENCRYPT: msg.append("ENCRYPT"); break;
     case ENCRYPTED: msg.append("ENCRYPTED"); break;
     case END: msg.append("END"); break;
-    case ENDCPP: msg.append("ENDCPP"); break;
+    case ENDCPP: msg.append("ENDC++"); break;
     case ENDEMBED: msg.append("ENDEMBED"); break;
     case ENTH: msg.append("ENTH"); break;
     case ENUM: msg.append("ENUM"); break;
@@ -10698,7 +10699,7 @@ static void getTokenText(StringBuffer & msg, int token)
     case ONLY: msg.append("ONLY"); break;
     case ONWARNING: msg.append("ONWARNING"); break;
     case OPT: msg.append("OPT"); break;
-    case OR : msg.append("OR "); break;
+    case OR : msg.append("OR"); break;
     case ORDER: msg.append("ORDER"); break;
     case ORDERED: msg.append("ORDERED"); break;
     case OUTER: msg.append("OUTER"); break;
@@ -11808,7 +11809,7 @@ void parseAttribute(IHqlScope * scope, IFileContents * contents, HqlLookupContex
     attrCtx.noteEndAttribute();
 }
 
-void testHqlInternals()
+int testHqlInternals()
 {
     printf("Sizes: const(%u) expr(%u) select(%u) dataset(%u) annotation(%u) prop(%u)\n",
             (unsigned)sizeof(CHqlConstant),
@@ -11865,12 +11866,33 @@ void testHqlInternals()
         }
     }
 
-    // 
-    // report test result
-    if (error)
-        printf("%d error%s found!\n", error, error<=1?"":"s");
-    else
-        printf("No errors\n");
+    return error;
+}
+
+int testReservedWords()
+{
+    printf("Testing --keywords is complete...\n");
+    int error = 0;
+
+    for (int token = 258; token < YY_LAST_TOKEN; token++)
+    {
+        try
+        {
+            StringBuffer tokenText;
+            getTokenText(tokenText, token);
+            tokenText.toLowerCase();
+            if (!searchReservedWords(tokenText.str()))
+            {
+                error++;
+                printf("   Error: '%s' is missing from reservedWords.cpp\n", tokenText.str());
+            }
+        }
+        catch (...)
+        {
+            printf("   Error: complete test not possible - getTokenText() does not handle expected: %d\n", token);
+        }
+    }
+    return error;
 }
 
 IHqlExpression *HqlGram::yyParse(bool _parsingTemplateAttribute, bool catchAbort)

ecl/hql/hqltrans.ipp

@@ -30,9 +30,6 @@
 
 #include "hqlexpr.hpp"
 #include "jset.hpp"
-#ifdef USE_TBB
-#include "tbb/scalable_allocator.h"
-#endif
 
 typedef MapOwnedToOwned<IHqlExpression, IHqlExpression> MapOwnedHqlToOwnedHql;
 
@@ -316,11 +313,6 @@ public:
 
     inline void setUnvisited() { lastPass = (byte)-1; }
 
-#ifdef USE_TBB
-    void *operator new(size32_t size) { return scalable_malloc(size); }
-    void operator delete(void *ptr) { return scalable_free(ptr); }
-#endif
-
 #ifdef OPTIMIZE_TRANSFORM_ALLOCATOR
     void *operator new(size32_t size, void * ptr) { return ptr; }                                                   
     void *operator new(size32_t size);      // cause a link error if called.    

ecl/eclcc/reservedwords.cpp

@@ -105,6 +105,7 @@ static const char * eclReserved3[] = { //Template language
     "#trace",
     "#uniquename",
     "#warning",
+    "#webservice",
     "#workunit",
     "loadxml",
     NULL
@@ -119,7 +120,9 @@ static const char * eclReserved4[] = { //String functions
     "length",
     "realformat",
     "regexfind",
+    "regexfindset",
     "regexreplace",
+    "tojson",
     "tounicode",
     "toxml",
     "trim",
@@ -161,13 +164,15 @@ static const char * eclReserved5[] = { //Math
     NULL
 };
 
-static const char * eclReserved6[] = {
+static const char * eclReserved6[] = { //bit ops
     "&",
     "|",
     //"^", ambiguous group location
     "bnot",
     "<<",
     ">>",
+    "(>",
+    "<)",
     NULL
 };
 
@@ -176,7 +181,8 @@ static const char * eclReserved7[] = { //Comparison ops
     "<",
     "<=",
     ">",
-    //">=",     //x.<y>= n  really wants to be processed as x.<y> = n, not x.<y >=
+    //">=", //x.<y>= n  really wants to be processed as x.<y> = n, not x.<y >= (this token has been added to the internalTokens list instead)
+
     "!=",
     "<>",
     "between",
@@ -196,6 +202,15 @@ static const char * eclReserved8[] = { //Binary ops
     NULL
 };
 
+static const char * eclReserved19[] = { //Misc ops
+    ":=",
+    "<?>",
+    "<\?\?>",
+    "..",
+    "=>",
+    NULL
+};
+
 static const char * eclReserved9[] = {//ECL fundamental types and associates
     "ascii",
     "big_endian",
@@ -261,6 +276,7 @@ static const char * eclReserved11[] = {//Activities
     "enth",
     "fetch",
     "fromxml",
+    "fromjson",
     "graph",
     "group",
     "having",
@@ -272,9 +288,11 @@ static const char * eclReserved11[] = {//Activities
     "map",
     "merge",
     "mergejoin",
+    "nocombine",
     "nonempty",
     "normalize",
     "parse",
+    "pattern",
     "process",
     "project",
     "quantile",
@@ -286,11 +304,13 @@ static const char * eclReserved11[] = {//Activities
     "rollup",
     "row",
     "sample",
+    "score",
     "sort",
     "stepped",
     "subsort",
     "table",
     "topn",
+    "trace",
     "ungroup",
     "which",
     "within",
@@ -318,6 +338,7 @@ static const char * eclReserved12[] = {//Attributes
     "best",
     "bitmap",
     "blob",
+    "c++",
     "choosen:all",
     "const",
     "counter",
@@ -398,12 +419,14 @@ static const char * eclReserved14[] = { //Attribute functions (some might actual
     "compressed",
     "default",
     "escape",
+    "format",
     "global",
     "groupby",
     "guard",
     "httpheader",
     "internal",
     "joined",
+    "json",
     "keep",
     "keyed",
     "limit",
@@ -425,6 +448,7 @@ static const char * eclReserved14[] = { //Attribute functions (some might actual
     "penalty",
     "prefetch",
     "proxyaddress",
+    "refresh",
     "repeat",
     "response",
     "retry",
@@ -446,6 +470,7 @@ static const char * eclReserved14[] = { //Attribute functions (some might actual
     "width",
     "wild",
     "xml",
+    "xmlns",
     "xmldefault",
     "xpath",
     "xmlproject",
@@ -455,6 +480,7 @@ static const char * eclReserved14[] = { //Attribute functions (some might actual
 };
 
 static const char * eclReserved15[] = { //Actions and statements
+    "action",
     "apply",
     "as",
     "build",
@@ -554,7 +580,8 @@ static const Keywords keywordList[] = {
    {15, eclReserved15},
    {16, eclReserved16},
    {17, eclReserved17},
-   {18, eclReserved18}
+   {18, eclReserved18},
+   {19, eclReserved19}
 };
 
 void printKeywordsToXml()
@@ -577,3 +604,64 @@ void printKeywordsToXml()
      buffer.append("</xml>");
      fprintf(stdout, "%s\n", buffer.str());
 }
+
+static const char * internalTokens[] = { //tokens present in the grammar that aren't in the language or are waiting to be deprecated
+    "",
+    ">=", //x.<y>= n  really wants to be processed as x.<y> = n, not x.<y >= (this token would otherwise belong to group7 - comparison ops)
+    "#elif",
+    "&&",
+    "constant",
+    "complex-macro",
+    "datarow",
+    "expression",
+    "feature-name",
+    "field list",
+    "field reference",
+    "function-name",
+    "hole",
+    "identifier",
+    "implements",
+    "inline",
+    "macro-name",
+    "module-name",
+    "number",
+    "of",
+    "or",
+    "order",
+    "pattern-name",
+    "physicalfilename",
+    "record-name",
+    "relationship",
+    "right2",
+    "skipped",
+    "swapped",
+    "transform-name",
+    "type name",
+    "type-name",
+    "unicode-string",
+    NULL
+};
+
+
+bool searchReservedWords(const char * tokenText)
+{
+    unsigned int nGroups = sizeof(keywordList)/sizeof(keywordList[0]);
+    for (unsigned i = 0; i < nGroups; ++i)
+    {
+        int j = 0;
+        while(keywordList[i].keywords[j])
+        {
+            if (strcmp(tokenText, keywordList[i].keywords[j]) == 0)
+                return true;
+            j++;
+        }
+    }
+    int i = 0;
+    while(internalTokens[i])
+    {
+        if (strcmp(tokenText, internalTokens[i]) == 0)
+            return true;
+        i++;
+    }
+    return false;
+}

ecl/eclcc/reservedwords.hpp

@@ -18,5 +18,6 @@
 #define RESERVEDWORDS_HPP
 
 void printKeywordsToXml();
+bool searchReservedWords(const char * tokenText);
 
 #endif

ecl/hqlcpp/hqlcerrors.hpp

@@ -221,6 +221,7 @@
 #define HQLERR_ConditionalAggregateVarOffset    4201
 #define HQLERR_AggregateDynamicOffset           4202
 #define HQLERR_ServiceDefinitionNotAllowed      4203
+#define HQLERR_BodyNotAllowedWithInline         4204
 
 //Warnings....
 #define HQLWRN_PersistDataNotLikely             4500
@@ -517,6 +518,7 @@
 #define HQLERR_ConditionalAggregateVarOffset_Text "Conditional aggregate '%s' cannot follow a variable length field"
 #define HQLERR_AggregateDynamicOffset_Text      "Aggregate assignment to '%s' cannot follow variable size aggregate"
 #define HQLERR_ServiceDefinitionNotAllowed_Text "Insufficient access rights to use SERVICE"
+#define HQLERR_BodyNotAllowedWithInline_Text    "#body not supported with INLINE attribute"
 
 //Warnings.
 #define HQLWRN_CannotRecreateDistribution_Text  "Cannot recreate the distribution for a persistent dataset"

ecl/hqlcpp/hqlcpp.cpp

@@ -11679,6 +11679,8 @@ void HqlCppTranslator::buildCppFunctionDefinition(BuildCtx &funcctx, IHqlExpress
     const char * separator = strstr(body, cppSeparatorText);
     if (separator)
     {
+        if (bodyCode->hasAttribute(inlineAtom))
+            throwError(HQLERR_BodyNotAllowedWithInline);
         text.setCharAt(separator-text.str(), 0);
         if (location)
             funcctx.addLine(locationFilename, startLine);
@@ -11943,7 +11945,16 @@ void HqlCppTranslator::buildFunctionDefinition(IHqlExpression * funcdef)
         if (languageAttr)
             buildScriptFunctionDefinition(funcctx, funcdef, proto);
         else
-            buildCppFunctionDefinition(funcctx, bodyCode, proto);
+        {
+            bool isInline = bodyCode->hasAttribute(inlineAtom);
+            if (isInline && options.spanMultipleCpp)
+            {
+                BuildCtx funcctx2(*code, parentHelpersAtom);
+                buildCppFunctionDefinition(funcctx2, bodyCode, proto);
+            }
+            else
+                buildCppFunctionDefinition(funcctx, bodyCode, proto);
+        }
     }
     else
     {

ecl/hqlcpp/hqlcpp.ipp

@@ -1963,6 +1963,7 @@ protected:
     unsigned            nextTypeId;
     unsigned            nextFieldId;
     unsigned            curWfid;
+    unsigned            implicitFunctionId = 0;
     HqlExprArray        internalFunctions;
     HqlExprArray        internalFunctionExternals;
     UniqueSequenceCounter spillSequence;

ecl/hqlcpp/hqlcppds.cpp

@@ -4489,8 +4489,15 @@ void HqlCppTranslator::buildRowAssign(BuildCtx & ctx, IReferenceSelector * targe
                 buildRowAssign(ctx, target, deserialized->queryChild(0));
             else if (!typeRequiresDeserialization(deserialized->queryType(), serializeForm))
                 buildRowAssign(ctx, target, deserialized);
-            else
+            else if (target->queryRootRow()->queryBuilder())
                 doBuildRowAssignSerializeRow(ctx, target, expr);
+            else
+            {
+                CHqlBoundExpr bound;
+                buildTempExpr(ctx, expr, bound);
+                OwnedHqlExpr translated = bound.getTranslatedExpr();
+                buildRowAssign(ctx, target, translated);
+            }
             return;
         }
     case no_if:

ecl/hqlcpp/hqlinline.cpp

@@ -1008,7 +1008,6 @@ void ParentExtract::beginCreateExtract(BuildCtx & ctx, bool doDeclare)
     //Collect a list of cursors together... NB these are in reverse order..
     gatherActiveRows(*buildctx);
 
-    serialization->BoundRow::setBuilder(queryExtractName());
     childSerialization->setBuilder(this);
 }
 

ecl/hqlcpp/hqlttcpp.cpp

@@ -1997,8 +1997,8 @@ static IHqlExpression * normalizeIndexBuild(IHqlExpression * expr, bool sortInde
 
 
 static HqlTransformerInfo thorHqlTransformerInfo("ThorHqlTransformer");
-ThorHqlTransformer::ThorHqlTransformer(HqlCppTranslator & _translator, ClusterType _targetClusterType, IConstWorkUnit * wu)
-: NewHqlTransformer(thorHqlTransformerInfo), translator(_translator), options(_translator.queryOptions())
+ThorHqlTransformer::ThorHqlTransformer(HqlCppTranslator & _translator, ClusterType _targetClusterType, IConstWorkUnit * wu, unsigned & _implicitFunctionId)
+: NewHqlTransformer(thorHqlTransformerInfo), translator(_translator), options(_translator.queryOptions()), implicitFunctionId(_implicitFunctionId)
 {
     targetClusterType = _targetClusterType;
     topNlimit = options.topnLimit;
@@ -2091,6 +2091,25 @@ IHqlExpression * ThorHqlTransformer::createTransformed(IHqlExpression * expr)
     case no_debug_option_value:
         //pick best engine etc. definitely done by now, so substitute any options that haven't been processed already
         return getDebugValueExpr(translator.wu(), expr);
+    case no_embedbody:
+        {
+            //Convert all definitions of non functional embeds to implicit functions
+            StringBuffer funcname;
+            funcname.append("userx").append(++implicitFunctionId);
+            HqlExprArray args;
+            args.append(*LINK(expr));
+            if (expr->hasAttribute(languageAtom))
+            {
+                args.append(*createAttribute(contextAtom));
+                if (expr->isDatarow())
+                    args.append(*createAttribute(allocatorAtom));
+            }
+            OwnedHqlExpr body = createWrapper(no_outofline, expr->queryType(), args);
+            IHqlExpression * formals = createValue(no_sortlist, makeSortListType(NULL));
+            OwnedHqlExpr funcdef = createFunctionDefinition(createIdAtom(funcname), body.getClear(), formals, NULL, NULL);
+            HqlExprArray actuals;
+            return createBoundFunction(NULL, funcdef, actuals, nullptr, true);
+        }
     }
 
     if (normalized && (normalized != transformed))
@@ -3772,7 +3791,7 @@ void HqlCppTranslator::convertLogicalToActivities(WorkflowItem & curWorkflow)
 {
     {
         cycle_t startCycles = get_cycles_now();
-        ThorHqlTransformer transformer(*this, targetClusterType, wu());
+        ThorHqlTransformer transformer(*this, targetClusterType, wu(), implicitFunctionId);
 
         HqlExprArray & exprs = curWorkflow.queryExprs();
         HqlExprArray transformed;

ecl/hqlcpp/hqlttcpp.ipp

@@ -183,7 +183,7 @@ protected:
 class ThorHqlTransformer : public NewHqlTransformer
 {
 public:
-    ThorHqlTransformer(HqlCppTranslator & _translator, ClusterType _targetClusterType, IConstWorkUnit * _wu);
+    ThorHqlTransformer(HqlCppTranslator & _translator, ClusterType _targetClusterType, IConstWorkUnit * _wu, unsigned & _implicitFunctionId);
 
 protected:
     virtual IHqlExpression * createTransformed(IHqlExpression * expr);
@@ -221,6 +221,7 @@ protected:
     const HqlCppOptions & options;
     ClusterType         targetClusterType;
     unsigned            topNlimit;
+    unsigned &          implicitFunctionId;
     bool                groupAllDistribute;
 };
 

ecl/hqlcpp/hqlwcpp.cpp

@@ -593,7 +593,8 @@ bool HqlCppWriter::generateFunctionPrototype(IHqlExpression * funcdef, const cha
     if (body->hasAttribute(_disallowed_Atom))
         throwError(HQLERR_ServiceDefinitionNotAllowed);
 
-    if (body->hasAttribute(includeAtom) || body->hasAttribute(ctxmethodAtom) || body->hasAttribute(gctxmethodAtom) || body->hasAttribute(methodAtom) || body->hasAttribute(sysAtom) || body->hasAttribute(omethodAtom))
+    if (body->hasAttribute(includeAtom) || body->hasAttribute(ctxmethodAtom) || body->hasAttribute(gctxmethodAtom) || body->hasAttribute(methodAtom)
+        || body->hasAttribute(sysAtom) || body->hasAttribute(omethodAtom) || body->hasAttribute(inlineAtom) )
         return false;
 
     IHqlExpression *proto = body->queryAttribute(prototypeAtom);
@@ -607,6 +608,10 @@ bool HqlCppWriter::generateFunctionPrototype(IHqlExpression * funcdef, const cha
     enum { ServiceApi, RtlApi, FastApi, CApi, CppApi, LocalApi } api = ServiceApi;
     bool isVirtual = funcdef->hasAttribute(virtualAtom);
     bool isLocal = body->hasAttribute(localAtom);
+
+    IHqlExpression *child = body->queryChild(0);
+    bool isInline = child?child->hasAttribute(inlineAtom):false;
+
     if (body->hasAttribute(eclrtlAtom))
         api = RtlApi;
     else if (body->hasAttribute(fastAtom))
@@ -618,7 +623,9 @@ bool HqlCppWriter::generateFunctionPrototype(IHqlExpression * funcdef, const cha
     else if (isLocal || isVirtual)
         api = LocalApi;
 
-    if (isVirtual)
+    if (isInline)
+        out.append("inline");
+    else if (isVirtual)
         out.append("virtual");
     else
         out.append("extern");

ecl/regress/cppbody3.ecl

@@ -20,21 +20,21 @@
 
 real8 s1 :=
 BEGINC++
-sqrt(2.0)
+return sqrt(2.0);
 ENDC++;
 s1p := s1 : persist('s1');
 output(s1p);
 
 real8 s2 :=
 BEGINC++
-sqrt(2.0)
+return sqrt(2.0);
 ENDC++;
 s2p := s2 : persist('s2');
 output(s2p);
 
 real8 s3 :=
 BEGINC++
-sqrt(3.0)
+return sqrt(3.0);
 ENDC++;
 s3p := s3 : persist('s3');
 output(s3p);

ecl/regress/cppbody7.ecl

@@ -19,19 +19,19 @@
 
 integer4 mkRandom1 :=
 BEGINC++
-rtlRandom()
+return rtlRandom();
 ENDC++;
 
 integer4 mkRandom2 :=
 BEGINC++
 #option pure
-rtlRandom()
+return rtlRandom();
 ENDC++;
 
 integer4 mkRandom3 :=
 BEGINC++
 #option action
-rtlRandom()
+return rtlRandom();
 ENDC++;
 
 output(mkRandom1 * mkRandom1);

ecl/regress/rkc42.ecl

@@ -21,7 +21,7 @@ return strdup(""); ENDC++;
 output(EmptyString1());
 
 VarString EmptyString2 := BEGINC++
-strdup("") ENDC++;
+return strdup(""); ENDC++;
 
 
 output(EmptyString2);

esp/logging/loggingagent/espserverloggingagent/CMakeLists.txt

@@ -33,7 +33,6 @@ include_directories (
     ${HPCC_SOURCE_DIR}/system/include
     ${HPCC_SOURCE_DIR}/system/jlib
     ${HPCC_SOURCE_DIR}/system/xmllib
-    ${HPCC_SOURCE_DIR}/system/boost/include
     ${HPCC_SOURCE_DIR}/system/security/shared
     ${HPCC_SOURCE_DIR}/esp/bindings
     ${HPCC_SOURCE_DIR}/esp/bindings/http/client

esp/logging/loggingmanager/CMakeLists.txt

@@ -38,7 +38,6 @@ set ( SRCS
 include_directories (
     ${HPCC_SOURCE_DIR}/esp/platform
     ${HPCC_SOURCE_DIR}/system/jlib
-    ${HPCC_SOURCE_DIR}/system/boost/include
     ${HPCC_SOURCE_DIR}/system/security/shared
     ${HPCC_SOURCE_DIR}/system/xmllib
     ${HPCC_SOURCE_DIR}/system/include

esp/services/ws_dfu/ws_dfuXRefService.cpp

@@ -532,7 +532,7 @@ inline void addLfnToUsedFileMap(MapStringTo<bool> &usedFileMap, const char *lfn)
         usedFileMap.setValue(lfn, true);
 }
 
-void addUsedFilesFromActivePackageMaps(MapStringTo<bool> &usedFileMap, const char *process)
+void addUsedFilesFromPackageMaps(MapStringTo<bool> &usedFileMap, const char *process)
 {
     Owned<IPropertyTree> packageSet = resolvePackageSetRegistry(process, true);
     if (!packageSet)
@@ -542,7 +542,7 @@ void addUsedFilesFromActivePackageMaps(MapStringTo<bool> &usedFileMap, const cha
     ForEach(*targets)
     {
         SCMStringBuffer target;
-        VStringBuffer xpath("PackageMap[@querySet='%s'][@active='1']", targets->str(target).str());
+        VStringBuffer xpath("PackageMap[@querySet='%s']", targets->str(target).str());
         Owned<IPropertyTreeIterator> activeMaps = packageSet->getElements(xpath);
         //Add files referenced in all active maps, for all targets configured for this process cluster
         ForEach(*activeMaps)
@@ -595,7 +595,7 @@ bool CWsDfuXRefEx::onDFUXRefUnusedFiles(IEspContext &context, IEspDFUXRefUnusedF
     ForEach(*roxieFiles)
         addLfnToUsedFileMap(usedFileMap, roxieFiles->query().queryProp("@name"));
     if (req.getCheckPackageMaps())
-        addUsedFilesFromActivePackageMaps(usedFileMap, process);
+        addUsedFilesFromPackageMaps(usedFileMap, process);
     StringArray unusedFiles;
     findUnusedFilesInDFS(unusedFiles, process, usedFileMap);
     resp.setUnusedFileCount(unusedFiles.length());

esp/services/ws_loggingservice/CMakeLists.txt

@@ -29,7 +29,6 @@ include_directories (
      ${HPCC_SOURCE_DIR}/system/security/shared
      ${HPCC_SOURCE_DIR}/system/security/securesocket
      ${HPCC_SOURCE_DIR}/common/environment
-     ${MYSQL_INCLUDE_DIR}
      ${HPCC_SOURCE_DIR}/dali/base
      ${HPCC_SOURCE_DIR}/esp/platform
      ${HPCC_SOURCE_DIR}/esp/bindings

esp/src/eclwatch/DFUQueryWidget.js

@@ -200,6 +200,9 @@ define([
         },
 
         _onCopyOk: function (event) {
+            var copyPreserveCompressionCheckbox = registry.byId(this.id + "CopyPreserveCompression");
+            var value = copyPreserveCompressionCheckbox.get("checked") ? 1 : 0;
+            
             if (this.copyForm.validate()) {
                 var context = this;
                 arrayUtil.forEach(this.copyGrid.store.data, function (item, idx) {
@@ -207,6 +210,7 @@ define([
                     var request = domForm.toObject(context.id + "CopyForm");
                     request.RenameSourceName = item.Name;
                     request.destLogicalName = item.targetCopyName;
+                    request.preserveCompression = value;
                     logicalFile.copy({
                         request: request
                     }).then(function (response) {

esp/src/eclwatch/ESPWorkunit.js

@@ -417,7 +417,7 @@ define([
             return this._action("Restore");
         },
 
-        publish: function (jobName, remoteDali, sourceProcess, priority, comment, allowForeign) {
+        publish: function (jobName, remoteDali, sourceProcess, priority, comment, allowForeign, updateSupers) {
             this._assertHasWuid();
             var context = this;
             WsWorkunits.WUPublishWorkunit({
@@ -429,6 +429,7 @@ define([
                     Priority: priority,
                     Comment: comment,
                     AllowForeignFiles: allowForeign,
+                    UpdateSuperFiles: updateSupers,
                     Activate: 1,
                     UpdateWorkUnitName: 1,
                     Wait: 5000

esp/src/eclwatch/WUDetailsWidget.js

@@ -192,6 +192,12 @@ define([
             } else {
                 allowForeign.value = 0;
             }
+            var updateSupers = registry.byId(this.id + "UpdateSuperFiles");
+            if (updateSupers.checked == true) {
+                updateSupers.value = 1;
+            } else {
+                updateSupers.value = 0;
+            }
             if (this.publishForm.validate()) {
                 registry.byId(this.id + "Publish").closeDropDown();
                 this.wu.publish(
@@ -200,7 +206,8 @@ define([
                     dom.byId(this.id + "SourceProcess").value,
                     registry.byId(this.id + "Priority").value,
                     dom.byId(this.id + "Comment").value,
-                    allowForeign.value
+                    allowForeign.value,
+                    updateSupers.value
                 );
             }
         },

esp/src/eclwatch/nls/hpcc.js

@@ -567,6 +567,7 @@ define({root:
     Unsuspend: "Unsuspend",
     Unsuspended: "Unsuspended",
     Up: "Up",
+    UpdateSuperFiles: "Update Super Files",
     Upload: "Upload",
     Usage: "Usage",
     Used: "Used",

esp/src/eclwatch/templates/WUDetailsWidget.html

@@ -35,7 +35,8 @@
                                         <option value="Low">${i18n.Low}</option>
                                         <option value="High">${i18n.High}</option>
                                     </select>
-                                    <input id="${id}AllowForeignFiles" title="${i18n.AllowForeignFiles}:" name="AllowForeignFiles" colspan="2" data-dojo-props="trim: true" checked data-dojo-type="dijit.form.CheckBox" />
+                                    <input id="${id}AllowForeignFiles" title="${i18n.AllowForeignFiles}:" name="AllowForeignFiles" colspan="2" checked data-dojo-type="dijit.form.CheckBox" />
+                                    <input id="${id}UpdateSuperFiles" title="${i18n.UpdateSuperFiles}:" name="UpdateSuperFiles" colspan="2" data-dojo-type="dijit.form.CheckBox" />
                                 </div>
                                 <div class="dijitDialogPaneActionBar">
                                     <button type="submit" data-dojo-attach-event="onClick:_onPublish" data-dojo-type="dijit.form.Button">${i18n.Submit}</button>

esp/test/httptest/httptest.cpp

@@ -151,7 +151,7 @@ HttpClient::HttpClient(int threads, int times, const char* host, int port, FILE*
     m_use_ssl = use_ssl;
     if(use_ssl)
     {
-#ifdef USE_OPENSSL
+#ifdef _USE_OPENSSL
         if(sslconfig != NULL)
             m_ssctx.setown(createSecureSocketContextEx2(sslconfig, ClientSocket));
         else
@@ -186,7 +186,7 @@ int HttpClient::getUrl(const char* url)
 
     if(m_use_ssl)
     {
-#if USE_OPENSSL
+#if _USE_OPENSSL
         if(m_ssctx.get() == NULL)
             m_ssctx.setown(createSecureSocketContext(ClientSocket));
 #else
@@ -283,7 +283,7 @@ int HttpClient::sendSoapRequest(const char* url, const char* soapaction, const c
 
     if(m_use_ssl)
     {
-#ifdef USE_OPENSSL
+#ifdef _USE_OPENSSL
         if(m_ssctx.get() == NULL)
             m_ssctx.setown(createSecureSocketContext(ClientSocket));
 #else
@@ -613,7 +613,7 @@ HttpServer::HttpServer(int port, const char* in, FILE* ofile, bool use_ssl, IPro
     m_recvDelay = m_sendDelay = m_closeDelay = 0;
     if(use_ssl)
     {
-#ifdef USE_OPENSSL
+#ifdef _USE_OPENSSL
         if(sslconfig != NULL)
             m_ssctx.setown(createSecureSocketContextEx2(sslconfig, ServerSocket));
         else
@@ -1183,7 +1183,7 @@ HttpProxy::HttpProxy(int localport, const char* host, int port, FILE* ofile, boo
     m_use_ssl = use_ssl;
     if(use_ssl)
     {
-#if USE_OPENSSL
+#if _USE_OPENSSL
         if(sslconfig != NULL)
             m_ssctx.setown(createSecureSocketContextEx2(sslconfig, ClientSocket));
         else

esp/tools/soapplus/http.cpp

@@ -461,7 +461,7 @@ HttpClient::HttpClient(IProperties* globals, const char* url, const char* inname
 
         if(stricmp(m_protocol.str(), "HTTPS") == 0)
         {
-#ifdef USE_OPENSSL
+#ifdef _USE_OPENSSL
             if(m_ssctx.get() == NULL)
                 m_ssctx.setown(createSecureSocketContext(ClientSocket));
 #else

esp/tools/soapplus/httpproxy.cpp

@@ -197,7 +197,7 @@ int COneServerHttpProxyThread::start()
         socket2.setown(ISocket::connect(ep));
         if(m_use_ssl && m_ssctx != NULL)
         {
-#ifdef USE_OPENSSL
+#ifdef _USE_OPENSSL
             Owned<ISecureSocket> securesocket = m_ssctx->createSecureSocket(socket2.getLink());
             int res = securesocket->secure_connect();
             if(res >= 0)
@@ -562,7 +562,7 @@ HttpProxy::HttpProxy(int localport, const char* url, FILE* ofile, const char* ur
 
         if(m_use_ssl)
         {
-#ifdef USE_OPENSSL
+#ifdef _USE_OPENSSL
             m_ssctx.setown(createSecureSocketContext(ClientSocket));
 #else
         throw MakeStringException(-1, "HttpProxy: failure to create SSL socket - OpenSSL not enabled in build");
@@ -582,7 +582,7 @@ HttpProxy::HttpProxy(int localport, const char* host, int port, FILE* ofile, boo
     m_use_ssl = use_ssl;
     if(use_ssl)
     {
-#ifdef USE_OPENSSL
+#ifdef _USE_OPENSSL
         if(sslconfig != NULL)
             m_ssctx.setown(createSecureSocketContextEx2(sslconfig, ClientSocket));
         else

plugins/memcached/CMakeLists.txt

@@ -25,7 +25,7 @@
 project( memcached )
 
 if (MEMCACHED)
-    ADD_PLUGIN(memcached PACKAGES MEMCACHED OPTION MAKE_MEMCACHED)
+    ADD_PLUGIN(memcached PACKAGES LIBMEMCACHED OPTION MAKE_MEMCACHED MINVERSION 1.0.10)
     if(MAKE_MEMCACHED)
         set(
             SRCS

plugins/memcached/README.md

@@ -17,7 +17,7 @@ The memcached daemon can be obtained via - [source](http://memcached.org/downloa
 sudo apt-get install memcached
 ```
 
-*Note:* **libmemcached** 0.53 or greater is required to use this plugin as intended. It is advised to use the newest version of **memcached** possible to you.
+*Note:* **libmemcached** 1.0.10 or greater is required to use this plugin as intended. It is advised to use the newest version of **memcached** possible to you.
 
 
 Getting started

plugins/memcached/memcachedplugin.cpp

@@ -296,62 +296,14 @@ void MemCachedPlugin::MCached::getVoidPtrLenPair(ICodeContext * ctx, const char
 
 MemCachedPlugin::MCached::MCached(ICodeContext * ctx, const char * _options) : connection(NULL), pool(NULL), typeMismatchCount(0)
 {
-    options.set(_options);
+#if (LIBMEMCACHED_VERSION_HEX < 0x01000010)
+    StringBuffer msg("Memcached Plugin: libmemcached version '");
+    msg.append(LIBMEMCACHED_VERSION_STRING).append("' incompatible with min version>=1.0.10");
+    rtlFail(0, msg.str());
+#endif
 
-#if (LIBMEMCACHED_VERSION_HEX<0x53000)
-    memcached_st *memc = memcached_create(NULL);
-    memcached_return_t rc;
-    memcached_server_st *servers = NULL;
-    try
-    {
-        unsigned pool_min = 1;
-        unsigned pool_max = 1;
-        StringArray optionStrings;
-        optionStrings.appendList(_options, " ");
-        ForEachItemIn(idx, optionStrings)
-        {
-            const char *opt = optionStrings.item(idx);
-            if (strncmp(opt, "--SERVER=", 9) ==0)
-            {
-                opt += 9;
-                StringArray splitPort;
-                splitPort.appendList(opt, ":");
-                unsigned port;
-                if (splitPort.ordinality()==2)
-                    port = atoi(splitPort.item(1));
-                else
-                    port = 11211;
-                servers = memcached_server_list_append(NULL, splitPort.item(0), port, &rc);
-                assertOnError(rc, "memcached_server_list_append failed - ");
-            }
-            else if (strncmp(opt, "--POOL-MIN=", 11) ==0)
-                pool_min = atoi(opt+11);
-            else if (strncmp(opt, "--POOL-MAX=", 11) ==0)
-                pool_max = atoi(opt+11);
-            else
-            {
-                VStringBuffer err("MemCachedPlugin: unsupported option string %s", opt);
-                rtlFail(0, err.str());
-            }
-        }
-        if (!servers)
-            rtlFail(0, "No servers specified");
-        rc = memcached_server_push(memc, servers);
-        memcached_server_list_free(servers);
-        assertOnError(rc, "memcached_server_push failed - ");
-        pool = memcached_pool_create(memc, pool_min, pool_max);  // takes ownership of memc
-    }
-    catch (...)
-    {
-        if (servers)
-            memcached_server_list_free(servers);
-        if (memc)
-            memcached_free(memc);
-        throw;
-    }
-#else
+    options.set(_options);
     pool = memcached_pool(_options, strlen(_options));
-#endif
     assertPool();
 
     setPoolSettings();
@@ -363,11 +315,8 @@ MemCachedPlugin::MCached::~MCached()
 {
     if (pool)
     {
-#if (LIBMEMCACHED_VERSION_HEX<0x53000)
-        memcached_pool_push(pool, connection);
-#else
+
         memcached_pool_release(pool, connection);
-#endif
         connection = NULL;//For safety (from changing this destructor) as not implicit in either the above or below.
         memcached_st *memc = memcached_pool_destroy(pool);
         if (memc)

roxie/ccd/CMakeLists.txt

@@ -36,6 +36,7 @@ set (   SRCS
         ccdkey.cpp 
         ccdlistener.cpp
         ccdmain.cpp
+        ccdprotocol.cpp
         ccdquery.cpp
         ccdqueue.cpp
         ccdsnmp.cpp 
@@ -50,10 +51,12 @@ set (   SRCS
         ccdfile.hpp
         ccdkey.hpp
         ccdlistener.hpp
+        ccdprotocol.hpp
         ccdquery.hpp
         ccdqueue.ipp
         ccdsnmp.hpp
         ccdstate.hpp 
+        hpccprotocol.hpp
         
                 sourcedoc.xml
     )

roxie/ccd/ccdcontext.cpp

@@ -1123,8 +1123,6 @@ protected:
     const IRoxieContextLogger &logctx;
 
 protected:
-    CriticalSection resultsCrit;
-    IPointerArrayOf<FlushingStringBuffer> resultMap;
     bool exceptionLogged;
     bool aborted;
     CriticalSection abortLock; // NOTE: we don't bother to get lock when just reading to see whether to abort
@@ -2546,10 +2544,12 @@ public:
 class CRoxieServerContext : public CRoxieContextBase, implements IRoxieServerContext, implements IGlobalCodeContext
 {
     const IQueryFactory *serverQueryFactory;
+    IHpccProtocolResponse *protocol;
+    IHpccProtocolResultsWriter *results;
+    IHpccNativeProtocolResponse *nativeProtocol;
     CriticalSection daliUpdateCrit;
     StringAttr querySetName;
 
-    TextMarkupFormat mlFmt;
     bool isRaw;
     bool sendHeartBeats;
     unsigned lastSocketCheckTime;
@@ -2559,34 +2559,23 @@ protected:
     Owned<CRoxieWorkflowMachine> workflow;
     Owned<ITimeReporter> myTimer;
     mutable MapStringToMyClass<IResolvedFile> fileCache;
-    SafeSocket *client;
     StringArray clusterNames;
     int clusterWidth = -1;
 
     bool isBlocked;
-    bool isHttp;
+    bool isNative;
     bool trim;
 
     void doPostProcess()
     {
-        CriticalBlock b(resultsCrit); // Probably not needed
+        if (!protocol)
+            return;
+
         if (!isRaw && !isBlocked)
-        {
-            ForEachItemIn(seq, resultMap)
-            {
-                FlushingStringBuffer *result = resultMap.item(seq);
-                if (result)
-                    result->flush(true);
-            }
-        }
+            protocol->flush();
 
         if (probeQuery)
         {
-            FlushingStringBuffer response(client, isBlocked, MarkupFmt_XML, false, isHttp, *this);
-
-            // create output stream
-            response.startDataset("_Probe", NULL, (unsigned) -1);  // initialize it
-
             // loop through all of the graphs and create a _Probe to output each xgmml
             Owned<IPropertyTreeIterator> graphs = probeQuery->getElements("Graph");
             ForEach(*graphs)
@@ -2595,8 +2584,7 @@ protected:
 
                 StringBuffer xgmml;
                 _toXML(&graph, xgmml, 0);
-                response.append("\n");
-                response.append(xgmml.str());
+                protocol->appendProbeGraph(xgmml.str());
             }
         }
     }
@@ -2608,13 +2596,10 @@ protected:
 
     void init()
     {
-        client = NULL;
         totSlavesReplyLen = 0;
-        mlFmt = MarkupFmt_XML;
         isRaw = false;
         isBlocked = false;
-        isHttp = false;
-        trim = false;
+        isNative = true;
         sendHeartBeats = false;
 
         lastSocketCheckTime = startTime;
@@ -2628,7 +2613,7 @@ protected:
         wu->subscribe(SubscribeOptionAbort);
         addTimeStamp(wu, SSTglobal, NULL, StWhenQueryStarted);
         if (!context->getPropBool("@outputToSocket", false))
-            client = NULL;
+            protocol = NULL;
         updateSuppliedXmlParams(wu);
         SCMStringBuffer wuParams;
         if (workUnit->getXmlParams(wuParams, false).length())
@@ -2657,9 +2642,9 @@ protected:
 
     void initDebugMode(bool breakAtStart, const char *debugUID)
     {
-        if (!debugPermitted || !ownEP.port)
+        if (!debugPermitted || !ownEP.port || !nativeProtocol)
             throw MakeStringException(ROXIE_ACCESS_ERROR, "Debug queries are not permitted on this system");
-        debugContext.setown(new CRoxieServerDebugContext(this, logctx, factory->cloneQueryXGMML(), *client));
+        debugContext.setown(new CRoxieServerDebugContext(this, logctx, factory->cloneQueryXGMML(), *nativeProtocol->querySafeSocket()));
         debugContext->debugInitialize(debugUID, factory->queryQueryName(), breakAtStart);
         if (workUnit)
         {
@@ -2677,7 +2662,7 @@ public:
     IMPLEMENT_IINTERFACE;
 
     CRoxieServerContext(const IQueryFactory *_factory, const IRoxieContextLogger &_logctx)
-        : CRoxieContextBase(_factory, _logctx), serverQueryFactory(_factory)
+        : CRoxieContextBase(_factory, _logctx), serverQueryFactory(_factory), results(NULL)
     {
         init();
         rowManager->setMemoryLimit(options.memoryLimit);
@@ -2686,7 +2671,7 @@ public:
     }
 
     CRoxieServerContext(IConstWorkUnit *_workUnit, const IQueryFactory *_factory, const ContextLogger &_logctx)
-        : CRoxieContextBase(_factory, _logctx), serverQueryFactory(_factory)
+        : CRoxieContextBase(_factory, _logctx), serverQueryFactory(_factory), results(NULL)
     {
         init();
         workUnit.set(_workUnit);
@@ -2700,18 +2685,20 @@ public:
         startWorkUnit();
     }
 
-    CRoxieServerContext(IPropertyTree *_context, const IQueryFactory *_factory, SafeSocket &_client, TextMarkupFormat _mlFmt, bool _isRaw, bool _isBlocked, HttpHelper &httpHelper, bool _trim, const ContextLogger &_logctx, PTreeReaderOptions _xmlReadFlags, const char *_querySetName)
-        : CRoxieContextBase(_factory, _logctx), serverQueryFactory(_factory), querySetName(_querySetName)
+    CRoxieServerContext(IPropertyTree *_context, IHpccProtocolResponse *_protocol, const IQueryFactory *_factory, unsigned flags, const ContextLogger &_logctx, PTreeReaderOptions _xmlReadFlags, const char *_querySetName)
+        : CRoxieContextBase(_factory, _logctx), serverQueryFactory(_factory), querySetName(_querySetName), protocol(_protocol), results(NULL)
     {
         init();
+        if (protocol)
+        {
+            nativeProtocol = dynamic_cast<IHpccNativeProtocolResponse*>(protocol);
+            results = protocol->queryHpccResultsSection();
+        }
         context.set(_context);
         options.setFromContext(context);
-        client = &_client;
-        mlFmt = _mlFmt;
-        isRaw = _isRaw;
-        isBlocked = _isBlocked;
-        isHttp = httpHelper.isHttp();
-        trim = _trim;
+        isNative = (flags & HPCC_PROTOCOL_NATIVE);
+        isRaw = (flags & HPCC_PROTOCOL_NATIVE_RAW);
+        isBlocked = (flags & HPCC_PROTOCOL_BLOCKED);
         xmlStoredDatasetReadFlags = _xmlReadFlags;
         sendHeartBeats = enableHeartBeat && isRaw && isBlocked && options.priority==0;
 
@@ -2738,7 +2725,7 @@ public:
         // MORE some of these might be appropriate in wu case too?
         rowManager->setActivityTracking(context->getPropBool("_TraceMemory", false));
         rowManager->setMemoryLimit(options.memoryLimit);
-        authToken.append(httpHelper.queryAuthToken());
+
         workflow.setown(_factory->createWorkflowMachine(workUnit, false, logctx));
     }
 
@@ -2805,14 +2792,14 @@ public:
                 }
             }
         }
-        if (client)
+        if (protocol)
         {
             if (socketCheckInterval)
             {
                 if (ticksNow - lastSocketCheckTime > socketCheckInterval)
                 {
                     CriticalBlock b(abortLock);
-                    if (!client->checkConnection())
+                    if (!protocol->checkConnection())
                         throw MakeStringException(ROXIE_CLIENT_CLOSED, "Client socket closed");
                     lastSocketCheckTime = ticksNow;
                 }
@@ -2823,7 +2810,7 @@ public:
                 if (hb > 30000)
                 {
                     lastHeartBeat = msTick();
-                    client->sendHeartBeat(*this);
+                    protocol->sendHeartBeat();
                 }
             }
         }
@@ -2945,31 +2932,15 @@ public:
         return this;
     }
 
-    // interface ICodeContext
-    virtual FlushingStringBuffer *queryResult(unsigned sequence)
-    {
-        if (!client && workUnit)
-            return NULL;    // when outputting to workunit only, don't output anything to stdout
-        CriticalBlock procedure(resultsCrit);
-        while (!resultMap.isItem(sequence))
-            resultMap.append(NULL);
-        FlushingStringBuffer *result = resultMap.item(sequence);
-        if (!result)
-        {
-            result = new FlushingStringBuffer(client, isBlocked, mlFmt, isRaw, isHttp, *this);
-            result->isSoap = isHttp;
-            result->trim = trim;
-            result->queryName.set(context->queryName());
-            resultMap.replace(result, sequence);
-        }
-        return result;
-    }
-
     virtual char *getDaliServers()
     {
         //MORE: Should this now be implemented using IRoxieDaliHelper?
         throwUnexpected();
     }
+    virtual IHpccProtocolResponse *queryProtocol()
+    {
+        return protocol;
+    }
 
     virtual void setResultBool(const char *name, unsigned sequence, bool value)
     {
@@ -2978,18 +2949,11 @@ public:
             CriticalBlock b(contextCrit);
             useContext(sequence).setPropBool(name, value);
         }
-        else
+        else if (results)
         {
-            FlushingStringBuffer *r = queryResult(sequence);
-            if (r)
-            {
-                r->startScalar(name, sequence);
-                if (isRaw)
-                    r->append(sizeof(value), (char *)&value);
-                else
-                    r->append(value ? "true" : "false");
-            }
+            results->setResultBool(name, sequence, value);
         }
+
         if (workUnit)
         {
             try
@@ -3024,14 +2988,9 @@ public:
             IPropertyTree &ctx = useContext(sequence);
             ctx.setProp(name, s.str());
         }
-        else
+        else if (results)
         {
-            FlushingStringBuffer *r = queryResult(sequence);
-            if (r)
-            {
-                r->startScalar(name, sequence);
-                r->encodeData(data, len);
-            }
+            results->setResultData(name, sequence, len, data);
         }
         if (workUnit)
         {
@@ -3125,17 +3084,9 @@ public:
             ctx.setPropBin(name, len, data);
             ctx.queryPropTree(name)->setProp("@format", "raw");
         }
-        else
+        else if (results)
         {
-            FlushingStringBuffer *r = queryResult(sequence);
-            if (r)
-            {
-                r->startScalar(name, sequence);
-                if (isRaw)
-                    r->append(len, (const char *) data);
-                else
-                    UNIMPLEMENTED;
-            }
+            results->setResultRaw(name, sequence, len, data);
         }
 
         if (workUnit)
@@ -3169,42 +3120,9 @@ public:
             ctx.queryPropTree(name)->setProp("@format", "raw");
             ctx.queryPropTree(name)->setPropBool("@isAll", isAll);
         }
-        else
+        else if (results)
         {
-            FlushingStringBuffer *r = queryResult(sequence);
-            if (r)
-            {
-                r->startScalar(name, sequence);
-                if (isRaw)
-                    r->append(len, (char *)data);
-                else if (mlFmt==MarkupFmt_XML)
-                {
-                    assertex(transformer);
-                    CommonXmlWriter writer(getXmlFlags()|XWFnoindent, 0);
-                    transformer->toXML(isAll, len, (byte *)data, writer);
-                    r->append(writer.str());
-                }
-                else if (mlFmt==MarkupFmt_JSON)
-                {
-                    assertex(transformer);
-                    CommonJsonWriter writer(getXmlFlags()|XWFnoindent, 0);
-                    transformer->toXML(isAll, len, (byte *)data, writer);
-                    r->append(writer.str());
-                }
-                else
-                {
-                    assertex(transformer);
-                    r->append('[');
-                    if (isAll)
-                        r->appendf("*]");
-                    else
-                    {
-                        SimpleOutputWriter x;
-                        transformer->toXML(isAll, len, (const byte *) data, x);
-                        r->appendf("%s]", x.str());
-                    }
-                }
-            }
+            results->setResultSet(name, sequence, isAll, len, data, transformer);
         }
 
         if (workUnit)
@@ -3243,24 +3161,9 @@ public:
             CriticalBlock b(contextCrit);
             useContext(sequence).setPropBin(name, m.length(), m.toByteArray());
         }
-        else
+        else if (results)
         {
-            FlushingStringBuffer *r = queryResult(sequence);
-            if (r)
-            {
-                r->startScalar(name, sequence);
-                if (isRaw)
-                    r->append(len, (char *)val);
-                else
-                {
-                    StringBuffer s;
-                    if (isSigned)
-                        outputXmlDecimal(val, len, precision, NULL, s);
-                    else
-                        outputXmlUDecimal(val, len, precision, NULL, s);
-                    r->append(s);
-                }
-            }
+            results->setResultDecimal(name, sequence, len, precision, isSigned, val);
         }
         if (workUnit)
         {
@@ -3290,21 +3193,10 @@ public:
             CriticalBlock b(contextCrit);
             useContext(sequence).setPropInt64(name, value);
         }
-        else
+        else if (results)
         {
-            FlushingStringBuffer *r = queryResult(sequence);
-            if (r)
-            {
-                if (isRaw)
-                {
-                    r->startScalar(name, sequence);
-                    r->append(sizeof(value), (char *)&value);
-                }
-                else
-                    r->setScalarInt(name, sequence, value, size);
-            }
+            results->setResultInt(name, sequence, value, size);
         }
-
         if (workUnit)
         {
             try
@@ -3334,19 +3226,9 @@ public:
             CriticalBlock b(contextCrit);
             useContext(sequence).setPropInt64(name, value);
         }
-        else
+        else if (results)
         {
-            FlushingStringBuffer *r = queryResult(sequence);
-            if (r)
-            {
-                if (isRaw)
-                {
-                    r->startScalar(name, sequence);
-                    r->append(sizeof(value), (char *)&value);
-                }
-                else
-                    r->setScalarUInt(name, sequence, value, size);
-            }
+            results->setResultUInt(name, sequence, value, size);
         }
 
         if (workUnit)
@@ -3378,14 +3260,9 @@ public:
             CriticalBlock b(contextCrit);
             useContext(sequence).setPropBin(name, sizeof(value), &value);
         }
-        else
+        else if (results)
         {
-            FlushingStringBuffer *r = queryResult(sequence);
-            if (r)
-            {
-                r->startScalar(name, sequence);
-                r->append(value);
-            }
+            results->setResultReal(name, sequence, value);
         }
         if (workUnit)
         {
@@ -3415,23 +3292,10 @@ public:
             CriticalBlock b(contextCrit);
             useContext(sequence).setPropBin(name, len, str);
         }
-        else
+        else if (results)
         {
-            FlushingStringBuffer *r = queryResult(sequence);
-            if (r)
-            {
-                r->startScalar(name, sequence);
-                if (r->isRaw)
-                {
-                    r->append(len, str);
-                }
-                else
-                {
-                    r->encodeString(str, len);
-                }
-            }
+            results->setResultString(name, sequence, len, str);
         }
-
         if (workUnit)
         {
             try
@@ -3463,26 +3327,10 @@ public:
             CriticalBlock b(contextCrit);
             useContext(sequence).setPropBin(name, bufflen, buff.getstr());
         }
-        else
+        else if (results)
         {
-            FlushingStringBuffer *r = queryResult(sequence);
-            if (r)
-            {
-                r->startScalar(name, sequence);
-                if (r->isRaw)
-                {
-                    r->append(len*2, (const char *) str);
-                }
-                else
-                {
-                    rtlDataAttr buff;
-                    unsigned bufflen = 0;
-                    rtlUnicodeToCodepageX(bufflen, buff.refstr(), len, str, "utf-8");
-                    r->encodeString(buff.getstr(), bufflen, true); // output as UTF-8
-                }
-            }
+            results->setResultUnicode(name, sequence, len, str);
         }
-
         if (workUnit)
         {
             try
@@ -3519,7 +3367,9 @@ public:
 
     virtual IWorkUnitRowReader *createStreamedRawRowReader(IEngineRowAllocator *rowAllocator, bool isGrouped, const char *id)
     {
-        return new StreamedRawDataReader(this, rowAllocator, isGrouped, logctx, *client, id);
+        if (!nativeProtocol)
+            throwUnexpected();
+        return new StreamedRawDataReader(this, rowAllocator, isGrouped, logctx, *nativeProtocol->querySafeSocket(), id);
     }
 
     virtual void printResults(IXmlWriter *output, const char *name, unsigned sequence)
@@ -3885,10 +3735,15 @@ public:
             superfileTransaction.setown(createDistributedFileTransaction(queryUserDescriptor(), queryCodeContext()));
         return superfileTransaction.get();
     }
-    virtual void flush(unsigned seqNo) { throwUnexpected(); }
+    virtual void finalize(unsigned seqNo)
+    {
+        if (!protocol)
+            throwUnexpected();
+        protocol->finalize(seqNo);
+    }
     virtual unsigned getPriority() const { return options.priority; }
     virtual IConstWorkUnit *queryWorkUnit() const { return workUnit; }
-    virtual bool outputResultsToSocket() const { return client != NULL; }
+    virtual bool outputResultsToSocket() const { return protocol != NULL; }
 
     virtual void selectCluster(const char * newCluster)
     {
@@ -3927,67 +3782,8 @@ private:
     StringAttr queryName;
 
 public:
-    CSoapRoxieServerContext(IPropertyTree *_context, const IQueryFactory *_factory, SafeSocket &_client, HttpHelper &httpHelper, const ContextLogger &_logctx, PTreeReaderOptions xmlReadFlags, const char *_querySetName)
-        : CRoxieServerContext(_context, _factory, _client, MarkupFmt_XML, false, false, httpHelper, httpHelper.getTrim(), _logctx, xmlReadFlags, _querySetName)
-    {
-        queryName.set(_context->queryName());
-    }
-
-    virtual void process()
-    {
-        EclProcessFactory pf = (EclProcessFactory) factory->queryDll()->getEntry("createProcess");
-        Owned<IEclProcess> p = pf();
-        if (workflow)
-            workflow->perform(this, p);
-        else
-            p->perform(this, 0);
-    }
-
-    virtual void flush(unsigned seqNo)
-    {
-        CriticalBlock b(resultsCrit);
-        CriticalBlock b1(client->queryCrit());
-
-        StringBuffer responseHead, responseTail;
-        responseHead.append("<").append(queryName).append("Response");
-        responseHead.append(" sequence=\"").append(seqNo).append("\"");
-        responseHead.append(" xmlns=\"urn:hpccsystems:ecl:").appendLower(queryName.length(), queryName.str()).append("\">");
-        responseHead.append("<Results><Result>");
-        unsigned len = responseHead.length();
-        client->write(responseHead.detach(), len, true);
-
-        ForEachItemIn(seq, resultMap)
-        {
-            FlushingStringBuffer *result = resultMap.item(seq);
-            if (result)
-            {
-                result->flush(true);
-                for(;;)
-                {
-                    size32_t length;
-                    void *payload = result->getPayload(length);
-                    if (!length)
-                        break;
-                    client->write(payload, length, true);
-                }
-            }
-        }
-
-        responseTail.append("</Result></Results>");
-        responseTail.append("</").append(queryName).append("Response>");
-        len = responseTail.length();
-        client->write(responseTail.detach(), len, true);
-    }
-};
-
-class CJsonRoxieServerContext : public CRoxieServerContext
-{
-private:
-    StringAttr queryName;
-
-public:
-    CJsonRoxieServerContext(IPropertyTree *_context, const IQueryFactory *_factory, SafeSocket &_client, HttpHelper &httpHelper, const ContextLogger &_logctx, PTreeReaderOptions xmlReadFlags, const char *_querySetName)
-        : CRoxieServerContext(_context, _factory, _client, MarkupFmt_JSON, false, false, httpHelper, httpHelper.getTrim(), _logctx, xmlReadFlags, _querySetName)
+    CSoapRoxieServerContext(IPropertyTree *_context, IHpccProtocolResponse *_protocol, const IQueryFactory *_factory, unsigned flags, const ContextLogger &_logctx, PTreeReaderOptions xmlReadFlags, const char *_querySetName)
+        : CRoxieServerContext(_context, _protocol, _factory, flags, _logctx, xmlReadFlags, _querySetName)
     {
         queryName.set(_context->queryName());
     }
@@ -4001,80 +3797,13 @@ public:
         else
             p->perform(this, 0);
     }
-
-    virtual void flush(unsigned seqNo)
-    {
-        CriticalBlock b(resultsCrit);
-        CriticalBlock b1(client->queryCrit());
-
-        StringBuffer responseHead, responseTail;
-        appendfJSONName(responseHead, "%sResponse", queryName.get()).append(" {");
-        appendJSONValue(responseHead, "sequence", seqNo);
-        appendJSONName(responseHead, "Results").append(" {");
-
-        unsigned len = responseHead.length();
-        client->write(responseHead.detach(), len, true);
-
-        bool needDelimiter = false;
-        ForEachItemIn(seq, resultMap)
-        {
-            FlushingStringBuffer *result = resultMap.item(seq);
-            if (result)
-            {
-                result->flush(true);
-                for(;;)
-                {
-                    size32_t length;
-                    void *payload = result->getPayload(length);
-                    if (!length)
-                        break;
-                    if (needDelimiter)
-                    {
-                        StringAttr s(","); //write() will take ownership of buffer
-                        size32_t len = s.length();
-                        client->write((void *)s.detach(), len, true);
-                        needDelimiter=false;
-                    }
-                    client->write(payload, length, true);
-                }
-                needDelimiter=true;
-            }
-        }
-
-        responseTail.append("}}");
-        len = responseTail.length();
-        client->write(responseTail.detach(), len, true);
-    }
-
-    virtual FlushingStringBuffer *queryResult(unsigned sequence)
-    {
-        if (!client && workUnit)
-            return NULL;    // when outputting to workunit only, don't output anything to stdout
-        CriticalBlock procedure(resultsCrit);
-        while (!resultMap.isItem(sequence))
-            resultMap.append(NULL);
-        FlushingStringBuffer *result = resultMap.item(sequence);
-        if (!result)
-        {
-            result = new FlushingJsonBuffer(client, isBlocked, isHttp, *this);
-            result->trim = trim;
-            result->queryName.set(context->queryName());
-            resultMap.replace(result, sequence);
-        }
-        return result;
-    }
 };
 
-IRoxieServerContext *createRoxieServerContext(IPropertyTree *context, const IQueryFactory *factory, SafeSocket &client, bool isXml, bool isRaw, bool isBlocked, HttpHelper &httpHelper, bool trim, const ContextLogger &_logctx, PTreeReaderOptions readFlags, const char *querySetName)
+IRoxieServerContext *createRoxieServerContext(IPropertyTree *context, IHpccProtocolResponse *protocol, const IQueryFactory *factory, unsigned flags, const ContextLogger &_logctx, PTreeReaderOptions readFlags, const char *querySetName)
 {
-    if (httpHelper.isHttp())
-    {
-        if (httpHelper.queryContentFormat()==MarkupFmt_JSON)
-            return new CJsonRoxieServerContext(context, factory, client, httpHelper, _logctx, readFlags, querySetName);
-        return new CSoapRoxieServerContext(context, factory, client, httpHelper, _logctx, readFlags, querySetName);
-    }
-    else
-        return new CRoxieServerContext(context, factory, client, isXml ? MarkupFmt_XML : MarkupFmt_Unknown, isRaw, isBlocked, httpHelper, trim, _logctx, readFlags, querySetName);
+    if (flags & HPCC_PROTOCOL_NATIVE)
+        return new CRoxieServerContext(context, protocol, factory, flags, _logctx, readFlags, querySetName);
+    return new CSoapRoxieServerContext(context, protocol, factory, flags, _logctx, readFlags, querySetName);
 }
 
 IRoxieServerContext *createOnceServerContext(const IQueryFactory *factory, const IRoxieContextLogger &_logctx)

roxie/ccd/ccdcontext.hpp

@@ -76,7 +76,6 @@ interface IRoxieSlaveContext : extends IRoxieContextLogger
 interface IRoxieServerContext : extends IInterface
 {
     virtual IGlobalCodeContext *queryGlobalCodeContext() = 0;
-    virtual FlushingStringBuffer *queryResult(unsigned sequence) = 0;
     virtual void setResultXml(const char *name, unsigned sequence, const char *xml) = 0;
     virtual void appendResultDeserialized(const char *name, unsigned sequence, size32_t count, byte **data, bool extend, IOutputMetaData *meta) = 0;
     virtual void appendResultRawContext(const char *name, unsigned sequence, int len, const void * data, int numRows, bool extend, bool saveInContext) = 0;
@@ -84,7 +83,7 @@ interface IRoxieServerContext : extends IInterface
 
     virtual void process() = 0;
     virtual void done(bool failed) = 0;
-    virtual void flush(unsigned seqNo) = 0;
+    virtual void finalize(unsigned seqNo) = 0;
     virtual unsigned getMemoryUsage() = 0;
     virtual unsigned getSlavesReplyLen() = 0;
 
@@ -94,6 +93,7 @@ interface IRoxieServerContext : extends IInterface
 
     virtual IRoxieDaliHelper *checkDaliConnection() = 0;
     virtual const IProperties *queryXmlns(unsigned seqNo) = 0;
+    virtual IHpccProtocolResponse *queryProtocol() = 0;
 };
 
 interface IDeserializedResultStore : public IInterface
@@ -109,7 +109,7 @@ class CRoxieWorkflowMachine;
 
 extern IDeserializedResultStore *createDeserializedResultStore();
 extern IRoxieSlaveContext *createSlaveContext(const IQueryFactory *factory, const SlaveContextLogger &logctx, IRoxieQueryPacket *packet, bool hasRemoteChildren);
-extern IRoxieServerContext *createRoxieServerContext(IPropertyTree *context, const IQueryFactory *factory, SafeSocket &client, bool isXml, bool isRaw, bool isBlocked, HttpHelper &httpHelper, bool trim, const ContextLogger &logctx, PTreeReaderOptions xmlReadFlags, const char *querySetName);
+extern IRoxieServerContext *createRoxieServerContext(IPropertyTree *context, IHpccProtocolResponse *protocol, const IQueryFactory *factory, unsigned flags, const ContextLogger &logctx, PTreeReaderOptions xmlReadFlags, const char *querySetName);
 extern IRoxieServerContext *createOnceServerContext(const IQueryFactory *factory, const IRoxieContextLogger &_logctx);
 extern IRoxieServerContext *createWorkUnitServerContext(IConstWorkUnit *wu, const IQueryFactory *factory, const ContextLogger &logctx);
 extern CRoxieWorkflowMachine *createRoxieWorkflowMachine(IPropertyTree *_workflowInfo, IConstWorkUnit *wu, bool doOnce, const IRoxieContextLogger &_logctx);

roxie/ccd/ccdlistener.hpp

@@ -19,25 +19,19 @@
 #define _CCDLISTENER_INCL
 
 #include <jlib.hpp>
+#include "ccdprotocol.hpp"
 
-interface IRoxieListener : extends IInterface
-{
-    virtual void start() = 0;
-    virtual bool stop(unsigned timeout) = 0;
-    virtual void stopListening() = 0;
-    virtual void disconnectQueue() = 0;
-    virtual void addAccess(bool allow, bool allowBlind, const char *ip, const char *mask, const char *query, const char *errMsg, int errCode) = 0;
-    virtual unsigned queryPort() const = 0;
-    virtual const SocketEndpoint &queryEndpoint() const = 0;
-    virtual bool suspend(bool suspendIt) = 0;
-
-    virtual void runOnce(const char *query) = 0;
-};
-
-extern IRoxieListener *createRoxieSocketListener(unsigned port, unsigned poolSize, unsigned listenQueue, bool suspended);
-extern IRoxieListener *createRoxieWorkUnitListener(unsigned poolSize, bool suspended);
+extern IHpccProtocolMsgSink *createRoxieProtocolMsgSink(const IpAddress &ip, unsigned short port, unsigned poolSize, bool suspended);
+
+extern IHpccProtocolListener *createRoxieWorkUnitListener(unsigned poolSize, bool suspended);
 extern bool suspendRoxieListener(unsigned port, bool suspended);
-extern IArrayOf<IRoxieListener> socketListeners;
+
+extern MapStringToMyClass<SharedObject> protocolDlls;
+extern MapStringToMyClass<IHpccProtocolPlugin> protocolPlugins;
+extern IArrayOf<IHpccProtocolListener> socketListeners;
+
+extern IHpccProtocolPlugin *ensureProtocolPlugin(IHpccProtocolPluginContext &protocolCtx, const char *soname);
+
 extern void disconnectRoxieQueues();
 extern void updateAffinity(unsigned __int64 affinity);
 

roxie/ccd/ccdmain.cpp

@@ -409,6 +409,24 @@ void saveTopology()
     }
 }
 
+class CHpccProtocolPluginCtx : public CInterface, implements IHpccProtocolPluginContext
+{
+public:
+    IMPLEMENT_IINTERFACE;
+    virtual int ctxGetPropInt(const char *propName, int defaultValue) const
+    {
+        return topology->getPropInt(propName, defaultValue);
+    }
+    virtual bool ctxGetPropBool(const char *propName, bool defaultValue) const
+    {
+        return topology->getPropBool(propName, defaultValue);
+    }
+    virtual const char *ctxQueryProp(const char *propName) const
+    {
+        return topology->queryProp(propName);
+    }
+};
+
 int STARTQUERY_API start_query(int argc, const char *argv[])
 {
     EnableSEHtoExceptionMapping();
@@ -1008,9 +1026,11 @@ int STARTQUERY_API start_query(int argc, const char *argv[])
 #endif
         EnableSEHtoExceptionMapping();
         setSEHtoExceptionHandler(&abortHandler);
+        Owned<IHpccProtocolPluginContext> protocolCtx = new CHpccProtocolPluginCtx();
         if (runOnce)
         {
-            Owned <IRoxieListener> roxieServer = createRoxieSocketListener(0, 1, 0, false);
+            Owned<IHpccProtocolPlugin> protocolPlugin = loadHpccProtocolPlugin(protocolCtx, NULL);
+            Owned<IHpccProtocolListener> roxieServer = protocolPlugin->createListener("runOnce", createRoxieProtocolMsgSink(getNodeAddress(myNodeIndex), 0, 1, false), 0, 0, NULL);
             try
             {
                 const char *format = globals->queryProp("format");
@@ -1046,18 +1066,26 @@ int STARTQUERY_API start_query(int argc, const char *argv[])
                 unsigned numThreads = roxieFarm.getPropInt("@numThreads", numServerThreads);
                 unsigned port = roxieFarm.getPropInt("@port", ROXIE_SERVER_PORT);
                 unsigned requestArrayThreads = roxieFarm.getPropInt("@requestArrayThreads", 5);
+                const IpAddress &ip = getNodeAddress(myNodeIndex);
                 if (!roxiePort)
                 {
                     roxiePort = port;
-                    ownEP.set(roxiePort, getNodeAddress(myNodeIndex));
+                    ownEP.set(roxiePort, ip);
                 }
                 bool suspended = roxieFarm.getPropBool("@suspended", false);
-                Owned <IRoxieListener> roxieServer;
+                Owned <IHpccProtocolListener> roxieServer;
                 if (port)
-                    roxieServer.setown(createRoxieSocketListener(port, numThreads, listenQueue, suspended));
+                {
+                    const char *protocol = roxieFarm.queryProp("@protocol");
+                    const char *soname =  roxieFarm.queryProp("@so");
+                    const char *config  = roxieFarm.queryProp("@config");
+                    Owned<IHpccProtocolPlugin> protocolPlugin = ensureProtocolPlugin(*protocolCtx, soname);
+                    roxieServer.setown(protocolPlugin->createListener(protocol ? protocol : "native", createRoxieProtocolMsgSink(ip, port, numThreads, suspended), port, listenQueue, config));
+                }
                 else
                     roxieServer.setown(createRoxieWorkUnitListener(numThreads, suspended));
 
+                IHpccProtocolMsgSink *sink = roxieServer->queryMsgSink();
                 const char *aclName = roxieFarm.queryProp("@aclName");
                 if (aclName && *aclName)
                 {
@@ -1069,7 +1097,7 @@ int STARTQUERY_API start_query(int argc, const char *argv[])
                         IPropertyTree &access = accesses->query();
                         try
                         {
-                            roxieServer->addAccess(access.getPropBool("@allow", true), access.getPropBool("@allowBlind", true), access.queryProp("@ip"), access.queryProp("@mask"), access.queryProp("@query"), access.queryProp("@error"), access.getPropInt("@errorCode"));
+                            sink->addAccess(access.getPropBool("@allow", true), access.getPropBool("@allowBlind", true), access.queryProp("@ip"), access.queryProp("@mask"), access.queryProp("@query"), access.queryProp("@error"), access.getPropInt("@errorCode"));
                         }
                         catch (IException *E)
                         {

roxie/ccd/ccdprotocol.hpp

@@ -0,0 +1,46 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2016 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#ifndef _CCDPROTOCOL_INCL
+#define _CCDPROTOCOL_INCL
+
+#include "hpccprotocol.hpp"
+
+#define HPCC_PROTOCOL_NATIVE           0x10000
+#define HPCC_PROTOCOL_NATIVE_RAW       0x20000
+#define HPCC_PROTOCOL_NATIVE_XML       0x40000
+#define HPCC_PROTOCOL_NATIVE_ASCII     0x80000
+
+interface IHpccNativeProtocolMsgSink : extends IHpccProtocolMsgSink
+{
+    virtual void onControlMsg(IHpccProtocolMsgContext *msgctx, IPropertyTree *msg, IHpccProtocolResponse *protocol) = 0;
+    virtual void onDebugMsg(IHpccProtocolMsgContext *msgctx, const char *uid, IPropertyTree *msg, IXmlWriter &out) = 0;
+};
+
+interface IHpccNativeProtocolResultsWriter : extends IHpccProtocolResultsWriter
+{
+    virtual void appendRawRow(unsigned sequence, unsigned len, const char *data) = 0;
+    virtual void appendSimpleRow(unsigned sequence, const char *str) = 0;
+    virtual void appendRaw(unsigned sequence, unsigned len, const char *data) = 0;
+};
+
+interface IHpccNativeProtocolResponse : extends IHpccProtocolResponse
+{
+    virtual SafeSocket *querySafeSocket() = 0; //still passed to debug context, and row streaming, for now.. tbd get rid of this
+};
+
+#endif

roxie/ccd/ccdquery.cpp

@@ -1487,7 +1487,7 @@ public:
         throwUnexpected();   // only implemented in derived slave class
     }
 
-    virtual IRoxieServerContext *createContext(IPropertyTree *xml, SafeSocket &client, TextMarkupFormat mlFmt, bool isRaw, bool isBlocked, HttpHelper &httpHelper, bool trim, const ContextLogger &_logctx, PTreeReaderOptions xmlReadFlags, const char *querySetName) const
+    virtual IRoxieServerContext *createContext(IPropertyTree *xml, IHpccProtocolResponse *protocol, unsigned flags, const ContextLogger &_logctx, PTreeReaderOptions xmlReadFlags, const char *querySetName) const
     {
         throwUnexpected();   // only implemented in derived server class
     }
@@ -1611,10 +1611,10 @@ public:
         return activities;
     }
 
-    virtual IRoxieServerContext *createContext(IPropertyTree *context, SafeSocket &client, TextMarkupFormat mlFmt, bool isRaw, bool isBlocked, HttpHelper &httpHelper, bool trim, const ContextLogger &_logctx, PTreeReaderOptions _xmlReadFlags, const char *_querySetName) const
+    virtual IRoxieServerContext *createContext(IPropertyTree *context, IHpccProtocolResponse *protocol, unsigned flags, const ContextLogger &_logctx, PTreeReaderOptions _xmlReadFlags, const char *_querySetName) const
     {
         checkSuspended();
-        return createRoxieServerContext(context, this, client, mlFmt==MarkupFmt_XML, isRaw, isBlocked, httpHelper, trim, _logctx, _xmlReadFlags, _querySetName);
+        return createRoxieServerContext(context, protocol, this, flags, _logctx, _xmlReadFlags, _querySetName);
     }
 
     virtual IRoxieServerContext *createContext(IConstWorkUnit *wu, const ContextLogger &_logctx) const

roxie/ccd/ccdquery.hpp

@@ -22,6 +22,7 @@
 #include "ccdserver.hpp"
 #include "ccdkey.hpp"
 #include "ccdfile.hpp"
+#include "ccdprotocol.hpp"
 #include "jhtree.hpp"
 #include "jisem.hpp"
 #include "dllserver.hpp"
@@ -165,7 +166,7 @@ interface IQueryFactory : extends IInterface
     virtual CRoxieWorkflowMachine *createWorkflowMachine(IConstWorkUnit *wu, bool isOnce, const IRoxieContextLogger &logctx) const = 0;
     virtual char *getEnv(const char *name, const char *defaultValue) const = 0;
 
-    virtual IRoxieServerContext *createContext(IPropertyTree *xml, SafeSocket &client, TextMarkupFormat mlFmt, bool isRaw, bool isBlocked, HttpHelper &httpHelper, bool trim, const ContextLogger &_logctx, PTreeReaderOptions xmlReadFlags, const char *querySetName) const = 0;
+    virtual IRoxieServerContext *createContext(IPropertyTree *xml, IHpccProtocolResponse *protocol, unsigned flags, const ContextLogger &_logctx, PTreeReaderOptions xmlReadFlags, const char *querySetName) const = 0;
     virtual IRoxieServerContext *createContext(IConstWorkUnit *wu, const ContextLogger &_logctx) const = 0;
     virtual void noteQuery(time_t startTime, bool failed, unsigned elapsed, unsigned memused, unsigned slavesReplyLen, unsigned bytesOut) = 0;
     virtual IPropertyTree *getQueryStats(time_t from, time_t to) = 0;

roxie/ccd/ccdserver.cpp

@@ -481,7 +481,7 @@ public:
         if (_processed)
         {
             CriticalBlock b(statsCrit);
-            processed += processed;
+            processed += _processed;
         }
     }
 
@@ -20356,29 +20356,25 @@ public:
             storedName = "Dataset";
 
         MemoryBuffer result;
-        FlushingStringBuffer *response = NULL;
         bool saveInContext = (int) sequence < 0 || isReread;
         if (!meta.queryOriginal()) // this is a bit of a hack - don't know why no meta on an output....
             meta.set(input->queryOutputMeta());
         Owned<IOutputRowSerializer> rowSerializer;
-        Owned<IXmlWriter> writer;
+        Owned<IXmlWriter> xmlwriter;
+        bool appendRaw = false;
+
+        IHpccProtocolResultsWriter *results = NULL;
+        IHpccNativeProtocolResultsWriter *nativeResults = NULL;
+        IHpccProtocolResponse *protocol = serverContext->queryProtocol();
+        unsigned protocolFlags = (protocol) ? protocol->getFlags() : 0;
         if ((int) sequence >= 0)
         {
-            response = serverContext->queryResult(sequence);
-            if (response)
+            results = (protocol) ? protocol->queryHpccResultsSection() : NULL;
+            if (results)
             {
-                const IProperties *xmlns = serverContext->queryXmlns(sequence);
-                response->startDataset("Dataset", helper.queryName(), sequence, (helper.getFlags() & POFextend) != 0, xmlns);
-                if (response->mlFmt==MarkupFmt_XML || response->mlFmt==MarkupFmt_JSON)
-                {
-                    unsigned int writeFlags = serverContext->getXmlFlags();
-                    if (response->mlFmt==MarkupFmt_JSON)
-                        writeFlags |= XWFnoindent;
-                    writer.setown(createIXmlWriterExt(writeFlags, 1, response, (response->mlFmt==MarkupFmt_JSON) ? WTJSON : WTStandard));
-                    writer->outputBeginArray(DEFAULTXMLROWTAG);
-                }
+                nativeResults = dynamic_cast<IHpccNativeProtocolResultsWriter*>(results);
+                xmlwriter.setown(results->addDataset(helper.queryName(), sequence, "Dataset", appendRaw, serverContext->getXmlFlags(), (helper.getFlags() & POFextend) != 0, serverContext->queryXmlns(sequence)));  //xmlwriter only returned if needed
             }
-
         }
         size32_t outputLimitBytes = 0;
         IConstWorkUnit *workunit = serverContext->queryWorkUnit();
@@ -20397,7 +20393,7 @@ public:
             assertex(outputLimit<=0x1000); // 32bit limit because MemoryBuffer/CMessageBuffers involved etc.
             outputLimitBytes = outputLimit * 0x100000;
         }
-        if (workunit != NULL || (response && response->isRaw))
+        if (workunit != NULL || (results && protocol->getFlags() & HPCC_PROTOCOL_NATIVE_RAW))
         {
             createRowAllocator();
             rowSerializer.setown(rowAllocator->createDiskSerializer(ctx->queryCodeContext()));
@@ -20416,13 +20412,18 @@ public:
             {
                 if (workunit)
                     result.append(row == NULL);
-                if (response)
+                if (results)
                 {
-                    if (response->isRaw)
-                        response->append((char)(row == NULL));
-                    else
+                    if (protocolFlags & HPCC_PROTOCOL_NATIVE_RAW && nativeResults)
                     {
-                        response->append("<Row __GroupBreak__=\"1\"/>");        // sensible, but need to handle on input
+                        char val = (char)(row == NULL);
+                        nativeResults->appendRaw(sequence, 1, &val);
+                    }
+                    else if (xmlwriter)
+                    {
+                        xmlwriter->outputBeginNested("Row", false);
+                        xmlwriter->outputCString("1", "@__GroupBreak__");
+                        xmlwriter->outputEndNested("Row");
                     }
                 }
             }
@@ -20440,31 +20441,30 @@ public:
                 CThorDemoRowSerializer serializerTarget(result);
                 rowSerializer->serialize(serializerTarget, (const byte *) row);
             }
-            if (response)
+            if ((int) sequence >= 0)
             {
-                if (response->isRaw)
+                if (appendRaw && nativeResults)
                 {
                     // MORE - should be able to serialize straight to the response...
                     MemoryBuffer rowbuff;
                     CThorDemoRowSerializer serializerTarget(rowbuff);
                     rowSerializer->serialize(serializerTarget, (const byte *) row);
-                    response->append(rowbuff.length(), rowbuff.toByteArray());
+                    nativeResults->appendRawRow(sequence, rowbuff.length(), rowbuff.toByteArray());
                 }
-                else if (writer)
+                else if (xmlwriter)
                 {
-                    writer->outputBeginNested(DEFAULTXMLROWTAG, false);
-                    helper.serializeXml((byte *) row, *writer);
-                    writer->outputEndNested(DEFAULTXMLROWTAG);
+                    xmlwriter->outputBeginNested(DEFAULTXMLROWTAG, false);
+                    helper.serializeXml((byte *) row, *xmlwriter);
+                    xmlwriter->outputEndNested(DEFAULTXMLROWTAG);
+                    results->finalizeXmlRow(sequence);
                 }
-                else
+                else if (nativeResults)
                 {
                     SimpleOutputWriter x;
                     helper.serializeXml((byte *) row, x);
                     x.newline();
-                    response->append(x.str());
+                    nativeResults->appendSimpleRow(sequence, x.str());
                 }
-                response->incrementRowCount();
-                response->flush(false);
             }
             ReleaseRoxieRow(row);
             if (outputLimitBytes && result.length() > outputLimitBytes)
@@ -20480,8 +20480,8 @@ public:
                 throw MakeStringExceptionDirect(0, errMsg.str());
             }
         }
-        if (writer)
-            writer->outputEndArray(DEFAULTXMLROWTAG);
+        if (xmlwriter)
+            xmlwriter->outputEndArray(DEFAULTXMLROWTAG);
         if (saveInContext)
             serverContext->appendResultDeserialized(storedName, sequence, builder.getcount(), builder.linkrows(), (helper.getFlags() & POFextend) != 0, LINK(meta.queryOriginal()));
         if (workunit)

roxie/ccd/hpccprotocol.hpp

@@ -0,0 +1,145 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2016 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#ifndef _HPCCPROTOCOL_INCL
+#define _HPCCPROTOCOL_INCL
+
+#include <jlib.hpp>
+#include "roxiehelper.hpp"
+
+#define HPCC_PROTOCOL_BLOCKED          0x0001
+#define HPCC_PROTOCOL_TRIM             0x0002
+
+interface IHpccProtocolMsgContext : extends IInterface
+{
+    virtual bool initQuery(StringBuffer &target, const char *queryname) = 0;
+    virtual void noteQueryActive() = 0;
+    virtual unsigned getQueryPriority() = 0;
+    virtual IContextLogger *queryLogContext() = 0;
+    virtual bool checkSetBlind(bool blind) = 0;
+    virtual void verifyAllowDebug() = 0;
+    virtual bool logFullQueries() = 0;
+    virtual bool trapTooManyActiveQueries() = 0;
+    virtual bool getStripWhitespace() = 0;
+    virtual int getBindCores() = 0;
+    virtual void setTraceLevel(unsigned val) = 0;
+    virtual void setIntercept(bool val) = 0;
+    virtual bool getIntercept() = 0;
+    virtual void outputLogXML(IXmlStreamFlusher &out) = 0;
+};
+
+interface IHpccProtocolResultsWriter : extends IInterface
+{
+    virtual IXmlWriter *addDataset(const char *name, unsigned sequence, const char *elementName, bool &appendRawData, unsigned xmlFlags, bool _extend, const IProperties *xmlns) = 0;
+    virtual void finalizeXmlRow(unsigned sequence) = 0;
+
+    virtual void setResultBool(const char *name, unsigned sequence, bool value) = 0;
+    virtual void setResultData(const char *name, unsigned sequence, int len, const void * data) = 0;
+    virtual void setResultDecimal(const char * stepname, unsigned sequence, int len, int precision, bool isSigned, const void *val) = 0;
+    virtual void setResultInt(const char *name, unsigned sequence, __int64 value, unsigned size) = 0;
+    virtual void setResultRaw(const char *name, unsigned sequence, int len, const void * data) = 0;
+    virtual void setResultReal(const char * stepname, unsigned sequence, double value) = 0;
+    virtual void setResultSet(const char *name, unsigned sequence, bool isAll, size32_t len, const void * data, ISetToXmlTransformer * transformer) = 0;
+    virtual void setResultString(const char *name, unsigned sequence, int len, const char * str) = 0;
+    virtual void setResultUInt(const char *name, unsigned sequence, unsigned __int64 value, unsigned size) = 0;
+    virtual void setResultUnicode(const char *name, unsigned sequence, int len, UChar const * str) = 0;
+    virtual void setResultVarString(const char * name, unsigned sequence, const char * value) = 0;
+    virtual void setResultVarUnicode(const char * name, unsigned sequence, UChar const * value) = 0;
+};
+
+interface IHpccProtocolResponse : extends IInterface
+{
+    virtual unsigned getFlags() = 0;
+    virtual IHpccProtocolResultsWriter *queryHpccResultsSection() = 0;
+
+    virtual void appendContent(TextMarkupFormat mlFmt, const char *content, const char *name=NULL) = 0; //will be transformed
+    virtual IXmlWriter *writeAppendContent(const char *name = NULL) = 0;
+    virtual void appendProbeGraph(const char *xml) = 0;
+
+    virtual void finalize(unsigned seqNo) = 0;
+
+    virtual bool checkConnection() = 0;
+    virtual void sendHeartBeat() = 0;
+    virtual void flush() = 0;
+};
+
+interface IHpccProtocolMsgSink : extends IInterface
+{
+    virtual CriticalSection &getActiveCrit() = 0;
+    virtual bool getIsSuspended() = 0;
+    virtual unsigned getActiveThreadCount() = 0;
+    virtual unsigned getPoolSize() = 0;
+    virtual unsigned getMaxActiveThreads() = 0;
+    virtual void setMaxActiveThreads(unsigned val) = 0;
+    virtual void incActiveThreadCount() = 0;
+    virtual void decActiveThreadCount() = 0;
+    virtual bool suspend(bool suspendIt) = 0;
+
+    virtual void addAccess(bool allow, bool allowBlind, const char *ip, const char *mask, const char *query, const char *errorMsg, int errorCode) = 0;
+    virtual void checkAccess(IpAddress &peer, const char *queryName, const char *queryText, bool isBlind) = 0;
+    virtual void queryAccessInfo(StringBuffer &info) = 0;
+
+    virtual IHpccProtocolMsgContext *createMsgContext(time_t startTime) = 0;
+    virtual StringArray &getTargetNames(StringArray &targets) = 0;
+
+    virtual void noteQuery(IHpccProtocolMsgContext *msgctx, const char *peer, bool failed, unsigned bytesOut, unsigned elapsed, unsigned priority, unsigned memused, unsigned slavesReplyLen, bool continuationNeeded) = 0;
+    virtual void onQueryMsg(IHpccProtocolMsgContext *msgctx, IPropertyTree *msg, IHpccProtocolResponse *protocol, unsigned flags, PTreeReaderOptions readFlags, const char *target, unsigned idx, unsigned &memused, unsigned &slaveReplyLen) = 0;
+};
+
+interface IHpccProtocolListener : extends IInterface
+{
+    virtual IHpccProtocolMsgSink *queryMsgSink() = 0;
+
+    virtual unsigned queryPort() const = 0;
+    virtual const SocketEndpoint &queryEndpoint() const = 0;
+
+    virtual void start() = 0;
+    virtual bool stop(unsigned timeout) = 0;
+    virtual void stopListening() = 0;
+    virtual void disconnectQueue() = 0;
+    virtual bool suspend(bool suspendIt) = 0;
+
+    virtual void runOnce(const char *query) = 0;
+};
+
+interface IHpccProtocolPluginContext : extends IInterface
+{
+    virtual int ctxGetPropInt(const char *propName, int defaultValue) const = 0;
+    virtual bool ctxGetPropBool(const char *propName, bool defaultValue) const = 0;
+    virtual const char *ctxQueryProp(const char *propName) const = 0;
+};
+
+interface IActiveQueryLimiter : extends IInterface
+{
+    virtual bool isAccepted() = 0;
+};
+
+interface IActiveQueryLimiterFactory : extends IInterface
+{
+    virtual IActiveQueryLimiter *create(IHpccProtocolListener *listener) = 0;
+};
+
+interface IHpccProtocolPlugin : extends IInterface
+{
+    virtual IHpccProtocolListener *createListener(const char *protocol, IHpccProtocolMsgSink *sink, unsigned port, unsigned listenQueue, const char *config)=0;
+};
+
+extern IHpccProtocolPlugin *loadHpccProtocolPlugin(IHpccProtocolPluginContext *ctx, IActiveQueryLimiterFactory *limiterFactory);
+typedef IHpccProtocolPlugin *(HpccProtocolInstallFunction)(IHpccProtocolPluginContext *ctx, IActiveQueryLimiterFactory *limiterFactory);
+
+
+#endif

rtl/eclrtl/CMakeLists.txt

@@ -68,13 +68,15 @@ include_directories (
 
 ADD_DEFINITIONS( -D_USRDLL -DECLRTL_EXPORTS )
 
-if (CMAKE_COMPILER_IS_GNUCXX OR CMAKE_COMPILER_IS_CLANGXX)
-    set_source_files_properties(eclrtl.cpp PROPERTIES COMPILE_FLAGS -std=c++98)
-endif ()
+if (USE_BOOST_REGEX)
+    if (CMAKE_COMPILER_IS_GNUCXX OR CMAKE_COMPILER_IS_CLANGXX)
+        set_source_files_properties(eclrtl.cpp PROPERTIES COMPILE_FLAGS -std=c++98)
+    endif ()
 
-if (WIN32)
-else ()
-    ADD_DEFINITIONS( -DBOOST_DYN_LINK )
+    if (WIN32)
+    else ()
+        ADD_DEFINITIONS( -DBOOST_DYN_LINK )
+    endif ()
 endif ()
 
 HPCC_ADD_LIBRARY( eclrtl SHARED ${SRCS} )

rtl/eclrtl/eclrtl.cpp

@@ -16,8 +16,10 @@
 ############################################################################## */
 
 #include "limits.h"
-#ifdef _USE_BOOST_REGEX
+#if defined(_USE_BOOST_REGEX)
 #include "boost/regex.hpp" // must precede platform.h ; n.b. this uses a #pragma comment(lib, ...) to link the appropriate .lib in MSVC
+#elif defined(_USE_C11_REGEX)
+#include <regex>
 #endif
 #include "platform.h"
 #include <math.h>
@@ -4830,18 +4832,34 @@ int rtlNewSearchUtf8Table(unsigned count, unsigned elemlen, char * * table, unsi
 }
 
 //---------------------------------------------------------------------------
-#ifdef _USE_BOOST_REGEX
+#if defined(_USE_BOOST_REGEX) || defined(_USE_C11_REGEX)
+
+#if defined(_USE_BOOST_REGEX)
+using boost::regex;
+using boost::regex_search;
+using boost::regex_replace;
+using boost::regex_iterator;
+using boost::cmatch;
+using boost::match_results;
+#else
+using std::regex;
+using std::regex_search;
+using std::regex_replace;
+using std::regex_iterator;
+using std::cmatch;
+using std::match_results;
+#endif
 
 class CStrRegExprFindInstance : implements IStrRegExprFindInstance
 {
 private:
     bool            matched;
-    const boost::regex * regEx;
-    boost::cmatch   subs;
+    const regex * regEx;
+    cmatch   subs;
     char *          sample; //only required if findstr/findvstr will be called
 
 public:
-    CStrRegExprFindInstance(const boost::regex * _regEx, const char * _str, size32_t _from, size32_t _len, bool _keep)
+    CStrRegExprFindInstance(const regex * _regEx, const char * _str, size32_t _from, size32_t _len, bool _keep)
         : regEx(_regEx)
     {
         matched = false;
@@ -4853,16 +4871,20 @@ public:
                 sample = (char *)rtlMalloc(_len + 1);  //required for findstr
                 memcpy(sample, _str + _from, _len);
                 sample[_len] = (char)NULL;
-                matched = boost::regex_search(sample, subs, *regEx);
+                matched = regex_search(sample, subs, *regEx);
             }
             else
             {
-                matched = boost::regex_search(_str + _from, _str + _len, subs, *regEx);
+                matched = regex_search(_str + _from, _str + _len, subs, *regEx);
             }
         }
         catch (const std::runtime_error & e)
         {
+#if defined(_USE_BOOST_REGEX)
             throw MakeStringException(0, "Error in regex search: %s (regex: %s)", e.what(), regEx->str().c_str());
+#else
+            throw MakeStringException(0, "Error in regex search: %s", e.what());
+#endif
         }
 
     }
@@ -4914,19 +4936,30 @@ public:
 class CCompiledStrRegExpr : implements ICompiledStrRegExpr
 {
 private:
-    boost::regex    regEx;
+    regex    regEx;
 
 public:
     CCompiledStrRegExpr(const char * _regExp, bool _isCaseSensitive = false) 
     {
         try
         {
+#if defined(_USE_BOOST_REGEX)
             if (_isCaseSensitive)
-                regEx.assign(_regExp, boost::regbase::perl);
+                regEx.assign(_regExp, regex::perl);
             else
-                regEx.assign(_regExp, boost::regbase::perl | boost::regbase::icase);                
+                regEx.assign(_regExp, regex::perl | regex::icase);
+#else
+            if (_isCaseSensitive)
+                regEx.assign(_regExp, regex::ECMAScript);
+            else
+                regEx.assign(_regExp, regex::ECMAScript | regex::icase);
+#endif
         }
+#if defined(_USE_BOOST_REGEX)
         catch(const boost::bad_expression & e)
+#else
+        catch(const std::regex_error & e)
+#endif
         {
             StringBuffer msg;
             msg.append("Bad regular expression: ").append(e.what()).append(": ").append(_regExp);
@@ -4944,11 +4977,19 @@ public:
         try
         {
 //          tgt = boost::regex_merge(src, cre->regEx, fmt, boost::format_perl); //Algorithm regex_merge has been renamed regex_replace, existing code will continue to compile, but new code should use regex_replace instead.
-            tgt = boost::regex_replace(src, regEx, fmt, boost::format_perl);
+#if defined(_USE_BOOST_REGEX)
+            tgt = regex_replace(src, regEx, fmt, boost::format_perl);
+#else
+            tgt = regex_replace(src, regEx, fmt);
+#endif
         }
         catch(const std::runtime_error & e)
         {
+#if defined(_USE_BOOST_REGEX)
             throw MakeStringException(0, "Error in regex replace: %s (regex: %s)", e.what(), regEx.str().c_str());
+#else
+            throw MakeStringException(0, "Error in regex replace: %s", e.what());
+#endif
         }
         outlen = tgt.length();
         out = (char *)rtlMalloc(outlen);
@@ -4967,11 +5008,11 @@ public:
         size32_t outBytes = 0;
         const char * search_end = _search+_srcLen;
 
-        boost::regex_iterator<const char *> cur(_search, search_end, regEx);
-        boost::regex_iterator<const char *> end; // Default contructor creates an end of list marker
+        regex_iterator<const char *> cur(_search, search_end, regEx);
+        regex_iterator<const char *> end; // Default contructor creates an end of list marker
         for (; cur != end; ++cur)
         {
-            const boost::match_results<const char *> &match = *cur;
+            const match_results<const char *> &match = *cur;
             if (match[0].first==search_end) break;
 
             const size32_t lenBytes = match[0].second - match[0].first;
@@ -5197,7 +5238,7 @@ ECLRTL_API void rtlDestroyUStrRegExprFindInstance(IUStrRegExprFindInstance * fin
         delete (CUStrRegExprFindInstance*)findInst;
 }
 
-#else // _USE_BOOST_REGEX not set
+#else // _USE_BOOST_REGEX or _USE_C11_REGEX not set
 ECLRTL_API ICompiledStrRegExpr * rtlCreateCompiledStrRegExpr(const char * regExpr, bool isCaseSensitive)
 {
     UNIMPLEMENTED_X("Boost regex disabled");
@@ -5223,7 +5264,7 @@ ECLRTL_API void rtlDestroyCompiledUStrRegExpr(ICompiledUStrRegExpr * compiledExp
 ECLRTL_API void rtlDestroyUStrRegExprFindInstance(IUStrRegExprFindInstance * findInst)
 {
 }
-#endif
+#endif // _USE_BOOST_REGEX or _USE_C11_REGEX
 //---------------------------------------------------------------------------
 
 ECLRTL_API int rtlQueryLocalFailCode(IException * e)

rtl/eclrtl/eclrtl_imp.hpp

@@ -100,7 +100,7 @@ public:
 };
 
 template <unsigned maxsize>
-class ECLRTL_API rtlFixedRowBuilder
+class rtlFixedRowBuilder
 {
 public:
     inline void ensureAvailable(size32_t size) {}

system/CMakeLists.txt

@@ -17,6 +17,7 @@ HPCC_ADD_SUBDIRECTORY (hrpc)
 HPCC_ADD_SUBDIRECTORY (include "PLATFORM")
 HPCC_ADD_SUBDIRECTORY (jhtree)
 HPCC_ADD_SUBDIRECTORY (jlib)
+HPCC_ADD_SUBDIRECTORY (lz4_sm)
 HPCC_ADD_SUBDIRECTORY (lzma)
 HPCC_ADD_SUBDIRECTORY (mp)
 HPCC_ADD_SUBDIRECTORY (security)

system/jlib/CMakeLists.txt

@@ -55,6 +55,7 @@ set (    SRCS
          jkeyboard.cpp 
          jlib.cpp 
          jlog.cpp 
+         jlz4.cpp
          jlzma.cpp 
          jlzw.cpp 
          jmalloc.cpp 
@@ -99,6 +100,7 @@ set (    INCLUDES
         jencrypt.hpp
         jerror.hpp
         jexcept.hpp
+        jfcmp.hpp
         jfile.hpp
         jfile.ipp
         jflz.hpp
@@ -118,6 +120,7 @@ set (    INCLUDES
         jliball.hpp
         jlog.hpp
         jlog.ipp
+        jlz4.hpp
         jlzma.hpp
         jlzw.hpp
         jlzw.ipp
@@ -172,6 +175,7 @@ include_directories (
          ../../system/win32 
          ../../system/include 
          ../../system/lzma
+         ../../system/lz4_sm/lz4/lib
          ${CMAKE_CURRENT_BINARY_DIR}  # for generated jelog.h file 
          ${CMAKE_BINARY_DIR}
          ${CMAKE_BINARY_DIR}/oss
@@ -183,6 +187,7 @@ HPCC_ADD_LIBRARY( jlib SHARED ${SRCS} ${INCLUDES} )
 
 target_link_libraries ( jlib
         lzma
+        lz4
        )
 
 if ( ${USE_TBB} )
@@ -201,7 +206,11 @@ if (WIN32)
  target_link_libraries ( jlib ws2_32 mpr.lib Winmm.lib psapi.lib )
 elseif (("${CMAKE_SYSTEM_NAME}" STREQUAL "Darwin" ))
 else ()
- target_link_libraries ( jlib rt numa)
+ target_link_libraries ( jlib rt )
+endif ()
+
+if (USE_NUMA)
+ target_link_libraries ( jlib numa )
 endif ()
 
 if (NOT PLUGIN)

system/jlib/jfcmp.hpp

@@ -0,0 +1,477 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2015 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#include "platform.h"
+#include "jlzw.hpp"
+
+#define COMMITTED ((size32_t)-1)
+
+#define FCMP_BUFFER_SIZE (0x100000)
+
+class jlib_decl CFcmpCompressor : public CSimpleInterfaceOf<ICompressor>
+{
+protected:
+    size32_t blksz;
+    size32_t bufalloc;
+    MemoryBuffer inma;      // equals blksize len
+    MemoryBuffer *outBufMb; // used when dynamic output buffer (when open() used)
+    size32_t outBufStart;
+    byte *inbuf;
+    size32_t inmax;         // remaining
+    size32_t inlen;
+    size32_t inlenblk;      // set to COMMITTED when so
+    bool trailing;
+    byte *outbuf;
+    size32_t outlen;
+    size32_t wrmax;
+    size32_t dynamicOutSz;
+
+    virtual void setinmax() = 0;
+    virtual void flushcommitted() = 0;
+
+    void initCommon()
+    {
+        blksz = inma.capacity();
+        *(size32_t *)outbuf = 0;
+        outlen = sizeof(size32_t);
+        inlen = 0;
+        inlenblk = COMMITTED;
+        setinmax();
+    }
+
+public:
+    CFcmpCompressor()
+    {
+        outlen = 0;
+        outbuf = NULL;      // only set on close
+        bufalloc = 0;
+        wrmax = 0;          // set at open
+        dynamicOutSz = 0;
+        outBufMb = NULL;
+        outBufStart = 0;
+        inbuf = NULL;
+    }
+
+    virtual ~CFcmpCompressor()
+    {
+        if (bufalloc)
+            free(outbuf);
+    }
+
+    virtual void open(void *buf,size32_t max)
+    {
+        if (max<1024)
+            throw MakeStringException(-1,"CFcmpCompressor::open - block size (%d) not large enough", blksz);
+        wrmax = max;
+        if (buf)
+        {
+            if (bufalloc)
+                free(outbuf);
+            bufalloc = 0;
+            outbuf = (byte *)buf;
+        }
+        else if (max>bufalloc)
+        {
+            if (bufalloc)
+                free(outbuf);
+            outbuf = (byte *)malloc(max);
+            if (!outbuf)
+                throw MakeStringException(-1,"CFcmpCompressor::open - out of memory, requesting %d bytes", max);
+            bufalloc = max;
+        }
+        outBufMb = NULL;
+        outBufStart = 0;
+        dynamicOutSz = 0;
+        inbuf = (byte *)inma.ensureCapacity(max);
+        initCommon();
+    }
+
+    virtual void open(MemoryBuffer &mb, size32_t initialSize)
+    {
+        if (!initialSize)
+            initialSize = FCMP_BUFFER_SIZE; // 1MB
+        if (initialSize<1024)
+            throw MakeStringException(-1,"CFcmpCompressor::open - block size (%d) not large enough", initialSize);
+        wrmax = initialSize;
+        if (bufalloc)
+        {
+            free(outbuf);
+            bufalloc = 0;
+        }
+        inbuf = (byte *)inma.ensureCapacity(initialSize);
+        outBufMb = &mb;
+        outBufStart = mb.length();
+        outbuf = (byte *)outBufMb->ensureCapacity(initialSize);
+        dynamicOutSz = outBufMb->capacity();
+        initCommon();
+    }
+
+    virtual void close()
+    {
+        if (inlenblk!=COMMITTED)
+        {
+            inlen = inlenblk; // transaction failed
+            inlenblk = COMMITTED;
+        }
+        flushcommitted();
+        size32_t totlen = outlen+sizeof(size32_t)+inlen;
+        assertex(blksz>=totlen);
+        size32_t *tsize = (size32_t *)(outbuf+outlen);
+        *tsize = inlen;
+        memcpy(tsize+1,inbuf,inlen);
+        outlen = totlen;
+        *(size32_t *)outbuf += inlen;
+        inbuf = NULL;
+        if (outBufMb)
+        {
+            outBufMb->setWritePos(outBufStart+outlen);
+            outBufMb = NULL;
+        }
+    }
+
+    size32_t write(const void *buf,size32_t len)
+    {
+        // no more than wrmax per write (unless dynamically sizing)
+        size32_t lenb = wrmax;
+        byte *b = (byte *)buf;
+        size32_t written = 0;
+        while (len)
+        {
+            if (len < lenb)
+                lenb = len;
+            if (lenb+inlen>inmax)
+            {
+                if (trailing)
+                    return written;
+                flushcommitted();
+                if (lenb+inlen>inmax)
+                {
+                    if (outBufMb) // sizing input buffer, but outBufMb!=NULL is condition of whether in use or not
+                    {
+                        blksz += len > FCMP_BUFFER_SIZE ? len : FCMP_BUFFER_SIZE;
+                        verifyex(inma.ensureCapacity(blksz));
+                        blksz = inma.capacity();
+                        inbuf = (byte *)inma.bufferBase();
+                        wrmax = blksz;
+                        setinmax();
+                    }
+                    lenb = inmax-inlen;
+                    if (len < lenb)
+                        lenb = len;
+                }
+            }
+            if (lenb == 0)
+                return written;
+            memcpy(inbuf+inlen,b,lenb);
+            b += lenb;
+            inlen += lenb;
+            len -= lenb;
+            written += lenb;
+        }
+        return written;
+    }
+
+    void * bufptr()
+    {
+        assertex(!inbuf);  // i.e. closed
+        return outbuf;
+    }
+
+    size32_t buflen()
+    {
+        assertex(!inbuf);  // i.e. closed
+        return outlen;
+    }
+
+    void startblock()
+    {
+        inlenblk = inlen;
+    }
+
+    void commitblock()
+    {
+        inlenblk = COMMITTED;
+    }
+
+};
+
+
+class jlib_decl CFcmpExpander : public CSimpleInterfaceOf<IExpander>
+{
+protected:
+    byte *outbuf;
+    size32_t outlen;
+    size32_t bufalloc;
+    const size32_t *in;
+
+public:
+    CFcmpExpander()
+    {
+        outbuf = NULL;
+        outlen = 0;
+        bufalloc = 0;
+    }
+
+    virtual ~CFcmpExpander()
+    {
+        if (bufalloc)
+            free(outbuf);
+    }
+
+    virtual size32_t init(const void *blk)
+    {
+        const size32_t *expsz = (const size32_t *)blk;
+        outlen = *expsz;
+        in = (expsz+1);
+        return outlen;
+    }
+
+    virtual void expand(void *buf)
+    {
+        if (!outlen)
+            return;
+        if (buf)
+        {
+            if (bufalloc)
+                free(outbuf);
+            bufalloc = 0;
+            outbuf = (unsigned char *)buf;
+        }
+        else if (outlen>bufalloc)
+        {
+            if (bufalloc)
+                free(outbuf);
+            bufalloc = outlen;
+            outbuf = (unsigned char *)malloc(bufalloc);
+            if (!outbuf)
+                throw MakeStringException(MSGAUD_operator,0, "Out of memory in FcmpExpander::expand, requesting %d bytes", bufalloc);
+        }
+        size32_t done = 0;
+        loop
+        {
+            const size32_t szchunk = *in;
+            in++;
+            if (szchunk+done<outlen)
+            {
+                memcpy((byte *)buf+done, in, szchunk);
+                size32_t written = szchunk;
+                done += written;
+                if (!written||(done>outlen))
+                    throw MakeStringException(0, "FcmpExpander - corrupt data(1) %d %d",written,szchunk);
+            }
+            else
+            {
+                if (szchunk+done!=outlen)
+                    throw MakeStringException(0, "FcmpExpander - corrupt data(2) %d %d",szchunk,outlen);
+                memcpy((byte *)buf+done,in,szchunk);
+                break;
+            }
+            in = (const size32_t *)(((const byte *)in)+szchunk);
+        }
+    }
+
+    virtual void *bufptr() { return outbuf;}
+    virtual size32_t buflen() { return outlen;}
+};
+
+struct FcmpCompressedFileTrailer
+{
+    offset_t        zfill1;             // must be first
+    offset_t        expandedSize;
+    __int64         compressedType;
+    unsigned        zfill2;             // must be last
+};
+
+class CFcmpStream : public CSimpleInterfaceOf<IFileIOStream>
+{
+protected:
+    Linked<IFileIO> baseio;
+    offset_t expOffset;     // expanded offset
+    offset_t cmpOffset;     // compressed offset in file
+    bool reading;
+    MemoryAttr ma;
+    size32_t bufsize;
+    size32_t bufpos;        // reading only
+    offset_t expSize;
+    __int64 compType;
+
+public:
+    CFcmpStream()
+    {
+        expOffset = 0;
+        cmpOffset = 0;
+        reading = true;
+        bufpos = 0;
+        bufsize = 0;
+    }
+
+    virtual ~CFcmpStream() { flush(); }
+
+    virtual bool load()
+    {
+        bufpos = 0;
+        bufsize = 0;
+        if (expOffset==expSize)
+            return false;
+        size32_t sz[2];
+        if (baseio->read(cmpOffset,sizeof(size32_t)*2,&sz)!=sizeof(size32_t)*2)
+            return false;
+        bufsize = sz[0];
+        if (!bufsize)
+            return false;
+        cmpOffset += sizeof(size32_t)*2;
+        if (ma.length()<bufsize)
+            ma.allocate(bufsize);
+        MemoryAttr cmpma;
+        byte *cmpbuf = (byte *)cmpma.allocate(sz[1]);
+        if (baseio->read(cmpOffset,sz[1],cmpbuf)!=sz[1])
+            throw MakeStringException(-1,"CFcmpStream: file corrupt.1");
+        memcpy(ma.bufferBase(), cmpbuf, sz[1]);
+        size32_t amnt = sz[1];
+        if (amnt!=bufsize)
+            throw MakeStringException(-1,"CFcmpStream: file corrupt.2");
+        cmpOffset += sz[1];
+        return true;
+    }
+
+    virtual void save()
+    {
+        if (bufsize)
+        {
+            MemoryAttr dstma;
+            byte *dst = (byte *)dstma.allocate(sizeof(size32_t)*2+bufsize);
+            memcpy((sizeof(size32_t)*2+dst), ma.get(), bufsize);
+            size32_t sz = bufsize;
+            memcpy(dst,&bufsize,sizeof(size32_t));
+            memcpy(dst+sizeof(size32_t),&sz,sizeof(size32_t));
+            baseio->write(cmpOffset,sz+sizeof(size32_t)*2,dst);
+            cmpOffset += sz+sizeof(size32_t)*2;
+        }
+        bufsize = 0;
+    }
+
+    virtual bool attach(IFileIO *_baseio)
+    {
+        baseio.set(_baseio);
+        expOffset = 0;
+        cmpOffset = 0;
+        reading = true;
+        bufpos = 0;
+        bufsize = 0;
+
+        FcmpCompressedFileTrailer trailer;
+        offset_t filesize = baseio->size();
+        if (filesize<sizeof(trailer))
+            return false;
+        baseio->read(filesize-sizeof(trailer),sizeof(trailer),&trailer);
+        expSize = trailer.expandedSize;
+        return trailer.compressedType==compType;
+    }
+
+    virtual void create(IFileIO *_baseio)
+    {
+        baseio.set(_baseio);
+        expOffset = 0;
+        cmpOffset = 0;
+        reading = false;
+        bufpos = 0;
+        bufsize = 0;
+        ma.allocate(FCMP_BUFFER_SIZE);
+        expSize = (offset_t)-1;
+    }
+
+    virtual void seek(offset_t pos, IFSmode origin)
+    {
+        if ((origin==IFScurrent)&&(pos==0))
+            return;
+        if ((origin==IFSbegin)||(pos!=0))
+            throw MakeStringException(-1,"CFcmpStream seek not supported");
+        expOffset = 0;
+        bufpos = 0;
+        bufsize = 0;
+    }
+
+    virtual offset_t size()
+    {
+        return (expSize==(offset_t)-1)?0:expSize;
+    }
+
+    virtual offset_t tell()
+    {
+        return expOffset;
+    }
+
+    virtual size32_t read(size32_t len, void * data)
+    {
+        if (!reading)
+            throw MakeStringException(-1,"CFcmpStream read to stream being written");
+        size32_t ret=0;
+        while (len)
+        {
+            size32_t cpy = bufsize-bufpos;
+            if (!cpy)
+            {
+                if (!load())
+                    break;
+                cpy = bufsize-bufpos;
+            }
+            if (cpy>len)
+                cpy = len;
+            memcpy(data,(const byte *)ma.get()+bufpos,cpy);
+            bufpos += cpy;
+            len -= cpy;
+            ret += cpy;
+        }
+        expOffset += ret;
+        return ret;
+    }
+
+    virtual size32_t write(size32_t len, const void * data)
+    {
+        if (reading)
+            throw MakeStringException(-1,"CFcmpStream write to stream being read");
+        size32_t ret = len;
+        while (len+bufsize>FCMP_BUFFER_SIZE)
+        {
+            size32_t cpy = FCMP_BUFFER_SIZE-bufsize;
+            memcpy((byte *)ma.bufferBase()+bufsize,data,cpy);
+            data = (const byte *)data+cpy;
+            len -= cpy;
+            bufsize = FCMP_BUFFER_SIZE;
+            save();
+        }
+        memcpy((byte *)ma.bufferBase()+bufsize,data,len);
+        bufsize += len;
+        expOffset += len;
+        return ret;
+    }
+
+    virtual void flush()
+    {
+        if (!reading&&(expSize!=expOffset))
+        {
+            save();
+            FcmpCompressedFileTrailer trailer;
+            memset(&trailer,0,sizeof(trailer));
+            trailer.compressedType = compType;
+            trailer.expandedSize = expOffset;
+            baseio->write(cmpOffset,sizeof(trailer),&trailer);
+            expSize = expOffset;
+        }
+    }
+
+};

system/jlib/jflz.cpp

@@ -47,9 +47,8 @@
 
 // adapted for jlib
 #include "platform.h"
-
+#include "jfcmp.hpp"
 #include "jflz.hpp"
-
 #include "jcrc.hpp"
 
 /*
@@ -593,49 +592,29 @@ static FASTLZ_INLINE int FASTLZ_DECOMPRESSOR(const void* input, int length, void
 
 #if defined(FASTLZ__JLIBCOMPRESSOR)
 
-
-#define COMMITTED ((size32_t)-1)
-
 /* Format:
     size32_t totalexpsize;
     { size32_t subcmpsize; bytes subcmpdata; }
     size32_t trailsize; bytes traildata;    // unexpanded
 */
 
-
-class jlib_decl CFastLZCompressor : public CInterface, public ICompressor
+class jlib_decl CFastLZCompressor : public CFcmpCompressor
 {
     HTAB_T ht;
-    size32_t blksz;
-    size32_t bufalloc;
-    MemoryBuffer inma;      // equals blksize len
-    MemoryBuffer *outBufMb; // used when dynamic output buffer (when open() used)
-    size32_t outBufStart;
-    byte *inbuf;
-    size32_t inmax;         // remaining
-    size32_t inlen;
-    size32_t inlenblk;      // set to COMMITTED when so
-    bool trailing;
-    byte *outbuf;
-    size32_t outlen;
-    size32_t wrmax;
-    size32_t dynamicOutSz;
-
-    inline void setinmax()
+
+    virtual void setinmax()
     {
         inmax = blksz-outlen-sizeof(size32_t);
         if (inmax<256)
             trailing = true;    // too small to bother compressing
-        else {
+        else
+        {
             trailing = false;
-            size32_t slack = inmax/17;
-            if (slack<66)
-                slack = 66;
-            inmax -= slack+sizeof(size32_t);
+            inmax -= (fastlzSlack(inmax) + sizeof(size32_t));
         }
     }
 
-    inline void flushcommitted()
+    virtual void flushcommitted()
     {
         // only does non trailing
         if (trailing)
@@ -675,203 +654,12 @@ class jlib_decl CFastLZCompressor : public CInterface, public ICompressor
         trailing = true;
     }
 
-    void initCommon()
-    {
-        blksz = inma.capacity();
-        *(size32_t *)outbuf = 0;
-        outlen = sizeof(size32_t);
-        inlen = 0;
-        inlenblk = COMMITTED;
-        setinmax();
-    }
-public:
-    IMPLEMENT_IINTERFACE;
-
-    CFastLZCompressor()
-    {
-        outlen = 0;
-        outbuf = NULL;      // only set on close
-        bufalloc = 0;
-        wrmax = 0;          // set at open
-        dynamicOutSz = 0;
-        outBufMb = NULL;
-        outBufStart = 0;
-        inbuf = NULL;
-    }
-
-    virtual ~CFastLZCompressor()
-    {
-        if (bufalloc)
-            free(outbuf);
-    }
-
-
-    virtual void open(void *buf,size32_t max)
-    {
-        if (max<1024)
-            throw MakeStringException(-1,"CFastLZCompressor::open - block size (%d) not large enough", blksz);
-        wrmax = max;
-        if (buf)
-        {
-            if (bufalloc)
-                free(outbuf);
-            bufalloc = 0;
-            outbuf = (byte *)buf;
-        }
-        else if (max>bufalloc)
-        {
-            if (bufalloc)
-                free(outbuf);
-            bufalloc = max;
-            outbuf = (byte *)malloc(bufalloc);
-        }
-        outBufMb = NULL;
-        outBufStart = 0;
-        dynamicOutSz = 0;
-        inbuf = (byte *)inma.ensureCapacity(max);
-        initCommon();
-    }
-
-    virtual void open(MemoryBuffer &mb, size32_t initialSize)
-    {
-        if (!initialSize)
-            initialSize = 0x100000; // 1MB
-        if (initialSize<1024)
-            throw MakeStringException(-1,"CFastLZCompressor::open - block size (%d) not large enough", initialSize);
-        wrmax = initialSize;
-        if (bufalloc)
-        {
-            free(outbuf);
-            bufalloc = 0;
-        }
-        inbuf = (byte *)inma.ensureCapacity(initialSize);
-        outBufMb = &mb;
-        outBufStart = mb.length();
-        outbuf = (byte *)outBufMb->ensureCapacity(initialSize);
-        dynamicOutSz = outBufMb->capacity();
-        initCommon();
-    }
-
-    virtual void close()
-    {
-        if (inlenblk!=COMMITTED) {
-            inlen = inlenblk; // transaction failed
-            inlenblk = COMMITTED;
-        }
-        flushcommitted();
-        size32_t totlen = outlen+sizeof(size32_t)+inlen;
-        assertex(blksz>=totlen);
-        size32_t *tsize = (size32_t *)(outbuf+outlen);
-        *tsize = inlen;
-        memcpy(tsize+1,inbuf,inlen);
-        outlen = totlen;
-        *(size32_t *)outbuf += inlen;
-        inbuf = NULL;
-        if (outBufMb)
-        {
-            outBufMb->setWritePos(outBufStart+outlen);
-            outBufMb = NULL;
-        }
-    }
-
-
-    size32_t write(const void *buf,size32_t len)
-    {
-        // no more than wrmax per write (unless dynamically sizing)
-        size32_t lenb = wrmax;
-        byte *b = (byte *)buf;
-        size32_t written = 0;
-        while (len)
-        {
-            if (len < lenb)
-                lenb = len;
-            if (lenb+inlen>inmax)
-            {
-                if (trailing)
-                    return written;
-                flushcommitted();
-                if (lenb+inlen>inmax)
-                {
-                    if (outBufMb) // sizing input buffer, but outBufMb!=NULL is condition of whether in use or not
-                    {
-                        blksz += len > 0x100000 ? len : 0x100000;
-                        verifyex(inma.ensureCapacity(blksz));
-                        blksz = inma.capacity();
-                        inbuf = (byte *)inma.bufferBase();
-                        wrmax = blksz;
-                        setinmax();
-                    }
-                    lenb = inmax-inlen;
-                    if (len < lenb)
-                        lenb = len;
-                }
-            }
-            if (lenb == 0)
-                return written;
-            memcpy(inbuf+inlen,b,lenb);
-            b += lenb;
-            inlen += lenb;
-            len -= lenb;
-            written += lenb;
-        }
-        return written;
-    }
-
-    void *  bufptr() 
-    { 
-        assertex(!inbuf);  // i.e. closed
-        return outbuf;
-    }
-    size32_t    buflen() 
-    { 
-        assertex(!inbuf);  // i.e. closed
-        return outlen;
-    }
-    void    startblock()
-    {
-        inlenblk = inlen;
-    }
-    void commitblock()
-    {
-        inlenblk = COMMITTED;
-    }
-
-
 };
 
 
-class jlib_decl CFastLZExpander : public CInterface, public IExpander
+class jlib_decl CFastLZExpander : public CFcmpExpander
 {
-
-    byte *outbuf;
-    size32_t outlen;
-    size32_t bufalloc;
-    const size32_t *in;  
-
 public:
-    IMPLEMENT_IINTERFACE;
-
-    CFastLZExpander()
-    {
-        outbuf = NULL;
-        outlen = 0;
-        bufalloc = 0;
-    }
-    ~CFastLZExpander()
-    {
-        if (bufalloc)
-            free(outbuf);
-
-    }
-
-    virtual size32_t  init(const void *blk)
-    {
-        const size32_t *expsz = (const size32_t *)blk;
-        outlen = *expsz;
-        in = (expsz+1);
-        return outlen;
-    }
-
     virtual void expand(void *buf)
     {
         if (!outlen)
@@ -910,8 +698,6 @@ public:
         }
     }
 
-    virtual void *bufptr() { return outbuf;}
-    virtual size32_t   buflen() { return outlen;}
 };
 
 void fastLZCompressToBuffer(MemoryBuffer & out, size32_t len, const void * src)
@@ -921,7 +707,8 @@ void fastLZCompressToBuffer(MemoryBuffer & out, size32_t len, const void * src)
     *sz = len;
     sz++;
     *sz = (len>16)?fastlz_compress(src, (int)len, sz+1):16;
-    if (*sz>=len) {
+    if (*sz>=len)
+    {
         *sz = len;
         memcpy(sz+1,src,len);
     }
@@ -999,30 +786,10 @@ IExpander *createFastLZExpander()
     return new CFastLZExpander;
 }
 
-#define FLZ_BUFFER_SIZE (0x100000)
-
-static const __uint64 FLZCOMPRESSEDFILEFLAG = U64C(0xc3518de42f15da57);
+static const __uint64 FLZSTRMCOMPRESSEDFILEFLAG = I64C(0xc3518de42f15da57);
 
-struct FlzCompressedFileTrailer
+class CFastLZStream : public CFcmpStream
 {
-    offset_t        zfill1;             // must be first
-    offset_t        expandedSize;
-    __uint64        compressedType;
-    unsigned        zfill2;             // must be last
-};
-
-
-class CFastLZStream : public CInterface, implements IFileIOStream
-{
-    Linked<IFileIO> baseio;
-    offset_t expOffset;     // expanded offset
-    offset_t cmpOffset;     // compressed offset in file
-    bool reading;
-    MemoryAttr ma;
-    size32_t bufsize;
-    size32_t bufpos;        // reading only
-    offset_t expSize;
-
     bool load()
     {
         bufpos = 0;
@@ -1064,133 +831,12 @@ class CFastLZStream : public CInterface, implements IFileIOStream
 
 
 public:
-    IMPLEMENT_IINTERFACE;
-
-    CFastLZStream() 
-    {
-        expOffset = 0;
-        cmpOffset = 0;
-        reading = true;
-        bufpos = 0;
-        bufsize = 0;
-    }
-
-    ~CFastLZStream()
-    {
-        flush();
-    }
-
-    bool attach(IFileIO *_baseio)
-    {
-        baseio.set(_baseio);
-        expOffset = 0;
-        cmpOffset = 0;
-        reading = true;
-        bufpos = 0;
-        bufsize = 0;
-
-        FlzCompressedFileTrailer trailer;
-        offset_t filesize = baseio->size();
-        if (filesize<sizeof(trailer))
-            return false;
-        baseio->read(filesize-sizeof(trailer),sizeof(trailer),&trailer);
-        expSize = trailer.expandedSize;
-        return trailer.compressedType==FLZCOMPRESSEDFILEFLAG;
-    }
-
-    void create(IFileIO *_baseio)
-    {
-        baseio.set(_baseio);
-        expOffset = 0;
-        cmpOffset = 0;
-        reading = false;
-        bufpos = 0;
-        bufsize = 0;
-        ma.allocate(FLZ_BUFFER_SIZE);
-        expSize = (offset_t)-1;
-    }
-
-    void seek(offset_t pos, IFSmode origin)
-    {
-        if ((origin==IFScurrent)&&(pos==0))
-            return;
-        if ((origin==IFSbegin)||(pos!=0))
-            throw MakeStringException(-1,"CFastLZStream seek not supported");
-        expOffset = 0;
-        bufpos = 0;
-        bufsize = 0;
-    }
-
-    offset_t size()
-    {
-        return (expSize==(offset_t)-1)?0:expSize;
-    }
-
-    offset_t tell()
-    {
-        return expOffset;
-    }
-
-
-    size32_t read(size32_t len, void * data)
-    {
-        if (!reading)
-            throw MakeStringException(-1,"CFastLZStream read to stream being written");
-        size32_t ret=0;
-        while (len) {
-            size32_t cpy = bufsize-bufpos;
-            if (!cpy) {
-                if (!load())
-                    break;
-                cpy = bufsize-bufpos;
-            }
-            if (cpy>len)
-                cpy = len;
-            memcpy(data,(const byte *)ma.get()+bufpos,cpy);
-            bufpos += cpy;
-            len -= cpy;
-            ret += cpy;
-        }
-        expOffset += ret;
-        return ret;
-    }
-
-    size32_t write(size32_t len, const void * data)
-    {
-        if (reading)
-            throw MakeStringException(-1,"CFastLZStream write to stream being read");
-        size32_t ret = len;
-        while (len+bufsize>FLZ_BUFFER_SIZE) {
-            size32_t cpy = FLZ_BUFFER_SIZE-bufsize;
-            memcpy((byte *)ma.bufferBase()+bufsize,data,cpy);
-            data = (const byte *)data+cpy;
-            len -= cpy;
-            bufsize = FLZ_BUFFER_SIZE;
-            save();
-        }
-        memcpy((byte *)ma.bufferBase()+bufsize,data,len);
-        bufsize += len;
-        expOffset += len;
-        return ret;
-    }
+    CFastLZStream() { compType = FLZSTRMCOMPRESSEDFILEFLAG; }
 
-    void flush()
-    {
-        if (!reading&&(expSize!=expOffset)) {
-            save();
-            FlzCompressedFileTrailer trailer;
-            memset(&trailer,0,sizeof(trailer));
-            trailer.compressedType = FLZCOMPRESSEDFILEFLAG;
-            trailer.expandedSize = expOffset;
-            baseio->write(cmpOffset,sizeof(trailer),&trailer);
-            expSize = expOffset;
-        }
-    }
+    virtual ~CFastLZStream() { flush(); }
 
 };
 
-
-
 IFileIOStream *createFastLZStreamRead(IFileIO *base)
 {
     Owned<CFastLZStream> strm = new CFastLZStream();
@@ -1206,7 +852,4 @@ IFileIOStream *createFastLZStreamWrite(IFileIO *base)
     return strm.getClear();
 }
 
-
 #endif
-
-

system/jlib/jflz.hpp

@@ -22,6 +22,7 @@
 
 #include "jlzw.hpp"
 
+#define FASTCOMPRESSEDFILEBLOCKSIZE (0x10000)
 
 extern jlib_decl ICompressor *createFastLZCompressor();
 extern jlib_decl IExpander *createFastLZExpander();

system/jlib/jlz4.cpp

@@ -0,0 +1,316 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#include "platform.h"
+#include "jfcmp.hpp"
+#include "jlz4.hpp"
+#include "lz4.h"
+
+/* Format:
+    size32_t totalexpsize;
+    { size32_t subcmpsize; bytes subcmpdata; }
+    size32_t trailsize; bytes traildata;    // unexpanded
+*/
+
+class jlib_decl CLZ4Compressor : public CFcmpCompressor
+{
+    virtual void setinmax()
+    {
+        inmax = blksz-outlen-sizeof(size32_t);
+        if (inmax<256)
+            trailing = true;    // too small to bother compressing
+        else
+        {
+            trailing = false;
+            size32_t slack = LZ4_COMPRESSBOUND(inmax) - inmax;
+            int inmax2 = inmax - (slack + sizeof(size32_t));
+            if (inmax2<256)
+                trailing = true;
+            else
+                inmax = inmax2;
+        }
+    }
+
+    virtual void flushcommitted()
+    {
+        // only does non trailing
+        if (trailing)
+            return;
+        size32_t toflush = (inlenblk==COMMITTED)?inlen:inlenblk;
+        if (toflush == 0)
+            return;
+
+        if (toflush < 256)
+        {
+            trailing = true;
+            return;
+        }
+
+        size32_t outSzRequired = outlen+sizeof(size32_t)*2+LZ4_COMPRESSBOUND(toflush);
+        if (!dynamicOutSz)
+            assertex(outSzRequired<=blksz);
+        else
+        {
+            if (outSzRequired>dynamicOutSz)
+            {
+                verifyex(outBufMb->ensureCapacity(outBufStart+outSzRequired));
+                dynamicOutSz = outBufMb->capacity();
+                outbuf = ((byte *)outBufMb->bufferBase()+outBufStart);
+            }
+        }
+        size32_t *cmpsize = (size32_t *)(outbuf+outlen);
+        byte *out = (byte *)(cmpsize+1);
+
+        *cmpsize = LZ4_compress_default((const char *)inbuf, (char *)out, toflush, LZ4_COMPRESSBOUND(toflush));
+        if (*cmpsize && *cmpsize<toflush)
+        {
+            *(size32_t *)outbuf += toflush;
+            outlen += *cmpsize+sizeof(size32_t);
+            if (inlenblk==COMMITTED)
+                inlen = 0;
+            else
+            {
+                inlen -= inlenblk;
+                memmove(inbuf,inbuf+toflush,inlen);
+            }
+            setinmax();
+            return;
+        }
+        trailing = true;
+    }
+
+};
+
+
+class jlib_decl CLZ4Expander : public CFcmpExpander
+{
+public:
+    virtual void expand(void *buf)
+    {
+        if (!outlen)
+            return;
+        if (buf)
+        {
+            if (bufalloc)
+                free(outbuf);
+            bufalloc = 0;
+            outbuf = (unsigned char *)buf;
+        }
+        else if (outlen>bufalloc)
+        {
+            if (bufalloc)
+                free(outbuf);
+            bufalloc = outlen;
+            outbuf = (unsigned char *)malloc(bufalloc);
+            if (!outbuf)
+                throw MakeStringException(MSGAUD_operator,0, "Out of memory in LZ4Expander::expand, requesting %d bytes", bufalloc);
+        }
+        size32_t done = 0;
+        loop
+        {
+            const size32_t szchunk = *in;
+            in++;
+            if (szchunk+done<outlen)
+            {
+                size32_t written = LZ4_decompress_safe((const char *)in, (char *)((byte *)buf+done), szchunk, outlen-done);
+                done += written;
+                if (!written||(done>outlen))
+                    throw MakeStringException(0, "LZ4Expander - corrupt data(1) %d %d",written,szchunk);
+            }
+            else
+            {
+                if (szchunk+done!=outlen)
+                    throw MakeStringException(0, "LZ4Expander - corrupt data(2) %d %d",szchunk,outlen);
+                memcpy((byte *)buf+done,in,szchunk);
+                break;
+            }
+            in = (const size32_t *)(((const byte *)in)+szchunk);
+        }
+    }
+
+};
+
+void LZ4CompressToBuffer(MemoryBuffer & out, size32_t len, const void * src)
+{
+    size32_t outbase = out.length();
+    size32_t *sz = (size32_t *)out.reserve(LZ4_COMPRESSBOUND(len)+sizeof(size32_t)*2);
+    *sz = len;
+    sz++;
+    if (len < 64)
+    {
+        *sz = len;
+        memcpy(sz+1,src,len);
+    }
+    else
+    {
+        *sz = LZ4_compress_default((const char *)src, (char *)(sz+1), len, LZ4_COMPRESSBOUND(len));
+        if (!*sz)
+        {
+            *sz = len;
+            memcpy(sz+1,src,len);
+        }
+    }
+    out.setLength(outbase+*sz+sizeof(size32_t)*2);
+}
+
+void LZ4DecompressToBuffer(MemoryBuffer & out, const void * src)
+{
+    size32_t *sz = (size32_t *)src;
+    size32_t expsz = *(sz++);
+    size32_t cmpsz = *(sz++);
+    void *o = out.reserve(expsz);
+    if (cmpsz!=expsz)
+    {
+        size32_t written = LZ4_decompress_safe((const char *)sz, (char *)o, cmpsz, expsz);
+        if (written!=expsz)
+            throw MakeStringException(0, "LZ4DecompressToBuffer - corrupt data(1) %d %d",written,expsz);
+    }
+    else
+        memcpy(o,sz,expsz);
+}
+
+void LZ4DecompressToBuffer(MemoryBuffer & out, MemoryBuffer & in)
+{
+    size32_t expsz;
+    size32_t cmpsz;
+    in.read(expsz).read(cmpsz);
+    void *o = out.reserve(expsz);
+    if (cmpsz!=expsz)
+    {
+        size32_t written = LZ4_decompress_safe((const char *)in.readDirect(cmpsz), (char *)o, cmpsz, expsz);
+        if (written!=expsz)
+            throw MakeStringException(0, "LZ4DecompressToBuffer - corrupt data(3) %d %d",written,expsz);
+    }
+    else
+        memcpy(o,in.readDirect(cmpsz),expsz);
+}
+
+void LZ4DecompressToAttr(MemoryAttr & out, const void * src)
+{
+    size32_t *sz = (size32_t *)src;
+    size32_t expsz = *(sz++);
+    size32_t cmpsz = *(sz++);
+    void *o = out.allocate(expsz);
+    if (cmpsz!=expsz)
+    {
+        size32_t written = LZ4_decompress_safe((const char *)sz, (char *)o, cmpsz, expsz);
+        if (written!=expsz)
+            throw MakeStringException(0, "LZ4DecompressToBuffer - corrupt data(2) %d %d",written,expsz);
+    }
+    else
+        memcpy(o,sz,expsz);
+}
+
+void LZ4DecompressToBuffer(MemoryAttr & out, MemoryBuffer & in)
+{
+    size32_t expsz;
+    size32_t cmpsz;
+    in.read(expsz).read(cmpsz);
+    void *o = out.allocate(expsz);
+    if (cmpsz!=expsz)
+    {
+        size32_t written = LZ4_decompress_safe((const char *)in.readDirect(cmpsz), (char *)o, cmpsz, expsz);
+        if (written!=expsz)
+            throw MakeStringException(0, "LZ4DecompressToBuffer - corrupt data(4) %d %d",written,expsz);
+    }
+    else
+        memcpy(o,in.readDirect(cmpsz),expsz);
+}
+
+
+ICompressor *createLZ4Compressor()
+{
+    return new CLZ4Compressor;
+}
+
+IExpander *createLZ4Expander()
+{
+    return new CLZ4Expander;
+}
+
+#define LZ4STRMCOMPRESSEDFILEFLAG (I64C(0xc129b02d53545e91))
+
+class CLZ4Stream : public CFcmpStream
+{
+    bool load()
+    {
+        bufpos = 0;
+        bufsize = 0;
+        if (expOffset==expSize)
+            return false;
+        size32_t sz[2];
+        if (baseio->read(cmpOffset,sizeof(size32_t)*2,&sz)!=sizeof(size32_t)*2)
+            return false;
+        bufsize = sz[0];
+        if (!bufsize)
+            return false;
+        cmpOffset += sizeof(size32_t)*2;
+        if (ma.length()<bufsize)
+            ma.allocate(bufsize);
+        MemoryAttr cmpma;
+        byte *cmpbuf = (byte *)cmpma.allocate(sz[1]);
+        if (baseio->read(cmpOffset,sz[1],cmpbuf)!=sz[1])
+            throw MakeStringException(-1,"CLZ4Stream: file corrupt.1");
+        size32_t amnt = LZ4_decompress_safe((const char *)cmpbuf, (char *)ma.bufferBase(), sz[1], bufsize);
+        if (amnt!=bufsize)
+            throw MakeStringException(-1,"CLZ4Stream: file corrupt.2");
+        cmpOffset += sz[1];
+        return true;
+    }
+
+    void save()
+    {
+        if (bufsize)
+        {
+            MemoryAttr dstma;
+            byte *dst = (byte *)dstma.allocate(sizeof(size32_t)*2+LZ4_COMPRESSBOUND(bufsize));
+            size32_t sz = LZ4_compress_default((const char *)ma.get(), (char *)(sizeof(size32_t)*2+dst), bufsize, LZ4_COMPRESSBOUND(bufsize));
+            if (!sz)
+            {
+                sz = bufsize;
+                memcpy((sizeof(size32_t)*2+dst), ma.get(), bufsize);
+            }
+            memcpy(dst,&bufsize,sizeof(size32_t));
+            memcpy(dst+sizeof(size32_t),&sz,sizeof(size32_t));
+            baseio->write(cmpOffset,sz+sizeof(size32_t)*2,dst);
+            cmpOffset += sz+sizeof(size32_t)*2;
+        }
+        bufsize = 0;
+    }
+
+
+public:
+    CLZ4Stream() { compType = LZ4STRMCOMPRESSEDFILEFLAG; }
+
+    virtual ~CLZ4Stream() { flush(); }
+
+};
+
+IFileIOStream *createLZ4StreamRead(IFileIO *base)
+{
+    Owned<CLZ4Stream> strm = new CLZ4Stream();
+    if (strm->attach(base))
+        return strm.getClear();
+    return NULL;
+}
+
+IFileIOStream *createLZ4StreamWrite(IFileIO *base)
+{
+    Owned<CLZ4Stream> strm = new CLZ4Stream();
+    strm->create(base);
+    return strm.getClear();
+}

system/jlib/jlz4.hpp

@@ -0,0 +1,37 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2015 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#ifndef JLZ4_INCL
+#define JLZ4_INCL
+
+#include "jlzw.hpp"
+
+#define LZ4COMPRESSEDFILEBLOCKSIZE (0x100000)
+
+extern jlib_decl ICompressor *createLZ4Compressor();
+extern jlib_decl IExpander   *createLZ4Expander();
+
+extern jlib_decl void LZ4CompressToBuffer(MemoryBuffer & out, size32_t len, const void * src);
+extern jlib_decl void LZ4DecompressToBuffer(MemoryBuffer & out, const void * src);
+extern jlib_decl void LZ4DecompressToBuffer(MemoryBuffer & out, MemoryBuffer & in);
+extern jlib_decl void LZ4DecompressToAttr(MemoryAttr & out, const void * src);
+extern jlib_decl void LZ4DecompressToBuffer(MemoryAttr & out, MemoryBuffer & in);
+
+extern jlib_decl IFileIOStream *createLZ4StreamRead(IFileIO *base);
+extern jlib_decl IFileIOStream *createLZ4StreamWrite(IFileIO *base);
+
+#endif

system/jlib/jlzw.cpp

@@ -24,6 +24,7 @@
 #include "jfile.hpp"
 #include "jencrypt.hpp"
 #include "jflz.hpp"
+#include "jlz4.hpp"
 
 #ifdef _WIN32
 #include <io.h>
@@ -1830,6 +1831,7 @@ typedef enum { ICFcreate, ICFread, ICFappend } ICFmode;
 static const __int64 COMPRESSEDFILEFLAG = I64C(0xc0528ce99f10da55);
 #define COMPRESSEDFILEBLOCKSIZE (0x10000)
 static const __int64 FASTCOMPRESSEDFILEFLAG = I64C(0xc1518de99f10da55);
+static const __int64 LZ4COMPRESSEDFILEFLAG = I64C(0xc1200e0b71321c73);
 
 #pragma pack(push,1)
 
@@ -1839,18 +1841,23 @@ struct CompressedFileTrailer
     offset_t        expandedSize;
     offset_t        indexPos;       // end of blocks
     size32_t        blockSize;
-    size32_t        recordSize;     // 0 is lzw compressed
-    __int64        compressedType;
+    size32_t        recordSize;     // 0 is lzw or fastlz or lz4
+    __int64         compressedType;
     unsigned        crc;                // must be last
     unsigned numBlocks() { return (unsigned)((indexPos+blockSize-1)/blockSize); }
     unsigned method()
     {
-        if (recordSize)
-            return COMPRESS_METHOD_ROWDIF;
-        if (compressedType==COMPRESSEDFILEFLAG)
-            return COMPRESS_METHOD_LZW;
         if (compressedType==FASTCOMPRESSEDFILEFLAG)
             return COMPRESS_METHOD_FASTLZ;
+        if (compressedType==LZ4COMPRESSEDFILEFLAG)
+            return COMPRESS_METHOD_LZ4;
+        if (compressedType==COMPRESSEDFILEFLAG)
+        {
+            if (recordSize)
+                return COMPRESS_METHOD_ROWDIF;
+            else
+                return COMPRESS_METHOD_LZW;
+        }
         return 0;
     }
 
@@ -1860,7 +1867,7 @@ struct CompressedFileTrailer
         tree.setPropInt64("@expandedSize",expandedSize);
         tree.setPropInt64("@indexPos",indexPos);
         tree.setPropInt("@blockSize",blockSize);
-        tree.setPropInt("@recordSize",recordSize);      // 0 is lzw compressed
+        tree.setPropInt("@recordSize",recordSize);      // 0 is lzw or fastlz or lz4
         tree.setPropInt64("@compressedType",compressedType);
         tree.setPropInt("@method",method());
         tree.setPropInt("@crc",crc);                
@@ -1876,7 +1883,7 @@ struct WinCompressedFileTrailer
     offset_t        expandedSize;
     offset_t        indexPos;       // end of blocks
     size32_t        blockSize;
-    size32_t        recordSize;     // 0 is lzw compressed
+    size32_t        recordSize;     // 0 is lzw or fastlz or lz4
     __int64         compressedType;
     unsigned        crc;            // must be last
     unsigned        filler2;
@@ -1924,6 +1931,7 @@ class CCompressedFile : public CInterface, implements ICompressedFileIO
     bool writeException;
     Owned<ICompressor> compressor;
     Owned<IExpander> expander;
+    unsigned compMethod;
 
     unsigned indexNum() { return indexbuf.length()/sizeof(offset_t); }
 
@@ -2024,7 +2032,7 @@ class CCompressedFile : public CInterface, implements ICompressedFileIO
     virtual void expand(const void *compbuf,MemoryBuffer &expbuf,size32_t expsize)
     {
         size32_t rs = trailer.recordSize;
-        if (rs) { // diff compress
+        if (rs) { // diff expand
             const byte *src = (const byte *)compbuf;
             byte *dst = (byte *)expbuf.reserve(expsize);
             if (expsize) {
@@ -2041,7 +2049,7 @@ class CCompressedFile : public CInterface, implements ICompressedFileIO
                 }
             }
         }
-        else { // lzw 
+        else { // lzw or fastlz or lz4
             assertex(expander.get());
             size32_t exp = expander->init(compbuf);
             if (exp!=expsize) {
@@ -2101,14 +2109,16 @@ class CCompressedFile : public CInterface, implements ICompressedFileIO
                 src += len;
             }
         }
-        else 
+        else // lzw or fastlz or lz4
+        {
             src += compressor->write(src, len);
+        }
         return (size32_t)(src-(const byte *)expbuf);
     }
 public:
     IMPLEMENT_IINTERFACE;
 
-    CCompressedFile(IFileIO *_fileio,IMemoryMappedFile *_mmfile,CompressedFileTrailer &_trailer,ICFmode _mode, bool _setcrc,ICompressor *_compressor,IExpander *_expander, bool fast)
+    CCompressedFile(IFileIO *_fileio,IMemoryMappedFile *_mmfile,CompressedFileTrailer &_trailer,ICFmode _mode, bool _setcrc,ICompressor *_compressor,IExpander *_expander, unsigned _compMethod)
         : fileio(_fileio), mmfile(_mmfile)
     {
         compressor.set(_compressor);
@@ -2119,6 +2129,7 @@ public:
         mode = _mode;
         curblockpos = 0;
         curblocknum = (unsigned)-1; // relies on wrap
+        compMethod = _compMethod;
         if (mode!=ICFread) {
             if (!_fileio&&_mmfile)
                 throw MakeStringException(-1,"Compressed Write not supported on memory mapped files");
@@ -2134,10 +2145,16 @@ public:
             if (trailer.recordSize==0) {
                 if (!compressor)
                 {
-                    if (fast)
+                    if (compMethod == COMPRESS_METHOD_FASTLZ)
                         compressor.setown(createFastLZCompressor());
-                    else
+                    else if (compMethod == COMPRESS_METHOD_LZ4)
+                        compressor.setown(createLZ4Compressor());
+                    else // fallback
+                    {
+                        compMethod = COMPRESS_METHOD_LZW;
+                        trailer.compressedType = COMPRESSEDFILEFLAG;
                         compressor.setown(createLZWCompressor(true));
+                    }
                 }
                 compressor->open(compblkptr, trailer.blockSize);
             }
@@ -2162,10 +2179,15 @@ public:
             }
             if (trailer.recordSize==0) {
                 if (!expander) {
-                    if (fast)
+                    if (compMethod == COMPRESS_METHOD_FASTLZ)
                         expander.setown(createFastLZExpander());
-                    else
+                    else if (compMethod == COMPRESS_METHOD_LZ4)
+                        expander.setown(createLZ4Expander());
+                    else // fallback
+                    {
+                        compMethod = COMPRESS_METHOD_LZW;
                         expander.setown(createLZWExpander(true));
+                    }
                 }
             }
         }
@@ -2316,11 +2338,19 @@ ICompressedFileIO *createCompressedFileReader(IFileIO *fileio,IExpander *expande
             CompressedFileTrailer trailer;
             if (fileio->read(fsize-sizeof(WinCompressedFileTrailer),sizeof(WinCompressedFileTrailer),&wintrailer)==sizeof(WinCompressedFileTrailer)) {
                 wintrailer.translate(trailer);
-                if ((trailer.compressedType==COMPRESSEDFILEFLAG)||(trailer.compressedType==FASTCOMPRESSEDFILEFLAG)) {
+                if ( (trailer.compressedType==COMPRESSEDFILEFLAG) ||
+                     (trailer.compressedType==FASTCOMPRESSEDFILEFLAG) ||
+                     (trailer.compressedType==LZ4COMPRESSEDFILEFLAG) )
+                {
                     if (expander&&(trailer.recordSize!=0)) {
                         throw MakeStringException(-1, "Compressed file format error(%d), Encrypted?",trailer.recordSize);
                     }
-                    CCompressedFile *cfile = new CCompressedFile(fileio,NULL,trailer,ICFread,false,NULL,expander,(trailer.compressedType==FASTCOMPRESSEDFILEFLAG));
+                    unsigned compMethod1 = COMPRESS_METHOD_LZW;
+                    if (trailer.compressedType == FASTCOMPRESSEDFILEFLAG)
+                        compMethod1 = COMPRESS_METHOD_FASTLZ;
+                    else if (trailer.compressedType == LZ4COMPRESSEDFILEFLAG)
+                        compMethod1 = COMPRESS_METHOD_LZ4;
+                    CCompressedFile *cfile = new CCompressedFile(fileio,NULL,trailer,ICFread,false,NULL,expander,compMethod1);
                     return cfile;
                 }
             }
@@ -2342,11 +2372,19 @@ ICompressedFileIO *createCompressedFileReader(IFile *file,IExpander *expander, b
                     CompressedFileTrailer trailer;
                     memcpy(&wintrailer,mmfile->base()+fsize-sizeof(WinCompressedFileTrailer),sizeof(WinCompressedFileTrailer));
                     wintrailer.translate(trailer);
-                    if ((trailer.compressedType==COMPRESSEDFILEFLAG)||(trailer.compressedType==FASTCOMPRESSEDFILEFLAG)) {
+                    if ( (trailer.compressedType==COMPRESSEDFILEFLAG) ||
+                         (trailer.compressedType==FASTCOMPRESSEDFILEFLAG) ||
+                         (trailer.compressedType==LZ4COMPRESSEDFILEFLAG) )
+                    {
                         if (expander&&(trailer.recordSize!=0)) {
                             throw MakeStringException(-1, "Compressed file format error(%d), Encrypted?",trailer.recordSize);
                         }
-                        CCompressedFile *cfile = new CCompressedFile(NULL,mmfile,trailer,ICFread,false,NULL,expander,(trailer.compressedType==FASTCOMPRESSEDFILEFLAG));
+                        unsigned compMethod1 = COMPRESS_METHOD_LZW;
+                        if (trailer.compressedType == FASTCOMPRESSEDFILEFLAG)
+                            compMethod1 = COMPRESS_METHOD_FASTLZ;
+                        else if (trailer.compressedType == LZ4COMPRESSEDFILEFLAG)
+                            compMethod1 = COMPRESS_METHOD_LZ4;
+                        CCompressedFile *cfile = new CCompressedFile(NULL,mmfile,trailer,ICFread,false,NULL,expander,compMethod1);
                         return cfile;
                     }
                 }
@@ -2362,7 +2400,7 @@ ICompressedFileIO *createCompressedFileReader(IFile *file,IExpander *expander, b
 
 
 
-ICompressedFileIO *createCompressedFileWriter(IFileIO *fileio,size32_t recordsize,bool _setcrc,ICompressor *compressor,bool fast)
+ICompressedFileIO *createCompressedFileWriter(IFileIO *fileio,size32_t recordsize,bool _setcrc,ICompressor *compressor, unsigned _compMethod)
 {
     CompressedFileTrailer trailer;
     offset_t fsize = fileio->size();
@@ -2373,7 +2411,20 @@ ICompressedFileIO *createCompressedFileWriter(IFileIO *fileio,size32_t recordsiz
                 CompressedFileTrailer trailer;
                 if (fileio->read(fsize-sizeof(WinCompressedFileTrailer),sizeof(WinCompressedFileTrailer),&wintrailer)==sizeof(WinCompressedFileTrailer)) {
                     wintrailer.translate(trailer);
-                    if ((trailer.compressedType==COMPRESSEDFILEFLAG)||(trailer.compressedType==FASTCOMPRESSEDFILEFLAG)) {
+                    if ( (trailer.compressedType==COMPRESSEDFILEFLAG) ||
+                         (trailer.compressedType==FASTCOMPRESSEDFILEFLAG) ||
+                         (trailer.compressedType==LZ4COMPRESSEDFILEFLAG) )
+                    {
+                        // check trailer.compressedType against _compMethod
+                        unsigned compMethod1 = 0;
+                        if (trailer.compressedType == COMPRESSEDFILEFLAG)
+                            compMethod1 = COMPRESS_METHOD_LZW;
+                        else if (trailer.compressedType == FASTCOMPRESSEDFILEFLAG)
+                            compMethod1 = COMPRESS_METHOD_FASTLZ;
+                        else if (trailer.compressedType == LZ4COMPRESSEDFILEFLAG)
+                            compMethod1 = COMPRESS_METHOD_LZ4;
+                        if (_compMethod != compMethod1)
+                            throw MakeStringException(-1,"Appending to file with different compression method");
                         if ((recordsize==trailer.recordSize)||!trailer.recordSize)
                             break;
                         throw MakeStringException(-1,"Appending to file with different record size (%d,%d)",recordsize,trailer.recordSize);
@@ -2386,24 +2437,40 @@ ICompressedFileIO *createCompressedFileWriter(IFileIO *fileio,size32_t recordsiz
     else {
         memset(&trailer,0,sizeof(trailer));
         trailer.crc = ~0U;
-        trailer.compressedType = fast?FASTCOMPRESSEDFILEFLAG:COMPRESSEDFILEFLAG;
-        trailer.blockSize = COMPRESSEDFILEBLOCKSIZE;
-        trailer.recordSize = recordsize;
+        if (_compMethod == COMPRESS_METHOD_FASTLZ)
+        {
+            trailer.compressedType = FASTCOMPRESSEDFILEFLAG;
+            trailer.blockSize = FASTCOMPRESSEDFILEBLOCKSIZE;
+            trailer.recordSize = 0;
+        }
+        else if (_compMethod == COMPRESS_METHOD_LZ4)
+        {
+            trailer.compressedType = LZ4COMPRESSEDFILEFLAG;
+            trailer.blockSize = LZ4COMPRESSEDFILEBLOCKSIZE;
+            trailer.recordSize = 0;
+        }
+        else // fallback
+        {
+            trailer.compressedType = COMPRESSEDFILEFLAG;
+            trailer.blockSize = COMPRESSEDFILEBLOCKSIZE;
+            trailer.recordSize = recordsize;
+        }
     }
+    // MCK - may present compatibility issue if passing in compressor and wanting row comp
     if (compressor)
         trailer.recordSize = 0; // force not row compressed if compressor specified
-    CCompressedFile *cfile = new CCompressedFile(fileio,NULL,trailer,fsize?ICFappend:ICFcreate,_setcrc,compressor,NULL,fast);
+    CCompressedFile *cfile = new CCompressedFile(fileio,NULL,trailer,fsize?ICFappend:ICFcreate,_setcrc,compressor,NULL,_compMethod);
     return cfile;
 }
 
-ICompressedFileIO *createCompressedFileWriter(IFile *file,size32_t recordsize,bool append,bool _setcrc,ICompressor *compressor,bool fast, IFEflags extraFlags)
+ICompressedFileIO *createCompressedFileWriter(IFile *file,size32_t recordsize,bool append,bool _setcrc,ICompressor *compressor, unsigned _compMethod, IFEflags extraFlags)
 {
     if (file) {
         if (append&&!file->exists())
             append = false;
         Owned<IFileIO> fileio = file->open(append?IFOreadwrite:IFOcreate, extraFlags);
         if (fileio) 
-            return createCompressedFileWriter(fileio,recordsize,_setcrc,compressor,fast);
+            return createCompressedFileWriter(fileio,recordsize,_setcrc,compressor,_compMethod);
     }
     return NULL;
 }
@@ -2603,7 +2670,10 @@ IPropertyTree *getBlockedFileDetails(IFile *file)
             CompressedFileTrailer trailer;
             if (fileio->read(fsize-sizeof(WinCompressedFileTrailer),sizeof(WinCompressedFileTrailer),&wintrailer)==sizeof(WinCompressedFileTrailer)) {
                 wintrailer.translate(trailer);
-                if ((trailer.compressedType==COMPRESSEDFILEFLAG)||(trailer.compressedType==FASTCOMPRESSEDFILEFLAG)) {
+                if ( (trailer.compressedType==COMPRESSEDFILEFLAG) ||
+                     (trailer.compressedType==FASTCOMPRESSEDFILEFLAG) ||
+                     (trailer.compressedType==LZ4COMPRESSEDFILEFLAG) )
+                {
                     trailer.setDetails(*tree);
                     unsigned nb = trailer.numBlocks();
                     MemoryAttr indexbuf;
@@ -2681,6 +2751,13 @@ MODULE_INIT(INIT_PRIORITY_STANDARD)
         virtual ICompressor *getCompressor(const char *options) { return createFastLZCompressor(); }
         virtual IExpander *getExpander(const char *options) { return createFastLZExpander(); }
     };
+    class CLZ4CompressHandler : public CCompressHandlerBase
+    {
+    public:
+        CLZ4CompressHandler() : CCompressHandlerBase("LZ4") { }
+        virtual ICompressor *getCompressor(const char *options) { return createLZ4Compressor(); }
+        virtual IExpander *getExpander(const char *options) { return createLZ4Expander(); }
+    };
     class CAESCompressHandler : public CCompressHandlerBase
     {
     public:
@@ -2715,6 +2792,7 @@ MODULE_INIT(INIT_PRIORITY_STANDARD)
     addCompressorHandler(new CAESCompressHandler());
     addCompressorHandler(new CDiffCompressHandler());
     addCompressorHandler(new CLZWCompressHandler());
+    addCompressorHandler(new CLZ4CompressHandler());
     defaultCompressor.set(flzCompressor);
     return true;
 }

system/jlib/jlzw.hpp

@@ -98,6 +98,7 @@ extern jlib_decl void appendToBuffer(MemoryBuffer & out, size32_t len, const voi
 #define COMPRESS_METHOD_LZW    2
 #define COMPRESS_METHOD_FASTLZ 3
 #define COMPRESS_METHOD_LZMA   4
+#define COMPRESS_METHOD_LZ4    5
 
 interface ICompressedFileIO: extends IFileIO
 {
@@ -111,8 +112,8 @@ interface ICompressedFileIO: extends IFileIO
 
 extern jlib_decl ICompressedFileIO *createCompressedFileReader(IFile *file,IExpander *expander=NULL, bool memorymapped=false, IFEflags extraFlags=IFEnone);
 extern jlib_decl ICompressedFileIO *createCompressedFileReader(IFileIO *fileio,IExpander *expander=NULL);
-extern jlib_decl ICompressedFileIO *createCompressedFileWriter(IFile *file,size32_t recordsize,bool append=false,bool setcrc=true,ICompressor *compressor=NULL,bool fast=false, IFEflags extraFlags=IFEnone);
-extern jlib_decl ICompressedFileIO *createCompressedFileWriter(IFileIO *fileio,size32_t recordsize,bool setcrc=true,ICompressor *compressor=NULL,bool fast=false);
+extern jlib_decl ICompressedFileIO *createCompressedFileWriter(IFile *file,size32_t recordsize,bool append=false,bool setcrc=true,ICompressor *compressor=NULL, unsigned compMethod=COMPRESS_METHOD_LZW, IFEflags extraFlags=IFEnone);
+extern jlib_decl ICompressedFileIO *createCompressedFileWriter(IFileIO *fileio,size32_t recordsize,bool setcrc=true,ICompressor *compressor=NULL, unsigned compMethod=COMPRESS_METHOD_LZW);
 
 #define COMPRESSEDFILECRC (~0U)
 

system/jlib/jthread.cpp

@@ -33,7 +33,7 @@
 #include <sys/syscall.h>
 #include <sys/types.h>
 #include <sys/resource.h>
-#ifndef __APPLE__
+#ifdef _USE_NUMA
 #include <numa.h>
 #endif
 #endif
@@ -2426,7 +2426,7 @@ void setProcessAffinity(const char * cpuList)
 
 void setAutoAffinity(unsigned curProcess, unsigned processPerMachine, const char * optNodes)
 {
-#if defined(CPU_ZERO) && !defined(__APPLE__)
+#if defined(CPU_ZERO) && defined(_USE_NUMA)
     if (processPerMachine <= 1)
         return;
 

system/lz4_sm/CMakeLists.txt

@@ -0,0 +1,44 @@
+################################################################################
+#    HPCC SYSTEMS software Copyright (C) 2015 HPCC Systems.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+################################################################################
+
+# Component: lz4
+
+#####################################################
+# Description:
+# ------------
+#    Cmake Input File for lz4
+#####################################################
+
+
+project( lz4 )
+
+set ( SRCS
+        lz4/lib/lz4.c
+)
+
+include_directories (
+        lz4/lib
+)
+
+ADD_DEFINITIONS( -D_LIB )
+
+SET_SOURCE_FILES_PROPERTIES( ${SRCS} PROPERTIES LANGUAGE C )
+
+if (CMAKE_COMPILER_IS_GNUCC OR CMAKE_COMPILER_IS_CLANG)
+    set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fPIC -std=c99")
+endif()
+
+HPCC_ADD_LIBRARY( lz4 STATIC ${SRCS} )

system/lz4_sm/lz4

@@ -0,0 +1 @@
+Subproject commit d86dc916771c126afb797637dda9f6421c0cb998

system/security/LdapSecurity/CMakeLists.txt

@@ -31,8 +31,6 @@ endif()
 set (    SRCS 
          ../shared/authmap.cpp 
          ../shared/caching.cpp 
-         ../shared/basesecurity.cpp
-         ../shared/defaultsecuritymanager.cpp
          aci.cpp 
          ldapconnection.cpp 
          ldapsecurity.cpp 

system/security/LdapSecurity/ldapsecurity.cpp

@@ -22,7 +22,6 @@
 #include "ldapsecurity.ipp"
 #include "ldapsecurity.hpp"
 #include "authmap.ipp"
-#include "defaultsecuritymanager.hpp"
 
 /**********************************************************
  *     CLdapSecUser                                       *
@@ -1317,16 +1316,6 @@ LDAPSECURITY_API ISecManager * newLdapSecManager(const char *serviceName, IPrope
     return new CLdapSecManager(serviceName, config);
 }
 
-LDAPSECURITY_API ISecManager * newDefaultSecManager(const char *serviceName, IPropertyTree &config)
-{
-    return new CDefaultSecurityManager(serviceName, &config);
-}
-
-LDAPSECURITY_API ISecManager * newLocalSecManager(const char *serviceName, IPropertyTree &config)
-{
-    return new CLocalSecurityManager(serviceName, &config);
-}
-
 LDAPSECURITY_API IAuthMap *newDefaultAuthMap(IPropertyTree* config)
 {
     CAuthMap* authmap = new CAuthMap(NULL);

system/security/LdapSecurity/ldapsecurity.hpp

@@ -33,8 +33,6 @@
 #endif 
 
 extern "C" LDAPSECURITY_API ISecManager * newLdapSecManager(const char *serviceName, IPropertyTree &config);
-extern "C" LDAPSECURITY_API ISecManager * newDefaultSecManager(const char *serviceName, IPropertyTree &config);
-extern "C" LDAPSECURITY_API ISecManager * newLocalSecManager(const char *serviceName, IPropertyTree &config);
 extern "C" LDAPSECURITY_API IAuthMap *newDefaultAuthMap(IPropertyTree* config);
 
 #endif

system/security/htpasswdSecurity/CMakeLists.txt

@@ -27,8 +27,6 @@ project( htpasswdSecurity )
 set (    SRCS
          ../shared/authmap.cpp
          ../shared/caching.cpp
-         ../shared/basesecurity.cpp
-         ../shared/defaultsecuritymanager.cpp
          ./htpasswdSecurity.cpp
     )
 

system/security/htpasswdSecurity/htpasswdSecurity.cpp

@@ -17,15 +17,15 @@
 
 #pragma warning( disable : 4786 )
 
-#include "defaultsecuritymanager.hpp"
+#include "basesecurity.hpp"
 #include "authmap.ipp"
 #include <apr_md5.h>
 #include "htpasswdSecurity.hpp"
 
-class CHtpasswdSecurityManager : public CDefaultSecurityManager
+class CHtpasswdSecurityManager : public CBaseSecurityManager
 {
 public:
-	CHtpasswdSecurityManager(const char *serviceName, IPropertyTree *authconfig) : CDefaultSecurityManager(serviceName, (IPropertyTree *)NULL)
+    CHtpasswdSecurityManager(const char *serviceName, IPropertyTree *authconfig) : CBaseSecurityManager(serviceName, (IPropertyTree *)NULL)
 	{
 		if (authconfig)
 			authconfig->getProp("@htpasswdFile", pwFile);
@@ -80,6 +80,7 @@ public:
 					unsigned requiredaccess = str2perm(required.str());
 					rs->setRequiredAccessFlags(requiredaccess);
 					rs->setDescription(description.str());
+                    rs->setAccessFlags(SecAccess_Full);//grant full access to authenticated users
 				}
 				loc_iter->next();
 			}
@@ -90,6 +91,7 @@ public:
 
 protected:
 
+    //ISecManager
 	bool IsPasswordValid(ISecUser& sec_user)
 	{
 		StringBuffer user;
@@ -113,6 +115,41 @@ protected:
 		return false;
 	}
 
+    const char * getDescription()
+    {
+        return "HTPASSWD Security Manager";
+    }
+
+    bool authorize(ISecUser & user, ISecResourceList * resources)
+    {
+        return IsPasswordValid(user);
+    }
+
+    unsigned getPasswordExpirationWarningDays()
+    {
+        return -2;//never expires
+    }
+
+    int authorizeEx(SecResourceType rtype, ISecUser & user, const char * resourcename)
+    {
+        return SecAccess_Full;//grant full access to authenticated users
+    }
+
+    int getAccessFlagsEx(SecResourceType rtype, ISecUser& sec_user, const char* resourcename)
+    {
+        return SecAccess_Full;//grant full access to authenticated users
+    }
+
+    int authorizeFileScope(ISecUser & user, const char * filescope)
+    {
+        return SecAccess_Full;//grant full access to authenticated users
+    }
+
+    int authorizeWorkunitScope(ISecUser & user, const char * filescope)
+    {
+        return SecAccess_Full;//grant full access to authenticated users
+    }
+
 private:
 
 	void initAPR()

system/security/htpasswdSecurity/htpasswdSecurity.hpp

@@ -32,8 +32,6 @@
 
 #endif 
 
-#include "basesecurity.hpp"
-
 extern "C" 
 {
     HTPASSWDSECURITY_API ISecManager * newHtpasswdSecManager(const char *serviceName, IPropertyTree &config);

system/security/shared/basesecurity.cpp

@@ -1,599 +0,0 @@
-/*##############################################################################
-
-    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems®.
-
-    Licensed under the Apache License, Version 2.0 (the "License");
-    you may not use this file except in compliance with the License.
-    You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-    Unless required by applicable law or agreed to in writing, software
-    distributed under the License is distributed on an "AS IS" BASIS,
-    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-    See the License for the specific language governing permissions and
-    limitations under the License.
-############################################################################## */
-
-#pragma warning (disable : 4786)
-#pragma warning (disable : 4018)
-#pragma warning (disable : 4146)
-#pragma warning (disable : 4275)
-
-#ifdef _WIN32
-#define AXA_API __declspec(dllexport)
-#endif
-//#include "ctconnection.h"
-#include "basesecurity.hpp"
-#include "jmd5.hpp"
-
-#define     cacheTimeout 30000
-//#ifdef _WIN32
-
-CBaseSecurityManager::CBaseSecurityManager(const char *serviceName, const char *config)
-{
-    Owned<IPropertyTree> cfg = createPTreeFromXMLString(config, ipt_caseInsensitive);
-    if(cfg.get() == NULL)
-        throw MakeStringException(-1, "createPTreeFromXMLString() failed for %s", config);
-    init(serviceName,cfg);
-    cfg->Release();
-}
-
-CBaseSecurityManager::CBaseSecurityManager(const char *serviceName, IPropertyTree *config)
-{
-    m_dbpasswordEncoding = SecPwEnc_unknown;
-    init(serviceName,config);
-}
-
-void CBaseSecurityManager::init(const char *serviceName, IPropertyTree *config)
-{
-    if(config == NULL)
-        return;
-    
-    m_config.set(config);
-
-    m_permissionsCache.setCacheTimeout( 60 * config->getPropInt("@cacheTimeout", 5) );
-    
-
-    m_dbserver.appendf("%s",config->queryProp("@serverName"));
-    m_dbuser.appendf("%s",config->queryProp("@systemUser"));
-    if(config->hasProp("@ConnectionPoolSize"))
-        m_poolsize = atoi(config->queryProp("@connectionPoolSize"));
-    else
-        m_poolsize = 2;
-
-    StringBuffer encodedPass,encryptedPass;
-    encodedPass.appendf("%s",config->queryProp("@systemPassword"));
-    decrypt(m_dbpassword, encodedPass.str());
-
-    m_dbpasswordEncoding = SecPwEnc_plain_text;
-
-    StringBuffer strPasswordEncoding;
-    const char* encodingType = config->queryProp("@encodePassword");
-    if(encodingType && strcmp(encodingType,"MD5") == 0)
-        m_dbpasswordEncoding=SecPwEnc_salt_md5;
-    else if (encodingType && strcmp(encodingType,"Rijndael") == 0)
-        m_dbpasswordEncoding=SecPwEnc_Rijndael;
-    else if (encodingType && strcmp(encodingType,"Accurint MD5") == 0)
-        m_dbpasswordEncoding = SecPwEnc_salt_accurint_md5;
-
-    if(m_dbserver.length() == 0 || m_dbuser.length() == 0)
-        throw MakeStringException(-1, "CBaseSecurityManager() - db server or user is missing");
-
-    IPropertyTree* pNonRestrictedIPTree = config->queryBranch("SafeIPList");
-    if(pNonRestrictedIPTree)
-    {
-        Owned<IPropertyTreeIterator> Itr = pNonRestrictedIPTree->getElements("ip");
-        for(Itr->first();Itr->isValid();Itr->next())
-        {
-            IPropertyTree& tree = Itr->query();
-            m_safeIPList[tree.queryProp("")]=true;
-        }
-    }
-
-    m_enableIPRoaming = config->getPropBool("@enableIPRoaming");
-    m_enableOTP = config->getPropBool("@enableOTP",false);
-    m_passwordExpirationWarningDays = config->getPropInt(".//@passwordExpirationWarningDays", 10); //Default to 10 days
-}
-
-CBaseSecurityManager::~CBaseSecurityManager()
-{
-    MapStrToUsers::iterator pos;
-    for(pos=m_userList.begin();pos!=m_userList.end();){
-        pos->second->Release();
-        pos++;
-    }
-    dbDisconnect();
-}
-
-
-
-
-//interface ISecManager : extends IInterface
-ISecUser * CBaseSecurityManager::createUser(const char * user_name)
-{
-    return (new CSecureUser(user_name, NULL));
-}
-
-ISecResourceList * CBaseSecurityManager::createResourceList(const char * rlname)
-{
-    return (new CSecurityResourceList(rlname));
-}
-
-bool CBaseSecurityManager::subscribe(ISecAuthenticEvents & events)
-{
-    m_subscriber.set(&events);
-    return true;
-}
-
-bool CBaseSecurityManager::unsubscribe(ISecAuthenticEvents & events)
-{
-    if (&events == m_subscriber.get())
-    {
-        m_subscriber.set(NULL);
-    }
-    return true;
-}
-
-bool CBaseSecurityManager::authorize(ISecUser & sec_user, ISecResourceList * Resources)
-{   
-    if(sec_user.getAuthenticateStatus() != AS_AUTHENTICATED)
-    {
-        bool bOk = ValidateUser(sec_user);
-        if(bOk == false)
-            return false;
-    }
-    return ValidateResources(sec_user,Resources);
-}
-
-bool CBaseSecurityManager::updateSettings(ISecUser &sec_user, ISecPropertyList* resources) 
-{
-    CSecurityResourceList * reslist = (CSecurityResourceList*)resources;
-    if(!reslist)
-        return true;
-    IArrayOf<ISecResource>& rlist = reslist->getResourceList();
-    int nResources = rlist.length();
-    if (nResources <= 0)
-        return true;
-
-    bool rc = false;
-    if (m_permissionsCache.isCacheEnabled()==false)
-        return updateSettings(sec_user, rlist);
-
-    bool* cached_found = (bool*)alloca(nResources*sizeof(bool));
-    int nFound = m_permissionsCache.lookup(sec_user, rlist, cached_found);
-    if (nFound >= nResources)
-        return true;
-    
-    IArrayOf<ISecResource> rlist2;
-    for (int i=0; i < nResources; i++)
-    {
-        if (*(cached_found+i) == false)
-        {
-            ISecResource& secRes = rlist.item(i);
-            secRes.Link();
-            rlist2.append(secRes);
-        }
-    }
-    rc = updateSettings(sec_user, rlist2);
-    if (rc)
-        m_permissionsCache.add(sec_user, rlist2);
-    return rc;
-}
-
-bool CBaseSecurityManager::updateSettings(ISecUser & sec_user,IArrayOf<ISecResource>& rlist) 
-{
-    CSecureUser* user = (CSecureUser*)&sec_user;
-    if(user == NULL)
-        return false;
-
-    int usernum = findUser(user->getName(),user->getRealm());
-    if(usernum < 0)
-    {
-        PrintLog("User number of %s can't be found", user->getName());
-        return false;
-    }
-    bool sqchecked = false, sqverified = false, otpchecked = false;
-    int otpok = -1;
-    ForEachItemIn(x, rlist)
-    {
-        ISecResource* secRes = (ISecResource*)(&(rlist.item(x)));
-        if(secRes == NULL)
-            continue;
-        //AccessFlags default value is -1. Set it to 0 so that the settings can be cached. AccessFlags is not being used for settings.
-        secRes->setAccessFlags(0);
-        if(secRes->getParameter("userprop") && *secRes->getParameter("userprop")!='\0')
-        {
-            //if we have a parameter in the user or company table it will have been added as a parameter to the ISecUser when 
-            // the authentication query was run. We should keep this messiness here so that the the end user is insulated....
-            dbValidateSetting(*secRes,sec_user);
-            continue;
-        }
-
-        const char* resource_name = secRes->getParameter("resource");
-        if(resource_name && *resource_name && 
-            (stricmp(resource_name, "SSN Masking") == 0 || stricmp(resource_name, "Driver License Masking") == 0))
-        {
-            //If OTP Enabled and OTP2FACTOR cookie not valid, mask
-            if(m_enableOTP)
-            {
-                if(!otpchecked)
-                {
-                    const char* otpcookie = sec_user.getProperty("OTP2FACTOR");
-                    // -1 means OTP is not enabled for the user. 0: failed verfication, 1: passed verification.
-                    otpok = validateOTP(&sec_user, otpcookie);
-                    otpchecked = true;
-                }
-                if(otpok == 0)
-                {
-                    CSecurityResource* cres = dynamic_cast<CSecurityResource*>(secRes);
-                    if(resource_name && *resource_name && cres)
-                    {
-                        if(stricmp(resource_name, "SSN Masking") == 0)
-                        {
-                            cres->setValue("All");
-                            continue;
-                        }
-                        else if(stricmp(resource_name, "Driver License Masking") == 0)
-                        {
-                            cres->setValue("1");
-                            continue;
-                        }
-                    }
-                }
-                else if(otpok == 1)
-                {
-                    CSecurityResource* cres = dynamic_cast<CSecurityResource*>(secRes);
-                    if(resource_name && *resource_name && cres)
-                    {
-                        if(stricmp(resource_name, "SSN Masking") == 0)
-                        {
-                            cres->setValue("None");
-                            continue;
-                        }
-                        else if(stricmp(resource_name, "Driver License Masking") == 0)
-                        {
-                            cres->setValue("0");
-                            continue;
-                        }
-                    }
-                }
-            }
-
-            if(m_enableIPRoaming && sec_user.getPropertyInt("IPRoaming") == 1)
-            {
-                if(!sqchecked)
-                {
-                    const char* sequest = sec_user.getProperty("SEQUEST");
-                    if(sequest && *sequest)
-                    {
-                        sqverified = validateSecurityQuestion(&sec_user, sequest);
-                    }
-                    sqchecked = true;
-                }
-                if(!sqverified)
-                {
-                    CSecurityResource* cres = dynamic_cast<CSecurityResource*>(secRes);
-                    if(resource_name && *resource_name && cres)
-                    {
-                        if(stricmp(resource_name, "SSN Masking") == 0)
-                        {
-                            cres->setValue("All");
-                            continue;
-                        }
-                        else if(stricmp(resource_name, "Driver License Masking") == 0)
-                        {
-                            cres->setValue("1");
-                            continue;
-                        }
-                    }
-                }
-            }
-
-        }
-
-        dbValidateSetting(*secRes,usernum,user->getRealm());
-    }
-    return true;
-}
-
-
-bool CBaseSecurityManager::ValidateResources(ISecUser & sec_user, ISecResourceList * resources)
-{
-    CSecurityResourceList * reslist = (CSecurityResourceList*)resources;
-    if(!reslist)
-        return true;
-    IArrayOf<ISecResource>& rlist = reslist->getResourceList();
-    int nResources = rlist.length();
-    if (nResources <= 0)
-        return true;
-
-    bool rc = false;
-    if (m_permissionsCache.isCacheEnabled()==false)
-        return ValidateResources(sec_user, rlist);
-
-    bool* cached_found = (bool*)alloca(nResources*sizeof(bool));
-    int nFound = m_permissionsCache.lookup(sec_user, rlist, cached_found);
-    if (nFound >= nResources)
-    {
-        return true;
-    }
-
-    IArrayOf<ISecResource> rlist2;
-    for (int i=0; i < nResources; i++)
-    {
-        if (*(cached_found+i) == false)
-        {
-            ISecResource& secRes = rlist.item(i);
-            secRes.Link();
-            rlist2.append(secRes);
-        }
-    }
-    rc = ValidateResources(sec_user, rlist2);
-    if (rc)
-    {
-        IArrayOf<ISecResource> rlistValid;
-        for (int i=0; i < rlist2.ordinality(); i++)
-        {
-            ISecResource& secRes = rlist2.item(i);
-            if(secRes.getAccessFlags() >= secRes.getRequiredAccessFlags() || secRes.getAccessFlags() == SecAccess_Unknown)
-            {
-                secRes.Link();
-                rlistValid.append(secRes);
-            }
-        }
-        m_permissionsCache.add(sec_user, rlistValid);
-    }
-        
-    return rc;
-}
-
-static bool stringDiff(const char* str1, const char* str2)
-{
-    if(!str1 || !*str1)
-    {
-        if(!str2 || !*str2)
-            return false;
-        else
-            return true;
-    }
-    else
-    {
-        if(!str2 || !*str2)
-            return true;
-        else
-            return (strcmp(str1, str2) != 0);
-    }
-}
-
-bool CBaseSecurityManager::ValidateUser(ISecUser & sec_user)
-{
-    StringBuffer clientip(sec_user.getPeer());
-    StringBuffer otpbuf, sqbuf;
-    if(m_enableOTP)
-    {
-        otpbuf.append(sec_user.getProperty("OTP2FACTOR"));
-    }
-    if(m_enableIPRoaming)
-    {
-        sqbuf.append(sec_user.getProperty("SEQUEST"));
-    }
-    if(m_permissionsCache.isCacheEnabled() && m_permissionsCache.lookup(sec_user))
-    {
-        bool bReturn = true;
-        if(IsIPRestricted(sec_user))
-        {
-            const char* cachedclientip = sec_user.getPeer();
-            if(clientip.length() > 0 && cachedclientip && strncmp(clientip.str(), cachedclientip , clientip.length()) != 0)
-            {
-                //we seem to be coming from a different peer... this is not good
-                WARNLOG("Found user %d in cache, but have to re-validate IP, because it was coming from %s but is now coming from %s",sec_user.getUserID(), cachedclientip, clientip.str());
-                sec_user.setAuthenticateStatus(AS_INVALID_CREDENTIALS);
-                sec_user.setPeer(clientip.str());
-                m_permissionsCache.removeFromUserCache(sec_user);
-                bReturn =  false;
-            }
-        }
-
-        if(m_enableOTP)
-        {
-            const char* old_otp = sec_user.getProperty("OTP2FACTOR");
-            if(stringDiff(old_otp, otpbuf.str()))
-                bReturn = false;
-        }
-        if(m_enableIPRoaming)
-        {
-            const char* old_sq = sec_user.getProperty("SEQUEST");
-            if(stringDiff(old_sq, sqbuf.str()))
-                bReturn = false;
-        }
-
-        if(bReturn)
-        {
-            sec_user.setAuthenticateStatus(AS_AUTHENTICATED);
-            return true;
-        }
-    }
-
-    if(!IsPasswordValid(sec_user))
-    {
-        ERRLOG("Password validation failed for user: %s",sec_user.getName());
-        return false;
-    }
-    else
-    {
-        if(IsIPRestricted(sec_user)==true)
-        {
-            if(ValidateSourceIP(sec_user,m_safeIPList)==false)
-            {
-                ERRLOG("IP check failed for user:%s coming from %s",sec_user.getName(),sec_user.getPeer());
-                sec_user.setAuthenticateStatus(AS_INVALID_CREDENTIALS);
-                return false;
-            }
-        }
-        if(m_permissionsCache.isCacheEnabled())
-            m_permissionsCache.add(sec_user);
-        sec_user.setAuthenticateStatus(AS_AUTHENTICATED);
-    }
-    return true;
-}
-
-bool CBaseSecurityManager::IsPasswordValid(ISecUser& sec_user)
-{
-    StringBuffer password(sec_user.credentials().getPassword());
-    EncodePassword(password);
-    StringBuffer SQLQuery;
-    buildAuthenticateQuery(sec_user.getName(),password.str(),sec_user.getRealm(),SQLQuery);
-    return dbauthenticate(sec_user , SQLQuery);
-}
-
-bool CBaseSecurityManager::IsIPRestricted(ISecUser& sec_user)
-{
-    const char* iprestricted = sec_user.getProperty("iprestricted");
-    if(iprestricted!=NULL && strncmp(iprestricted,"1",1)==0)
-        return true;
-    return false;
-}
-
-
-void CBaseSecurityManager::EncodePassword(StringBuffer& password)
-{
-    StringBuffer encodedPassword;
-    switch (m_dbpasswordEncoding)
-    {
-        case SecPwEnc_salt_md5:
-            md5_string(password,encodedPassword);
-            password.clear().append(encodedPassword.str());
-            break;
-        case SecPwEnc_Rijndael:
-            encrypt(encodedPassword,password.str());
-            password.clear().append(encodedPassword.str());
-            break;
-        case SecPwEnc_salt_accurint_md5:
-            password.toUpperCase();
-            md5_string(password,encodedPassword);
-            password.clear().append(encodedPassword.str());
-            break;
-    }
-}
-
-
-
-bool CBaseSecurityManager::addResources(ISecUser & sec_user, ISecResourceList * Resources)
-{
-    return false;
-}
-bool CBaseSecurityManager::addUser(ISecUser & user)
-{
-    return false;
-}
-
-int CBaseSecurityManager::getUserID(ISecUser& user)
-{
-    return findUser(user.getName(),user.getRealm());
-}
-
-bool CBaseSecurityManager::ValidateResources(ISecUser & sec_user,IArrayOf<ISecResource>& rlist)
-{
-    CSecureUser* user = (CSecureUser*)&sec_user;
-    if(user == NULL)
-        return false;
-
-    int usernum = findUser(user->getName(),user->getRealm());
-    if(usernum < 0)
-    {
-        PrintLog("User number of %s can't be found", user->getName());
-        return false;
-    }
-
-    ForEachItemIn(x, rlist)
-    {
-        ISecResource* res = (ISecResource*)(&(rlist.item(x)));
-        if(res == NULL)
-            continue;
-        dbValidateResource(*res,usernum,user->getRealm());
-    }
-
-    return true;
-}
-
-bool CBaseSecurityManager::updateResources(ISecUser & user, ISecResourceList * resources)
-{   
-    //("CBaseSecurityManager::updateResources");
-    if(!resources)
-        return false;
-
-    const char* username = user.getName();
-    //const char* realm = user.getRealm();
-    const char* realm = NULL;
-
-    int usernum = findUser(username,realm);
-    if(usernum <= 0)
-    {
-        PrintLog("User number of %s can't be found", username);
-        return false;
-    }
-    CSecurityResourceList * reslist = (CSecurityResourceList*)resources;
-    if (reslist)
-    {
-        IArrayOf<ISecResource>& rlist = reslist->getResourceList();
-        ForEachItemIn(x, rlist)
-        {
-            ISecResource* res = (ISecResource*)(&(rlist.item(x)));
-            if(res == NULL)
-                continue;
-            dbUpdateResource(*res,usernum,realm);
-        }
-    }
-    return true;
-}
-
-bool CBaseSecurityManager::updateUserPassword(ISecUser& user, const char* newPassword, const char* currPassword)
-{
-    //("CBaseSecurityManager::updateUser");
-    if(!newPassword)
-        return false;
-    StringBuffer password(newPassword);
-    EncodePassword(password);
-    const char* realm = NULL;
-    bool bReturn =  dbUpdatePasswrd(user.getName(),realm,password.str());
-    if(bReturn == true)
-        user.credentials().setPassword(password.str());
-    //need to flush the users info from the cache....
-    if(m_permissionsCache.isCacheEnabled())
-        m_permissionsCache.removeFromUserCache(user);
-    return bReturn;
-}
-
-
-void CBaseSecurityManager::logon_failed(const char* user, const char* msg) 
-{
-    PrintLog("%s: %s", user, msg);
-}
-
-int CBaseSecurityManager::findUser(const char* user,const char* realm)
-{
-    if(user == NULL)
-        return -1;
-    synchronized block(m_usermap_mutex);
-    int* uidptr = m_usermap.getValue(user);
-    if(uidptr != NULL)
-    {
-        return *uidptr;
-    }
-    else
-    {
-        int uid = dbLookupUser(user,realm);
-        if(uid >= 0)
-        {
-            m_usermap.setValue(user, uid);
-        }
-        return uid;
-    }
-}
-
-
-
-
-
-//#endif //_WIN32

system/security/shared/basesecurity.hpp

@@ -18,9 +18,6 @@
 #ifndef BASESECURITY_INCL
 #define BASESECURITY_INCL
 
-#pragma warning (disable : 4786)
-#pragma warning (disable : 4018)
-
 #include <stdlib.h>
 #include "seclib.hpp"
 #include "jliball.hpp"
@@ -28,358 +25,279 @@
 
 #include "SecureUser.hpp"
 #include "SecurityResource.hpp"
-// to avoid warning about macro max/min
-#undef max
-#undef min
-#include "caching.hpp"
 #include "SecurityResourceList.hpp"
 
-#include <map>
-#include <string>
-
-
-typedef MapStringTo<int> MapStrToInt;
-
-const char* const def_ExpirationDate = "ExpirationDate";
-
-typedef std::map<std::string, CSecurityResourceList* > MapStrToResList;
-
-typedef std::map<std::string, bool> IPList;
-
-class UserInfo : public CInterface
+class CBaseSecurityManager : public CInterface,
+    implements ISecManager
 {
-private:
-    ISecUser* _UserInfo;
-    MapStrToResList _resList;
-    unsigned _timeCreated;
-    unsigned _timeOut;
-
 public:
-    IMPLEMENT_IINTERFACE;
+    IMPLEMENT_IINTERFACE
 
-    UserInfo(ISecUser& userInfo, unsigned TimeoutPeriod=60000)
+    CBaseSecurityManager()
     {
-        _UserInfo = &userInfo;
-        if(_UserInfo)
-            _UserInfo->Link();
-        _timeCreated = msTick();
-        _timeOut = TimeoutPeriod; 
     }
 
-    virtual ~UserInfo()
+
+    CBaseSecurityManager(const char *serviceName, IPropertyTree *config)
     {
-        MapStrToResList::iterator pos;
-        for(pos=_resList.begin();pos!=_resList.end();){
-            pos->second->Release();
-            pos++;
-        }
-        if(_UserInfo)
-            _UserInfo->Release();
     }
 
-    ISecResourceList* queryList(const char* ListName)
+    ~CBaseSecurityManager()
     {
-        return (ISecResourceList*)_resList[ListName];
     }
 
-    void appendList(const char* ListName, ISecResourceList* list)
+
+    //ISecManager
+    ISecUser * createUser(const char * user_name)
     {
-        if(list && ListName)
-            _resList[ListName] = (CSecurityResourceList*)list;
+        return new CSecureUser(user_name, NULL);
     }
 
-    virtual bool IsValid()
+    ISecResourceList * createResourceList(const char * rlname)
     {
-        return (msTick() > (_timeOut + _timeCreated)) ? false : true;
+        return new CSecurityResourceList(rlname);
     }
 
-    void CopyTo(ISecUser & sec_user)
+    bool subscribe(ISecAuthenticEvents & events)
     {
-        if(_UserInfo)
-            _UserInfo->copyTo(sec_user);
+        UNIMPLEMENTED;
+        return false;
     }
-};
 
-
-typedef std::map<std::string, UserInfo*> MapStrToUsers;
-
-
-class CBaseSecurityManager : public CInterface,
-    implements ISecManager
-{
-private:
-    Owned<ISecAuthenticEvents> m_subscriber;
-    StringBuffer               m_dbserver;
-    StringBuffer               m_dbuser;
-    StringBuffer               m_dbpassword;
-    int                        m_poolsize;
-    SecPasswordEncoding        m_dbpasswordEncoding;
-    MapStrToInt                m_usermap;
-    Mutex                      m_usermap_mutex;
-    MapStrToUsers              m_userList;  
-    Owned<IProperties>         m_extraparams;
-    CPermissionsCache          m_permissionsCache;
-    unsigned                   m_passwordExpirationWarningDays;
-
-protected:
-    CriticalSection             crit;
-    Owned<IPropertyTree>        m_config;
-    CriticalSection             _cache_Section;
-    IPList                      m_safeIPList;
-    bool                        m_enableIPRoaming;
-    bool                        m_enableOTP;
-public:
-    IMPLEMENT_IINTERFACE
-
-    CBaseSecurityManager(const char *serviceName, const char *config);
-    CBaseSecurityManager(const char *serviceName, IPropertyTree *config);
-    virtual ~CBaseSecurityManager();
-
-//interface ISecManager : extends IInterface
-    ISecUser * createUser(const char * user_name);
-    ISecResourceList * createResourceList(const char * rlname);
-    bool subscribe(ISecAuthenticEvents & events);
-    bool unsubscribe(ISecAuthenticEvents & events);
-    bool virtual authorize(ISecUser & sec_user, ISecResourceList * Resources);
-    bool authorizeEx(SecResourceType rtype, ISecUser& sec_user, ISecResourceList * Resources)
+    bool unsubscribe(ISecAuthenticEvents & events)
     {
-        return authorize(sec_user, Resources);
+        UNIMPLEMENTED;
+        return false;
     }
-    int authorizeEx(SecResourceType rtype, ISecUser& sec_user, const char* resourcename)
-    {
-        if(!resourcename || !*resourcename)
-            return SecAccess_Full;
 
-        Owned<ISecResourceList> rlist;
-        rlist.setown(createResourceList("resources"));
-        rlist->addResource(resourcename);
-        
-        bool ok = authorizeEx(rtype, sec_user, rlist.get());
-        if(ok)
-            return rlist->queryResource(0)->getAccessFlags();
-        else
-            return -1;
-    }   
-    virtual int getAccessFlagsEx(SecResourceType rtype, ISecUser& sec_user, const char* resourcename)
+    bool authorize(ISecUser & user, ISecResourceList * resources)
     {
         UNIMPLEMENTED;
+        return false;
     }
-    virtual int authorizeFileScope(ISecUser & user, const char * filescope)
+
+    bool authorizeEx(SecResourceType rtype, ISecUser & user, ISecResourceList * resources)
     {
         UNIMPLEMENTED;
+        return false;
     }
-    virtual bool authorizeFileScope(ISecUser & user, ISecResourceList * resources)
+
+    int authorizeEx(SecResourceType rtype, ISecUser & user, const char * resourcename)
     {
         UNIMPLEMENTED;
+        return 0;
     }
-    virtual int authorizeWorkunitScope(ISecUser & user, const char * filescope)
+
+    int getAccessFlagsEx(SecResourceType rtype, ISecUser & user, const char * resourcename)
     {
         UNIMPLEMENTED;
+        return 0;
     }
-    virtual bool authorizeWorkunitScope(ISecUser & user, ISecResourceList * resources)
+
+    int authorizeFileScope(ISecUser & user, const char * filescope)
     {
         UNIMPLEMENTED;
+        return 0;
     }
-    virtual bool ValidateSourceIP(ISecUser & user,IPList& SafeIPList)
+
+    bool authorizeFileScope(ISecUser & user, ISecResourceList * resources)
     {
-        return true;
+        UNIMPLEMENTED;
+        return false;
     }
-    bool addResourcesEx(SecResourceType rtype, ISecUser& sec_user, ISecResourceList * resources, SecPermissionType ptype = PT_ADMINISTRATORS_ONLY, const char* basedn=NULL)
+
+    bool addResources(ISecUser & user, ISecResourceList * resources)
     {
-        return addResources(sec_user, resources);
+        UNIMPLEMENTED;
+        return false;
     }
-    bool addResourceEx(SecResourceType rtype, ISecUser& user, const char* resourcename, SecPermissionType ptype = PT_ADMINISTRATORS_ONLY, const char* basedn=NULL)
-    {
-        Owned<ISecResourceList> rlist;
-        rlist.setown(createResourceList("resources"));
-        rlist->addResource(resourcename);
 
-        return addResourcesEx(rtype, user, rlist.get(), ptype, basedn);
+    bool addResourcesEx(SecResourceType rtype, ISecUser & user, ISecResourceList * resources, SecPermissionType ptype, const char * basedn)
+    {
+        UNIMPLEMENTED;
+        return false;
     }
 
-    bool addResources(ISecUser & user, ISecResourceList * resources);
-    bool updateResources(ISecUser & user, ISecResourceList * resources);
-    
-
-
-    virtual bool updateSettings(ISecUser &user, ISecPropertyList* resources) ;
-    
+    bool addResourceEx(SecResourceType rtype, ISecUser & user, const char * resourcename, SecPermissionType ptype, const char * basedn)
+    {
+        UNIMPLEMENTED;
+        return false;
+    }
 
-    virtual bool getResources(SecResourceType rtype, const char * basedn, IArrayOf<ISecResource> & resources)
+    bool getResources(SecResourceType rtype, const char * basedn, IResourceArray & resources)
     {
         UNIMPLEMENTED;
+        return false;
     }
-    virtual bool addUser(ISecUser & user);
-    virtual ISecUser * lookupUser(unsigned uid)
+
+    bool updateResources(ISecUser & user, ISecResourceList * resources)
     {
-        return NULL;
+        UNIMPLEMENTED;
+        return false;
     }
-    virtual ISecUser * findUser(const char * username)
+
+    bool updateSettings(ISecUser & user, ISecPropertyList * resources)
     {
-        return NULL;
+        UNIMPLEMENTED;
+        return false;
     }
 
-    virtual bool initUser(ISecUser& user)
+    bool addUser(ISecUser & user)
     {
+        UNIMPLEMENTED;
         return false;
     }
 
-    virtual ISecUserIterator * getAllUsers()
+    ISecUser * findUser(const char * username)
     {
+        UNIMPLEMENTED;
         return NULL;
     }
 
-    virtual void setExtraParam(const char * name, const char * value)
+    ISecUser * lookupUser(unsigned uid)
     {
-        if(name == NULL || name[0] == '\0')
-            return;
-
-        if (!m_extraparams)
-            m_extraparams.setown(createProperties(false));
-        m_extraparams->setProp(name, value);
+        UNIMPLEMENTED;
+        return NULL;
     }
-    virtual IAuthMap * createAuthMap(IPropertyTree * authconfig) {return NULL;}
-    virtual IAuthMap * createFeatureMap(IPropertyTree * authconfig) {return NULL;}
-    virtual IAuthMap * createSettingMap(IPropertyTree * authconfig) {return NULL;}
 
-    virtual bool updateUserPassword(ISecUser& user, const char* newPassword, const char* currPassword = NULL);
-    virtual bool IsPasswordExpired(ISecUser& user){return false;}
-    void getAllGroups(StringArray & groups, StringArray & managedBy, StringArray & descriptions) { UNIMPLEMENTED;}
-    
-    virtual void deleteResource(SecResourceType rtype, const char * name, const char * basedn)
+    ISecUserIterator * getAllUsers()
     {
         UNIMPLEMENTED;
+        return NULL;
     }
-    virtual void renameResource(SecResourceType rtype, const char * oldname, const char * newname, const char * basedn)
+
+    void getAllGroups(StringArray & groups, StringArray & managedBy, StringArray & descriptions )
     {
-        //UNIMPLEMENTED;
+        UNIMPLEMENTED;
     }
-    virtual void copyResource(SecResourceType rtype, const char * oldname, const char * newname, const char * basedn)
+
+    bool updateUserPassword(ISecUser & user, const char * newPassword, const char* currPassword = 0)
     {
         UNIMPLEMENTED;
+        return false;
     }
-    virtual void cacheSwitch(SecResourceType rtype, bool on)
+
+    bool initUser(ISecUser & user)
     {
         UNIMPLEMENTED;
+        return false;
     }
 
-    virtual SecUserStatus getUserStatus(const char* StatusFlag)
+    void setExtraParam(const char * name, const char * value)
     {
         UNIMPLEMENTED;
     }
 
-    virtual SecPasswordEncoding getPasswordEncoding()
+    IAuthMap * createAuthMap(IPropertyTree * authconfig)
     {
-        return m_dbpasswordEncoding;
+        UNIMPLEMENTED;
+        return NULL;
     }
-    virtual bool authTypeRequired(SecResourceType rtype) {return false;};
 
-    virtual const char* getDescription()
+    IAuthMap * createFeatureMap(IPropertyTree * authconfig)
     {
+        UNIMPLEMENTED;
         return NULL;
     }
 
-    virtual unsigned getPasswordExpirationWarningDays()
+    IAuthMap * createSettingMap(IPropertyTree * authconfig)
     {
-        return m_passwordExpirationWarningDays;
+        UNIMPLEMENTED;
+        return NULL;
     }
 
-    virtual bool createUserScopes() {UNIMPLEMENTED; return false;}
-    virtual aindex_t getManagedFileScopes(IArrayOf<ISecResource>& scopes) {UNIMPLEMENTED; }
-    virtual int queryDefaultPermission(ISecUser& user) {UNIMPLEMENTED; }
-    virtual bool clearPermissionsCache(ISecUser& user) {return false;}
-    virtual bool authenticateUser(ISecUser & user, bool &superUser) {return false;}
-protected:
-    const char* getServer(){return m_dbserver.str();}
-    const char* getUser(){return m_dbuser.str();}
-    const char* getPassword(){return m_dbpassword.str();}
-    int getPoolsize() { return m_poolsize;}
-    void setUserMap(const char* user,int uid){synchronized block(m_usermap_mutex); m_usermap.setValue(user, uid);}
-    int getUserID(ISecUser& user);
-
-    void logon_failed(const char* user, const char* msg);
-    int findUser(const char* user,const char* realm);
-    void init(const char *serviceName, IPropertyTree *config);
-
-    void EncodePassword(StringBuffer& password);
-    bool ValidateResources(ISecUser & sec_user,IArrayOf<ISecResource>& rlist);
-    bool updateSettings(ISecUser & sec_user,IArrayOf<ISecResource>& rlist);
-
-    virtual bool ValidateUser(ISecUser & sec_user);
-    virtual bool ValidateResources(ISecUser & sec_user, ISecResourceList * Resources);
-
-    virtual StringBuffer& buildAuthenticateQuery(const char* user,const char* password,const char* realm, StringBuffer& SQLQuery){return SQLQuery;}
+    void deleteResource(SecResourceType rtype, const char * name, const char * basedn)
+    {
+        UNIMPLEMENTED;
+    }
 
-    virtual bool dbValidateResource(ISecResource& res,int usernum,const char* realm)
+    void renameResource(SecResourceType rtype, const char * oldname, const char * newname, const char * basedn)
     {
-        return false;
+        UNIMPLEMENTED;
     }
-    virtual bool dbValidateSetting(ISecResource& res,int usernum,const char* realm)
+
+    void copyResource(SecResourceType rtype, const char * oldname, const char * newname, const char * basedn)
     {
-        return false;
+        UNIMPLEMENTED;
     }
-    virtual bool dbValidateSetting(ISecResource& res,ISecUser& User)
+
+    void cacheSwitch(SecResourceType rtype, bool on)
     {
-        return false;
+        UNIMPLEMENTED;
     }
 
-    virtual bool dbUpdateResource(ISecResource& res,int usernum,const char* Realm)
+    bool authTypeRequired(SecResourceType rtype)
     {
+        UNIMPLEMENTED;
         return false;
     }
 
-    virtual StringBuffer& ExecuteScalar(const char* Query,const char* FieldName, StringBuffer & ReturnVal){return ReturnVal;}
-
+    int authorizeWorkunitScope(ISecUser & user, const char * filescope)
+    {
+        UNIMPLEMENTED;
+        return 0;
+    }
 
-    virtual bool dbauthenticate(StringBuffer &user, StringBuffer &password,StringBuffer &realm,StringBuffer& status,int& ExpirationDate)
+    bool authorizeWorkunitScope(ISecUser & user, ISecResourceList * resources)
     {
+        UNIMPLEMENTED;
         return false;
     }
 
-    virtual bool dbauthenticate(ISecUser& User, StringBuffer& SQLQuery)
+    const char * getDescription()
     {
-        return false;
+        UNIMPLEMENTED;
+        return NULL;
     }
 
-    virtual int dbLookupUser(const char* user,const char* realm)
+    unsigned getPasswordExpirationWarningDays()
     {
+        UNIMPLEMENTED;
         return 0;
     }
-    virtual bool dbUpdatePasswrd(const char* user,const char* realm,const char* password)
+
+    bool createUserScopes()
     {
+        UNIMPLEMENTED;
         return false;
     }
-    virtual StringBuffer& dbGetEffectiveAccess(int usernum, const char * resource, const char * member, const char * objclass,StringBuffer& returnValue)
-    {
-        returnValue.appendf("%d",-1);
-        return returnValue;
-    }
-    bool IsIPRestricted(ISecUser& User);
-    virtual bool IsPasswordValid(ISecUser& sec_user);
 
-    virtual void dbConnect()
+    aindex_t getManagedFileScopes(IArrayOf<ISecResource>& scopes)
     {
+        UNIMPLEMENTED;
+        return 0;
     }
-    virtual void dbDisconnect()
+
+    int queryDefaultPermission(ISecUser& user)
     {
+        UNIMPLEMENTED;
+        return 0;
     }
-    virtual bool inHierarchy(const char* username)
+
+    bool clearPermissionsCache(ISecUser & user)
     {
+        UNIMPLEMENTED;
         return false;
     }
-    virtual bool inHierarchy(int usernum)
+
+    bool authenticateUser(ISecUser & user, bool &superUser)
     {
+        UNIMPLEMENTED;
         return false;
     }
 
-    virtual bool validateSecurityQuestion(ISecUser* user, const char* token)
+    secManagerType querySecMgrType()
     {
-        return false;
+        UNIMPLEMENTED;
+        return (secManagerType)0;
     }
-    virtual int validateOTP(ISecUser* user, const char* cookie)
+
+    const char* querySecMgrTypeName()
     {
-        return 0;
+        UNIMPLEMENTED;
+        return NULL;
     }
+
 };
 
 #endif // BASESECURITY_INCL

system/security/shared/defaultsecuritymanager.cpp

@@ -1,106 +0,0 @@
-/*##############################################################################
-
-    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems®.
-
-    Licensed under the Apache License, Version 2.0 (the "License");
-    you may not use this file except in compliance with the License.
-    You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-    Unless required by applicable law or agreed to in writing, software
-    distributed under the License is distributed on an "AS IS" BASIS,
-    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-    See the License for the specific language governing permissions and
-    limitations under the License.
-############################################################################## */
-
-#pragma warning( disable : 4786 )
-
-#include "defaultsecuritymanager.hpp"
-#include "authmap.ipp"
-
-
-//#ifdef _WIN32
-CDefaultSecurityManager::CDefaultSecurityManager(const char *serviceName, const char *config) : CBaseSecurityManager(serviceName,config)
-{
-}
-
-CDefaultSecurityManager::CDefaultSecurityManager(const char *serviceName, IPropertyTree *config) : CBaseSecurityManager(serviceName,config)
-{
-}
-
-
-//========Local Secuirty Manager==========
-CLocalSecurityManager::CLocalSecurityManager(const char *serviceName, const char *config) : CDefaultSecurityManager(serviceName, config)
-{
-}
-
-CLocalSecurityManager::CLocalSecurityManager(const char *serviceName, IPropertyTree *config) : CDefaultSecurityManager(serviceName, config)
-{
-}
-
-CLocalSecurityManager::~CLocalSecurityManager()
-{
-}
-
-bool CLocalSecurityManager::IsPasswordValid(ISecUser& sec_user)
-{
-    IAuthenticatedUser* au = createAuthenticatedUser();
-    StringBuffer userbuf;
-#ifdef _WIN32
-    const char* realm = sec_user.getRealm();
-    if(realm&&*realm)
-        userbuf.append(realm).append("\\");
-#endif
-    userbuf.append(sec_user.getName());
-    return au->login(userbuf.str(), sec_user.credentials().getPassword());
-}
-
-
-IAuthMap * CLocalSecurityManager::createAuthMap(IPropertyTree * authconfig)
-{
-    CAuthMap* authmap = new CAuthMap(this);
-
-    IPropertyTreeIterator *loc_iter = NULL;
-    loc_iter = authconfig->getElements(".//Location");
-    if (loc_iter != NULL)
-    {
-        IPropertyTree *location = NULL;
-        loc_iter->first();
-        while(loc_iter->isValid())
-        {
-            location = &loc_iter->query();
-            if (location)
-            {
-                StringBuffer pathstr, rstr, required, description;
-                location->getProp("@path", pathstr);
-                location->getProp("@resource", rstr);
-                location->getProp("@required", required);
-                location->getProp("@description", description);
-                
-                if(pathstr.length() == 0)
-                    throw MakeStringException(-1, "path empty in Authenticate/Location");
-                if(rstr.length() == 0)
-                    throw MakeStringException(-1, "resource empty in Authenticate/Location");
-
-                ISecResourceList* rlist = authmap->queryResourceList(pathstr.str());
-                if(rlist == NULL)
-                {
-                    rlist = createResourceList("localsecurity");                        
-                    authmap->add(pathstr.str(), rlist);
-                }
-                ISecResource* rs = rlist->addResource(rstr.str());
-                unsigned requiredaccess = str2perm(required.str());
-                rs->setRequiredAccessFlags(requiredaccess);
-                rs->setDescription(description.str());
-            }
-            loc_iter->next();
-        }
-        loc_iter->Release();
-        loc_iter = NULL;
-    }
-
-    return authmap;
-}
-

system/security/shared/defaultsecuritymanager.hpp

@@ -1,106 +0,0 @@
-/*##############################################################################
-
-    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems®.
-
-    Licensed under the Apache License, Version 2.0 (the "License");
-    you may not use this file except in compliance with the License.
-    You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-    Unless required by applicable law or agreed to in writing, software
-    distributed under the License is distributed on an "AS IS" BASIS,
-    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-    See the License for the specific language governing permissions and
-    limitations under the License.
-############################################################################## */
-
-#ifndef DEFAULTSECURITY_INCL
-#define DEFAULTSECURITY_INCL
-
-#include "basesecurity.hpp"
-
-class CDefaultSecurityManager : public CBaseSecurityManager
-{
-protected:
-
-public:
-    CDefaultSecurityManager(const char *serviceName, const char *config);
-    CDefaultSecurityManager(const char *serviceName, IPropertyTree *config);
-    virtual ~CDefaultSecurityManager(){};
-    virtual bool dbauthenticate(ISecUser& User, StringBuffer& SQLQuery){return true;}
-    virtual bool dbValidateResource(ISecResource& res,int usernum,const char* realm)
-    {
-        CSecurityResource * tmpResource =  (CSecurityResource*)(&res);
-        if(tmpResource)
-            tmpResource->setAccessFlags(SecAccess_Full);
-        return true;
-    }
-
-    virtual int getAccessFlagsEx(SecResourceType rtype, ISecUser& sec_user, const char* resourcename)
-    {
-        return SecAccess_Full;
-    }
-    virtual int authorizeFileScope(ISecUser & user, const char * filescope)
-    {
-        return SecAccess_Full;
-    }
-    virtual bool authorizeFileScope(ISecUser & user, ISecResourceList * resources)
-    {
-        if(resources)
-        {
-            int cnt = resources->count();
-            for(int i = 0; i < cnt; i++)
-            {
-                ISecResource* r = resources->queryResource(i);
-                if(r)
-                    r->setAccessFlags(SecAccess_Full);
-            }
-        }
-
-        return true;
-    }
-    virtual int authorizeWorkunitScope(ISecUser & user, const char * filescope)
-    {
-        return SecAccess_Full;
-    }
-    virtual bool authorizeWorkunitScope(ISecUser & user, ISecResourceList * resources)
-    {
-        if(resources)
-        {
-            int cnt = resources->count();
-            for(int i = 0; i < cnt; i++)
-            {
-                ISecResource* r = resources->queryResource(i);
-                if(r)
-                    r->setAccessFlags(SecAccess_Full);
-            }
-        }
-
-        return true;
-    }
-    virtual bool createUserScopes() { return false; }
-    virtual aindex_t getManagedFileScopes(IArrayOf<ISecResource>& scopes) { return 0; }
-    virtual int queryDefaultPermission(ISecUser& user) { return SecAccess_Full; }
-    virtual secManagerType querySecMgrType() { return SMT_Default; }
-    inline virtual const char* querySecMgrTypeName() { return "Default"; }
-};
-
-class CLocalSecurityManager : public CDefaultSecurityManager
-{
-public:
-    CLocalSecurityManager(const char *serviceName, const char *config);
-    CLocalSecurityManager(const char *serviceName, IPropertyTree *config);
-    virtual ~CLocalSecurityManager();
-    IAuthMap * createAuthMap(IPropertyTree * authconfig);
-protected:
-    virtual bool IsPasswordValid(ISecUser& sec_user);
-    virtual secManagerType querySecMgrType() { return SMT_Local; }
-    inline virtual const char* querySecMgrTypeName() { return "Local"; }
-};
-
-
-#endif // DEFAULTSECURITY_INCL
-//end
-
-

system/security/shared/seclib.hpp

@@ -265,7 +265,8 @@ enum secManagerType
     SMT_Default,
     SMT_Local,
     SMT_LDAP,
-    SMT_HTPasswd
+    SMT_HTPasswd,
+    SMT_HTPluggable
 };
 interface ISecManager : extends IInterface
 {

system/security/shared/secloader.hpp

@@ -92,36 +92,6 @@ public:
             else
                 throw MakeStringException(-1, "procedure newLdapSecManager of %s can't be loaded", realName.str());
         }
-        else if(stricmp(model_name, "Local") == 0)
-        {
-            realName.append(SharedObjectPrefix).append(LDAPSECLIB).append(SharedObjectExtension);
-            HINSTANCE ldapseclib = LoadSharedObject(realName.str(), true, false);
-            if(ldapseclib == NULL)
-                throw MakeStringException(-1, "can't load library %s", realName.str());
-
-            newSecManager_t_ xproc = NULL;
-            xproc = (newSecManager_t_)GetSharedProcedure(ldapseclib, "newLocalSecManager");
-
-            if (xproc)
-                return xproc(servicename, *cfg);
-            else
-                throw MakeStringException(-1, "procedure newLocalSecManager of %s can't be loaded", realName.str());
-        }
-        else if(stricmp(model_name, "Default") == 0)
-        {
-            realName.append(SharedObjectPrefix).append(LDAPSECLIB).append(SharedObjectExtension);
-            HINSTANCE ldapseclib = LoadSharedObject(realName.str(), true, false);
-            if(ldapseclib == NULL)
-                throw MakeStringException(-1, "can't load library %s", realName.str());
-
-            newSecManager_t_ xproc = NULL;
-            xproc = (newSecManager_t_)GetSharedProcedure(ldapseclib, "newDefaultSecManager");
-
-            if (xproc)
-                return xproc(servicename, *cfg);
-            else
-                throw MakeStringException(-1, "procedure newDefaultSecManager of %s can't be loaded", realName.str());
-        }
         else if(stricmp(model_name, "htpasswd") == 0)
         {
             realName.append(SharedObjectPrefix).append(HTPASSWDSECLIB).append(SharedObjectExtension);

testing/regress/ecl/embedcppinline.ecl

@@ -0,0 +1,34 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2016 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//class=embedded
+
+BOOLEAN isUpper(const string mystring) := EMBED(C++ : inline)
+  size_t i=0;
+  while (i < lenMystring)
+  {
+    if (!isupper((byte)mystring[i]))
+        return false;
+    i++;
+  }
+  return true;
+ENDEMBED;
+
+OUTPUT(IsUpper('TOM'));
+
+ds := dataset(4, transform({string1 id}, SELF.id := 'aBcD'[COUNTER]));
+output(TABLE(ds, { isUpper(id) }));

testing/regress/ecl/key/embedcppinline.xml

@@ -0,0 +1,9 @@
+<Dataset name='Result 1'>
+ <Row><Result_1>true</Result_1></Row>
+</Dataset>
+<Dataset name='Result 2'>
+ <Row><isupper>false</isupper></Row>
+ <Row><isupper>true</isupper></Row>
+ <Row><isupper>false</isupper></Row>
+ <Row><isupper>true</isupper></Row>
+</Dataset>

testing/regress/ecl/key/memcachederrortests.xml

@@ -0,0 +1,3 @@
+<Dataset name='Result 1'>
+ <Row><value>Memcached Plugin: Failed connecting to ALL servers. Check memcached on all servers and &quot;memcached -B ascii&quot; not used.</value></Row>
+</Dataset>

testing/regress/ecl/memcachederrortests.ecl

@@ -0,0 +1,29 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2016 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//class=embedded
+//class=3rdparty
+//nothor
+
+IMPORT memcached FROM lib_memcached;
+
+//Test failure to connect
+STRING downServer := '--SERVER=127.0.0.1:6379';
+    ds := DATASET(NOFOLD(1), TRANSFORM({STRING value}, SELF.value := memcached.GetString('connectionTest', downServer)));
+SEQUENTIAL(
+    OUTPUT(CATCH(ds, ONFAIL(TRANSFORM({ STRING value }, SELF.value := FAILMESSAGE))));
+    );

thorlcr/activities/filter/thfilterslave.cpp

@@ -242,7 +242,7 @@ class CFilterGroupSlaveActivity : public CFilterSlaveActivityBase, public CThorS
     IHThorFilterGroupArg *helper;
     Owned<IThorRowLoader> groupLoader;
     Owned<IRowStream> groupStream;
-    bool compressSpills;
+    unsigned spillCompInfo;
 
 public:
     IMPLEMENT_IINTERFACE_USING(CSimpleInterface);
@@ -251,7 +251,13 @@ public:
     {
         groupLoader.setown(createThorRowLoader(*this, NULL, stableSort_none, rc_allMem));
         helper = NULL;
-        compressSpills = getOptBool(THOROPT_COMPRESS_SPILLS, true);
+        spillCompInfo = 0x0;
+        if (getOptBool(THOROPT_COMPRESS_SPILLS, true))
+        {
+            StringBuffer compType;
+            getOpt(THOROPT_COMPRESS_SPILL_TYPE, compType);
+            setCompFlag(compType, spillCompInfo);
+        }
     }
     void init(MemoryBuffer &data, MemoryBuffer &slaveData)
     {
@@ -299,7 +305,7 @@ public:
                 {
                     CThorSpillableRowArray spillableRows(*this, this);
                     spillableRows.transferFrom(rows);
-                    groupStream.setown(spillableRows.createRowStream(SPILL_PRIORITY_SPILLABLE_STREAM, compressSpills));
+                    groupStream.setown(spillableRows.createRowStream(SPILL_PRIORITY_SPILLABLE_STREAM, spillCompInfo));
                 }
                 // else read next group
             }

thorlcr/activities/hashdistrib/thhashdistribslave.cpp

@@ -2184,7 +2184,12 @@ public:
                 MemoryAttr ma;
                 activity->startInput(in);
                 if (activity->getOptBool(THOROPT_COMPRESS_SPILLS, true))
+                {
                     rwFlags |= rw_compress;
+                    StringBuffer compType;
+                    activity->getOpt(THOROPT_COMPRESS_SPILL_TYPE, compType);
+                    setCompFlag(compType, rwFlags);
+                }
                 Owned<IExtRowWriter> out = createRowWriter(tempfile, activity, rwFlags);
                 if (!out)
                     throw MakeStringException(-1,"Could not created file %s",tempname.str());
@@ -2551,7 +2556,12 @@ public:
         OwnedIFile iFile = createIFile(tempname.str());
         spillFile.setown(new CFileOwner(iFile.getLink()));
         if (owner.getOptBool(THOROPT_COMPRESS_SPILLS, true))
+        {
             rwFlags |= rw_compress;
+            StringBuffer compType;
+            owner.getOpt(THOROPT_COMPRESS_SPILL_TYPE, compType);
+            setCompFlag(compType, rwFlags);
+        }
         writer = createRowWriter(iFile, rowIf, rwFlags);
     }
     IRowStream *getReader(rowcount_t *_count=NULL) // NB: also detatches ownership of 'fileOwner'