Merge branch 'candidate-5.2.0'

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

cmake_modules/FindREDIS.cmake

@@ -0,0 +1,42 @@
+################################################################################
+#    HPCC SYSTEMS software Copyright (C) 2015 HPCC Systems.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+################################################################################
+
+# - Try to find the libhiredislibrary
+# Once done this will define
+#
+#  LIBHEDIS_FOUND - system has the libhiredis library
+#  LIBHIREDIS_INCLUDE_DIR - the libhiredis include directory(s)
+#  LIBHIREDIS_LIBRARY - The library needed to use hiredis
+
+IF (NOT LIBREDIS_FOUND)
+  IF (WIN32)
+    SET (libhiredis "libhiredis")
+  ELSE()
+    SET (libhiredis "hiredis")
+  ENDIF()
+
+  FIND_PATH(LIBHIREDIS_INCLUDE_DIR hiredis/hiredis.h PATHS /usr/include /usr/share/include /usr/local/include PATH_SUFFIXES hiredis)
+  FIND_LIBRARY(LIBHIREDIS_LIBRARY NAMES ${libhiredis} PATHS /usr/lib /usr/share /usr/lib64 /usr/local/lib /usr/local/lib64)
+
+  include(FindPackageHandleStandardArgs)
+  find_package_handle_standard_args(redis DEFAULT_MSG
+    LIBHIREDIS_LIBRARY
+    LIBHIREDIS_INCLUDE_DIR
+  )
+
+  MARK_AS_ADVANCED(LIBHIREDIS_INCLUDE_DIR LIBHIREDIS_LIBRARY)
+ENDIF()
+

cmake_modules/commonSetup.cmake

@@ -89,10 +89,11 @@ IF ("${COMMONSETUP_DONE}" STREQUAL "")
   option(USE_JNI "Enable Java JNI support" ON)
   option(USE_RINSIDE "Enable R support" ON)
   option(USE_MEMCACHED "Enable Memcached support" ON)
+  option(USE_REDIS "Enable Redis support" ON)
 
   option(USE_OPTIONAL "Automatically disable requested features with missing dependencies" ON)
 
-  if ( USE_PYTHON OR USE_V8 OR USE_JNI OR USE_RINSIDE OR USE_SQLITE3 OR USE_MYSQL OR USE_CASSANDRA OR USE_MEMCACHED)
+  if ( USE_PYTHON OR USE_V8 OR USE_JNI OR USE_RINSIDE OR USE_SQLITE3 OR USE_MYSQL OR USE_CASSANDRA OR USE_MEMCACHED OR USE_REDIS)
       set( WITH_PLUGINS ON )
   endif()
 

common/deftype/deftype.cpp

@@ -1947,17 +1947,18 @@ extern DEFTYPE_API ITypeInfo *makeGroupedTableType(ITypeInfo *basetype)
     return commonUpType(new CGroupedTableTypeInfo(basetype));
 }
 
-extern DEFTYPE_API ITypeInfo *makeFunctionType(ITypeInfo *basetype, IInterface * parameters, IInterface * defaults)
+extern DEFTYPE_API ITypeInfo *makeFunctionType(ITypeInfo *basetype, IInterface * parameters, IInterface * defaults, IInterface * attrs)
 {
     assertex(basetype->getTypeCode() != type_function); // not just yet anyway
     if (!basetype || !parameters)
     {
-        basetype->Release();
-        parameters->Release();
-        defaults->Release();
+        ::Release(basetype);
+        ::Release(parameters);
+        ::Release(defaults);
+        ::Release(attrs);
         throwUnexpected();
     }
-    return commonUpType(new CFunctionTypeInfo(basetype, parameters, defaults));
+    return commonUpType(new CFunctionTypeInfo(basetype, parameters, defaults, attrs));
 }
 
 /* In basetype: linked. Return: linked */

common/deftype/deftype.hpp

@@ -212,8 +212,9 @@ private:
 
 interface IFunctionTypeExtra : public IInterface
 {
-    virtual IInterface * queryParameters() = 0;
-    virtual IInterface * queryDefaults() = 0;
+    virtual IInterface * queryParameters() const = 0;
+    virtual IInterface * queryDefaults() const = 0;
+    virtual IInterface * queryAttributes() const = 0;
 };
 
 interface IEnumeratedTypeBuilder : public IInterface
@@ -254,7 +255,7 @@ extern DEFTYPE_API ITypeInfo *makeRowType(ITypeInfo *basetype);
 extern DEFTYPE_API ITypeInfo *makeSetType(ITypeInfo *basetype);
 extern DEFTYPE_API ITypeInfo *makeTransformType(ITypeInfo *basetype);
 extern DEFTYPE_API ITypeInfo *makeSortListType(ITypeInfo *basetype);
-extern DEFTYPE_API ITypeInfo *makeFunctionType(ITypeInfo *basetype, IInterface * args, IInterface * defaults);
+extern DEFTYPE_API ITypeInfo *makeFunctionType(ITypeInfo *basetype, IInterface * args, IInterface * defaults, IInterface * attrs);
 
 extern DEFTYPE_API ITypeInfo * makePointerType(ITypeInfo * basetype);
 extern DEFTYPE_API ITypeInfo * makeArrayType(ITypeInfo * basetype, unsigned size=0);

common/deftype/deftype.ipp

@@ -773,9 +773,10 @@ class CFunctionTypeInfo : public CBasedTypeInfo, implements IFunctionTypeExtra
 private:
     Owned<IInterface> parameters;
     Owned<IInterface> defaults;
+    Owned<IInterface> attrs;
 public:
-    CFunctionTypeInfo(ITypeInfo * _basetype, IInterface * _parameters, IInterface * _defaults) 
-        : CBasedTypeInfo(_basetype, UNKNOWN_LENGTH), parameters(_parameters), defaults(_defaults) 
+    CFunctionTypeInfo(ITypeInfo * _basetype, IInterface * _parameters, IInterface * _defaults, IInterface *_attrs)
+        : CBasedTypeInfo(_basetype, UNKNOWN_LENGTH), parameters(_parameters), defaults(_defaults), attrs(_attrs)
     {}
     IMPLEMENT_IINTERFACE_USING(CBasedTypeInfo)
 
@@ -791,8 +792,9 @@ public:
     virtual bool equals(const CTypeInfo & other) const;
 
 //IFunctionTypeExtra
-    virtual IInterface * queryParameters() { return parameters; }
-    virtual IInterface * queryDefaults() { return defaults; }
+    virtual IInterface * queryParameters() const { return parameters; }
+    virtual IInterface * queryDefaults() const { return defaults; }
+    virtual IInterface * queryAttributes() const { return attrs; }
 };
 
 

common/remote/hooks/libarchive/archive.cpp

@@ -184,8 +184,12 @@ public:
         archive_read_support_compression_bzip2(archive);
 #else
         archive_read_support_format_all(archive);
+#if (ARCHIVE_VERSION_NUMBER >= 3000000)
+        archive_read_support_filter_all(archive);
+#else
         archive_read_support_compression_all(archive);
 #endif
+#endif
         int retcode = archive_read_open_filename(archive, container, 10240);
         if (retcode == ARCHIVE_OK)
         {
@@ -204,7 +208,11 @@ public:
     }
     ~ArchiveFileIO()
     {
+#if (ARCHIVE_VERSION_NUMBER >= 3000000)
+        archive_read_free(archive);
+#else
         archive_read_finish(archive);
+#endif
     }
 
     virtual size32_t read(offset_t pos, size32_t len, void * _data)
@@ -486,8 +494,12 @@ public:
         archive_read_support_compression_bzip2(archive);
 #else
         archive_read_support_format_all(archive);
+#if (ARCHIVE_VERSION_NUMBER >= 3000000)
+        archive_read_support_filter_all(archive);
+#else
         archive_read_support_compression_all(archive);
 #endif
+#endif
         int retcode = archive_read_open_filename(archive, container, 10240);
         if (retcode == ARCHIVE_OK)
         {
@@ -522,7 +534,11 @@ public:
             }
             archive_entry_free(entry);
         }
+#if (ARCHIVE_VERSION_NUMBER >= 3000000)
+        archive_read_free(archive);
+#else
         archive_read_finish(archive);
+#endif
         return next();
     }
 

common/remote/rmtfile.cpp

@@ -35,12 +35,11 @@
 
 static class CSecuritySettings
 {
-    bool useSSL;
     unsigned short daliServixPort;
 public:
     CSecuritySettings()
     {
-        querySecuritySettings(&useSSL, &daliServixPort, NULL, NULL);
+        querySecuritySettings(NULL, &daliServixPort, NULL, NULL);
     }
 
     unsigned short queryDaliServixPort() { return daliServixPort; }

common/remote/sockfile.cpp

@@ -175,13 +175,20 @@ static unsigned maxConnectTime = 0;
 static unsigned maxReceiveTime = 0;
 
 //Security and default port attributes
-#define PORT_UNKNOWN -1
-static unsigned short configepPort = PORT_UNKNOWN;
-static struct _ClientCertificate
+static class _securitySettings
 {
-    const char * certificate;
-    const char * privateKey;
-} clientCertificate;
+public:
+    bool            useSSL;
+    unsigned short  daFileSrvPort;
+    const char *    certificate;
+    const char *    privateKey;
+
+    _securitySettings()
+    {
+        querySecuritySettings(&useSSL, &daFileSrvPort, &certificate, &privateKey);
+    }
+} securitySettings;
+
 
 static CriticalSection              secureContextCrit;
 static Owned<ISecureSocketContext>  secureContext;
@@ -192,8 +199,8 @@ static ISecureSocket *createSecureSocket(ISocket *sock,SecureSocketType type)
         CriticalBlock b(secureContextCrit);
         if (!secureContext)
         {
-            if (clientCertificate.certificate)
-                secureContext.setown(createSecureSocketContextEx(clientCertificate.certificate,clientCertificate.privateKey, NULL, type));
+            if (securitySettings.certificate)
+                secureContext.setown(createSecureSocketContextEx(securitySettings.certificate,securitySettings.privateKey, NULL, type));
             else
                 secureContext.setown(createSecureSocketContext(type));
         }
@@ -433,9 +440,7 @@ void setDafsEndpointPort(SocketEndpoint &ep)
     }
     if (ep.port==0)
     {
-        if (configepPort == (unsigned short)PORT_UNKNOWN)
-            querySecuritySettings(NULL, &configepPort, &clientCertificate.certificate, &clientCertificate.privateKey);
-        ep.port = configepPort;
+        ep.port = securitySettings.daFileSrvPort;
     }
 }
 
@@ -1163,21 +1168,28 @@ protected: friend class CRemoteFileIO;
                 }
             }
             if (!socket) {
+                bool tryNonSecure = true;
                 if (useSSL) {
                     try {
                         connectSocket(tep);//first try secure connect
+                        tryNonSecure = false;
                     }
-                    catch(...) {
+                    catch (IException *e) {
+
+                        StringBuffer s;
+                        e->errorMessage(s);
+                        e->Release();
+                        WARNLOG("Secure connect failed, retrying on legacy port (%s)",s.str());
+
                         useSSL = false;
-                        tep.port = DAFILESRV_PORT;
-                        PROGLOG("Secure connect failed, retrying on legacy port");
-                        connectSocket(tep);
+                        tep.port = DAFILESRV_PORT;//retry on nonsecure port
+                        tryNonSecure = true;
                     }
                 }
-                else
+
+                if (tryNonSecure)
                     connectSocket(tep);
             }
-
         }
 
         unsigned errCode;
@@ -1317,8 +1329,7 @@ public:
         : filename(_filename)
     {
         ep = _ep;
-        //Determine whether or not client should use SSL
-        querySecuritySettings(&useSSL, &configepPort, &clientCertificate.certificate, &clientCertificate.privateKey);
+        useSSL = securitySettings.useSSL;
     }
 
 
@@ -3023,7 +3034,7 @@ class CRemoteFileServer : public CInterface, implements IRemoteFileServer, imple
     int                 lasthandle;
     CriticalSection     sect;
     Owned<ISocket>      acceptsock;
-    Owned<ISocket>      rejectsock;
+    Owned<ISocket>      rejectsock;//used to immediately reject nonsecure connection requests when in secure mode
     Owned<ISocketSelectHandler> selecthandler;
     Owned<IThreadPool>  threads;    // for commands
     bool stopping;
@@ -4432,18 +4443,18 @@ public:
             acceptsock.setown(ISocket::create_ip(listenep.port,ips.str()));
         }
         if (useSSL) {
-            querySecuritySettings(&useSSL, &configepPort, &clientCertificate.certificate, &clientCertificate.privateKey);
-            if (!clientCertificate.certificate)
+            if (!securitySettings.certificate)
                 throw createDafsException(DAFSERR_connection_failed,"SSL Certificate information not found in environment.conf");
             if (listenep.port <= 0)
             {
-                assertex(FALSE);
-                listenep.port = configepPort;
+                assertex(FALSE);//should never get here
+                listenep.port = securitySettings.daFileSrvPort;
             }
             //Create unsecure socket to reject non-ssl client requests
             if (listenep.isNull())
                 rejectsock.setown(ISocket::create(DAFILESRV_PORT));
-            else {
+            else
+            {
                 StringBuffer ips;
                 listenep.getIpText(ips);
                 rejectsock.setown(ISocket::create_ip(DAFILESRV_PORT,ips.str()));
@@ -4457,7 +4468,8 @@ public:
         selecthandler->start();
 
         UnsignedArray readSocks;
-        if (useSSL) {
+        if (useSSL)
+        {
             readSocks.append(acceptsock->OShandle());
             readSocks.append(rejectsock->OShandle());
         }

dali/ft/daftformat.cpp

@@ -113,7 +113,7 @@ void CPartitioner::commonCalcPartitions()
         //Don't add an empty block on the start of the this chunk to transfer.
         if ((split != firstSplit) || (inputOffset != startInputOffset))
         {
-            results.append(*new PartitionPoint(whichInput, split-1, startInputOffset-thisOffset+thisHeaderSize, inputOffset - startInputOffset, cursor.outputOffset-startOutputOffset));
+            results.append(*new PartitionPoint(whichInput, split-1, startInputOffset-thisOffset+thisHeaderSize, inputOffset - startInputOffset - cursor.trimLength, cursor.outputOffset-startOutputOffset));
             startInputOffset = inputOffset;
             startOutputOffset = cursor.outputOffset;
         }
@@ -178,6 +178,12 @@ void CPartitioner::getRecordStructure(StringBuffer & _recordStructure)
 
 //----------------------------------------------------------------------------
 
+CSimpleFixedPartitioner::CSimpleFixedPartitioner(unsigned _recordSize, bool _noTranslation)
+{
+    LOG(MCdebugProgressDetail, unknownJob, "CSimpleFixedPartitioner::CSimpleFixedPartitioner( _recordSize:%d, _noTranslation:%d)", _recordSize, _noTranslation);
+    recordSize = _recordSize;
+    noTranslation = _noTranslation;
+}
 
 void CSimpleFixedPartitioner::findSplitPoint(offset_t splitOffset, PartitionCursor & cursor)
 {
@@ -200,6 +206,7 @@ void CSimpleFixedPartitioner::setPartitionRange(offset_t _totalSize, offset_t _t
 
 CSimpleBlockedPartitioner::CSimpleBlockedPartitioner(bool _noTranslation) : CSimpleFixedPartitioner(EFX_BLOCK_SIZE, _noTranslation)
 {
+    LOG(MCdebugProgressDetail, unknownJob, "CSimpleBlockedPartitioner::CSimpleBlockedPartitioner( _noTranslation:%d)", _noTranslation);
 }
 
 //----------------------------------------------------------------------------
@@ -420,6 +427,7 @@ unsigned CInputBasePartitioner::transformBlock(offset_t endOffset, TransformCurs
 
 CFixedPartitioner::CFixedPartitioner(size32_t _recordSize) : CInputBasePartitioner(0, _recordSize)
 {
+    LOG(MCdebugProgressDetail, unknownJob, "CFixedPartitioner::CFixedPartitioner( recordSize:%d)", recordSize);
     recordSize = _recordSize;
 }
 
@@ -438,6 +446,7 @@ size32_t CFixedPartitioner::getTransformRecordSize(const byte * record, unsigned
 
 CBlockedPartitioner::CBlockedPartitioner() : CFixedPartitioner(EFX_BLOCK_SIZE)
 {
+    LOG(MCdebugProgressDetail, unknownJob, "CBlockedPartitioner::CBlockedPartitioner()");
 }
 
 
@@ -451,6 +460,7 @@ void CBlockedPartitioner::setTarget(IOutputProcessor * _target)
 
 CVariablePartitioner::CVariablePartitioner(bool _bigendian) : CInputBasePartitioner(sizeof(varLenType), EXPECTED_VARIABLE_LENGTH)
 {
+    LOG(MCdebugProgressDetail, unknownJob, "CVariablePartitioner::CVariablePartitioner(_bigendian:%d)", _bigendian);
     assertex(sizeof(varLenType) == 4);
     bigendian = _bigendian;
 }
@@ -491,6 +501,7 @@ void CVariablePartitioner::setTarget(IOutputProcessor * _target)
 
 CRECFMvbPartitioner::CRECFMvbPartitioner(bool blocked) : CInputBasePartitioner(blocked?BDW_SIZE:RDW_SIZE, EXPECTED_VARIABLE_LENGTH)
 {
+    LOG(MCdebugProgressDetail, unknownJob, "CRECFMvbPartitioner::CRECFMvbPartitioner(blocked:%d)", blocked);
     isBlocked = blocked;
 }
 
@@ -631,6 +642,12 @@ unsigned CRECFMvbPartitioner::transformBlock(offset_t endOffset, TransformCursor
 
 CCsvPartitioner::CCsvPartitioner(const FileFormat & _format) : CInputBasePartitioner(_format.maxRecordSize, _format.maxRecordSize)
 {
+    LOG(MCdebugProgressDetail, unknownJob, "CCsvPartitioner::CCsvPartitioner(_format :'%s', maxRecordSize:%d)", _format.getFileFormatTypeString(), _format.maxRecordSize);
+    LOG(MCdebugProgressDetail, unknownJob, "        separator :'%s'", _format.separate.get());
+    LOG(MCdebugProgressDetail, unknownJob, "        quote     :'%s'", _format.quote.get());
+    LOG(MCdebugProgressDetail, unknownJob, "        terminator:'%s'", _format.terminate.get());
+    LOG(MCdebugProgressDetail, unknownJob, "        escape    :'%s'", _format.escape.get());
+
     maxElementLength = 1;
     format.set(_format);
     const char * separator = format.separate.get();
@@ -871,6 +888,12 @@ void CCsvPartitioner::setTarget(IOutputProcessor * _target)
 
 // A quick version of the csv partitioner that jumps to the split offset, and then searches for a terminator.
 
+CCsvQuickPartitioner::CCsvQuickPartitioner(const FileFormat & _format, bool _noTranslation) : CCsvPartitioner(_format)
+{
+    LOG(MCdebugProgressDetail, unknownJob, "CCsvQuickPartitioner::CCsvQuickPartitioner(_format.type :'%s', _noTranslation:%d)", _format.getFileFormatTypeString(), _noTranslation);
+    noTranslation = _noTranslation;
+}
+
 void CCsvQuickPartitioner::findSplitPoint(offset_t splitOffset, PartitionCursor & cursor)
 {
     const byte *buffer = bufferBase();
@@ -960,6 +983,11 @@ void CCsvQuickPartitioner::findSplitPoint(offset_t splitOffset, PartitionCursor
 
 CUtfPartitioner::CUtfPartitioner(const FileFormat & _format) : CInputBasePartitioner(_format.maxRecordSize, _format.maxRecordSize)
 {
+    LOG(MCdebugProgressDetail, unknownJob, "CUtfPartitioner::CUtfPartitioner(_format.type :'%s', maxRecordSize:%d)", _format.getFileFormatTypeString(), _format.maxRecordSize);
+    LOG(MCdebugProgressDetail, unknownJob, "        separator :'%s'", _format.separate.get());
+    LOG(MCdebugProgressDetail, unknownJob, "        quote     :'%s'", _format.quote.get());
+    LOG(MCdebugProgressDetail, unknownJob, "        terminator:'%s'", _format.terminate.get());
+
     maxElementLength = 1;
     format.set(_format);
     utfFormat = getUtfFormatType(format.type);
@@ -1197,6 +1225,12 @@ void CUtfPartitioner::setTarget(IOutputProcessor * _target)
 
 // A quick version of the Utf partitioner that jumps to the split offset, and then searches for a terminator.
 
+CUtfQuickPartitioner::CUtfQuickPartitioner(const FileFormat & _format, bool _noTranslation) : CUtfPartitioner(_format)
+{
+    LOG(MCdebugProgressDetail, unknownJob, "CUtfQuickPartitioner::CUtfQuickPartitioner(_format.type :'%s', _noTranslation:%d)", _format.getFileFormatTypeString(), _noTranslation);
+    noTranslation = _noTranslation;
+}
+
 void CUtfQuickPartitioner::findSplitPoint(offset_t splitOffset, PartitionCursor & cursor)
 {
     const byte *buffer = bufferBase();
@@ -1324,6 +1358,7 @@ size32_t BufferedDirectReader::ensure(size32_t required)
 
 XmlSplitter::XmlSplitter(const FileFormat & format)
 {
+    LOG(MCdebugProgressDetail, unknownJob, "XmlSplitter::XmlSplitter(_format.type :'%s', format.rowTag:'%s')", format.getFileFormatTypeString(), format.rowTag.get());
     maxElementLength = 1;
     utfFormat = getUtfFormatType(format.type);
     StringBuffer openTag, closeTag, endTag, endCloseTag;
@@ -1545,8 +1580,69 @@ offset_t XmlSplitter::getFooterLength(BufferedDirectReader & reader, offset_t si
 }
 
 
+CJsonInputPartitioner::CJsonInputPartitioner(const FileFormat & _format)
+{
+    LOG(MCdebugProgressDetail, unknownJob, "CJsonInputPartitioner::CJsonInputPartitioner(format.type :'%s', unitSize:%d)", _format.getFileFormatTypeString(), _format.getUnitSize());
+
+    format.set(_format);
+    CriticalBlock block(openfilecachesect);
+    if (!openfilecache)
+        openfilecache = createFileIOCache(16);
+    else
+        openfilecache->Link();
+}
+
+IFileIOCache *CJsonInputPartitioner::openfilecache = NULL;
+CriticalSection CJsonInputPartitioner::openfilecachesect;
+
+CJsonInputPartitioner::~CJsonInputPartitioner()
+{
+    json.clear();
+    inStream.clear();
+    if (openfilecache) {
+        CriticalBlock block(openfilecachesect);
+        if (openfilecache->Release())
+            openfilecache = NULL;
+    }
+}
+
+void CJsonInputPartitioner::setSource(unsigned _whichInput, const RemoteFilename & _fullPath, bool _compressedInput, const char *_decryptKey)
+{
+    CPartitioner::setSource(_whichInput, _fullPath, _compressedInput,_decryptKey);
+    Owned<IFileIO> inIO;
+    Owned<IFile> inFile = createIFile(inputName);
+    if (!inFile->exists()) {
+        StringBuffer tmp;
+        inputName.getRemotePath(tmp);
+        throwError1(DFTERR_CouldNotOpenFilePart, tmp.str());
+    }
+    inIO.setown(openfilecache->addFile(inputName,IFOread));
+
+    if (_compressedInput) {
+        Owned<IExpander> expander;
+        if (_decryptKey&&*_decryptKey) {
+            StringBuffer key;
+            decrypt(key,_decryptKey);
+            expander.setown(createAESExpander256(key.length(),key.str()));
+        }
+        inIO.setown(createCompressedFileReader(inIO,expander));
+    }
+
+    inStream.setown(createIOStream(inIO));
+    json.setown(new JsonSplitter(format, *inStream));
+    json->getHeaderLength();
+}
+
+CJsonPartitioner::CJsonPartitioner(const FileFormat & _format) : CJsonInputPartitioner(_format)
+{
+    unitSize = format.getUnitSize();
+    utfFormat = getUtfFormatType(format.type);
+}
+
+
 CXmlPartitioner::CXmlPartitioner(const FileFormat & _format) : CInputBasePartitioner(_format.maxRecordSize, _format.maxRecordSize), splitter(_format)
 {
+    LOG(MCdebugProgressDetail, unknownJob, "CXmlPartitioner::CXmlPartitioner(_format.type :'%s', unitSize:%d)", _format.getFileFormatTypeString(), format.getUnitSize());
     format.set(_format);
     unitSize = format.getUnitSize();
     utfFormat = getUtfFormatType(format.type);
@@ -1576,6 +1672,12 @@ void CXmlPartitioner::setTarget(IOutputProcessor * _target)
 
 // A quick version of the Utf partitioner that jumps to the split offset, and then searches for a terminator.
 
+CXmlQuickPartitioner::CXmlQuickPartitioner(const FileFormat & _format, bool _noTranslation) : CXmlPartitioner(_format)
+{
+    LOG(MCdebugProgressDetail, unknownJob, "CXmlQuickPartitioner::CXmlQuickPartitioner(_format.type :'%s', _noTranslation:%d)", _format.getFileFormatTypeString(), _noTranslation);
+    noTranslation = _noTranslation;
+}
+
 void CXmlQuickPartitioner::findSplitPoint(offset_t splitOffset, PartitionCursor & cursor)
 {
     const byte *buffer = bufferBase();
@@ -1632,6 +1734,7 @@ void CXmlQuickPartitioner::findSplitPoint(offset_t splitOffset, PartitionCursor
 CRemotePartitioner::CRemotePartitioner(const SocketEndpoint & _ep, const FileFormat & _srcFormat, const FileFormat & _tgtFormat, const char * _slave, const char *_wuid)
     : wuid(_wuid)
 {
+    LOG(MCdebugProgressDetail, unknownJob, "CRemotePartitioner::CRemotePartitioner(_srcFormat.type :'%s', _tgtFormat.type:'%s', _slave:'%s', _wuid:'%s')", _srcFormat.getFileFormatTypeString(), _tgtFormat.getFileFormatTypeString(), _slave, _wuid);
     ep.set(_ep);
     srcFormat.set(_srcFormat);
     tgtFormat.set(_tgtFormat);
@@ -2054,6 +2157,7 @@ IFormatProcessor * createFormatProcessor(const FileFormat & srcFormat, const Fil
 {
     IFormatProcessor * partitioner;
     bool sameFormats = srcFormat.equals(tgtFormat);
+    LOG(MCdebugProgressDetail, unknownJob, "createFormatProcessor(srcFormat:'%s', tgtFormat:'%s', calcOutput:%d, sameFormats:%d)", srcFormat.getFileFormatTypeString(), tgtFormat.getFileFormatTypeString(), calcOutput, sameFormats);
     switch (srcFormat.type)
     {
     case FFTfixed:
@@ -2087,13 +2191,15 @@ IFormatProcessor * createFormatProcessor(const FileFormat & srcFormat, const Fil
             partitioner = new CCsvQuickPartitioner(srcFormat, sameFormats);
         break;
     case FFTutf8: case FFTutf8n: case FFTutf16: case FFTutf16be: case FFTutf16le: case FFTutf32: case FFTutf32be: case FFTutf32le:
-        if (srcFormat.rowTag)
+        if (srcFormat.hasXmlMarkup())
         {
             if (calcOutput && !sameFormats)
                 partitioner = new CXmlPartitioner(srcFormat);
             else
                 partitioner = new CXmlQuickPartitioner(srcFormat, sameFormats);
         }
+        else if (srcFormat.hasJsonMarkup())
+            partitioner = new CJsonPartitioner(srcFormat);
         else
         {
             if (calcOutput && !sameFormats)
@@ -2111,6 +2217,7 @@ IFormatProcessor * createFormatProcessor(const FileFormat & srcFormat, const Fil
 
 IOutputProcessor * createOutputProcessor(const FileFormat & format)
 {
+    LOG(MCdebugProgressDetail, unknownJob, "createOutputProcessor(format.type:'%s')", format.getFileFormatTypeString());
     switch (format.type)
     {
     case FFTfixed:
@@ -2132,6 +2239,7 @@ IOutputProcessor * createOutputProcessor(const FileFormat & format)
 IFormatPartitioner * createFormatPartitioner(const SocketEndpoint & ep, const FileFormat & srcFormat, const FileFormat & tgtFormat, bool calcOutput, const char * slave, const char *wuid)
 {
     bool sameFormats = sameEncoding(srcFormat, tgtFormat);
+    LOG(MCdebugProgressDetail, unknownJob, "createFormatProcessor(srcFormat.type:'%s', tgtFormat.type:'%s', calcOutput:%d, sameFormats:%d)", srcFormat.getFileFormatTypeString(), tgtFormat.getFileFormatTypeString(), calcOutput, sameFormats);
     if (sameFormats)
     {
         switch (srcFormat.type)
@@ -2147,15 +2255,13 @@ IFormatPartitioner * createFormatPartitioner(const SocketEndpoint & ep, const Fi
                 return new CCsvQuickPartitioner(srcFormat, sameFormats);
             break;
         case FFTutf: case FFTutf8: case FFTutf8n: case FFTutf16: case FFTutf16be: case FFTutf16le: case FFTutf32: case FFTutf32be: case FFTutf32le:
-            if (srcFormat.rowTag)
+            if (srcFormat.hasXmlMarkup())
                 return new CXmlQuickPartitioner(srcFormat, sameFormats);
-            else
-            {
-                if (srcFormat.hasQuote() && srcFormat.hasQuotedTerminator())
-                    return new CUtfPartitioner(srcFormat);
-                else
-                    return new CUtfQuickPartitioner(srcFormat, sameFormats);
-            }
+            if (srcFormat.hasJsonMarkup())
+                return new CJsonPartitioner(srcFormat);
+            if (srcFormat.hasQuote() && srcFormat.hasQuotedTerminator())
+                return new CUtfPartitioner(srcFormat);
+            return new CUtfQuickPartitioner(srcFormat, sameFormats);
         }
     }
     if (!calcOutput)
@@ -2177,10 +2283,11 @@ IFormatPartitioner * createFormatPartitioner(const SocketEndpoint & ep, const Fi
         case FFTcsv:
             return new CCsvQuickPartitioner(srcFormat, sameFormats);
         case FFTutf: case FFTutf8: case FFTutf8n: case FFTutf16: case FFTutf16be: case FFTutf16le: case FFTutf32: case FFTutf32be: case FFTutf32le:
-            if (srcFormat.rowTag)
+            if (srcFormat.hasXmlMarkup())
                 return new CXmlQuickPartitioner(srcFormat, sameFormats);
-            else
-                return new CUtfQuickPartitioner(srcFormat, sameFormats);
+            if (srcFormat.hasJsonMarkup())
+                return new CJsonPartitioner(srcFormat);
+            return new CUtfQuickPartitioner(srcFormat, sameFormats);
         default:
             throwError(DFTERR_UnknownFileFormatType);
             break;

dali/ft/daftformat.hpp

@@ -33,11 +33,12 @@
 struct PartitionCursor
 {
 public:
-    PartitionCursor(offset_t _inputOffset)  { inputOffset = nextInputOffset = _inputOffset; outputOffset = 0; }
+    PartitionCursor(offset_t _inputOffset)  { inputOffset = nextInputOffset = _inputOffset; outputOffset = 0; trimLength = 0; }
     
     offset_t        inputOffset;
     offset_t        nextInputOffset;
     offset_t        outputOffset;
+    offset_t        trimLength;
 };
 
 struct TransformCursor

dali/ft/daftformat.ipp

@@ -20,9 +20,10 @@
 
 #include "filecopy.hpp"
 #include "ftbase.ipp"
-#include "daft.hpp"
+#include "daftmc.hpp"
 #include "daftformat.hpp"
 #include "rmtpass.hpp"
+#include "jptree.hpp"
 
 //---------------------------------------------------------------------------
 
@@ -81,7 +82,7 @@ public:
 class DALIFT_API CSimpleFixedPartitioner : public CSimplePartitioner
 {
 public:
-    CSimpleFixedPartitioner(unsigned _recordSize, bool _noTranslation) { recordSize = _recordSize; noTranslation = _noTranslation; }
+    CSimpleFixedPartitioner(unsigned _recordSize, bool _noTranslation);
 
     virtual void setPartitionRange(offset_t _totalSize, offset_t _thisOffset, offset_t _thisSize, unsigned _thisHeaderSize, unsigned _numParts);
 
@@ -268,11 +269,7 @@ protected:
 class DALIFT_API CCsvQuickPartitioner : public CCsvPartitioner
 {
 public:
-    CCsvQuickPartitioner(const FileFormat & _format, bool _noTranslation) 
-        : CCsvPartitioner(_format)
-    {
-        noTranslation = _noTranslation;
-    }
+    CCsvQuickPartitioner(const FileFormat & _format, bool _noTranslation);
 
 protected:
     virtual void findSplitPoint(offset_t curOffset, PartitionCursor & cursor);
@@ -324,7 +321,7 @@ protected:
 class DALIFT_API CUtfQuickPartitioner : public CUtfPartitioner
 {
 public:
-    CUtfQuickPartitioner(const FileFormat & _format, bool _noTranslation) : CUtfPartitioner(_format) { noTranslation = _noTranslation; }
+    CUtfQuickPartitioner(const FileFormat & _format, bool _noTranslation);
 
 protected:
     virtual void findSplitPoint(offset_t curOffset, PartitionCursor & cursor);
@@ -359,6 +356,226 @@ protected:
     Linked<IFileIOStream> stream;
 };
 
+class DALIFT_API JsonSplitter : public CInterface, implements IPTreeNotifyEvent
+{
+public:
+    IMPLEMENT_IINTERFACE;
+
+    JsonSplitter(const FileFormat & format, IFileIOStream &stream) : headerLength(0), pathPos(0), tangent(0), rowDepth(0), rowStart(0), rowEnd(0), footerLength(0), newRowSet(true)
+    {
+        LOG(MCdebugProgressDetail, unknownJob, "JsonSplitter::JsonSplitter(format.type :'%s', rowPath:'%s')", format.getFileFormatTypeString(), format.rowTag.get());
+
+        size = stream.size();
+        const char *rowPath = format.rowTag;
+        while (*rowPath=='/')
+            rowPath++;
+        pathNodes.appendList(rowPath, "/");
+        noPath = !pathNodes.length();
+        reader.setown(createPullJSONStreamReader(stream, *this, ptr_noRoot));
+    }
+    virtual void beginNode(const char *tag, offset_t startOffset)
+    {
+        if (tangent)
+        {
+            tangent++;
+            return;
+        }
+        if (rowDepth)
+        {
+            rowDepth++;
+            return;
+        }
+        if (!pathPos)
+        {
+            if (streq(tag, "__array__")) //ignore root array
+                return;
+            if (!noPath && streq(tag, "__object__")) //paths start inside this object, with no path this starts a row
+                return;
+        }
+        if (noPath || streq(tag, pathNodes.item(pathPos))) //closer to a row
+        {
+            if (!noPath)
+                pathPos++;
+            if (noPath || pathPos==pathNodes.ordinality()) //start a row
+            {
+                rowDepth=1;
+                rowStart=startOffset;
+            }
+            return;
+        }
+        //off on a tangent
+        tangent=1;
+    }
+    virtual void newAttribute(const char *name, const char *value){}
+    virtual void beginNodeContent(const char *tag){}
+    virtual void endNode(const char *tag, unsigned length, const void *value, bool binary, offset_t endOffset)
+    {
+        if (rowDepth)
+        {
+            rowDepth--;
+            if (!rowDepth)
+            {
+                rowEnd=endOffset;
+                if (newRowSet)
+                    newRowSet = false;  //individual row ended, but track whether the rowset itself ends before next row start
+                pathPos--;
+            }
+        }
+        else if (tangent)
+            tangent--;
+        else if (pathPos)
+        {
+            if (!newRowSet)
+                newRowSet=true;
+            pathPos--;
+        }
+    }
+
+    bool findNextRow()
+    {
+        if (rowDepth && !exitRow())
+            return false;
+        while (reader->next())
+            if (rowDepth==1)
+                return true;
+        return false;
+    }
+
+    bool exitRow()
+    {
+        if (!rowDepth)
+            return false;
+        while (reader->next())
+            if (rowDepth==0)
+                return true;
+        return false;
+    }
+    bool findNextRowEnd()
+    {
+        if (rowDepth)
+            return exitRow();
+        while (reader->next())
+            if (rowDepth==1)
+                return exitRow();
+        return false;
+    }
+    bool findRowEnd(offset_t splitOffset, offset_t &prevRowEnd)
+    {
+        while (rowEnd < splitOffset + headerLength)
+        {
+            prevRowEnd = rowEnd;
+            if (!findNextRowEnd())
+                return false;
+        }
+        return true;
+    }
+    offset_t getHeaderLength()
+    {
+        if (!headerLength)
+        {
+            while (!rowStart && reader->next());
+            if (!rowStart)
+                throw MakeStringException(DFTERR_CannotFindFirstJsonRecord, "Could not find first json record (check path)");
+            else
+                headerLength = rowStart-1;
+        }
+        return headerLength;
+    }
+    offset_t getFooterLength()
+    {
+        if (!footerLength)
+        {
+            while (reader->next());
+            if (rowEnd)
+                footerLength = size + 1 - rowEnd;
+        }
+        return footerLength;
+    }
+    offset_t getRowOffset()
+    {
+        if (rowStart <= headerLength)
+            return 0;
+        return rowStart - headerLength - 1;
+    }
+
+public:
+    Owned<IFileIOStream> inStream;
+    Owned<IPullPTreeReader> reader;
+    StringArray pathNodes;
+    offset_t rowStart;
+    offset_t rowEnd;
+    offset_t headerLength;
+    offset_t footerLength;
+    offset_t size;
+    unsigned pathPos;
+    unsigned tangent;
+    unsigned rowDepth;
+    bool noPath;
+    bool newRowSet;
+};
+
+class DALIFT_API CJsonInputPartitioner : public CPartitioner
+{
+public:
+    CJsonInputPartitioner(const FileFormat & _format);
+    ~CJsonInputPartitioner();
+
+    virtual void setSource(unsigned _whichInput, const RemoteFilename & _fullPath, bool compressedInput, const char *decryptKey);
+
+protected:
+    virtual void findSplitPoint(offset_t splitOffset, PartitionCursor & cursor)
+    {
+        if (!splitOffset) //header + 0 is first offset
+            return;
+
+        offset_t prevRowEnd;
+        json->findRowEnd(splitOffset, prevRowEnd);
+        if (!json->rowStart)
+            return;
+        if (!json->newRowSet) //get rid of extra delimiter if we haven't closed and reopened in the meantime
+            cursor.trimLength = json->rowStart - prevRowEnd;
+        cursor.inputOffset = json->getRowOffset();
+        if (json->findNextRow())
+            cursor.nextInputOffset = json->getRowOffset();
+        else
+            cursor.nextInputOffset = cursor.inputOffset;  //eof
+    }
+
+protected:
+    FileFormat      format;
+    Owned<IFileIOStream>   inStream;
+    Owned<JsonSplitter> json;
+    static IFileIOCache    *openfilecache;
+    static CriticalSection openfilecachesect;
+};
+
+
+class DALIFT_API CJsonPartitioner : public CJsonInputPartitioner
+{
+public:
+    CJsonPartitioner(const FileFormat & _format);
+
+    virtual void setTarget(IOutputProcessor * _target){UNIMPLEMENTED;}
+
+    //Processing.
+    virtual void beginTransform(offset_t thisOffset, offset_t thisLength, TransformCursor & cursor){UNIMPLEMENTED;}
+    virtual void endTransform(TransformCursor & cursor){UNIMPLEMENTED;}
+    virtual crc32_t getInputCRC(){return 0;}
+    virtual void setInputCRC(crc32_t value){UNIMPLEMENTED;}
+    virtual unsigned transformBlock(offset_t endOffset, TransformCursor & cursor){UNIMPLEMENTED;}
+    virtual void killBuffer(){}
+
+protected:
+    virtual size32_t getSplitRecordSize(const byte * record, unsigned maxToRead, bool processFullBuffer){UNIMPLEMENTED;}
+    virtual size32_t getTransformRecordSize(const byte * record, unsigned maxToRead){UNIMPLEMENTED;}
+
+protected:
+    size32_t recordSize;
+    unsigned unitSize;
+    UtfReader::UtfFormat utfFormat;
+};
+
+
 class DALIFT_API XmlSplitter
 {
 public:
@@ -403,7 +620,7 @@ protected:
 class DALIFT_API CXmlQuickPartitioner : public CXmlPartitioner
 {
 public:
-    CXmlQuickPartitioner(const FileFormat & _format, bool _noTranslation) : CXmlPartitioner(_format) { noTranslation = _noTranslation; }
+    CXmlQuickPartitioner(const FileFormat & _format, bool _noTranslation);
 
 protected:
     virtual void findSplitPoint(offset_t curOffset, PartitionCursor & cursor);

dali/ft/filecopy.cpp

@@ -1041,17 +1041,31 @@ void FileSprayer::calculateSplitPrefixPartition(const char * splitPrefix)
     }
 }
 
-
 void FileSprayer::calculateMany2OnePartition()
 {
     LOG(MCdebugProgressDetail, job, "Setting up many2one partition");
-
+    bool isJSON = srcFormat.hasJsonMarkup();
+    offset_t lastContentLength = 0;
     ForEachItemIn(idx, sources)
     {
         FilePartInfo & cur = sources.item(idx);
         RemoteFilename curFilename;
         curFilename.set(cur.filename);
         setCanAccessDirectly(curFilename);
+        if (isJSON)
+        {
+            offset_t contentLength = cur.size - cur.xmlHeaderLength - cur.xmlFooterLength;
+            if (contentLength)
+            {
+                if (lastContentLength)
+                {
+                    PartitionPoint &part = createLiteral(1, ",", (unsigned) -1);
+                    part.whichOutput = 0;
+                    partition.append(part);
+                }
+                lastContentLength = contentLength;
+            }
+        }
         partition.append(*new PartitionPoint(idx, 0, cur.headerSize, cur.size, cur.size));
     }
 }
@@ -1531,6 +1545,15 @@ void FileSprayer::analyseFileHeaders(bool setcurheadersize)
 void FileSprayer::locateXmlHeader(IFileIO * io, unsigned headerSize, offset_t & xmlHeaderLength, offset_t & xmlFooterLength)
 {
     Owned<IFileIOStream> in = createIOStream(io);
+
+    if (srcFormat.hasJsonMarkup())
+    {
+        JsonSplitter jsplitter(srcFormat, *in);
+        xmlHeaderLength = jsplitter.getHeaderLength();
+        xmlFooterLength = jsplitter.getFooterLength();
+        return;
+    }
+
     XmlSplitter splitter(srcFormat);
     BufferedDirectReader reader;
 
@@ -2763,7 +2786,7 @@ void FileSprayer::spray()
         insertHeaders();
     }
     addEmptyFilesToPartition();
-    
+
     derivePartitionExtra();
     if (partition.ordinality() < 1000)
         displayPartition();

dali/ft/filecopy.hpp

@@ -35,6 +35,18 @@ typedef enum
     FFTrecfmvb, FFTrecfmv, FFTvariablebigendian,
     FFTlast
  } FileFormatType;
+
+static const char * FileFormatTypeStr[] =
+{       "FFTunknown",
+        "FFTfixed", "FFTvariable", "FFTblocked",
+        "FFTcsv",
+        "FFTutf",                             // any format, default to utf-8n
+        "FFTutf8", "FFTutf8n",
+        "FFTutf16", "FFTutf16be", "FFTutf16le",
+        "FFTutf32", "FFTutf32be", "FFTutf32le",
+        "FFTrecfmvb", "FFTrecfmv", "FFTvariablebigendian",
+        "FFTlast"
+};
 enum { FTactionpull, FTactionpush, FTactionpartition, FTactiondirectory, FTactionsize, FTactionpcopy };
 
 
@@ -59,6 +71,9 @@ public:
     void set(const FileFormat & src);
     bool hasQuote() const                           { return (quote == NULL) || (*quote != '\0'); }
     bool hasQuotedTerminator() const                { return quotedTerminator; }
+    const char * getFileFormatTypeString() const        { return FileFormatTypeStr[type]; }
+    bool hasJsonMarkup() const {return (rowTag.length() && *rowTag.get()=='/');} //need to add a more explicit markup indicator
+    bool hasXmlMarkup() const {return (rowTag.length() && *rowTag.get()!='/');}
 
 public:
     FileFormatType      type;

dali/ft/fterror.hpp

@@ -79,6 +79,7 @@
 #define DFTERR_WrongRECFMvbBlockSize            8106
 #define DFTERR_WrongRECFMvRecordSize            8107
 #define DFTERR_WrongSplitRecordSize             8108
+#define DFTERR_CannotFindFirstJsonRecord        8109
 
 //Internal errors
 #define DFTERR_UnknownFormatType                8190

docs/ECLProgrammersGuide/PRG_Mods/PrG_Roxie_Overview.xml

@@ -245,19 +245,4 @@ END;</programlisting>
     tested it on hThor, except the new service will appear under your Roxie in
     the tree list.</para>
   </sect2>
-
-  <sect2 id="Restrictions">
-    <title>Restrictions</title>
-
-    <para>Roxie queries may <emphasis role="underline">not</emphasis> contain
-    any code that would write a file to disk, such as:</para>
-
-    <para>OUTPUT actions that write to disk files</para>
-
-    <para>BUILD (or BUILDINDEX) actions</para>
-
-    <para>PERSISTed attributes</para>
-
-    <para></para>
-  </sect2>
 </sect1>

docs/HPCCDataTutorial/DataTutorial.xml

@@ -19,7 +19,7 @@
 
     <legalnotice>
       <para>We welcome your comments and feedback about this document via
-      email to <email>docfeedback@hpccsystems.com</email> </para>
+      email to <email>docfeedback@hpccsystems.com</email></para>
 
       <para>Please include <emphasis role="bold">Documentation
       Feedback</emphasis> in the subject line and reference the document name,
@@ -27,13 +27,13 @@
       message.</para>
 
       <para>LexisNexis and the Knowledge Burst logo are registered trademarks
-      of Reed Elsevier Properties Inc., used under license. </para>
+      of Reed Elsevier Properties Inc., used under license.</para>
 
       <para>HPCC Systems is a registered trademark of LexisNexis Risk Data
       Management Inc.</para>
 
       <para>Other products and services may be trademarks or registered
-      trademarks of their respective companies. </para>
+      trademarks of their respective companies.</para>
 
       <para>All names and example data used in this manual are fictitious. Any
       similarity to actual persons, living or dead, is purely
@@ -800,6 +800,8 @@ COUNT(TutorialYourName.File_OriginalPerson);
 
             <para><emphasis role="bold">Note: </emphasis>The modified portion
             is shown in <emphasis role="bold">bold</emphasis>.</para>
+
+            <para> </para>
           </listitem>
 
           <listitem>
@@ -837,7 +839,7 @@ COUNT(TutorialYourName.File_OriginalPerson);
         </orderedlist>
       </sect2>
 
-      <sect2 id="Process_the_Data">
+      <sect2 id="Process_the_Data" role="brk">
         <title>Process the Data</title>
 
         <para>In this section, we will write code to convert the original data

docs/Installing_and_RunningTheHPCCPlatform/Inst-Mods/UnityLauncher.xml

@@ -2,7 +2,7 @@
 <!DOCTYPE book PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN"
 "http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd">
 <sect1 id="Unity_Launcher">
-  <title><emphasis role="bold">Unity Launcher Icon</emphasis></title>
+  <title>Unity Launcher Icon</title>
 
   <para>The HPCC platform supports an Ubuntu Unity Launcher icon.</para>
 

docs/Installing_and_RunningTheHPCCPlatform/Inst-Mods/UserSecurityMaint.xml

@@ -13,8 +13,8 @@
 
     <para>HPCC systems maintains security in a number of ways. HPCC Systems
     can be configured to manage users' security rights by pointing either at
-    Microsoft’s Active Directory on a Windows system, or an OpenLDAP-based
-    software on Linux systems.</para>
+    Microsoft’s Active Directory on a Windows system, or a 389Directory Server
+    on Linux systems.</para>
 
     <para>Using the Permissions interface in ECL Watch, administrators can
     control access to features in ECL IDE, ECL Watch, ECL Plus, DFU Plus, and
@@ -460,7 +460,7 @@
                   <listitem>
                     <para>The name of the default Administrator group could
                     vary. For example, in Active Directory, it is
-                    "Administrators", in Open LDAP it is "Directory
+                    "Administrators", in LDAP it is "Directory
                     Administrators".</para>
                   </listitem>
                 </varlistentry>

docs/Installing_and_RunningTheHPCCPlatform/Inst-Mods/hpcc_ldap.xml

@@ -413,6 +413,22 @@
             </listitem>
 
             <listitem>
+              <para>Choose the LDAP server type from the serverType attribute
+              drop box.</para>
+
+              <para><variablelist>
+                  <varlistentry>
+                    <term>NOTE:</term>
+
+                    <listitem>
+                      <para>Support for OpenLDAP has been deprecated. The
+                      option is included only for legacy purposes.</para>
+                    </listitem>
+                  </varlistentry>
+                </variablelist></para>
+            </listitem>
+
+            <listitem>
               <para>Click on the disk icon to save.</para>
             </listitem>
           </orderedlist></para>

docs/Installing_and_RunningTheHPCCPlatform/Inst-Mods/ssl-esp.xml

@@ -2,8 +2,7 @@
 <!DOCTYPE book PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN"
 "http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd">
 <sect1 id="ssl4esp">
-  <title><emphasis role="bold">Configuring ESP Server to use HTTPS
-  (SSL)</emphasis></title>
+  <title>Configuring ESP Server to use HTTPS (SSL)</title>
 
   <para>The HPCC Enterprise Services Platform server (ESP) supports Secure
   Sockets Layer (SSL), a protocol used to send and receive private data or

docs/Installing_and_RunningTheHPCCPlatform/Installing_and_RunningTheHPCCPlatform.xml

@@ -2891,7 +2891,7 @@ OUTPUT(ValidWords)
                 <row>
                   <entry>-t</entry>
 
-                  <entry>Target file or directory. </entry>
+                  <entry>Target file or directory.</entry>
                 </row>
 
                 <row>

ecl/ecl-bundle/ecl-bundle.cpp

@@ -281,7 +281,8 @@ StringBuffer & fetchURL(const char *bundleName, StringBuffer &fetchedLocation)
     // If the bundle name looks like a url, fetch it somewhere temporary first...
     if (isUrl(bundleName))
     {
-        // Put it into a temp directory - we need the filename to be right
+        //Put it into a temp directory - we need the filename to be right
+        //I don't think there is any way to disable the following warning....
         const char *tmp = tmpnam(NULL);
         recursiveCreateDirectory(tmp);
         deleteOnCloseDown.append(tmp);

ecl/hql/hqlexpr.cpp

@@ -158,6 +158,7 @@ static IHqlExpression * cachedNullRecord;
 static IHqlExpression * cachedNullRowRecord;
 static IHqlExpression * cachedOne;
 static IHqlExpression * cachedLocalAttribute;
+static IHqlExpression * cachedContextAttribute;
 static IHqlExpression * constantTrue;
 static IHqlExpression * constantFalse;
 static IHqlExpression * defaultSelectorSequenceExpr;
@@ -238,6 +239,7 @@ MODULE_INIT(INIT_PRIORITY_HQLINTERNAL)
     cachedNullRowRecord = createRecord(nonEmptyAttr);
     cachedOne = createConstant(1);
     cachedLocalAttribute = createAttribute(localAtom);
+    cachedContextAttribute = createAttribute(contextAtom);
     constantTrue = createConstant(createBoolValue(true));
     constantFalse = createConstant(createBoolValue(false));
     defaultSelectorSequenceExpr = createAttribute(_selectorSequence_Atom);
@@ -271,6 +273,7 @@ MODULE_EXIT()
     constantFalse->Release();
     constantTrue->Release();
     blank->Release();
+    cachedContextAttribute->Release();
     cachedLocalAttribute->Release();
     cachedOne->Release();
     cachedActiveTableExpr->Release();
@@ -7790,13 +7793,27 @@ IHqlExpression * createFunctionDefinition(IIdAtom * id, IHqlExpression * value,
     return createFunctionDefinition(id, args);
 }
 
+bool functionBodyUsesContext(IHqlExpression * body)
+{
+    //All functions are assumed to require the context, unless it is an external c++ function without a context attr
+    switch (body->getOperator())
+    {
+    case no_external:
+        return (body->queryAttribute(contextAtom) != NULL);
+    default:
+        return true;
+    }
+}
+
 IHqlExpression * createFunctionDefinition(IIdAtom * id, HqlExprArray & args)
 {
     IHqlExpression * body = &args.item(0);
     IHqlExpression * formals = &args.item(1);
     IHqlExpression * defaults = args.isItem(2) ? &args.item(2) : NULL;
+    OwnedHqlExpr attrs;
+    if (functionBodyUsesContext(body))
+        attrs.set(cachedContextAttribute);
 
-#if 1
     //This is a bit of a waste of time, but need to improve assignableFrom for a function type to ignore the uids.
     QuickExpressionReplacer defaultReplacer;
     HqlExprArray normalized;
@@ -7813,11 +7830,7 @@ IHqlExpression * createFunctionDefinition(IIdAtom * id, HqlExprArray & args)
     OwnedHqlExpr newFormals = formals->clone(normalized);
     OwnedHqlExpr newDefaults = defaults ? defaultReplacer.transform(defaults) : NULL;
 
-    ITypeInfo * type = makeFunctionType(body->getType(), newFormals.getClear(), newDefaults.getClear());
-#else
-    ITypeInfo * type = makeFunctionType(body->getType(), LINK(formals), LINK(defaults));
-#endif
-
+    ITypeInfo * type = makeFunctionType(body->getType(), newFormals.getClear(), newDefaults.getClear(), attrs.getClear());
     return createNamedValue(no_funcdef, type, id, args);
 }
 
@@ -12050,6 +12063,8 @@ IHqlExpression * createExternalFuncdefFromInternal(IHqlExpression * funcdef)
         attrs.append(*createAttribute(actionAtom));
     if (body->getInfoFlags() & HEFcontextDependentException)
         attrs.append(*createAttribute(contextSensitiveAtom));
+    if (functionBodyUsesContext(body))
+        attrs.append(*LINK(cachedContextAttribute));
 
     ITypeInfo * returnType = funcdef->queryType()->queryChildType();
     OwnedHqlExpr externalExpr = createExternalReference(funcdef->queryId(), LINK(returnType), attrs);

ecl/hql/hqlexpr.hpp

@@ -1806,6 +1806,7 @@ inline int boolToInt(bool x)                    { return x ? 1 : 0; }
 extern HQL_API IHqlExpression * createFunctionDefinition(IIdAtom * name, IHqlExpression * value, IHqlExpression * parms, IHqlExpression * defaults, IHqlExpression * attrs);
 extern HQL_API IHqlExpression * createFunctionDefinition(IIdAtom * name, HqlExprArray & args);
 extern HQL_API IHqlExpression * queryNonDelayedBaseAttribute(IHqlExpression * expr);
+extern HQL_API bool functionBodyUsesContext(IHqlExpression * body);
 
 #define NO_AGGREGATE        \
          no_count:          \

ecl/hql/hqlgram.y

@@ -102,6 +102,11 @@ static void eclsyntaxerror(HqlGram * parser, const char * s, short yystate, int
 /* remember to add any new tokens to the error reporter and lexer too! */
 /* If they clash with other #defines etc then use TOK_ as a prefix */
 
+// NB: Very occassionally the same keyword in the source (e.g., MERGE, PARTITION may return a different token
+// (MERGE_ATTR, PARTITION_ATTR) depending on the context it is used in.  This is because there would be a s/r
+// error, so the _ATTR form is only allowed in the situations where the attibute is valid - enabled by a
+// call to enableAttributes() from a production in the grammar.
+
   ABS
   ACOS
   AFTER
@@ -212,10 +217,9 @@ static void eclsyntaxerror(HqlGram * parser, const char * s, short yystate, int
   FIRST
   TOK_FIXED
   FLAT
-  FROM
-  FORMAT
   FORMAT_ATTR
   FORWARD
+  FROM
   FROMJSON
   FROMUNICODE
   FROMXML
@@ -1612,15 +1616,15 @@ failure
                             parser->normalizeExpression($5, type_string, true);
                             $$.setExpr(createValueF(no_persist, makeVoidType(), $3.getExpr(), $5.getExpr(), $6.getExpr(), NULL), $1);
                         }
-    | STORED '(' expression ',' fewMany optStoredFieldFormat ')'
+    | STORED '(' startStoredAttrs expression ',' fewMany optStoredFieldFormat ')'
                         {
-                            parser->normalizeStoredNameExpression($3);
-                            $$.setExpr(createValue(no_stored, makeVoidType(), $3.getExpr(), $5.getExpr(), $6.getExpr()), $1);
+                            parser->normalizeStoredNameExpression($4);
+                            $$.setExpr(createValue(no_stored, makeVoidType(), $4.getExpr(), $6.getExpr(), $7.getExpr()), $1);
                         }
-    | STORED '(' expression optStoredFieldFormat ')'
+    | STORED '(' startStoredAttrs expression optStoredFieldFormat ')'
                         {
-                            parser->normalizeStoredNameExpression($3);
-                            $$.setExpr(createValue(no_stored, makeVoidType(), $3.getExpr(), $4.getExpr()), $1);
+                            parser->normalizeStoredNameExpression($4);
+                            $$.setExpr(createValue(no_stored, makeVoidType(), $4.getExpr(), $5.getExpr()), $1);
                         }
     | CHECKPOINT '(' constExpression ')'
                         {
@@ -1732,7 +1736,7 @@ optStoredFieldFormat
     :                   {
                             $$.setNullExpr();
                         }
-    | ',' FORMAT '(' hintList ')'
+    | ',' FORMAT_ATTR '(' hintList ')'
                         {
                             HqlExprArray args;
                             $4.unwindCommaList(args);
@@ -12386,6 +12390,7 @@ featureModifiers
 
 startDistributeAttrs :  { parser->enableAttributes(DISTRIBUTE); $$.clear(); } ;
 startHeadingAttrs:      { parser->enableAttributes(HEADING); $$.clear(); } ;
+startStoredAttrs:       { parser->enableAttributes(STORED); $$.clear(); } ;
 
 
 //================================== end of syntax section ==========================

ecl/hql/hqlgram2.cpp

@@ -152,6 +152,8 @@ MODULE_INIT(INIT_PRIORITY_STANDARD)
 
     setAttributes(DISTRIBUTE, MERGE_ATTR, PARTITION_ATTR, 0);
     setAttributes(HEADING, FORMAT_ATTR, 0);
+    setAttributes(STORED, FORMAT_ATTR, 0);
+
     alreadyAssignedNestedTag = createAttribute(_alreadyAssignedNestedTag_Atom);
     return true;
 }
@@ -890,15 +892,7 @@ IHqlExpression * HqlGram::convertToOutOfLineFunction(const ECLlocation & errpos,
     if (expr->getOperator() != no_outofline)
     {
         if (queryParametered())
-        {
-            OwnedHqlExpr mapped = convertWorkflowToImplicitParmeters(defineScopes.tos().activeParameters, defineScopes.tos().activeDefaults, expr);
-            if (containsWorkflow(mapped))
-            {
-                reportError(ERR_USER_FUNC_NO_WORKFLOW, errpos, "Out of line user functions cannot contain workflow/stored");
-                return mapped.getClear();
-            }
-            return createWrapper(no_outofline, mapped.getClear());
-        }
+            return createWrapper(no_outofline, LINK(expr));
     }
     return LINK(expr);
 }
@@ -996,7 +990,6 @@ IHqlExpression * HqlGram::processEmbedBody(const attribute & errpos, IHqlExpress
                 }
             }
         }
-
         return createWrapper(no_outofline, result->queryType(), args);
     }
     return result.getClear();
@@ -1674,8 +1667,10 @@ IHqlExpression * HqlGram::forceEnsureExprType(IHqlExpression * expr, ITypeInfo *
     //or the out of line node should be added much later.
     if (expr->getOperator() == no_outofline)
     {
-        OwnedHqlExpr ret = forceEnsureExprType(expr->queryChild(0), type);
-        return createWrapper(no_outofline, LINK(ret));
+        HqlExprArray args;
+        args.append(*forceEnsureExprType(expr->queryChild(0), type));
+        unwindChildren(args, expr, 1);
+        return expr->clone(args);
     }
         
     OwnedHqlExpr ret = ensureExprType(expr, type);
@@ -6079,7 +6074,7 @@ void HqlGram::addFunctionParameter(const attribute & errpos, IIdAtom * name, ITy
 
     HqlExprArray attrs;
     endList(attrs);
-    Owned<ITypeInfo> funcType = makeFunctionType(type, LINK(formals), defaults.getClear());
+    Owned<ITypeInfo> funcType = makeFunctionType(type, LINK(formals), defaults.getClear(), NULL);
     addActiveParameterOwn(errpos, createParameter(name, nextParameterIndex(), LINK(funcType), attrs), defValue);
 }
 
@@ -10507,7 +10502,6 @@ static void getTokenText(StringBuffer & msg, int token)
     case FIRST: msg.append("FIRST"); break;
     case TOK_FIXED: msg.append("FIXED"); break;
     case FLAT: msg.append("FLAT"); break;
-    case FORMAT: msg.append("FORMAT"); break;
     case FORMAT_ATTR: msg.append("FORMAT"); break;
     case FORWARD: msg.append("FORWARD"); break;
     case FROM: msg.append("FROM"); break;

ecl/hql/hqllex.l

@@ -705,10 +705,9 @@ FILTERED            { RETURNSYM(FILTERED); }
 FIRST               { RETURNSYM(FIRST); }
 FIXED               { RETURNSYM(TOK_FIXED); }
 FLAT                { RETURNSYM(FLAT); }
-FORMAT              { RETURNSYM(FORMAT); }
-FROM                { RETURNSYM(FROM); }
-FORMAT              { RETURNSYM(FORMAT_ATTR); }
+FORMAT              { RETURNSYM(FORMAT_ATTR); }             // Dynamically enabled based on the context
 FORWARD             { RETURNSYM(FORWARD); }
+FROM                { RETURNSYM(FROM); }
 FROMUNICODE         { RETURNSYM(FROMUNICODE); }
 FROMJSON            { RETURNSYM(FROMJSON); }
 FROMXML             { RETURNSYM(FROMXML); }
@@ -785,7 +784,7 @@ MAX                 { RETURNSYM(MAX); }
 MAXCOUNT            { RETURNSYM(MAXCOUNT); }
 MAXLENGTH           { RETURNSYM(MAXLENGTH); }
 MAXSIZE             { RETURNSYM(MAXSIZE); }
-MERGE               { RETURNSYM(MERGE); }
+MERGE               { RETURNSYM(MERGE); }                   // May be dynamically converted to MERGE_ATTR
 MERGEJOIN           { RETURNSYM(MERGEJOIN); }
 MIN                 { RETURNSYM(MIN); }
 MODULE              { RETURNHARD(MODULE); }
@@ -831,7 +830,7 @@ __OWNED__           { RETURNSYM(__OWNED__); }
 PACKED              { RETURNSYM(PACKED); }
 PARALLEL            { RETURNSYM(PARALLEL); }
 PARSE               { RETURNSYM(PARSE); }
-PARTITION           { RETURNSYM(PARTITION); }
+PARTITION           { RETURNSYM(PARTITION); }               // May be dynamically converted to PARTITION_ATTR
 PATTERN             { RETURNSYM(TOK_PATTERN); }
 PENALTY             { RETURNSYM(PENALTY); }
 PERSIST             { RETURNSYM(PERSIST); }

ecl/hql/hqltrans.cpp

@@ -4632,45 +4632,6 @@ void verifySplitConsistency(IHqlExpression * expr)
 
 //---------------------------------------------------------------------------
 
-static HqlTransformerInfo globalToParameterTransformerInfo("GlobalToParameterTransformer");
-class GlobalToParameterTransformer : public QuickHqlTransformer
-{
-public:
-    GlobalToParameterTransformer(HqlExprArray & _parameters, HqlExprArray & _defaults) 
-        : QuickHqlTransformer(globalToParameterTransformerInfo, NULL), parameters(_parameters), defaults(_defaults)
-    {}
-
-    virtual IHqlExpression * createTransformedBody(IHqlExpression * expr);
-
-protected:
-    HqlExprArray & parameters;
-    HqlExprArray & defaults;
-};
-
-IHqlExpression * GlobalToParameterTransformer::createTransformedBody(IHqlExpression * expr)
-{
-    if (expr->getOperator() != no_colon)
-        return QuickHqlTransformer::createTransformedBody(expr);
-
-    StringBuffer paramName;
-    paramName.append("_implicit_hidden_").append(parameters.ordinality());
-    HqlExprArray attrs;
-    attrs.append(*createAttribute(_hidden_Atom));
-    IHqlExpression * param = createParameter(createIdAtom(paramName.str()), parameters.ordinality(), expr->getType(), attrs);
-    parameters.append(*param);
-    defaults.append(*LINK(expr));
-    return LINK(param);
-}
-
-IHqlExpression * convertWorkflowToImplicitParmeters(HqlExprArray & parameters, HqlExprArray & defaults, IHqlExpression * expr)
-{
-    GlobalToParameterTransformer transformer(parameters, defaults);
-    return transformer.transform(expr);
-}
-
-
-//---------------------------------------------------------------------------
-
 static HqlTransformerInfo createRowSelectorExpanderInfo("CreateRowSelectorExpander");
 class CreateRowSelectorExpander : public NewHqlTransformer
 {

ecl/hql/hqlutil.cpp

@@ -2684,6 +2684,50 @@ void checkSelectConsistency(IHqlExpression * expr)
 
 //---------------------------------------------------------------------------
 
+static HqlTransformerInfo parameterDependencyCheckerInfo("ParameterDependencyChecker");
+class ParameterDependencyChecker  : public NewHqlTransformer
+{
+public:
+    ParameterDependencyChecker() : NewHqlTransformer(parameterDependencyCheckerInfo), foundParameter(false)
+    {
+    }
+
+    virtual void analyseExpr(IHqlExpression * expr)
+    {
+        if (expr->isFullyBound() || alreadyVisited(expr) || foundParameter)
+            return;
+
+        if (expr->getOperator() == no_param)
+        {
+            foundParameter = true;
+            return;
+        }
+
+        NewHqlTransformer::analyseExpr(expr);
+    }
+
+    bool isDependent(IHqlExpression * expr)
+    {
+        analyse(expr, 0);
+        return foundParameter;
+    }
+
+protected:
+    bool foundParameter;
+};
+
+
+//Is 'expr' really dependent on a parameter - expr->isFullyBound() can give false negatives.
+bool isDependentOnParameter(IHqlExpression * expr)
+{
+    if (expr->isFullyBound())
+        return false;
+    ParameterDependencyChecker checker;
+    return checker.isDependent(expr);
+}
+
+//---------------------------------------------------------------------------
+
 void DependencyGatherer::doGatherDependencies(IHqlExpression * expr)
 {
     if (expr->queryTransformExtra())
@@ -6725,7 +6769,7 @@ void ErrorSeverityMapper::exportMappings(IWorkUnit * wu) const
     for (unsigned i=firstActiveMapping; i < max; i++)
     {
         IHqlExpression & cur = severityMappings.item(i);
-        wu->setWarningSeverity(getIntValue(cur.queryChild(0)), getCheckSeverity(cur.queryChild(1)->queryName()));
+        wu->setWarningSeverity((unsigned)getIntValue(cur.queryChild(0)), getCheckSeverity(cur.queryChild(1)->queryName()));
     }
 }
 
@@ -7147,7 +7191,7 @@ public:
         StringBuffer mangledReturnParameters;
         mangleFunctionReturnType(mangledReturn, mangledReturnParameters, retType);
 
-        if (body->hasAttribute(contextAtom))
+        if (functionBodyUsesContext(body))
             mangled.append("P12ICodeContext");
         else if (body->hasAttribute(globalContextAtom) )
             mangled.append("P18IGlobalCodeContext");
@@ -7394,7 +7438,7 @@ public:
 
         mangled.append(mangledReturn);
 
-        if (body->hasAttribute(contextAtom))
+        if (functionBodyUsesContext(body))
             mangled.append("PVICodeContext@@");
         else if (body->hasAttribute(globalContextAtom) )
             mangled.append("PVIGlobalCodeContext@@");

ecl/hql/hqlutil.hpp

@@ -197,6 +197,8 @@ extern HQL_API IHqlExpression * createTransformForField(IHqlExpression * field,
 extern HQL_API IHqlExpression * convertScalarToRow(IHqlExpression * value, ITypeInfo * fieldType);
 extern HQL_API bool splitResultValue(SharedHqlExpr & dataset, SharedHqlExpr & attribute, IHqlExpression * value);
 
+//Is 'expr' really dependent on a parameter - expr->isFullyBound() can give false negatives.
+extern HQL_API bool isDependentOnParameter(IHqlExpression * expr);
 
 inline void extendConditionOwn(SharedHqlExpr & cond, node_operator op, IHqlExpression * r)
 {

ecl/hqlcpp/hqlcerrors.hpp

@@ -261,6 +261,7 @@
 #define HQLWRN_GlobalActionDependendOnScope     4543
 #define HQLWRN_NoThorContextDependent           4544
 #define HQLWRN_OnlyLocalMergeJoin               4545
+#define HQLWRN_WorkflowDependParameter          4546
 
 //Temporary errors
 #define HQLERR_OrderOnVarlengthStrings          4601
@@ -537,6 +538,7 @@
 #define HQLWRN_GlobalActionDependendOnScope_Text "Global action appears to be context dependent - this may cause a dataset not active error"
 #define HQLWRN_NoThorContextDependent_Text      "NOTHOR expression%s appears to access a parent dataset - this may cause a dataset not active error"
 #define HQLWRN_OnlyLocalMergeJoin_Text          "Only LOCAL versions of %s are currently supported on THOR"
+#define HQLWRN_WorkflowDependParameter_Text     "Workflow action %s appears to be dependent upon a parameter"
 
 #define HQLERR_DistributionVariableLengthX_Text "DISTRIBUTION does not support variable length field '%s'"
 #define HQLERR_DistributionUnsupportedTypeXX_Text "DISTRIBUTION does not support field '%s' with type %s"

ecl/hqlcpp/hqlcpp.cpp

@@ -2009,6 +2009,8 @@ IHqlExpression * HqlCppTranslator::bindFunctionCall(IIdAtom * name, HqlExprArray
     OwnedHqlExpr function = needFunction(name);
     useFunction(function);
     assertex(function->getOperator() == no_funcdef);
+    IFunctionTypeExtra * funcTypeExtra = queryFunctionTypeExtra(function->queryType());
+    assertex(funcTypeExtra);
     IHqlExpression * body = function->queryChild(0);
 
     HqlExprArray bodyArgs;
@@ -2017,7 +2019,7 @@ IHqlExpression * HqlCppTranslator::bindFunctionCall(IIdAtom * name, HqlExprArray
     HqlExprArray funcArgs;
     funcArgs.append(*createValue(body->getOperator(), LINK(newType), bodyArgs));
     unwindChildren(funcArgs, function, 1);
-    ITypeInfo * funcType = makeFunctionType(LINK(newType), LINK(function->queryChild(1)), LINK(function->queryChild(2)));
+    ITypeInfo * funcType = makeFunctionType(LINK(newType), LINK(function->queryChild(1)), LINK(function->queryChild(2)), LINK(funcTypeExtra->queryAttributes()));
     OwnedHqlExpr newFunction = createValue(function->getOperator(), funcType, funcArgs);
     return bindFunctionCall(newFunction, args);
 }
@@ -5709,6 +5711,7 @@ void HqlCppTranslator::doBuildCall(BuildCtx & ctx, const CHqlBoundTarget * tgt,
             attrs.append(*LINK(params));
             if (defaults)
                 attrs.append(*LINK(defaults));
+            attrs.append(*createAttribute(contextAtom));
 
             ITypeInfo * returnType = type->queryChildType();
             OwnedHqlExpr externalExpr = createExternalReference(def->queryId(), LINK(returnType), attrs);
@@ -11869,8 +11872,11 @@ void HqlCppTranslator::buildFunctionDefinition(IHqlExpression * funcdef)
         funcctx.addQuotedCompound(proto);
         //MORE: Need to work out how to handle functions that require the context.
         //Need to create a class instead.
-        assertex(!outofline->hasAttribute(contextAtom));
-
+        if (functionBodyUsesContext(outofline))
+        {
+            funcctx.associateExpr(codeContextMarkerExpr, codeContextMarkerExpr);
+            funcctx.associateExpr(globalContextMarkerExpr, globalContextMarkerExpr);
+        }
         OwnedHqlExpr newCode = replaceInlineParameters(funcdef, bodyCode);
         newCode.setown(foldHqlExpression(newCode));
         ITypeInfo * returnType = funcdef->queryType()->queryChildType();

ecl/hqlcpp/hqlcse.cpp

@@ -106,6 +106,7 @@ bool canCreateTemporary(IHqlExpression * expr)
     case no_thisnode:
     case no_libraryscopeinstance:
     case no_loopbody:
+    case no_external:
         return false;
     }
     ITypeInfo * type = expr->queryType();
@@ -1223,6 +1224,8 @@ static bool canHoistInvariant(IHqlExpression * expr)
     }
     if (!expr->isPure())
         return false;
+    if (expr->isFunction())
+        return false;
     switch (expr->getOperator())
     {
     case no_list:

ecl/hqlcpp/hqlttcpp.cpp

@@ -5851,6 +5851,13 @@ IHqlExpression * WorkflowTransformer::extractWorkflow(IHqlExpression * untransfo
         }
     }
 
+    if (isDependentOnParameter(expr))
+    {
+        StringBuffer s;
+        getStoredDescription(s, info.sequence, info.originalLabel, true);
+        translator.reportWarning(CategoryMistake, SeverityUnknown, queryActiveLocation(expr), HQLWRN_WorkflowDependParameter, HQLWRN_WorkflowDependParameter_Text, s.str());
+    }
+
     OwnedHqlExpr setValue;
     OwnedHqlExpr getValue;
     bool done = false;
@@ -6094,7 +6101,9 @@ IHqlExpression * WorkflowTransformer::transformInternalFunction(IHqlExpression *
 
     WorkflowItem * item = new WorkflowItem(namedFuncDef);
     workflowOut->append(*item);
-    return createExternalFuncdefFromInternal(namedFuncDef);
+    OwnedHqlExpr external = createExternalFuncdefFromInternal(namedFuncDef);
+    copyDependencies(queryBodyExtra(namedFuncDef), queryBodyExtra(external));
+    return external.getClear();
 }
 
 IHqlExpression * WorkflowTransformer::transformInternalCall(IHqlExpression * transformed)

ecl/hqlcpp/hqlwcpp.cpp

@@ -536,9 +536,13 @@ void HqlCppWriter::generateType(ITypeInfo * type, const char * name)
                 StringBuffer parameterText;
                 IFunctionTypeExtra * extra = queryFunctionTypeExtra(type);
                 IHqlExpression * args = static_cast<IHqlExpression *>(extra->queryParameters());
+                IHqlExpression * attrs = static_cast<IHqlExpression *>(extra->queryAttributes());
+                if (queryAttributeInList(contextAtom, attrs))
+                    parameterText.append("ICodeContext * ctx");
+
                 ForEachChild(i, args)
                 {
-                    if (i)
+                    if (parameterText.length())
                         parameterText.append(", ");
                     generateExprCpp(parameterText, args->queryChild(i));
                 }
@@ -656,7 +660,7 @@ bool HqlCppWriter::generateFunctionPrototype(IHqlExpression * funcdef, const cha
 
     bool firstParam = true;
     out.append('(');
-    if (body->hasAttribute(contextAtom))
+    if (functionBodyUsesContext(body))
     {
         out.append("ICodeContext * ctx");
         firstParam = false;
@@ -1170,7 +1174,7 @@ StringBuffer & HqlCppWriter::generateExprCpp(IHqlExpression * expr)
                 else
                     out.append(funcdef->queryBody()->queryId()->str());
                 out.append('(');
-                if (props->hasAttribute(contextAtom))
+                if (functionBodyUsesContext(props))
                 {
                     out.append("ctx");
                     if (numArgs)

ecl/regress/format.ecl

@@ -0,0 +1,5 @@
+import lib_stringlib;
+
+format := 3;
+
+output(format);

ecllibrary/std/Metaphone3.ecl

@@ -0,0 +1,40 @@
+/*##############################################################################
+## HPCC SYSTEMS software Copyright (C) 2015 HPCC Systems.  All rights reserved.
+############################################################################## */
+
+
+EXPORT Metaphone3 := MODULE
+
+IMPORT lib_metaphone3;
+
+/**
+ * Returns the primary metaphone value
+ *
+ * @param src           The string whose metphone is to be calculated.
+ * @see                 http://en.wikipedia.org/wiki/Metaphone#Double_Metaphone
+ */
+
+EXPORT String primary(STRING src, boolean encodeVowels=false, boolean encodeExact=false, unsigned4 maxLength=0) :=
+  lib_metaphone3.Metaphone3Lib.Metaphone3(src, encodeVowels, encodeExact, maxLength);
+
+/**
+ * Returns the secondary metaphone value
+ *
+ * @param src           The string whose metphone is to be calculated.
+ * @see                 http://en.wikipedia.org/wiki/Metaphone#Double_Metaphone
+ */
+
+EXPORT String secondary(STRING src, boolean encodeVowels=false, boolean encodeExact=false, unsigned4 maxLength=0) :=
+  lib_metaphone3.Metaphone3Lib.Metaphone3Alt(src, encodeVowels, encodeExact, maxLength);
+
+/**
+ * Returns the double metaphone value (primary and secondary concatenated
+ *
+ * @param src           The string whose metphone is to be calculated.
+ * @see                 http://en.wikipedia.org/wiki/Metaphone#Double_Metaphone
+ */
+
+EXPORT String double(STRING src, boolean encodeVowels=false, boolean encodeExact=false, unsigned4 maxLength=0) :=
+  lib_metaphone3.Metaphone3Lib.Metaphone3Both(src, encodeVowels, encodeExact, maxLength);
+
+END;

esp/esdllib/CMakeLists.txt

@@ -47,7 +47,7 @@ HPCC_ADD_LIBRARY( esdllib SHARED ${SRCS}
 
 install ( TARGETS esdllib RUNTIME DESTINATION bin LIBRARY DESTINATION lib )
 
-add_dependencies ( esdllib jlib)
+add_dependencies ( esdllib jlib espscm)
 
 target_link_libraries ( esdllib
     jlib

esp/esdllib/esdl_transformer2.cpp

@@ -1317,8 +1317,8 @@ void Esdl2Response::process(Esdl2TransformerContext &ctx, const char *out_name,
 
                     if (ctx.schemaLocation.length() > 0 )
                     {
-                        ctx.writer->outputXmlns("xmlns:xsi", "http://www.w3.org/2001/XMLSchema-instance");
-                        ctx.writer->outputXmlns("xsi:schemaLocation", ctx.schemaLocation.str());
+                        ctx.writer->outputXmlns("xsi", "http://www.w3.org/2001/XMLSchema-instance");
+                        ctx.writer->outputCString(ctx.schemaLocation.str(), "@xsi:schemaLocation");
                     }
 
                     ctx.do_output_ns=false;
@@ -1789,7 +1789,11 @@ void Esdl2Transformer::processHPCCResult(IEspContext &ctx, IEsdlDefMethod &mthde
     {
         while((dataset = gotoNextHPCCDataset(*xpp, stag)) != NULL)
         {
-            if (strieq(dataset, resdsname))
+            if ( strieq(dataset, resdsname)
+                ||(subresdsname && strieq(dataset, subresdsname)) //Only allow correctly named dataset?
+                || stricmp(dataset, "FinalResults")==0
+                || stricmp(dataset, "Results")==0
+               )
             {
                 Esdl2TransformerContext tctx(*this, writer, *xpp, ctx.getClientVersion(), ctx.queryRequestParameters(), EsdlResponseMode, 0, ns,schema_location);
                 tctx.flags = flags | ESDL_TRANS_ROW_IN;
@@ -1800,24 +1804,12 @@ void Esdl2Transformer::processHPCCResult(IEspContext &ctx, IEsdlDefMethod &mthde
                 {
                     Esdl2Response *resp = dynamic_cast<Esdl2Response *>(root);
                     if (resp)
-                        resp->process(tctx, restype);
-                }
-            }
-            else if ((subresdsname && strieq(dataset, subresdsname)) //Only allow correctly named dataset?
-                     || stricmp(dataset, "FinalResults")==0
-                     || stricmp(dataset, "Results")==0
-                     )
-            {
-                Esdl2TransformerContext tctx(*this, writer, *xpp, ctx.getClientVersion(), ctx.queryRequestParameters(), EsdlResponseMode, 0, ns,schema_location);
-                tctx.flags = flags | ESDL_TRANS_ROW_IN;
-                tctx.skip_root = !(flags & ESDL_TRANS_OUTPUT_ROOT);
-                tctx.root_type.set(restype);
-                Esdl2Base* root = queryType(restype);
-                if (root)
-                {
-                    Esdl2Response *resp = dynamic_cast<Esdl2Response *>(root);
-                    if (resp)
-                        resp->processChildNamedResponse(tctx, restype);
+                    {
+                        if (subresdsname)
+                            resp->processChildNamedResponse(tctx, restype);
+                        else
+                            resp->process(tctx, restype);
+                    }
                 }
             }
             else if (strnicmp(dataset, "VendorGatewayRecords", 20)==0)

esp/scm/ws_fs.ecm

@@ -92,6 +92,7 @@ ESPStruct [nil_remove] GroupNode
 {
     string Name;
     string ClusterType;
+    bool ReplicateOutputs;
 };
 
 ESPStruct DFUException
@@ -366,6 +367,8 @@ ESPrequest [nil_remove] SprayVariable
     [min_ver("1.09")] bool recordStructurePresent(false);
 
     [min_ver("1.10")] bool quotedTerminator(true);
+    [min_ver("1.11")] string sourceRowPath;
+    [min_ver("1.11")] bool isJSON(false);
 };
 
 ESPresponse [exceptions_inline] 
@@ -626,7 +629,7 @@ ESPresponse [exceptions_inline, nil_remove] GetSprayTargetsResponse
 };
 
 ESPservice [
-    version("1.10"), default_client_version("1.10"),
+    version("1.11"), default_client_version("1.10"),
     exceptions_inline("./smc_xslt/exceptions.xslt")] FileSpray
 {
     ESPuses ESPstruct DFUWorkunit;

esp/services/ecldirect/CMakeLists.txt

@@ -61,7 +61,6 @@ target_link_libraries ( EclDirect
          jlib
          xmllib 
          esphttp 
-         LdapSecurity
          mp 
          hrpc 
          remote 
@@ -78,4 +77,7 @@ target_link_libraries ( EclDirect
          fileview2 
          securesocket 
     )
+IF (USE_OPENLDAP)
+  target_link_libraries ( EclDirect LdapSecurity )
+ENDIF(USE_OPENLDAP)
 

esp/services/ws_fs/ws_fsService.cpp

@@ -2072,9 +2072,19 @@ bool CFileSprayEx::onSprayVariable(IEspContext &context, IEspSprayVariable &req,
         source->setMaxRecordSize(req.getSourceMaxRecordSize());
         source->setFormat((DFUfileformat)req.getSourceFormat());
 
-        // if rowTag specified, it means it's xml format, otherwise it's csv
-        const char* rowtag = req.getSourceRowTag();
-        if(rowtag != NULL && *rowtag != '\0')
+        StringBuffer rowtag;
+        if (req.getIsJSON())
+        {
+            const char *srcRowPath = req.getSourceRowPath();
+            if (!srcRowPath || *srcRowPath != '/')
+                rowtag.append("/");
+            rowtag.append(srcRowPath);
+        }
+        else
+            rowtag.append(req.getSourceRowTag());
+
+        // if rowTag specified, it means it's xml or json format, otherwise it's csv
+        if(rowtag.length())
         {
             source->setRowTag(rowtag);
             options->setKeepHeader(true);
@@ -3189,11 +3199,14 @@ bool CFileSprayEx::onDeleteDropZoneFiles(IEspContext &context, IEspDeleteDropZon
     return true;
 }
 
-void CFileSprayEx::appendGroupNode(IArrayOf<IEspGroupNode>& groupNodes, const char* nodeName, const char* clusterType)
+void CFileSprayEx::appendGroupNode(IArrayOf<IEspGroupNode>& groupNodes, const char* nodeName, const char* clusterType,
+    bool replicateOutputs)
 {
     Owned<IEspGroupNode> node = createGroupNode();
     node->setName(nodeName);
     node->setClusterType(clusterType);
+    if (replicateOutputs)
+        node->setReplicateOutputs(replicateOutputs);
     groupNodes.append(*node.getClear());
 }
 
@@ -3225,7 +3238,7 @@ bool CFileSprayEx::onGetSprayTargets(IEspContext &context, IEspGetSprayTargetsRe
 
             bool* found = uniqueThorClusterGroupNames.getValue(thorClusterGroupName.str());
             if (!found || !*found)
-                appendGroupNode(sprayTargets, thorClusterGroupName.str(), "thor");
+                appendGroupNode(sprayTargets, thorClusterGroupName.str(), "thor", cluster.getPropBool("@replicateOutputs", false));
         }
 
         //Fetch all the group names for all the hthor instances
@@ -3254,7 +3267,7 @@ bool CFileSprayEx::onGetSprayTargets(IEspContext &context, IEspGetSprayTargetsRe
                 if (ins>1)
                     gname.append('_').append(ins);
 
-                appendGroupNode(sprayTargets, gname.str(), "hthor");
+                appendGroupNode(sprayTargets, gname.str(), "hthor", false);
             }
         }
 

esp/services/ws_fs/ws_fsService.hpp

@@ -112,7 +112,7 @@ protected:
     bool ParseLogicalPath(const char * pLogicalPath, StringBuffer &title);
     bool ParseLogicalPath(const char * pLogicalPath, const char *group, const char* cluster, StringBuffer &folder, StringBuffer &title, StringBuffer &defaultFolder, StringBuffer &defaultReplicateFolder);
     StringBuffer& getAcceptLanguage(IEspContext& context, StringBuffer& acceptLanguage);
-    void appendGroupNode(IArrayOf<IEspGroupNode>& groupNodes, const char* nodeName, const char* clusterType);
+    void appendGroupNode(IArrayOf<IEspGroupNode>& groupNodes, const char* nodeName, const char* clusterType, bool replicateOutputs);
 };
 
 #endif //_ESPWIZ_FileSpray_HPP__

esp/services/ws_workunits/ws_workunitsService.cpp

@@ -879,18 +879,6 @@ bool CWsWorkunitsEx::onWUResubmit(IEspContext &context, IEspWUResubmitRequest &r
                 if(!cw)
                     throw MakeStringException(ECLWATCH_CANNOT_OPEN_WORKUNIT,"Cannot open workunit %s.",wuid.str());
 
-                //Don't allow resubmit of someone else's workunit
-                if (context.querySecManager())
-                {
-                    IUserDescriptor * owner = cw->queryUserDescriptor();
-                    if (!owner)
-                        throw MakeStringException(ECLWATCH_CANNOT_SUBMIT_WORKUNIT,"Workunit User Descriptor missing on %s", wuid.str());
-                    StringBuffer ownerUserName;
-                    owner->getUserName(ownerUserName);
-                    if (strcmp(context.queryUser()->getName(), ownerUserName.str()))
-                        throw MakeStringException(ECLWATCH_CANNOT_SUBMIT_WORKUNIT,"Cannot resubmit another user's workunit %s.", wuid.str());
-                }
-
                 WsWuHelpers::submitWsWorkunit(context, cw, NULL, NULL, 0, req.getRecompile(), req.getResetWorkflow(), false);
 
                 if (version < 1.40)

esp/src/eclwatch/LZBrowseWidget.js

@@ -89,6 +89,9 @@ define([
             this.sprayXmlForm = registry.byId(this.id + "SprayXmlForm");
             this.sprayXmlDestinationSelect = registry.byId(this.id + "SprayXmlDestinationSelect");
             this.sprayXmlGrid = registry.byId(this.id + "SprayXmlGrid");
+            this.sprayJsonForm = registry.byId(this.id + "SprayJsonForm");
+            this.sprayJsonDestinationSelect = registry.byId(this.id + "SprayJsonDestinationSelect");
+            this.sprayJsonGrid = registry.byId(this.id + "SprayJsonGrid");
             this.sprayVariableForm = registry.byId(this.id + "SprayVariableForm");
             this.sprayVariableDestinationSelect = registry.byId(this.id + "SprayVariableDestination");
             this.sprayVariableGrid = registry.byId(this.id + "SprayVariableGrid");
@@ -387,6 +390,21 @@ define([
             });
         },
 
+        _onSprayJson: function(event) {
+            var context = this;
+            this._spraySelectedOneAtATime("SprayJsonDropDown", "SprayJsonForm", function (request, item) {
+                lang.mixin(request, {
+                    sourceRowPath: item.targetRowPath,
+                    isJSON: true
+                });
+                FileSpray.SprayVariable({
+                    request: request
+                }).then(function (response) {
+                    context._handleResponse("SprayResponse.wuid", response);
+                });
+            });
+        },
+
         _onSprayVariable: function (event) {
             var context = this;
             this._spraySelectedOneAtATime("SprayVariableDropDown", "SprayVariableForm", function (request, item) {
@@ -427,6 +445,9 @@ define([
             this.sprayXmlDestinationSelect.init({
                 SprayTargets: true
             });
+            this.sprayJsonDestinationSelect.init({
+                Groups: true
+            });
             this.sprayVariableDestinationSelect.init({
                 SprayTargets: true
             });
@@ -577,6 +598,24 @@ define([
                 }
             });
 
+            this.sprayJsonGrid.createGrid({
+                idProperty: "calculatedID",
+                columns: {
+                    targetName: editor({
+                        label: this.i18n.TargetName,
+                        width: 144,
+                        autoSave: true,
+                        editor: "text"
+                    }),
+                    targetRowPath: editor({
+                        label: this.i18n.RowPath,
+                        width: 72,
+                        autoSave: true,
+                        editor: "text"
+                    })
+                }
+            });
+
             this.sprayVariableGrid.createGrid({
                 idProperty: "calculatedID",
                 columns: {
@@ -622,6 +661,7 @@ define([
             registry.byId(this.id + "SprayFixedDropDown").set("disabled", !hasSelection);
             registry.byId(this.id + "SprayDelimitedDropDown").set("disabled", !hasSelection);
             registry.byId(this.id + "SprayXmlDropDown").set("disabled", !hasSelection);
+            registry.byId(this.id + "SprayJsonDropDown").set("disabled", !hasSelection);
             registry.byId(this.id + "SprayVariableDropDown").set("disabled", !hasSelection);
             registry.byId(this.id + "SprayBlobDropDown").set("disabled", !hasSelection);
 
@@ -632,13 +672,15 @@ define([
                     lang.mixin(item, lang.mixin({
                         targetName: item.displayName,
                         targetRecordLength: "",
-                        targetRowTag: context.i18n.tag
+                        targetRowTag: context.i18n.tag,
+                        targetRowPath: "/"
                     }, item));
                     data.push(item);
                 });
                 this.sprayFixedGrid.setData(data);
                 this.sprayDelimitedGrid.setData(data);
                 this.sprayXmlGrid.setData(data);
+                this.sprayJsonGrid.setData(data);
                 this.sprayVariableGrid.setData(data);
                 this.sprayBlobGrid.setData(data);
             }

esp/src/eclwatch/SFDetailsWidget.js

@@ -48,6 +48,7 @@ define([
     "hpcc/_TabContainerWidget",
     "hpcc/ESPUtil",
     "hpcc/ESPLogicalFile",
+    "hpcc/DelayLoadWidget",
 
     "dojo/text!../templates/SFDetailsWidget.html",
 
@@ -56,7 +57,7 @@ define([
                 BorderContainer, TabContainer, ContentPane, Toolbar, ToolbarSeparator, TooltipDialog, Form, SimpleTextarea, TextBox, Button, DropDownButton, TitlePane, registry,
                 selector,
                 _TabContainerWidget,
-                ESPUtil, ESPLogicalFile,
+                ESPUtil, ESPLogicalFile, DelayLoadWidget,
                 template) {
     exports.fixCircularDependency = declare("SFDetailsWidget", [_TabContainerWidget], {
         templateString: template,
@@ -97,6 +98,19 @@ define([
         _onRemove: function (event) {
             this.logicalFile.removeSubfiles(this.subfilesGrid.getSelected());
         },
+        _onOpen: function (event) {
+            var selections = this.subfilesGrid.getSelected();
+            var firstTab = null;
+            for (var i = selections.length - 1; i >= 0; --i) {
+                var tab = this.ensureLFPane(selections[i].Name, selections[i]);
+                if (i == 0) {
+                    firstTab = tab;
+                }
+            }
+            if (firstTab) {
+                this.selectChild(firstTab, true);
+            }
+        },
         _onCopyOk: function (event) {
             this.logicalFile.copy({
                 request: domForm.toObject(this.id + "CopyDialog")
@@ -194,7 +208,12 @@ define([
                             return "";
                         }
                     },
-                    Name: { label: this.i18n.LogicalName },
+                    Name: {
+                        label: this.i18n.LogicalName,
+                        formatter: function (name, row) {
+                            return "<a href='#' class='dgrid-row-url'>" + name + "</a>";
+                        }
+                    },
                     Owner: { label: this.i18n.Owner, width: 72 },
                     Description: { label: this.i18n.Description, width: 153 },
                     RecordCount: { label: this.i18n.Records, width: 72, sortable: false },
@@ -204,14 +223,30 @@ define([
                 },
                 store: this.subfilesStore
             }, this.id + "SubfilesGrid");
-            this.subfilesGrid.onSelectionChanged(function (event) {
-                context.refreshActionState();
+            var context = this;
+            this.subfilesGrid.on(".dgrid-row-url:click", function (evt) {
+                var item = context.subfilesGrid.row(evt).data;
+                var tab = context.ensureLFPane(item.Name, item);
+                context.selectChild(tab, true);
+            });
+            this.subfilesGrid.on(".dgrid-row:dblclick", function (evt) {
+                var item = context.subfilesGrid.row(evt).data;
+                var tab = context.ensureLFPane(item.Name, item);
+                context.selectChild(tab, true);
             });
             this.subfilesGrid.startup();
         },
 
         initTab: function () {
             var currSel = this.getSelectedChild();
+            if (currSel && !currSel.initalized) {
+                if (currSel.id === this.summaryWidget.id) {
+                } else {
+                    if (!currSel.initalized) {
+                        currSel.init(currSel._hpccParams);
+                    }
+                }
+            }
         },
 
         showMessage: function (msg) {
@@ -261,9 +296,23 @@ define([
             }
         },
 
-        refreshActionState: function () {
-            var selection = this.subfilesGrid.getSelected();
-            registry.byId(this.id + "Remove").set("disabled", !selection.length);
+        ensureLFPane: function (id, params) {
+            id = this.createChildTabID(id);
+            var retVal = registry.byId(id);
+            if (!retVal) {
+                retVal = new DelayLoadWidget({
+                    id: id,
+                    title: params.Name,
+                    closable: true,
+                    delayWidget: "LFDetailsWidget",
+                    _hpccParams: {
+                        NodeGroup: params.NodeGroup,
+                        Name: params.Name
+                    }
+                });
+                this.addChild(retVal, 1);
+            }
+            return retVal;
         }
     });
 });

esp/src/eclwatch/nls/hpcc.js

@@ -34,6 +34,7 @@ define({root:
     ArchivedWarning: "Warning: please specify a small date range. If not, it may take some time to retrieve the workunits and the browser may be timed out.",
     BinaryInstalls: "Binary Installs",
     AutoRefresh: "Auto Refresh",
+    Back: "Back",
     BannerColor: "Banner Colour",
     BannerMessage: "Banner Message",
     BannerScroll: "Banner Scroll",
@@ -171,6 +172,7 @@ define({root:
     Fixed: "Fixed",
     Folder: "Folder",
     Format: "Format",
+    Forward: "Forward",
     FromDate: "From Date",
     FromSizes: "From Sizes",
     FullName: "Full Name",
@@ -400,6 +402,7 @@ define({root:
     RetainSuperfileStructure: "Retain Superfile Structure",
     RetypePassword: "Retype Password",
     Rows: "Rows",
+    RowPath: "Row Path",
     RowTag: "Row Tag",
     RoxieCluster: "Roxie Cluster",
     Sample: "Sample",

esp/src/eclwatch/templates/GraphWidget.html

@@ -3,8 +3,8 @@
         <div id="${id}ToolbarContentPane" class="${baseClass}ToolbarContentPane" style="padding: 0px; overflow: hidden" data-dojo-props="region: 'top'" data-dojo-type="dijit.layout.ContentPane">
             <div id="${id}Toolbar" class="topPanel dijit dijitToolbar" role="toolbar">
                 <div data-dojo-attach-event="onClick:_onClickRefresh" data-dojo-props="iconClass:'iconRefresh', showLabel:false" data-dojo-type="dijit.form.Button">${i18n.Refresh}</div>
-                <div id="${id}Previous" data-dojo-attach-event="onClick:_onClickPrevious" data-dojo-props="iconClass:'iconLeft', showLabel:false" data-dojo-type="dijit.form.Button">${i18n.Refresh}</div>
-                <div id="${id}Next" data-dojo-attach-event="onClick:_onClickNext" data-dojo-props="iconClass:'iconRight', showLabel:false" data-dojo-type="dijit.form.Button">${i18n.Refresh}</div>
+                <div id="${id}Previous" data-dojo-attach-event="onClick:_onClickPrevious" data-dojo-props="iconClass:'iconLeft', showLabel:false" data-dojo-type="dijit.form.Button">${i18n.Back}</div>
+                <div id="${id}Next" data-dojo-attach-event="onClick:_onClickNext" data-dojo-props="iconClass:'iconRight', showLabel:false" data-dojo-type="dijit.form.Button">${i18n.Forward}</div>
                 <span data-dojo-type="dijit.ToolbarSeparator"></span>
                 <div data-dojo-attach-event="onClick:_onClickZoomOrig" data-dojo-props="iconClass:'iconZoomOrig', showLabel:false" data-dojo-type="dijit.form.Button">${i18n.Zoom100Pct}</div>
                 <div data-dojo-attach-event="onClick:_onClickZoomAll" data-dojo-props="iconClass:'iconZoomAll', showLabel:false" data-dojo-type="dijit.form.Button">${i18n.ZoomAll}</div>

esp/src/eclwatch/templates/LZBrowseWidget.html

@@ -145,6 +145,46 @@
                             </div>
                         </div>
                     </div>
+                    <div id="${id}SprayJsonDropDown" data-dojo-type="dijit.form.DropDownButton">
+                        <span>${i18n.JSON}</span>
+                        <div data-dojo-type="dijit.TooltipDialog">
+                            <div id="${id}SprayJsonForm" style="width: 530px;" data-dojo-type="dijit.form.Form">
+                                <div data-dojo-type="dijit.Fieldset">
+                                    <legend>${i18n.Target}</legend>
+                                    <div data-dojo-type="hpcc.TableContainer">
+                                        <input id="${id}SprayJsonDestinationSelect" title="${i18n.Group}:" style="width: 95%;" name="destGroup" data-dojo-type="TargetSelectWidget" />
+                                        <input title="${i18n.NamePrefix}:" style="width: 95%;" name="namePrefix" data-dojo-props="trim: true, placeHolder:'${i18n.NamePrefixPlaceholder}'" data-dojo-type="dijit.form.TextBox" />
+                                    </div>
+                                    <div id="${id}SprayJsonGrid" data-dojo-type="SelectionGridWidget">
+                                    </div>
+                                </div>
+                                <div data-dojo-type="dijit.Fieldset">
+                                    <legend>${i18n.Options}</legend>
+                                    <div data-dojo-props="cols:2" data-dojo-type="hpcc.TableContainer">
+                                        <select id="${id}jsonsourceFormat" title="${i18n.Format}:" name="sourceFormat" colspan="2" data-dojo-type="dijit.form.Select">
+                                            <option value="2">UTF-8</option>
+                                            <option value="3">UTF-8N</option>
+                                            <option value="4">UTF-16</option>
+                                            <option value="5">UTF-16LE</option>
+                                            <option value="6">UTF-16BE</option>
+                                            <option value="7">UTF-32</option>
+                                            <option value="8">UTF-32LE</option>
+                                            <option value="9">UTF-32BE</option>
+                                        </select>
+                                        <input id="${id}SprayJsonMaxRecordLength" title="${i18n.MaxRecordLength}:" value="8192" style="width: 95%;" name="sourceMaxRecordSize" required="true" colspan="2" data-dojo-props="trim: true, placeHolder:'8192'" data-dojo-type="dijit.form.ValidationTextBox" />
+                                        <input title="${i18n.Overwrite}:" name="overwrite" data-dojo-type="dijit.form.CheckBox" />
+                                        <input title="${i18n.Replicate}:" name="replicate" data-dojo-type="dijit.form.CheckBox" />
+                                        <input title="${i18n.NoSplit}:" name="nosplit" data-dojo-type="dijit.form.CheckBox" />
+                                        <input title="${i18n.Compress}:" name="compress" data-dojo-type="dijit.form.CheckBox" />
+                                        <input title="${i18n.FailIfNoSourceFile}:" name="failIfNoSourceFile" data-dojo-type="dijit.form.CheckBox" />
+                                    </div>
+                                </div>
+                                <div class="dijitDialogPaneActionBar">
+                                    <button data-dojo-attach-event="onClick:_onSprayJson" data-dojo-type="dijit.form.Button">${i18n.Spray}</button>
+                                </div>
+                            </div>
+                        </div>
+                    </div>
                     <div id="${id}SprayVariableDropDown" data-dojo-type="dijit.form.DropDownButton">
                         <span>${i18n.Variable}</span>
                         <div data-dojo-type="dijit.TooltipDialog">

esp/src/eclwatch/templates/SFDetailsWidget.html

@@ -29,7 +29,9 @@
                 </div>
                 <div style="width: 100%; height: 66%" data-dojo-props="region: 'bottom', splitter: true, minSize: 120" data-dojo-type="dijit.layout.BorderContainer">
                     <div class="topPanel" data-dojo-props="region: 'top'" data-dojo-type="dijit.Toolbar">
-                        <div id="${id}Remove" data-dojo-attach-event="onClick:_onRemove" data-dojo-type="dijit.form.Button">${i18n.RemoveSubfiles}</div>
+                        <div id="${id}Open" data-dojo-attach-event="onClick:_onOpen" data-dojo-type="dijit.form.Button">${i18n.Open}</div>
+                        <div id="${id}Remove" data-dojo-attach-event="onClick:_onRemove" data-dojo-type="dijit.form.Button">${i18n.Remove}</div>
+                        <span data-dojo-type="dijit.ToolbarSeparator"></span>
                     </div>
                     <div id="${id}SubfilesGridCP" style="padding: 0px; border:0px; border-color:none" data-dojo-props="region: 'center'" data-dojo-type="dijit.layout.ContentPane">
                         <div id="${id}SubfilesGrid">

initfiles/bash/etc/init.d/hpcc_common.in

@@ -442,7 +442,7 @@ startCmd() {
       MIN_Hr_rtprio="4"
       MIN_Hl_memlock="unlimited" )
 
-    i=0
+    local i=0
     for element in "${limits[@]}"; do
       flag="-${element:4:2}"
       value=${element##*"="}
@@ -490,17 +490,31 @@ startCmd() {
     fi
     eval $startcmd
 
+
     local WAITTIME=120
     local RCSTART=0
+    local COMPONENT_HAS_STARTED=0
+    local SENTINEL_CHECK=1
+
+    if [ ${compType} = "dafilesrv" ]; then
+      SENTINEL_CHECK=0
+    fi
 
     while [ ${WAITTIME} -gt 0 ]
     do
         WAITTIME=`expr ${WAITTIME} - 1`
-        check_status ${PIDPATH} ${LOCKPATH} ${COMPPIDPATH} 1
+        check_status ${PIDPATH} ${LOCKPATH} ${COMPPIDPATH} ${SENTINEL_CHECK}
         RCSTART=$?
         if [ ${RCSTART} -eq 0 ]; then
           log_success_msg
           return 0;
+        elif [ ${RCSTART} -eq 2 ]; then
+          COMPONENT_HAS_STARTED=1
+        else
+          if [ ${COMPONENT_HAS_STARTED} -eq 1 ]; then
+            log_failure_msg "${compName} failed to start cleanly"
+            return 0;
+          fi
         fi
         sleep 1
     done

initfiles/bash/etc/init.d/pid.sh

@@ -138,7 +138,7 @@ checkPidExist() {
 #       1: Stopped Healthy
 #       2: Running w/ no sentinel file
 #       3: Stopped except sentinel orphaned
-#       4: Unhealthy, details in debug mode
+#       4: not yet in up state, details in debug mode
 check_status() {
     PIDFILEPATH=$1
     LOCKFILEPATH=$2
@@ -155,6 +155,8 @@ check_status() {
     local initRunning=$__pidExists
     checkPidExist $COMPPIDFILEPATH
     local compRunning=$__pidExists
+    checkSentinelFile
+    local sentinelFlag=$?
 
     # check if running and healthy
     if [ $pidfilepathExists -eq 1 ] && [ $comppidfilepathExists -eq 1 ] && [ $componentLocked -eq 1 ] && [ $initRunning -eq 1 ] && [ $compRunning -eq 1 ]; then
@@ -162,15 +164,14 @@ check_status() {
         echo "everything is up except sentinel"
       fi
       if [ ${SENTINELFILECHK} -eq 1 ]; then
-        checkSentinelFile
-        if [ $? -eq 0 ]; then
+        if [ ${sentinelFlag} -eq 0 ]; then
           if [ ${DEBUG} != "NO_DEBUG" ]; then
             echo "Sentinel is now up"
           fi
           return 0
         else
           if [ ${DEBUG} != "NO_DEBUG" ]; then
-            echo "Sentinel is currently down"
+            echo "Sentinel not yet located, process currently unhealthy"
           fi
           return 2
         fi
@@ -180,8 +181,7 @@ check_status() {
     # check if shutdown and healthy
     elif [ $pidfilepathExists -eq 0 ] && [ $comppidfilepathExists -eq 0 ] && [ $componentLocked -eq 0 ] && [ $initRunning -eq 0 ] && [ $compRunning -eq 0 ]; then
       if [ ${SENTINELFILECHK} -eq 1 ]; then
-        checkSentinelFile
-        if [ $? -eq 0 ]; then
+        if [ ${sentinelFlag} -eq 0 ]; then
           if [ ${DEBUG} != "NO_DEBUG" ]; then
             echo "Sentinel is up but orphaned"
           fi

plugins/CMakeLists.txt

@@ -32,3 +32,4 @@ add_subdirectory (javaembed)
 add_subdirectory (Rembed)
 add_subdirectory (cassandra)
 add_subdirectory (memcached)
+add_subdirectory (redis)

plugins/Rembed/CMakeLists.txt

@@ -50,7 +50,7 @@ if (USE_RINSIDE)
     HPCC_ADD_LIBRARY( Rembed SHARED ${SRCS} )
     if (${CMAKE_VERSION} VERSION_LESS "2.8.9")
       message("WARNING: Cannot set NO_SONAME. shlibdeps will give warnings when package is installed")
-    else()
+    elif(NOT APPLE)
       set_target_properties( Rembed PROPERTIES NO_SONAME 1 )
     endif()
 

plugins/cassandra/CMakeLists.txt

@@ -94,7 +94,7 @@ if (USE_CASSANDRA)
     HPCC_ADD_LIBRARY( cassandraembed SHARED ${SRCS} )
     if (${CMAKE_VERSION} VERSION_LESS "2.8.9")
       message("WARNING: Cannot set NO_SONAME. shlibdeps will give warnings when package is installed")
-    else()
+    elif(NOT APPLE)
       set_target_properties( cassandraembed PROPERTIES NO_SONAME 1 )
     endif()
 

plugins/javaembed/CMakeLists.txt

@@ -50,7 +50,7 @@ if (USE_JNI)
     HPCC_ADD_LIBRARY( javaembed SHARED ${SRCS} )
     if (${CMAKE_VERSION} VERSION_LESS "2.8.9")
       message("WARNING: Cannot set NO_SONAME. shlibdeps will give warnings when package is installed")
-    else()
+    elif(NOT APPLE)
       set_target_properties( javaembed PROPERTIES NO_SONAME 1 )
     endif()
 

plugins/memcached/CMakeLists.txt

@@ -50,7 +50,7 @@ if (USE_MEMCACHED)
     HPCC_ADD_LIBRARY( memcached SHARED ${SRCS} )
     if (${CMAKE_VERSION} VERSION_LESS "2.8.9")
       message("WARNING: Cannot set NO_SONAME. shlibdeps will give warnings when package is installed")
-    else()
+    elif(NOT APPLE)
       set_target_properties( memcached PROPERTIES NO_SONAME 1 )
     endif()
 

plugins/memcached/lib_memcached.ecllib

@@ -15,26 +15,26 @@
     limitations under the License.
 ############################################################################## */
 
-export memcached := SERVICE : plugin('memcached')
-  SetUnicode(CONST VARSTRING options, CONST VARSTRING key, CONST UNICODE value, CONST VARSTRING partitionKey = '', UNSIGNED4 expire = 0) : cpp,action,context,entrypoint='MSet';
-  SetString(CONST VARSTRING options, CONST VARSTRING key, CONST STRING value, CONST VARSTRING partitionKey = '', UNSIGNED4 expire = 0) : cpp,action,context,entrypoint='MSet';
-  SetUtf8(CONST VARSTRING options, CONST VARSTRING key, CONST UTF8 value, CONST VARSTRING partitionKey = '', UNSIGNED4 expire = 0) : cpp,action,context,entrypoint='MSetUtf8';
-  SetBoolean(CONST VARSTRING options, CONST VARSTRING key, BOOLEAN value, CONST VARSTRING partitionKey = '', UNSIGNED4 expire = 0) : cpp,action,context,entrypoint='MSet';
-  SetReal(CONST VARSTRING options, CONST VARSTRING key, REAL value, CONST VARSTRING partitionKey = '', UNSIGNED4 expire = 0) : cpp,action,context,entrypoint='MSet';
-  SetInteger(CONST VARSTRING options, CONST VARSTRING key, INTEGER value, CONST VARSTRING partitionKey = '', UNSIGNED4 expire = 0) : cpp,action,context,entrypoint='MSet';
-  SetUnsigned(CONST VARSTRING options, CONST VARSTRING key, UNSIGNED value,CONST VARSTRING partitionKey = '',  UNSIGNED4 expire = 0) : cpp,action,context,entrypoint='MSet';
-  SetData(CONST VARSTRING options, CONST VARSTRING key, CONST DATA value, CONST VARSTRING partitionKey = '', UNSIGNED4 expire = 0) : cpp,action,context,entrypoint='MSetData';
+export memcached := SERVICE : plugin('memcached'), namespace('MemCachedPlugin')
+  SetUnicode(CONST VARSTRING key, CONST UNICODE value, CONST VARSTRING options, CONST VARSTRING partitionKey = '', UNSIGNED4 expire = 0) : cpp,action,context,entrypoint='MSet';
+  SetString(CONST VARSTRING key, CONST STRING value, CONST VARSTRING options, CONST VARSTRING partitionKey = '', UNSIGNED4 expire = 0) : cpp,action,context,entrypoint='MSet';
+  SetUtf8(CONST VARSTRING key, CONST UTF8 value, CONST VARSTRING options, CONST VARSTRING partitionKey = '', UNSIGNED4 expire = 0) : cpp,action,context,entrypoint='MSetUtf8';
+  SetBoolean(CONST VARSTRING key, BOOLEAN value, CONST VARSTRING options, CONST VARSTRING partitionKey = '', UNSIGNED4 expire = 0) : cpp,action,context,entrypoint='MSet';
+  SetReal(CONST VARSTRING key, REAL value, CONST VARSTRING options, CONST VARSTRING partitionKey = '', UNSIGNED4 expire = 0) : cpp,action,context,entrypoint='MSet';
+  SetInteger(CONST VARSTRING key, INTEGER value, CONST VARSTRING options, CONST VARSTRING partitionKey = '', UNSIGNED4 expire = 0) : cpp,action,context,entrypoint='MSet';
+  SetUnsigned(CONST VARSTRING key, UNSIGNED value, CONST VARSTRING options, CONST VARSTRING partitionKey = '',  UNSIGNED4 expire = 0) : cpp,action,context,entrypoint='MSet';
+  SetData(CONST VARSTRING key, CONST DATA value, CONST VARSTRING options, CONST VARSTRING partitionKey = '', UNSIGNED4 expire = 0) : cpp,action,context,entrypoint='MSetData';
 
-  INTEGER8 GetInteger(CONST VARSTRING options, CONST VARSTRING key, CONST VARSTRING partitionKey = '') : cpp,action,context,entrypoint='MGetInt8';
-  UNSIGNED8 GetUnsigned(CONST VARSTRING options, CONST VARSTRING key, CONST VARSTRING partitionKey = '') : cpp,action,context,entrypoint='MGetUint8';
-  STRING GetString(CONST VARSTRING options, CONST VARSTRING key, CONST VARSTRING partitionKey = '') : cpp,action,context,entrypoint='MGetStr';
-  UNICODE GetUnicode(CONST VARSTRING options, CONST VARSTRING key, CONST VARSTRING partitionKey = '') : cpp,action,context,entrypoint='MGetUChar';
-  UTF8 GetUtf8(CONST VARSTRING options, CONST VARSTRING key, CONST VARSTRING partitionKey = '') : cpp,action,context,entrypoint='MGetUtf8';
-  BOOLEAN GetBoolean(CONST VARSTRING options, CONST VARSTRING key, CONST VARSTRING partitionKey = '') : cpp,action,context,entrypoint='MGetBool';
-  REAL GetReal(CONST VARSTRING options, CONST VARSTRING key, CONST VARSTRING partitionKey = '') : cpp,action,context,entrypoint='MGetDouble';
-  DATA GetData(CONST VARSTRING options, CONST VARSTRING key, CONST VARSTRING partitionKey = '') : cpp,action,context,entrypoint='MGetData';
+  INTEGER8 GetInteger(CONST VARSTRING key, CONST VARSTRING options, CONST VARSTRING partitionKey = '') : cpp,action,context,entrypoint='MGetInt8';
+  UNSIGNED8 GetUnsigned(CONST VARSTRING key, CONST VARSTRING options, CONST VARSTRING partitionKey = '') : cpp,action,context,entrypoint='MGetUint8';
+  STRING GetString(CONST VARSTRING key, CONST VARSTRING options, CONST VARSTRING partitionKey = '') : cpp,action,context,entrypoint='MGetStr';
+  UNICODE GetUnicode(CONST VARSTRING key, CONST VARSTRING options, CONST VARSTRING partitionKey = '') : cpp,action,context,entrypoint='MGetUChar';
+  UTF8 GetUtf8(CONST VARSTRING key, CONST VARSTRING options, CONST VARSTRING partitionKey = '') : cpp,action,context,entrypoint='MGetUtf8';
+  BOOLEAN GetBoolean(CONST VARSTRING key, CONST VARSTRING options, CONST VARSTRING partitionKey = '') : cpp,action,context,entrypoint='MGetBool';
+  REAL GetReal(CONST VARSTRING key, CONST VARSTRING options, CONST VARSTRING partitionKey = '') : cpp,action,context,entrypoint='MGetDouble';
+  DATA GetData(CONST VARSTRING key, CONST VARSTRING options, CONST VARSTRING partitionKey = '') : cpp,action,context,entrypoint='MGetData';
 
-  BOOLEAN Exists(CONST VARSTRING options, CONST VARSTRING key, CONST VARSTRING partitionKey = '') : cpp,action,context,entrypoint='MExists';
-  CONST VARSTRING KeyType(CONST VARSTRING options, CONST VARSTRING key, CONST VARSTRING partitionKey = '') : cpp,action,context,entrypoint='MKeyType'; //NOTE: calls get
+  BOOLEAN Exists(CONST VARSTRING key, CONST VARSTRING options, CONST VARSTRING partitionKey = '') : cpp,action,context,entrypoint='MExists';
+  CONST VARSTRING KeyType(CONST VARSTRING key, CONST VARSTRING options, CONST VARSTRING partitionKey = '') : cpp,action,context,entrypoint='MKeyType'; //NOTE: calls get
   Clear(CONST VARSTRING options) : cpp,action,context,entrypoint='MClear';
 END;

plugins/memcached/memcachedplugin.cpp

@@ -20,6 +20,7 @@
 #include "eclrtl.hpp"
 #include "jexcept.hpp"
 #include "jstring.hpp"
+#include "jthread.hpp"
 #include <libmemcached/memcached.hpp>
 #include <libmemcached/util.h>
 
@@ -40,7 +41,6 @@ ECL_MEMCACHED_API bool getECLPluginDefinition(ECLPluginDefinitionBlock *pb)
 }
 
 namespace MemCachedPlugin {
-IPluginContext * parentCtx = NULL;
 static const unsigned unitExpire = 86400;//1 day (secs)
 
 enum eclDataType {
@@ -125,26 +125,50 @@ private :
     unsigned typeMismatchCount;
 };
 
-#define OwnedMCached Owned<MemCachedPlugin::MCached>
+typedef Owned<MCached> OwnedMCached;
 
-#define MAX_TYPEMISMATCHCOUNT 10
+static const unsigned MAX_TYPEMISMATCHCOUNT = 10;
 
-static CriticalSection crit;
-static OwnedMCached cachedConnection;
+static __thread MCached * cachedConnection;
+static __thread ThreadTermFunc threadHookChain;
 
+//The following class is here to ensure destruction of the cachedConnection within the main thread
+//as this is not handled by the thread hook mechanism.
+static class mainThreadCachedConnection
+{
+public :
+    mainThreadCachedConnection() { }
+    ~mainThreadCachedConnection()
+    {
+        if (cachedConnection)
+            cachedConnection->Release();
+    }
+} mainThread;
+
+static void releaseContext()
+{
+    if (cachedConnection)
+        cachedConnection->Release();
+    if (threadHookChain)
+    {
+        (*threadHookChain)();
+        threadHookChain = NULL;
+    }
+}
 MCached * createConnection(ICodeContext * ctx, const char * options)
 {
-    CriticalBlock block(crit);
     if (!cachedConnection)
     {
-        cachedConnection.setown(new MemCachedPlugin::MCached(ctx, options));
+        cachedConnection = new MemCachedPlugin::MCached(ctx, options);
+        threadHookChain = addThreadTermFunc(releaseContext);
         return LINK(cachedConnection);
     }
 
     if (cachedConnection->isSameConnection(options))
         return LINK(cachedConnection);
 
-    cachedConnection.setown(new MemCachedPlugin::MCached(ctx, options));
+    cachedConnection->Release();
+    cachedConnection = new MemCachedPlugin::MCached(ctx, options);
     return LINK(cachedConnection);
 }
 
@@ -176,7 +200,6 @@ void MGetVoidPtrLenPair(ICodeContext * ctx, const char * options, const char * p
     OwnedMCached serverPool = createConnection(ctx, options);
     serverPool->getVoidPtrLenPair(ctx, partitionKey, key, returnLength, returnValue, eclType);
 }
-}//close namespace
 
 //----------------------------------SET----------------------------------------
 template<class type> void MemCachedPlugin::MCached::set(ICodeContext * ctx, const char * partitionKey, const char * key, type value, unsigned expire, eclDataType eclType)
@@ -263,8 +286,6 @@ void MemCachedPlugin::MCached::getVoidPtrLenPair(ICodeContext * ctx, const char
     returnValue = reinterpret_cast<void*>(cpy(value, returnLength));
 }
 
-ECL_MEMCACHED_API void setPluginContext(IPluginContext * ctx) { MemCachedPlugin::parentCtx = ctx; }
-
 MemCachedPlugin::MCached::MCached(ICodeContext * ctx, const char * _options)
 {
     alreadyInitialized = false;
@@ -594,12 +615,12 @@ ECL_MEMCACHED_API void ECL_MEMCACHED_CALL MClear(ICodeContext * ctx, const char
     OwnedMCached serverPool = MemCachedPlugin::createConnection(ctx, options);
     serverPool->clear(ctx, 0);
 }
-ECL_MEMCACHED_API bool ECL_MEMCACHED_CALL MExists(ICodeContext * ctx, const char * options, const char * key, const char * partitionKey)
+ECL_MEMCACHED_API bool ECL_MEMCACHED_CALL MExists(ICodeContext * ctx, const char * key, const char * options, const char * partitionKey)
 {
     OwnedMCached serverPool = MemCachedPlugin::createConnection(ctx, options);
     return serverPool->exists(ctx, key, partitionKey);
 }
-ECL_MEMCACHED_API const char * ECL_MEMCACHED_CALL MKeyType(ICodeContext * ctx, const char * options, const char * key, const char * partitionKey)
+ECL_MEMCACHED_API const char * ECL_MEMCACHED_CALL MKeyType(ICodeContext * ctx, const char * key, const char * options, const char * partitionKey)
 {
     OwnedMCached serverPool = MemCachedPlugin::createConnection(ctx, options);
     const char * keyType = enumToStr(serverPool->getKeyType(key, partitionKey));
@@ -607,84 +628,85 @@ ECL_MEMCACHED_API const char * ECL_MEMCACHED_CALL MKeyType(ICodeContext * ctx, c
 }
 //-----------------------------------SET------------------------------------------
 //NOTE: These were all overloaded by 'value' type, however; this caused problems since ecl implicitly casts and doesn't type check.
-ECL_MEMCACHED_API void ECL_MEMCACHED_CALL MSet(ICodeContext * ctx, const char * options, const char * key, size32_t valueLength, const char * value, const char * partitionKey, unsigned expire /* = 0 (ECL default)*/)
+ECL_MEMCACHED_API void ECL_MEMCACHED_CALL MSet(ICodeContext * ctx, const char * key, size32_t valueLength, const char * value, const char * options, const char * partitionKey, unsigned expire /* = 0 (ECL default)*/)
 {
     MemCachedPlugin::MSet(ctx, options, partitionKey, key, valueLength, value, expire, MemCachedPlugin::ECL_STRING);
 }
-ECL_MEMCACHED_API void ECL_MEMCACHED_CALL MSet(ICodeContext * ctx, const char * options, const char * key, size32_t valueLength, const UChar * value, const char * partitionKey, unsigned expire /* = 0 (ECL default)*/)
+ECL_MEMCACHED_API void ECL_MEMCACHED_CALL MSet(ICodeContext * ctx, const char * key, size32_t valueLength, const UChar * value, const char * options, const char * partitionKey, unsigned expire /* = 0 (ECL default)*/)
 {
     MemCachedPlugin::MSet(ctx, options, partitionKey, key, (valueLength)*sizeof(UChar), value, expire, MemCachedPlugin::ECL_UNICODE);
 }
-ECL_MEMCACHED_API void ECL_MEMCACHED_CALL MSet(ICodeContext * ctx, const char * options, const char * key, signed __int64 value, const char * partitionKey, unsigned expire /* = 0 (ECL default)*/)
+ECL_MEMCACHED_API void ECL_MEMCACHED_CALL MSet(ICodeContext * ctx, const char * key, signed __int64 value, const char * options, const char * partitionKey, unsigned expire /* = 0 (ECL default)*/)
 {
     MemCachedPlugin::MSet(ctx, options, partitionKey, key, value, expire, MemCachedPlugin::ECL_INTEGER);
 }
-ECL_MEMCACHED_API void ECL_MEMCACHED_CALL MSet(ICodeContext * ctx, const char * options, const char * key, unsigned __int64 value, const char * partitionKey, unsigned expire /* = 0 (ECL default)*/)
+ECL_MEMCACHED_API void ECL_MEMCACHED_CALL MSet(ICodeContext * ctx, const char * key, unsigned __int64 value, const char * options, const char * partitionKey, unsigned expire /* = 0 (ECL default)*/)
 {
     MemCachedPlugin::MSet(ctx, options, partitionKey, key, value, expire, MemCachedPlugin::ECL_UNSIGNED);
 }
-ECL_MEMCACHED_API void ECL_MEMCACHED_CALL MSet(ICodeContext * ctx, const char * options, const char * key, double value, const char * partitionKey, unsigned expire /* = 0 (ECL default)*/)
+ECL_MEMCACHED_API void ECL_MEMCACHED_CALL MSet(ICodeContext * ctx, const char * key, double value, const char * options, const char * partitionKey, unsigned expire /* = 0 (ECL default)*/)
 {
     MemCachedPlugin::MSet(ctx, options, partitionKey, key, value, expire, MemCachedPlugin::ECL_REAL);
 }
-ECL_MEMCACHED_API void ECL_MEMCACHED_CALL MSet(ICodeContext * ctx, const char * options, const char * key, bool value, const char * partitionKey, unsigned expire)
+ECL_MEMCACHED_API void ECL_MEMCACHED_CALL MSet(ICodeContext * ctx, const char * key, bool value, const char * options, const char * partitionKey, unsigned expire)
 {
     MemCachedPlugin::MSet(ctx, options, partitionKey, key, value, expire, MemCachedPlugin::ECL_BOOLEAN);
 }
-ECL_MEMCACHED_API void ECL_MEMCACHED_CALL MSetData(ICodeContext * ctx, const char * options, const char * key, size32_t valueLength, const void * value, const char * partitionKey, unsigned expire)
+ECL_MEMCACHED_API void ECL_MEMCACHED_CALL MSetData(ICodeContext * ctx, const char * key, size32_t valueLength, const void * value, const char * options, const char * partitionKey, unsigned expire)
 {
     MemCachedPlugin::MSet(ctx, options, partitionKey, key, valueLength, value, expire, MemCachedPlugin::ECL_DATA);
 }
-ECL_MEMCACHED_API void ECL_MEMCACHED_CALL MSetUtf8(ICodeContext * ctx, const char * options, const char * key, size32_t valueLength, const char * value, const char * partitionKey, unsigned expire /* = 0 (ECL default)*/)
+ECL_MEMCACHED_API void ECL_MEMCACHED_CALL MSetUtf8(ICodeContext * ctx, const char * key, size32_t valueLength, const char * value, const char * options, const char * partitionKey, unsigned expire /* = 0 (ECL default)*/)
 {
     MemCachedPlugin::MSet(ctx, options, partitionKey, key, rtlUtf8Size(valueLength, value), value, expire, MemCachedPlugin::ECL_UTF8);
 }
 //-------------------------------------GET----------------------------------------
-ECL_MEMCACHED_API bool ECL_MEMCACHED_CALL MGetBool(ICodeContext * ctx, const char * options, const char * key, const char * partitionKey)
+ECL_MEMCACHED_API bool ECL_MEMCACHED_CALL MGetBool(ICodeContext * ctx, const char * key, const char * options, const char * partitionKey)
 {
     bool value;
     MemCachedPlugin::MGet(ctx, options, partitionKey, key, value, MemCachedPlugin::ECL_BOOLEAN);
     return value;
 }
-ECL_MEMCACHED_API double ECL_MEMCACHED_CALL MGetDouble(ICodeContext * ctx, const char * options, const char * key, const char * partitionKey)
+ECL_MEMCACHED_API double ECL_MEMCACHED_CALL MGetDouble(ICodeContext * ctx, const char * key, const char * options, const char * partitionKey)
 {
     double value;
     MemCachedPlugin::MGet(ctx, options, partitionKey, key, value, MemCachedPlugin::ECL_REAL);
     return value;
 }
-ECL_MEMCACHED_API signed __int64 ECL_MEMCACHED_CALL MGetInt8(ICodeContext * ctx, const char * options, const char * key, const char * partitionKey)
+ECL_MEMCACHED_API signed __int64 ECL_MEMCACHED_CALL MGetInt8(ICodeContext * ctx, const char * key, const char * options, const char * partitionKey)
 {
     signed __int64 value;
     MemCachedPlugin::MGet(ctx, options, partitionKey, key, value, MemCachedPlugin::ECL_INTEGER);
     return value;
 }
-ECL_MEMCACHED_API unsigned __int64 ECL_MEMCACHED_CALL MGetUint8(ICodeContext * ctx, const char * options, const char * key, const char * partitionKey)
+ECL_MEMCACHED_API unsigned __int64 ECL_MEMCACHED_CALL MGetUint8(ICodeContext * ctx, const char * key, const char * options, const char * partitionKey)
 {
     unsigned __int64 value;
     MemCachedPlugin::MGet(ctx, options, partitionKey, key, value, MemCachedPlugin::ECL_UNSIGNED);
     return value;
 }
-ECL_MEMCACHED_API void ECL_MEMCACHED_CALL MGetStr(ICodeContext * ctx, size32_t & returnLength, char * & returnValue, const char * options, const char * key, const char * partitionKey)
+ECL_MEMCACHED_API void ECL_MEMCACHED_CALL MGetStr(ICodeContext * ctx, size32_t & returnLength, char * & returnValue, const char * key, const char * options, const char * partitionKey)
 {
     size_t _returnLength;
     MemCachedPlugin::MGet(ctx, options, partitionKey, key, _returnLength, returnValue, MemCachedPlugin::ECL_STRING);
     returnLength = static_cast<size32_t>(_returnLength);
 }
-ECL_MEMCACHED_API void ECL_MEMCACHED_CALL MGetUChar(ICodeContext * ctx, size32_t & returnLength, UChar * & returnValue,  const char * options, const char * key, const char * partitionKey)
+ECL_MEMCACHED_API void ECL_MEMCACHED_CALL MGetUChar(ICodeContext * ctx, size32_t & returnLength, UChar * & returnValue,  const char * key, const char * options, const char * partitionKey)
 {
     size_t _returnSize;
     MemCachedPlugin::MGet(ctx, options, partitionKey, key, _returnSize, returnValue, MemCachedPlugin::ECL_UNICODE);
     returnLength = static_cast<size32_t>(_returnSize/sizeof(UChar));
 }
-ECL_MEMCACHED_API void ECL_MEMCACHED_CALL MGetUtf8(ICodeContext * ctx, size32_t & returnLength, char * & returnValue, const char * options, const char * key, const char * partitionKey)
+ECL_MEMCACHED_API void ECL_MEMCACHED_CALL MGetUtf8(ICodeContext * ctx, size32_t & returnLength, char * & returnValue, const char * key, const char * options, const char * partitionKey)
 {
     size_t returnSize;
     MemCachedPlugin::MGet(ctx, options, partitionKey, key, returnSize, returnValue, MemCachedPlugin::ECL_UTF8);
     returnLength = static_cast<size32_t>(rtlUtf8Length(returnSize, returnValue));
 }
-ECL_MEMCACHED_API void ECL_MEMCACHED_CALL MGetData(ICodeContext * ctx, size32_t & returnLength, void * & returnValue, const char * options, const char * key, const char * partitionKey)
+ECL_MEMCACHED_API void ECL_MEMCACHED_CALL MGetData(ICodeContext * ctx, size32_t & returnLength, void * & returnValue, const char * key, const char * options, const char * partitionKey)
 {
     size_t _returnLength;
     MemCachedPlugin::MGetVoidPtrLenPair(ctx, options, partitionKey, key, _returnLength, returnValue, MemCachedPlugin::ECL_DATA);
     returnLength = static_cast<size32_t>(_returnLength);
 }
+}//close namespace

plugins/memcached/memcachedplugin.hpp

@@ -42,27 +42,29 @@ extern "C"
 //NB: LIBMEMCACHED_API already used by libmemcached
 extern "C++"
 {
+namespace MemCachedPlugin {
     //--------------------------SET----------------------------------------
-    ECL_MEMCACHED_API void ECL_MEMCACHED_CALL MSet    (ICodeContext * _ctx, const char * options, const char * key, bool value, const char * partitionKey, unsigned expire);
-    ECL_MEMCACHED_API void ECL_MEMCACHED_CALL MSet    (ICodeContext * _ctx, const char * options, const char * key, signed __int64 value, const char * partitionKey, unsigned expire);
-    ECL_MEMCACHED_API void ECL_MEMCACHED_CALL MSet    (ICodeContext * _ctx, const char * options, const char * key, unsigned __int64 value, const char * partitionKey, unsigned expire);
-    ECL_MEMCACHED_API void ECL_MEMCACHED_CALL MSet    (ICodeContext * _ctx, const char * options, const char * key, double value, const char * partitionKey, unsigned expire);
-    ECL_MEMCACHED_API void ECL_MEMCACHED_CALL MSetUtf8(ICodeContext * _ctx, const char * options, const char * key, size32_t valueLength, const char * value, const char * partitionKey, unsigned expire);
-    ECL_MEMCACHED_API void ECL_MEMCACHED_CALL MSet    (ICodeContext * _ctx, const char * options, const char * key, size32_t valueLength, const char * value, const char * partitionKey, unsigned expire);
-    ECL_MEMCACHED_API void ECL_MEMCACHED_CALL MSet    (ICodeContext * _ctx, const char * options, const char * key, size32_t valueLength, const UChar * value, const char * partitionKey, unsigned expire);
-    ECL_MEMCACHED_API void ECL_MEMCACHED_CALL MSetData(ICodeContext * _ctx, const char * options, const char * key, size32_t valueLength, const void * value, const char * partitionKey, unsigned expire);
+    ECL_MEMCACHED_API void ECL_MEMCACHED_CALL MSet    (ICodeContext * _ctx, const char * key, bool value, const char * options, const char * partitionKey, unsigned expire);
+    ECL_MEMCACHED_API void ECL_MEMCACHED_CALL MSet    (ICodeContext * _ctx, const char * key, signed __int64 value, const char * options, const char * partitionKey, unsigned expire);
+    ECL_MEMCACHED_API void ECL_MEMCACHED_CALL MSet    (ICodeContext * _ctx, const char * key, unsigned __int64 value, const char * options, const char * partitionKey, unsigned expire);
+    ECL_MEMCACHED_API void ECL_MEMCACHED_CALL MSet    (ICodeContext * _ctx, const char * key, double value, const char * options, const char * partitionKey, unsigned expire);
+    ECL_MEMCACHED_API void ECL_MEMCACHED_CALL MSetUtf8(ICodeContext * _ctx, const char * key, size32_t valueLength, const char * value, const char * options, const char * partitionKey, unsigned expire);
+    ECL_MEMCACHED_API void ECL_MEMCACHED_CALL MSet    (ICodeContext * _ctx, const char * key, size32_t valueLength, const char * value, const char * options, const char * partitionKey, unsigned expire);
+    ECL_MEMCACHED_API void ECL_MEMCACHED_CALL MSet    (ICodeContext * _ctx, const char * key, size32_t valueLength, const UChar * value, const char * options, const char * partitionKey, unsigned expire);
+    ECL_MEMCACHED_API void ECL_MEMCACHED_CALL MSetData(ICodeContext * _ctx, const char * key, size32_t valueLength, const void * value, const char * options, const char * partitionKey, unsigned expire);
     //--------------------------GET----------------------------------------
-    ECL_MEMCACHED_API bool             ECL_MEMCACHED_CALL MGetBool  (ICodeContext * _ctx, const char * options, const char * key, const char * partitionKey);
-    ECL_MEMCACHED_API signed __int64   ECL_MEMCACHED_CALL MGetInt8  (ICodeContext * _ctx, const char * options, const char * key, const char * partitionKey);
-    ECL_MEMCACHED_API unsigned __int64 ECL_MEMCACHED_CALL MGetUint8 (ICodeContext * _ctx, const char * options, const char * key, const char * partitionKey);
-    ECL_MEMCACHED_API double           ECL_MEMCACHED_CALL MGetDouble(ICodeContext * _ctx, const char * options, const char * key, const char * partitionKey);
-    ECL_MEMCACHED_API void             ECL_MEMCACHED_CALL MGetUtf8  (ICodeContext * _ctx, size32_t & valueLength, char * & returnValue, const char * options, const char * key, const char * partitionKey);
-    ECL_MEMCACHED_API void             ECL_MEMCACHED_CALL MGetStr   (ICodeContext * _ctx, size32_t & valueLength, char * & returnValue, const char * options, const char * key, const char * partitionKey);
-    ECL_MEMCACHED_API void             ECL_MEMCACHED_CALL MGetUChar (ICodeContext * _ctx, size32_t & valueLength, UChar * & returnValue, const char * options, const char * key, const char * partitionKey);
-    ECL_MEMCACHED_API void             ECL_MEMCACHED_CALL MGetData  (ICodeContext * _ctx,size32_t & returnLength, void * & returnValue, const char * options, const char * key, const char * partitionKey);
+    ECL_MEMCACHED_API bool             ECL_MEMCACHED_CALL MGetBool  (ICodeContext * _ctx, const char * key, const char * options, const char * partitionKey);
+    ECL_MEMCACHED_API signed __int64   ECL_MEMCACHED_CALL MGetInt8  (ICodeContext * _ctx, const char * key, const char * options, const char * partitionKey);
+    ECL_MEMCACHED_API unsigned __int64 ECL_MEMCACHED_CALL MGetUint8 (ICodeContext * _ctx, const char * key, const char * options, const char * partitionKey);
+    ECL_MEMCACHED_API double           ECL_MEMCACHED_CALL MGetDouble(ICodeContext * _ctx, const char * key, const char * options, const char * partitionKey);
+    ECL_MEMCACHED_API void             ECL_MEMCACHED_CALL MGetUtf8  (ICodeContext * _ctx, size32_t & valueLength, char * & returnValue, const char * key, const char * options, const char * partitionKey);
+    ECL_MEMCACHED_API void             ECL_MEMCACHED_CALL MGetStr   (ICodeContext * _ctx, size32_t & valueLength, char * & returnValue, const char * key, const char * options, const char * partitionKey);
+    ECL_MEMCACHED_API void             ECL_MEMCACHED_CALL MGetUChar (ICodeContext * _ctx, size32_t & valueLength, UChar * & returnValue, const char * key, const char * options, const char * partitionKey);
+    ECL_MEMCACHED_API void             ECL_MEMCACHED_CALL MGetData  (ICodeContext * _ctx,size32_t & returnLength, void * & returnValue, const char * key, const char * options, const char * partitionKey);
     //--------------------------------AUXILLARIES---------------------------
-    ECL_MEMCACHED_API bool             ECL_MEMCACHED_CALL MExists (ICodeContext * _ctx, const char * options, const char * key, const char * partitionKey);
-    ECL_MEMCACHED_API const char *     ECL_MEMCACHED_CALL MKeyType(ICodeContext * _ctx, const char * options, const char * key, const char * partitionKey);
+    ECL_MEMCACHED_API bool             ECL_MEMCACHED_CALL MExists (ICodeContext * _ctx, const char * key, const char * options, const char * partitionKey);
+    ECL_MEMCACHED_API const char *     ECL_MEMCACHED_CALL MKeyType(ICodeContext * _ctx, const char * key, const char * options, const char * partitionKey);
     ECL_MEMCACHED_API void             ECL_MEMCACHED_CALL MClear  (ICodeContext * _ctx, const char * options);
+}//close namespace
 }
 #endif

plugins/mysql/CMakeLists.txt

@@ -48,7 +48,7 @@ if (USE_MYSQL)
     HPCC_ADD_LIBRARY( mysqlembed SHARED ${SRCS} )
     if (${CMAKE_VERSION} VERSION_LESS "2.8.9")
       message("WARNING: Cannot set NO_SONAME. shlibdeps will give warnings when package is installed")
-    else()
+    elif(NOT APPLE)
       set_target_properties( mysqlembed PROPERTIES NO_SONAME 1 )
     endif()
 

plugins/proxies/lib_metaphone.ecllib

@@ -15,7 +15,7 @@
     limitations under the License.
 ############################################################################## */
 
-/* Proxy plugin definition for (EE-only) dmetaphone library version DMETAPHONE 1.1.05 */
+/* Proxy plugin definition for dmetaphone library version DMETAPHONE 1.1.05 */
 
 export MetaphoneLib := SERVICE : plugin('dmetaphone')
   string DMetaphone1(const string src) : c,pure,entrypoint='mpDMetaphone1';

plugins/proxies/lib_metaphone3.ecllib

@@ -0,0 +1,25 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2015 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+/* Proxy plugin definition for (EE only) metaphone3 library */
+
+export Metaphone3Lib := SERVICE : plugin('metaphone3')
+  string Metaphone3(const string src, boolean encodeVowels=true, boolean encodeExact=true, unsigned4 maxLength=0) : c,pure,entrypoint='m3DMetaphone1';
+  string Metaphone3Alt(const string src, boolean encodeVowels=true, boolean encodeExact=true, unsigned4 maxLength=0) : c,pure,entrypoint='m3DMetaphone2';
+  string Metaphone3Both(const string src, boolean encodeVowels=true, boolean encodeExact=true, unsigned4 maxLength=0) : c,pure,entrypoint='m3DMetaphoneBoth';
+END;
+

plugins/pyembed/CMakeLists.txt

@@ -54,7 +54,7 @@ if (USE_PYTHON)
     HPCC_ADD_LIBRARY( pyembed SHARED ${SRCS} )
     if (${CMAKE_VERSION} VERSION_LESS "2.8.9")
       message("WARNING: Cannot set NO_SONAME. shlibdeps will give warnings when package is installed")
-    else()
+    elif(NOT APPLE)
       set_target_properties( pyembed PROPERTIES NO_SONAME 1 )
     endif()
 

plugins/redis/CMakeLists.txt

@@ -0,0 +1,67 @@
+################################################################################
+#    HPCC SYSTEMS software Copyright (C) 2015 HPCC Systems.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+################################################################################
+
+# Component: redis
+
+#####################################################
+# Description:
+# ------------
+#    Cmake Input File for redis
+#####################################################
+
+project( redis )
+
+if (USE_REDIS)
+  ADD_PLUGIN(redis PACKAGES REDIS OPTION MAKE_REDIS)
+  if ( MAKE_REDIS )
+    set (    SRCS
+             redisplugin.hpp
+             redissync.hpp
+
+             redisplugin.cpp
+             redissync.cpp
+        )
+
+    include_directories (
+             ./../../system/include
+             ./../../rtl/eclrtl
+             ./../../rtl/include
+             ./../../common/deftype
+             ./../../system/jlib
+             ${LIBHIREDIS_INCLUDE_DIR}
+        )
+
+    ADD_DEFINITIONS( -D_USRDLL -DECL_REDIS_EXPORTS)
+
+    HPCC_ADD_LIBRARY( redis SHARED ${SRCS} )
+    if (${CMAKE_VERSION} VERSION_LESS "2.8.9")
+      message("WARNING: Cannot set NO_SONAME. shlibdeps will give warnings when package is installed")
+    elif(NOT APPLE)
+      set_target_properties( redis PROPERTIES NO_SONAME 1 )
+    endif()
+
+    install ( TARGETS redis DESTINATION plugins)
+
+    target_link_libraries ( redis
+        eclrtl
+        jlib
+        ${LIBHIREDIS_LIBRARY}
+        )
+  endif()
+endif()
+
+#Even if not making the redis plugin, we want to install the header
+install ( FILES ${CMAKE_CURRENT_SOURCE_DIR}/lib_redis.ecllib DESTINATION plugins COMPONENT Runtime)

plugins/redis/lib_redis.ecllib

@@ -0,0 +1,73 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2015 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the License);
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an AS IS BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+
+EXPORT sync := SERVICE : plugin('redis'), namespace('RedisPlugin')
+  SetUnicode( CONST VARSTRING key, CONST UNICODE value, CONST VARSTRING options, UNSIGNED database = 0, UNSIGNED4 expire = 0, CONST VARSTRING password = '', unsigned timeout = 1000000) : cpp,action,context,entrypoint='SyncRSetUChar';
+  SetString(  CONST VARSTRING key, CONST STRING value,  CONST VARSTRING options, UNSIGNED database = 0, UNSIGNED4 expire = 0, CONST VARSTRING password = '', unsigned timeout = 1000000) : cpp,action,context,entrypoint='SyncRSetStr';
+  SetUtf8(    CONST VARSTRING key, CONST UTF8 value,    CONST VARSTRING options, UNSIGNED database = 0, UNSIGNED4 expire = 0, CONST VARSTRING password = '', unsigned timeout = 1000000) : cpp,action,context,entrypoint='SyncRSetUtf8';
+  SetBoolean( CONST VARSTRING key, BOOLEAN value,       CONST VARSTRING options, UNSIGNED database = 0, UNSIGNED4 expire = 0, CONST VARSTRING password = '', unsigned timeout = 1000000) : cpp,action,context,entrypoint='SyncRSetBool';
+  SetReal(    CONST VARSTRING key, REAL value,          CONST VARSTRING options, UNSIGNED database = 0, UNSIGNED4 expire = 0, CONST VARSTRING password = '', unsigned timeout = 1000000) : cpp,action,context,entrypoint='SyncRSetReal';
+  SetInteger( CONST VARSTRING key, INTEGER value,       CONST VARSTRING options, UNSIGNED database = 0, UNSIGNED4 expire = 0, CONST VARSTRING password = '', unsigned timeout = 1000000) : cpp,action,context,entrypoint='SyncRSetInt';
+  SetUnsigned(CONST VARSTRING key, UNSIGNED value,      CONST VARSTRING options, UNSIGNED database = 0, UNSIGNED4 expire = 0, CONST VARSTRING password = '', unsigned timeout = 1000000) : cpp,action,context,entrypoint='SyncRSetUInt';
+  SetData(    CONST VARSTRING key, CONST DATA value,    CONST VARSTRING options, UNSIGNED database = 0, UNSIGNED4 expire = 0, CONST VARSTRING password = '', unsigned timeout = 1000000) : cpp,action,context,entrypoint='SyncRSetData';
+
+  INTEGER8   GetInteger(CONST VARSTRING key, CONST VARSTRING options, UNSIGNED database = 0, CONST VARSTRING password = '', unsigned timeout = 1000000) : cpp,once,context,entrypoint='SyncRGetInt8';
+  UNSIGNED8 GetUnsigned(CONST VARSTRING key, CONST VARSTRING options, UNSIGNED database = 0, CONST VARSTRING password = '', unsigned timeout = 1000000) : cpp,once,context,entrypoint='SyncRGetUint8';
+  STRING      GetString(CONST VARSTRING key, CONST VARSTRING options, UNSIGNED database = 0, CONST VARSTRING password = '', unsigned timeout = 1000000) : cpp,once,context,entrypoint='SyncRGetStr';
+  UNICODE    GetUnicode(CONST VARSTRING key, CONST VARSTRING options, UNSIGNED database = 0, CONST VARSTRING password = '', unsigned timeout = 1000000) : cpp,once,context,entrypoint='SyncRGetUChar';
+  UTF8          GetUtf8(CONST VARSTRING key, CONST VARSTRING options, UNSIGNED database = 0, CONST VARSTRING password = '', unsigned timeout = 1000000) : cpp,once,context,entrypoint='SyncRGetUtf8';
+  BOOLEAN    GetBoolean(CONST VARSTRING key, CONST VARSTRING options, UNSIGNED database = 0, CONST VARSTRING password = '', unsigned timeout = 1000000) : cpp,once,context,entrypoint='SyncRGetBool';
+  REAL          GetReal(CONST VARSTRING key, CONST VARSTRING options, UNSIGNED database = 0, CONST VARSTRING password = '', unsigned timeout = 1000000) : cpp,once,context,entrypoint='SyncRGetDouble';
+  DATA          GetData(CONST VARSTRING key, CONST VARSTRING options, UNSIGNED database = 0, CONST VARSTRING password = '', unsigned timeout = 1000000) : cpp,once,context,entrypoint='SyncRGetData';
+
+  BOOLEAN Exists(CONST VARSTRING key, CONST VARSTRING options, UNSIGNED database = 0, CONST VARSTRING password = '', unsigned timeout = 1000000) : cpp,once,context,entrypoint='RExist';
+  FlushDB(CONST VARSTRING options, UNSIGNED database = 0, CONST VARSTRING password = '', unsigned timeout = 1000000) : cpp,action,context,entrypoint='RClear';
+  Del(CONST VARSTRING key, CONST VARSTRING options, UNSIGNED database = 0, CONST VARSTRING password = '', unsigned timeout = 1000000) : cpp,action,context,entrypoint='RDel';
+  Delete(CONST VARSTRING key, CONST VARSTRING options, UNSIGNED database = 0, CONST VARSTRING password = '', unsigned timeout = 1000000) : cpp,action,context,entrypoint='RDel';
+  Persist(CONST VARSTRING key, CONST VARSTRING options, UNSIGNED database = 0, CONST VARSTRING password = '', unsigned timeout = 1000000) : cpp,action,context,entrypoint='RPersist';
+  Expire(CONST VARSTRING key, CONST VARSTRING options, UNSIGNED4 expire, UNSIGNED database = 0, CONST VARSTRING password = '', unsigned timeout = 1000000) : cpp,action,context,entrypoint='RExpire';
+  INTEGER DBSize(CONST VARSTRING options, UNSIGNED database = 0, CONST VARSTRING password = '', unsigned timeout = 1000000) : cpp,once,context,entrypoint='RDBSize';
+END;
+
+EXPORT RedisSync    (CONST VARSTRING options, CONST VARSTRING password = '', unsigned timeout = 1000000) := MODULE
+  EXPORT  SetUnicode(CONST VARSTRING key, CONST UNICODE value,  UNSIGNED database = 0, UNSIGNED4 expire = 0) := sync.SetUnicode (key, value, options, database, expire, password, timeout);
+  EXPORT   SetString(CONST VARSTRING key, CONST STRING value,   UNSIGNED database = 0, UNSIGNED4 expire = 0) := sync.SetString  (key, value, options, database, expire, password, timeout);
+  EXPORT     SetUtf8(CONST VARSTRING key, CONST UTF8 value,     UNSIGNED database = 0, UNSIGNED4 expire = 0) := sync.SetUtf8    (key, value, options, database, expire, password, timeout);
+  EXPORT  SetBoolean(CONST VARSTRING key, CONST BOOLEAN value,  UNSIGNED database = 0, UNSIGNED4 expire = 0) := sync.SetBoolean (key, value, options, database, expire, password, timeout);
+  EXPORT     SetReal(CONST VARSTRING key, CONST REAL value,     UNSIGNED database = 0, UNSIGNED4 expire = 0) := sync.SetReal    (key, value, options, database, expire, password, timeout);
+  EXPORT  SetInteger(CONST VARSTRING key, CONST INTEGER value,  UNSIGNED database = 0, UNSIGNED4 expire = 0) := sync.SetInteger (key, value, options, database, expire, password, timeout);
+  EXPORT SetUnsigned(CONST VARSTRING key, CONST UNSIGNED value, UNSIGNED database = 0, UNSIGNED4 expire = 0) := sync.SetUnsigned(key, value, options, database, expire, password, timeout);
+  EXPORT     SetData(CONST VARSTRING key, CONST DATA value,     UNSIGNED database = 0, UNSIGNED4 expire = 0) := sync.SetData    (key, value, options, database, expire, password, timeout);
+
+  EXPORT  GetUnicode(CONST VARSTRING key, UNSIGNED database = 0) :=  sync.GetUnicode(key, options, database, password, timeout);
+  EXPORT   GetString(CONST VARSTRING key, UNSIGNED database = 0) :=   sync.GetString(key, options, database, password, timeout);
+  EXPORT     GetUtf8(CONST VARSTRING key, UNSIGNED database = 0) :=     sync.GetUtf8(key, options, database, password, timeout);
+  EXPORT  GetBoolean(CONST VARSTRING key, UNSIGNED database = 0) :=  sync.GetBoolean(key, options, database, password, timeout);
+  EXPORT     GetReal(CONST VARSTRING key, UNSIGNED database = 0) :=     sync.GetReal(key, options, database, password, timeout);
+  EXPORT  GetInteger(CONST VARSTRING key, UNSIGNED database = 0) :=  sync.GetInteger(key, options, database, password, timeout);
+  EXPORT GetUnsigned(CONST VARSTRING key, UNSIGNED database = 0) := sync.GetUnsigned(key, options, database, password, timeout);
+  EXPORT     GetData(CONST VARSTRING key, UNSIGNED database = 0) :=     sync.GetData(key, options, database, password, timeout);
+
+  EXPORT Exists(CONST VARSTRING key, UNSIGNED database = 0) := sync.Exists(key, options, database, password, timeout);
+  EXPORT FlushDB(UNSIGNED database = 0) := sync.FlushDB(options, database, password, timeout);
+  EXPORT Del(CONST VARSTRING key, UNSIGNED database = 0) := sync.Del(key, options, database, password, timeout);
+  EXPORT Delete(CONST VARSTRING key, UNSIGNED database = 0) := sync.Delete(key, options, database, password, timeout);
+  EXPORT Persist(CONST VARSTRING key, UNSIGNED database = 0) := sync.Persist(key, options, database, password, timeout);
+  EXPORT Expire(CONST VARSTRING key, UNSIGNED4 expire, UNSIGNED database = 0)  := sync.Expire(key, options, expire, database, password, timeout);
+  EXPORT DBSize(UNSIGNED database = 0) := sync.DBSize(options, database, password, timeout);
+END;

plugins/redis/redisplugin.cpp

@@ -0,0 +1,114 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2015 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#include "platform.h"
+#include "eclrtl.hpp"
+#include "jstring.hpp"
+#include "redisplugin.hpp"
+
+#define REDIS_VERSION "redis plugin 1.0.0"
+
+ECL_REDIS_API bool getECLPluginDefinition(ECLPluginDefinitionBlock *pb)
+{
+    if (pb->size != sizeof(ECLPluginDefinitionBlock))
+        return false;
+
+    pb->magicVersion = PLUGIN_VERSION;
+    pb->version = REDIS_VERSION;
+    pb->moduleName = "lib_redis";
+    pb->ECL = NULL;
+    pb->flags = PLUGIN_IMPLICIT_MODULE;
+    pb->description = "ECL plugin library for the C API hiredis\n";
+    return true;
+}
+
+namespace RedisPlugin {
+
+StringBuffer & appendExpire(StringBuffer & buffer, unsigned expire)
+{
+    if (expire > 0)
+        buffer.append(" EX ").append(expire);
+    return buffer;
+}
+
+void RedisServer::parseOptions(ICodeContext * ctx, const char * _options)
+{
+    StringArray optionStrings;
+    optionStrings.appendList(_options, " ");
+    ForEachItemIn(idx, optionStrings)
+    {
+        const char *opt = optionStrings.item(idx);
+        if (strncmp(opt, "--SERVER=", 9) == 0)
+        {
+            opt += 9;
+            StringArray splitPort;
+            splitPort.appendList(opt, ":");
+            if (splitPort.ordinality()==2)
+            {
+                ip.set(splitPort.item(0));
+                port = atoi(splitPort.item(1));
+            }
+        }
+        else
+        {
+            VStringBuffer err("RedisPlugin: unsupported option string %s", opt);
+            rtlFail(0, err.str());
+        }
+    }
+    if (ip.isEmpty())
+    {
+        ip.set("localhost");
+        port = 6379;
+        if (ctx)
+        {
+            VStringBuffer msg("Redis Plugin: WARNING - using default server (%s:%d)", ip.str(), port);
+            ctx->logString(msg.str());
+        }
+    }
+    return;
+}
+Connection::Connection(ICodeContext * ctx, const char * _options, const char * pswd, unsigned __int64 _timeout) : alreadyInitialized(false), database(0), timeout(_timeout)
+{
+    server.setown(new RedisServer(ctx, _options, pswd));
+}
+Connection::Connection(ICodeContext * ctx, RedisServer * _server) : alreadyInitialized(false), database(0), timeout(0)
+{
+    server.setown(_server);
+}
+bool Connection::isSameConnection(ICodeContext * ctx, unsigned hash) const
+{
+    return server->isSame(ctx, hash);
+}
+void * Connection::allocateAndCopy(const char * src, size_t size)
+{
+    void * value = rtlMalloc(size);
+    return memcpy(value, src, size);
+}
+const char * Connection::appendIfKeyNotFoundMsg(const redisReply * reply, const char * key, StringBuffer & target) const
+{
+    if (reply && reply->type == REDIS_REPLY_NIL)
+        target.append("(key: '").append(key).append("') ");
+    return target.str();
+}
+void Connection::init(ICodeContext * ctx)
+{
+    logServerStats(ctx);
+    alreadyInitialized = true;
+}
+}//close namespace
+
+

plugins/redis/redisplugin.hpp

@@ -0,0 +1,126 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2015 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#ifndef ECL_REDIS_INCL
+#define ECL_REDIS_INCL
+
+#ifdef _WIN32
+#define ECL_REDIS_CALL _cdecl
+#ifdef ECL_REDIS_EXPORTS
+#define ECL_REDIS_API __declspec(dllexport)
+#else
+#define ECL_REDIS_API __declspec(dllimport)
+#endif
+#else
+#define ECL_REDIS_CALL
+#define ECL_REDIS_API
+#endif
+
+#include "jhash.hpp"
+#include "hqlplugins.hpp"
+#include "eclhelper.hpp"
+#include "jexcept.hpp"
+#include "hiredis/hiredis.h"
+
+extern "C"
+{
+    ECL_REDIS_API bool getECLPluginDefinition(ECLPluginDefinitionBlock *pb);
+    ECL_REDIS_API void setPluginContext(IPluginContext * _ctx);
+}
+
+class StringBuffer;
+
+namespace RedisPlugin {
+#define setFailMsg "'Set' request failed - "
+#define getFailMsg "'Get<type>' request failed - "
+
+StringBuffer & appendExpire(StringBuffer & buffer, unsigned expire);
+
+class RedisServer : public CInterface
+{
+public :
+    RedisServer(ICodeContext * ctx, const char * _options, const char * pswd)
+    {
+        serverIpPortPasswordHash = hashc((const unsigned char*)pswd, strlen(pswd), 0);
+        serverIpPortPasswordHash = hashc((const unsigned char*)_options, strlen(_options), serverIpPortPasswordHash);
+        options.set(_options);
+        parseOptions(ctx, _options);
+    }
+    bool isSame(ICodeContext * ctx, unsigned hash) const
+    {
+        return (serverIpPortPasswordHash == hash);
+    }
+    const char * getIp() { return ip.str(); }
+    int getPort() { return port; }
+    void parseOptions(ICodeContext * ctx, const char * _options);
+
+private :
+    unsigned serverIpPortPasswordHash;
+    StringAttr options;
+    StringAttr ip;
+    int port;
+};
+class Connection : public CInterface
+{
+public :
+    Connection(ICodeContext * ctx, const char * _options, const char * pswd, unsigned __int64 _timeout);
+    Connection(ICodeContext * ctx, RedisServer * _server);
+
+    bool isSameConnection(ICodeContext * ctx, unsigned hash) const;
+    const char * ip() const { return server->getIp(); }
+    int port() const { return server->getPort(); }
+
+protected :
+    virtual void assertOnError(const redisReply * reply, const char * _msg) { }
+    virtual void assertConnection() { }
+    virtual void logServerStats(ICodeContext * ctx) { }
+    virtual void updateTimeout(unsigned __int64 _timeout) { }
+
+    const char * appendIfKeyNotFoundMsg(const redisReply * reply, const char * key, StringBuffer & target) const;
+    void * allocateAndCopy(const char * src, size_t size);
+    void init(ICodeContext * ctx);
+
+protected :
+    Owned<RedisServer> server;
+    unsigned __int64 timeout;
+    unsigned __int64 database;
+    bool alreadyInitialized;
+};
+
+class Reply : public CInterface
+{
+public :
+    inline Reply() { reply = NULL; };
+    inline Reply(void * _reply) { reply = (redisReply*)_reply; }
+    inline Reply(redisReply * _reply) { reply = _reply; }
+    inline ~Reply()
+    {
+        if (reply)
+            freeReplyObject(reply);
+    }
+
+    static Reply * createReply(void * _reply) { return new Reply(_reply); }
+    inline const redisReply * query() const { return reply; }
+
+private :
+    redisReply * reply;
+};
+typedef Owned<RedisPlugin::Reply> OwnedReply;
+
+}//close namespace
+
+#endif

plugins/redis/redissync.cpp

@@ -0,0 +1,418 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2015 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#include "platform.h"
+#include "jthread.hpp"
+#include "jhash.hpp"
+#include "eclrtl.hpp"
+#include "jstring.hpp"
+#include "redissync.hpp"
+
+namespace RedisPlugin {
+static __thread SyncConnection * cachedConnection;
+static __thread ThreadTermFunc threadHookChain;
+
+//The following class is here to ensure destruction of the cachedConnection within the main thread
+//as this is not handled by the thread hook mechanism.
+static class mainThreadCachedConnection
+{
+public :
+    mainThreadCachedConnection() { }
+    ~mainThreadCachedConnection()
+    {
+        if (cachedConnection)
+            cachedConnection->Release();
+    }
+} mainThread;
+
+static void releaseContext()
+{
+    if (cachedConnection)
+        cachedConnection->Release();
+    if (threadHookChain)
+    {
+        (*threadHookChain)();
+        threadHookChain = NULL;
+    }
+}
+
+SyncConnection::SyncConnection(ICodeContext * ctx, const char * _options, unsigned __int64 _database, const char * pswd, unsigned __int64 _timeout)
+  : Connection(ctx, _options, pswd, _timeout)
+{
+    connect(ctx, _database, pswd);
+}
+SyncConnection::SyncConnection(ICodeContext * ctx, RedisServer * _server, unsigned __int64 _database, const char * pswd)
+  : Connection(ctx, _server)
+{
+    connect(ctx, _database, pswd);
+}
+void SyncConnection::connect(ICodeContext * ctx, unsigned __int64 _database, const char * pswd)
+{
+    struct timeval to = { timeout/1000000, timeout%1000000 };
+    context = redisConnectWithTimeout(server->getIp(), server->getPort(), to);
+    assertConnection();
+    authenticate(ctx, pswd);
+    selectDB(ctx, _database);
+    init(ctx);
+}
+void SyncConnection::authenticate(ICodeContext * ctx, const char * pswd)
+{
+    if (strlen(pswd) > 0)
+    {
+        OwnedReply reply = Reply::createReply(redisCommand(context, "AUTH %b", pswd, strlen(pswd)));
+        assertOnError(reply->query(), "server authentication failed");
+    }
+}
+void SyncConnection::resetContextErr()
+{
+    if (context)
+        context->err = REDIS_OK;
+}
+SyncConnection * SyncConnection::createConnection(ICodeContext * ctx, const char * options, unsigned __int64 _database, const char * pswd, unsigned __int64 _timeout)
+{
+    if (!cachedConnection)
+    {
+        cachedConnection = new SyncConnection(ctx, options, _database, pswd, _timeout);
+        threadHookChain = addThreadTermFunc(releaseContext);
+        return LINK(cachedConnection);
+    }
+
+    unsigned optionsPswdHash = hashc((const unsigned char*)options, strlen(options), hashc((const unsigned char*)pswd, strlen(pswd), 0));
+    if (cachedConnection->isSameConnection(ctx, optionsPswdHash))
+    {
+        //MORE: need to check that the connection has not expired (think hiredis REDIS_KEEPALIVE_INTERVAL is defaulted to 15s).
+        //At present updateTimeout calls assertConnection.
+        cachedConnection->resetContextErr();//reset the context err to allow reuse when an error previously occurred.
+        cachedConnection->updateTimeout(_timeout);
+        cachedConnection->selectDB(ctx, _database);
+        return LINK(cachedConnection);
+    }
+
+    cachedConnection->Release();
+    cachedConnection = new SyncConnection(ctx, options, _database, pswd, _timeout);
+    return LINK(cachedConnection);
+}
+void SyncConnection::selectDB(ICodeContext * ctx, unsigned __int64 _database)
+{
+    if (database == _database)
+        return;
+    database = _database;
+    VStringBuffer cmd("SELECT %" I64F "u", database);
+    OwnedReply reply = Reply::createReply(redisCommand(context, cmd.str()));
+    assertOnError(reply->query(), "'SELECT' request failed");
+}
+void SyncConnection::updateTimeout(unsigned __int64 _timeout)
+{
+    if (timeout == _timeout)
+        return;
+    assertConnection();
+    timeout = _timeout;
+    struct timeval to = { timeout/1000000, timeout%1000000 };
+    if (redisSetTimeout(context, to) != REDIS_OK)
+    {
+        if (context->err)
+        {
+            VStringBuffer msg("RedisPlugin: failed to set timeout - %s", context->errstr);
+            rtlFail(0, msg.str());
+        }
+        else
+            rtlFail(0, "RedisPlugin: failed to set timeout - no message available");
+    }
+}
+void SyncConnection::logServerStats(ICodeContext * ctx)
+{
+    OwnedReply reply = Reply::createReply(redisCommand(context, "INFO"));
+    assertOnError(reply->query(), "'INFO' request failed");
+    StringBuffer stats("Redis Plugin : Server stats - ");
+    stats.newline().append(reply->query()->str).newline();
+    ctx->logString(stats.str());
+}
+void SyncConnection::assertOnError(const redisReply * reply, const char * _msg)
+{
+    if (!reply)
+    {
+        //There should always be a context error if no reply error
+        assertConnection();
+        VStringBuffer msg("Redis Plugin: %s - %s", _msg, "neither 'reply' nor connection error available");
+        rtlFail(0, msg.str());
+    }
+    else if (reply->type == REDIS_REPLY_ERROR)
+    {
+        if (strncmp(reply->str, "NOAUTH", 6) == 0)
+        {
+            VStringBuffer msg("Redis Plugin: server authentication failed - %s", reply->str);
+            rtlFail(0, msg.str());
+        }
+        else
+        {
+            VStringBuffer msg("Redis Plugin: %s - %s", _msg, reply->str);
+            rtlFail(0, msg.str());
+        }
+    }
+}
+void SyncConnection::assertConnection()
+{
+    if (!context)
+        rtlFail(0, "Redis Plugin: 'redisConnect' failed - no error available.");
+    else if (context->err)//This check requires that context->err be reset after a non terminating error was encountered. However, the nature of context->err implies a runtime exception.
+    {
+        VStringBuffer msg("Redis Plugin: Connection failed - %s for %s:%u", context->errstr, ip(), port());
+        rtlFail(0, msg.str());
+    }
+}
+
+void SyncConnection::clear(ICodeContext * ctx)
+{
+    //NOTE: flush is the actual cache flush/clear/delete and not an io buffer flush.
+    OwnedReply reply = Reply::createReply(redisCommand(context, "FLUSHDB"));//NOTE: FLUSHDB deletes current database where as FLUSHALL deletes all dbs.
+    //NOTE: documented as never failing, but in case
+    assertOnError(reply->query(), "'FlushDB' request failed");
+}
+void SyncConnection::del(ICodeContext * ctx, const char * key)
+{
+    OwnedReply reply = Reply::createReply(redisCommand(context, "DEL %b", key, strlen(key)));
+    assertOnError(reply->query(), "'Del' request failed");
+}
+void SyncConnection::persist(ICodeContext * ctx, const char * key)
+{
+    OwnedReply reply = Reply::createReply(redisCommand(context, "PERSIST %b", key, strlen(key)));
+    assertOnError(reply->query(), "'Persist' request failed");
+}
+void SyncConnection::expire(ICodeContext * ctx, const char * key, unsigned _expire)
+{
+    OwnedReply reply = Reply::createReply(redisCommand(context, "DEL %b %u", key, strlen(key), _expire));
+    assertOnError(reply->query(), "'Expire' request failed");
+}
+bool SyncConnection::exists(ICodeContext * ctx, const char * key)
+{
+    OwnedReply reply = Reply::createReply(redisCommand(context, "EXISTS %b", key, strlen(key)));
+    assertOnError(reply->query(), "'Exists' request failed");
+    return (reply->query()->integer != 0);
+}
+unsigned __int64 SyncConnection::dbSize(ICodeContext * ctx)
+{
+    OwnedReply reply = Reply::createReply(redisCommand(context, "DBSIZE"));
+    assertOnError(reply->query(), "'DBSIZE' request failed");
+    return reply->query()->integer;
+}
+//-------------------------------------------SET-----------------------------------------
+//--OUTER--
+template<class type> void SyncRSet(ICodeContext * ctx, const char * _options, const char * key, type value, unsigned __int64 database, unsigned expire, const char * pswd, unsigned __int64 _timeout)
+{
+    Owned<SyncConnection> master = SyncConnection::createConnection(ctx, _options, database, pswd, _timeout);
+    master->set(ctx, key, value, expire);
+}
+//Set pointer types
+template<class type> void SyncRSet(ICodeContext * ctx, const char * _options, const char * key, size32_t valueLength, const type * value, unsigned __int64 database, unsigned expire, const char * pswd, unsigned __int64 _timeout)
+{
+    Owned<SyncConnection> master = SyncConnection::createConnection(ctx, _options, database, pswd, _timeout);
+    master->set(ctx, key, valueLength, value, expire);
+}
+//--INNER--
+template<class type> void SyncConnection::set(ICodeContext * ctx, const char * key, type value, unsigned expire)
+{
+    const char * _value = reinterpret_cast<const char *>(&value);//Do this even for char * to prevent compiler complaining
+    const char * msg = setFailMsg;
+
+    StringBuffer cmd("SET %b %b");
+    appendExpire(cmd, expire);
+
+    OwnedReply reply = Reply::createReply(redisCommand(context, cmd.str(), key, strlen(key), _value, sizeof(type)));
+    assertOnError(reply->query(), msg);
+}
+template<class type> void SyncConnection::set(ICodeContext * ctx, const char * key, size32_t valueLength, const type * value, unsigned expire)
+{
+    const char * _value = reinterpret_cast<const char *>(value);//Do this even for char * to prevent compiler complaining
+    const char * msg = setFailMsg;
+
+    StringBuffer cmd("SET %b %b");
+    appendExpire(cmd, expire);
+    OwnedReply reply = Reply::createReply(redisCommand(context, cmd.str(), key, strlen(key), _value, (size_t)valueLength));
+    assertOnError(reply->query(), msg);
+}
+//-------------------------------------------GET-----------------------------------------
+//--OUTER--
+template<class type> void SyncRGet(ICodeContext * ctx, const char * options, const char * key, type & returnValue, unsigned __int64 database, const char * pswd, unsigned __int64 _timeout)
+{
+    Owned<SyncConnection> master = SyncConnection::createConnection(ctx, options, database, pswd, _timeout);
+    master->get(ctx, key, returnValue);
+}
+template<class type> void SyncRGet(ICodeContext * ctx, const char * options, const char * key, size_t & returnLength, type * & returnValue, unsigned __int64 database, const char * pswd, unsigned __int64 _timeout)
+{
+    Owned<SyncConnection> master = SyncConnection::createConnection(ctx, options, database, pswd, _timeout);
+    master->get(ctx, key, returnLength, returnValue);
+}
+void SyncRGetVoidPtrLenPair(ICodeContext * ctx, const char * options, const char * key, size_t & returnLength, void * & returnValue, unsigned __int64 database, const char * pswd, unsigned __int64 _timeout)
+{
+    Owned<SyncConnection> master = SyncConnection::createConnection(ctx, options, database, pswd, _timeout);
+    master->getVoidPtrLenPair(ctx, key, returnLength, returnValue);
+}
+//--INNER--
+template<class type> void SyncConnection::get(ICodeContext * ctx, const char * key, type & returnValue)
+{
+    OwnedReply reply = Reply::createReply(redisCommand(context, "GET %b", key, strlen(key)));
+
+    StringBuffer keyMsg = getFailMsg;
+    assertOnError(reply->query(), appendIfKeyNotFoundMsg(reply->query(), key, keyMsg));
+
+    size_t returnSize = reply->query()->len;
+    if (sizeof(type)!=returnSize)
+    {
+        VStringBuffer msg("RedisPlugin: ERROR - Requested type of different size (%uB) from that stored (%uB).", (unsigned)sizeof(type), (unsigned)returnSize);
+
+        rtlFail(0, msg.str());
+    }
+    memcpy(&returnValue, reply->query()->str, returnSize);
+}
+template<class type> void SyncConnection::get(ICodeContext * ctx, const char * key, size_t & returnLength, type * & returnValue)
+{
+    OwnedReply reply = Reply::createReply(redisCommand(context, "GET %b", key, strlen(key)));
+
+    StringBuffer keyMsg = getFailMsg;
+    assertOnError(reply->query(), appendIfKeyNotFoundMsg(reply->query(), key, keyMsg));
+
+    returnLength = reply->query()->len;
+    size_t returnSize = returnLength;
+
+    returnValue = reinterpret_cast<type*>(allocateAndCopy(reply->query()->str, returnSize));
+}
+void SyncConnection::getVoidPtrLenPair(ICodeContext * ctx, const char * key, size_t & returnLength, void * & returnValue)
+{
+    OwnedReply reply = Reply::createReply(redisCommand(context, "GET %b", key, strlen(key)));
+    StringBuffer keyMsg = getFailMsg;
+    assertOnError(reply->query(), appendIfKeyNotFoundMsg(reply->query(), key, keyMsg));
+
+    returnLength = reply->query()->len;
+    returnValue = reinterpret_cast<void*>(allocateAndCopy(reply->query()->str, reply->query()->len));
+}
+
+//--------------------------------------------------------------------------------
+//                           ECL SERVICE ENTRYPOINTS
+//--------------------------------------------------------------------------------
+ECL_REDIS_API void ECL_REDIS_CALL RClear(ICodeContext * ctx, const char * options, unsigned __int64 database, const char * pswd, unsigned __int64 timeout)
+{
+    Owned<SyncConnection> master = SyncConnection::createConnection(ctx, options, database, pswd, timeout);
+    master->clear(ctx);
+}
+ECL_REDIS_API bool ECL_REDIS_CALL RExist(ICodeContext * ctx, const char * key, const char * options, unsigned __int64 database, const char * pswd, unsigned __int64 timeout)
+{
+    Owned<SyncConnection> master = SyncConnection::createConnection(ctx, options, database, pswd, timeout);
+    return master->exists(ctx, key);
+}
+ECL_REDIS_API void ECL_REDIS_CALL RDel(ICodeContext * ctx, const char * key, const char * options, unsigned __int64 database, const char * pswd, unsigned __int64 timeout)
+{
+    Owned<SyncConnection> master = SyncConnection::createConnection(ctx, options, database, pswd, timeout);
+    master->del(ctx, key);
+}
+ECL_REDIS_API void ECL_REDIS_CALL RPersist(ICodeContext * ctx, const char * key, const char * options, unsigned __int64 database, const char * pswd, unsigned __int64 timeout)
+{
+    Owned<SyncConnection> master = SyncConnection::createConnection(ctx, options, database, pswd, timeout);
+    master->persist(ctx, key);
+}
+ECL_REDIS_API void ECL_REDIS_CALL RExpire(ICodeContext * ctx, const char * key, const char * options, unsigned _expire, unsigned __int64 database, const char * pswd, unsigned __int64 timeout)
+{
+    Owned<SyncConnection> master = SyncConnection::createConnection(ctx, options, database, pswd, timeout);
+    master->expire(ctx, key, _expire);
+}
+ECL_REDIS_API unsigned __int64 ECL_REDIS_CALL RDBSize(ICodeContext * ctx, const char * options, unsigned __int64 database, const char * pswd, unsigned __int64 timeout)
+{
+    Owned<SyncConnection> master = SyncConnection::createConnection(ctx, options, database, pswd, timeout);
+    return master->dbSize(ctx);
+}
+//-----------------------------------SET------------------------------------------
+ECL_REDIS_API void ECL_REDIS_CALL SyncRSetStr(ICodeContext * ctx, const char * key, size32_t valueLength, const char * value, const char * options, unsigned __int64 database, unsigned expire, const char * pswd, unsigned __int64 timeout)
+{
+    SyncRSet(ctx, options, key, valueLength, value, database, expire, pswd, timeout);
+}
+ECL_REDIS_API void ECL_REDIS_CALL SyncRSetUChar(ICodeContext * ctx, const char * key, size32_t valueLength, const UChar * value, const char * options, unsigned __int64 database, unsigned expire, const char * pswd, unsigned __int64 timeout)
+{
+    SyncRSet(ctx, options, key, (valueLength)*sizeof(UChar), value, database, expire, pswd, timeout);
+}
+ECL_REDIS_API void ECL_REDIS_CALL SyncRSetInt(ICodeContext * ctx, const char * key, signed __int64 value, const char * options, unsigned __int64 database, unsigned expire, const char * pswd, unsigned __int64 timeout)
+{
+    SyncRSet(ctx, options, key, value, database, expire, pswd, timeout);
+}
+ECL_REDIS_API void ECL_REDIS_CALL SyncRSetUInt(ICodeContext * ctx, const char * key, unsigned __int64 value, const char * options, unsigned __int64 database, unsigned expire, const char * pswd, unsigned __int64 timeout)
+{
+    SyncRSet(ctx, options, key, value, database, expire, pswd, timeout);
+}
+ECL_REDIS_API void ECL_REDIS_CALL SyncRSetReal(ICodeContext * ctx, const char * key, double value, const char * options, unsigned __int64 database, unsigned expire, const char * pswd, unsigned __int64 timeout)
+{
+    SyncRSet(ctx, options, key, value, database, expire, pswd, timeout);
+}
+ECL_REDIS_API void ECL_REDIS_CALL SyncRSetBool(ICodeContext * ctx, const char * key, bool value, const char * options, unsigned __int64 database, unsigned expire, const char * pswd, unsigned __int64 timeout)
+{
+    SyncRSet(ctx, options, key, value, database, expire, pswd, timeout);
+}
+ECL_REDIS_API void ECL_REDIS_CALL SyncRSetData(ICodeContext * ctx, const char * key, size32_t valueLength, const void * value, const char * options, unsigned __int64 database, unsigned expire, const char * pswd, unsigned __int64 timeout)
+{
+    SyncRSet(ctx, options, key, valueLength, value, database, expire, pswd, timeout);
+}
+ECL_REDIS_API void ECL_REDIS_CALL SyncRSetUtf8(ICodeContext * ctx, const char * key, size32_t valueLength, const char * value, const char * options, unsigned __int64 database, unsigned expire, const char * pswd, unsigned __int64 timeout)
+{
+    SyncRSet(ctx, options, key, rtlUtf8Size(valueLength, value), value, database, expire, pswd, timeout);
+}
+//-------------------------------------GET----------------------------------------
+ECL_REDIS_API bool ECL_REDIS_CALL SyncRGetBool(ICodeContext * ctx, const char * key, const char * options, unsigned __int64 database, const char * pswd, unsigned __int64 timeout)
+{
+    bool value;
+    SyncRGet(ctx, options, key, value, database, pswd, timeout);
+    return value;
+}
+ECL_REDIS_API double ECL_REDIS_CALL SyncRGetDouble(ICodeContext * ctx, const char * key, const char * options, unsigned __int64 database, const char * pswd, unsigned __int64 timeout)
+{
+    double value;
+    SyncRGet(ctx, options, key, value, database, pswd, timeout);
+    return value;
+}
+ECL_REDIS_API signed __int64 ECL_REDIS_CALL SyncRGetInt8(ICodeContext * ctx, const char * key, const char * options, unsigned __int64 database, const char * pswd, unsigned __int64 timeout)
+{
+    signed __int64 value;
+    SyncRGet(ctx, options, key, value, database, pswd, timeout);
+    return value;
+}
+ECL_REDIS_API unsigned __int64 ECL_REDIS_CALL SyncRGetUint8(ICodeContext * ctx, const char * key, const char * options, unsigned __int64 database, const char * pswd, unsigned __int64 timeout)
+{
+    unsigned __int64 value;
+    SyncRGet(ctx, options, key, value, database, pswd, timeout);
+    return value;
+}
+ECL_REDIS_API void ECL_REDIS_CALL SyncRGetStr(ICodeContext * ctx, size32_t & returnLength, char * & returnValue, const char * key, const char * options, unsigned __int64 database, const char * pswd, unsigned __int64 timeout)
+{
+    size_t _returnLength;
+    SyncRGet(ctx, options, key, _returnLength, returnValue, database, pswd, timeout);
+    returnLength = static_cast<size32_t>(_returnLength);
+}
+ECL_REDIS_API void ECL_REDIS_CALL SyncRGetUChar(ICodeContext * ctx, size32_t & returnLength, UChar * & returnValue,  const char * key, const char * options, unsigned __int64 database, const char * pswd, unsigned __int64 timeout)
+{
+    size_t _returnLength;
+    SyncRGet(ctx, options, key, _returnLength, returnValue, database, pswd, timeout);
+    returnLength = static_cast<size32_t>(_returnLength/sizeof(UChar));
+}
+ECL_REDIS_API void ECL_REDIS_CALL SyncRGetUtf8(ICodeContext * ctx, size32_t & returnLength, char * & returnValue, const char * key, const char * options, unsigned __int64 database, const char * pswd, unsigned __int64 timeout)
+{
+    size_t _returnLength;
+    SyncRGet(ctx, options, key, _returnLength, returnValue, database, pswd, timeout);
+    returnLength = static_cast<size32_t>(rtlUtf8Length(_returnLength, returnValue));
+}
+ECL_REDIS_API void ECL_REDIS_CALL SyncRGetData(ICodeContext * ctx, size32_t & returnLength, void * & returnValue, const char * key, const char * options, unsigned __int64 database, const char * pswd, unsigned __int64 timeout)
+{
+    size_t _returnLength;
+    SyncRGetVoidPtrLenPair(ctx, options, key, _returnLength, returnValue, database, pswd, timeout);
+    returnLength = static_cast<size32_t>(_returnLength);
+}
+}//close namespace

plugins/redis/redissync.hpp

@@ -0,0 +1,98 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2015 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#ifndef ECL_REDIS_SYNC_INCL
+#define ECL_REDIS_SYNC_INCL
+
+#include "redisplugin.hpp"
+
+namespace RedisPlugin
+{
+class SyncConnection : public Connection
+{
+public :
+    SyncConnection(ICodeContext * ctx, const char * options, unsigned __int64 database, const char * pswd, unsigned __int64 _timeout);
+    SyncConnection(ICodeContext * ctx, RedisServer * _server, unsigned __int64 database, const char * pswd);
+    ~SyncConnection()
+    {
+        if (context)
+            redisFree(context);
+    }
+    static SyncConnection * createConnection(ICodeContext * ctx, const char * options, unsigned __int64 database, const char * pswd, unsigned __int64 _timeout);
+
+    //set
+    template <class type> void set(ICodeContext * ctx, const char * key, type value, unsigned expire);
+    template <class type> void set(ICodeContext * ctx, const char * key, size32_t valueLength, const type * value, unsigned expire);
+    //get
+    template <class type> void get(ICodeContext * ctx, const char * key, type & value);
+    template <class type> void get(ICodeContext * ctx, const char * key, size_t & valueLength, type * & value);
+    void getVoidPtrLenPair(ICodeContext * ctx, const char * key, size_t & valueLength, void * & value);
+    void persist(ICodeContext * ctx, const char * key);
+    void expire(ICodeContext * ctx, const char * key, unsigned _expire);
+    void del(ICodeContext * ctx, const char * key);
+    void clear(ICodeContext * ctx);
+    unsigned __int64 dbSize(ICodeContext * ctx);
+    bool exists(ICodeContext * ctx, const char * key);
+
+protected :
+    void connect(ICodeContext * ctx, unsigned __int64 _database, const char * pswd);
+    void selectDB(ICodeContext * ctx, unsigned __int64 _database);
+    void authenticate(ICodeContext * ctx, const char * pswd);
+    void resetContextErr();
+
+    virtual void updateTimeout(unsigned __int64 _timeout);
+    virtual void assertOnError(const redisReply * reply, const char * _msg);
+    virtual void assertConnection();
+    virtual void logServerStats(ICodeContext * ctx);
+
+protected :
+    redisContext * context;
+};
+}//close namespace
+
+extern "C++"
+{
+namespace RedisPlugin {
+    //--------------------------SET----------------------------------------
+    ECL_REDIS_API void ECL_REDIS_CALL SyncRSetBool (ICodeContext * _ctx, const char * key, bool value, const char * options, unsigned __int64 database, unsigned expire, const char * pswd, unsigned timeout);
+    ECL_REDIS_API void ECL_REDIS_CALL SyncRSetInt  (ICodeContext * _ctx, const char * key, signed __int64 value, const char * options, unsigned __int64 database, unsigned expire, const char * pswd, unsigned timeout);
+    ECL_REDIS_API void ECL_REDIS_CALL SyncRSetUInt (ICodeContext * _ctx, const char * key, unsigned __int64 value, const char * options, unsigned __int64 database, unsigned expire, const char * pswd, unsigned timeout);
+    ECL_REDIS_API void ECL_REDIS_CALL SyncRSetReal (ICodeContext * _ctx, const char * key, double value, const char * options, unsigned __int64 database, unsigned expire, const char * pswd, unsigned timeout);
+    ECL_REDIS_API void ECL_REDIS_CALL SyncRSetUtf8 (ICodeContext * _ctx, const char * key, size32_t valueLength, const char * value, const char * options, unsigned __int64 database, unsigned expire, const char * pswd, unsigned timeout);
+    ECL_REDIS_API void ECL_REDIS_CALL SyncRSetStr  (ICodeContext * _ctx, const char * key, size32_t valueLength, const char * value, const char * options, unsigned __int64 database, unsigned expire, const char * pswd, unsigned timeout);
+    ECL_REDIS_API void ECL_REDIS_CALL SyncRSetUChar(ICodeContext * _ctx, const char * key, size32_t valueLength, const UChar * value, const char * options, unsigned __int64 database, unsigned expire, const char * pswd, unsigned timeout);
+    ECL_REDIS_API void ECL_REDIS_CALL SyncRSetData (ICodeContext * _ctx, const char * key, size32_t valueLength, const void * value, const char * options, unsigned __int64 database, unsigned expire, const char * pswd, unsigned timeout);
+    //--------------------------GET----------------------------------------
+    ECL_REDIS_API bool             ECL_REDIS_CALL SyncRGetBool  (ICodeContext * _ctx, const char * key, const char * options, unsigned __int64 database, const char * pswd, unsigned timeout);
+    ECL_REDIS_API signed __int64   ECL_REDIS_CALL SyncRGetInt8  (ICodeContext * _ctx, const char * key, const char * options, unsigned __int64 database, const char * pswd, unsigned timeout);
+    ECL_REDIS_API unsigned __int64 ECL_REDIS_CALL SyncRGetUint8 (ICodeContext * _ctx, const char * key, const char * options, unsigned __int64 database, const char * pswd, unsigned timeout);
+    ECL_REDIS_API double           ECL_REDIS_CALL SyncRGetDouble(ICodeContext * _ctx, const char * key, const char * options, unsigned __int64 database, const char * pswd, unsigned timeout);
+    ECL_REDIS_API void             ECL_REDIS_CALL SyncRGetUtf8  (ICodeContext * _ctx, size32_t & returnLength, char * & returnValue, const char * key, const char * options, unsigned __int64 database, const char * pswd, unsigned timeout);
+    ECL_REDIS_API void             ECL_REDIS_CALL SyncRGetStr   (ICodeContext * _ctx, size32_t & returnLength, char * & returnValue, const char * key, const char * options, unsigned __int64 database, const char * pswd, unsigned timeout);
+    ECL_REDIS_API void             ECL_REDIS_CALL SyncRGetUChar (ICodeContext * _ctx, size32_t & returnLength, UChar * & returnValue, const char * key, const char * options, unsigned __int64 database, const char * pswd, unsigned timeout);
+    ECL_REDIS_API void             ECL_REDIS_CALL SyncRGetData  (ICodeContext * _ctx,size32_t & returnLength, void * & returnValue, const char * key, const char * options, unsigned __int64 database, const char * pswd, unsigned timeout);
+
+    //--------------------------------AUXILLARIES---------------------------
+    ECL_REDIS_API bool             ECL_REDIS_CALL RExist  (ICodeContext * _ctx, const char * key, const char * options, unsigned __int64 database, const char * pswd, unsigned timeout);
+    ECL_REDIS_API void             ECL_REDIS_CALL RClear  (ICodeContext * _ctx, const char * options, unsigned __int64 database, const char * pswd, unsigned timeout);
+    ECL_REDIS_API void             ECL_REDIS_CALL RDel    (ICodeContext * _ctx, const char * key, const char * options, unsigned __int64 database, const char * pswd, unsigned timeout);
+    ECL_REDIS_API void             ECL_REDIS_CALL RPersist(ICodeContext * _ctx, const char * key, const char * options, unsigned __int64 database, const char * pswd, unsigned timeout);
+    ECL_REDIS_API void             ECL_REDIS_CALL RExpire (ICodeContext * _ctx, const char * key, const char * options, unsigned expire, unsigned __int64 database, const char * pswd, unsigned timeout);
+    ECL_REDIS_API unsigned __int64 ECL_REDIS_CALL RDBSize (ICodeContext * _ctx, const char * options, unsigned __int64 database, const char * pswd, unsigned timeout);
+}
+}
+#endif

plugins/sqlite3/CMakeLists.txt

@@ -47,7 +47,7 @@ if (USE_SQLITE3)
     HPCC_ADD_LIBRARY( sqlite3embed SHARED ${SRCS} )
     if (${CMAKE_VERSION} VERSION_LESS "2.8.9")
       message("WARNING: Cannot set NO_SONAME. shlibdeps will give warnings when package is installed")
-    else()
+    elif(NOT APPLE)
       set_target_properties( sqlite3embed PROPERTIES NO_SONAME 1 )
     endif()
 

plugins/v8embed/CMakeLists.txt

@@ -48,7 +48,7 @@ if (USE_V8)
     HPCC_ADD_LIBRARY( v8embed SHARED ${SRCS} )
     if (${CMAKE_VERSION} VERSION_LESS "2.8.9")
       message("WARNING: Cannot set NO_SONAME. shlibdeps will give warnings when package is installed")
-    else()
+    elif(NOT APPLE)
       set_target_properties( v8embed PROPERTIES NO_SONAME 1 )
     endif()
 

roxie/ccd/ccdfile.cpp

@@ -2734,7 +2734,8 @@ protected:
 
         f = open("test.buddy", _O_WRONLY | _O_CREAT | _O_TRUNC, _S_IREAD | _S_IWRITE);
         val = 2;
-        write(f, &val, sizeof(int));
+        ssize_t numwritten = write(f, &val, sizeof(int));
+        CPPUNIT_ASSERT(numwritten == sizeof(int));
         close(f);
 
         // Reading it should still read 1...
@@ -2753,7 +2754,8 @@ protected:
         // And the data in the file should be 2
         f = open("test.local", _O_RDONLY);
         val = 0;
-        read(f, &val, sizeof(int));
+        ssize_t numread = read(f, &val, sizeof(int));
+        CPPUNIT_ASSERT(numread == sizeof(int));
         close(f);
         CPPUNIT_ASSERT(val==2);
 

system/jlib/jptree.cpp

@@ -6587,12 +6587,12 @@ public:
                     readChild(tagName.str(), true);
                     break;
                 case '{':  //treat unnamed objects like we're in a noroot array
-                    readObject("__item__");
+                    readObject("__object__");
                     break;
                 case '[':  //treat unnamed arrays like we're in a noroot array
-                    iEvent->beginNode("__item__", curOffset);
+                    iEvent->beginNode("__array__", curOffset);
                     readArray("__item__");
-                    iEvent->endNode("__item__", 0, "", false, curOffset);
+                    iEvent->endNode("__array__", 0, "", false, curOffset);
                     break;
                 default:
                     expecting("{[ or \"");
@@ -6775,11 +6775,11 @@ public:
         }
     }
 
-    inline const char *arrayItemName()
+    inline const char *arrayItemName(const char *defaultName)
     {
         if (stack.ordinality()>1)
             return stateInfo->wnsTag;
-        return "__item__";
+        return defaultName;
     }
 
     bool arrayItem(offset_t offset)
@@ -6797,18 +6797,18 @@ public:
         case '{':
             state=objAttributes;
             readNext();
-            beginNode(arrayItemName(), offset, elementTypeObject);
+            beginNode(arrayItemName("__object__"), offset, elementTypeObject);
             break;
         case '[':
             state=valueStart;
             readNext();
-            beginNode(arrayItemName(), offset, elementTypeArray, true);
+            beginNode(arrayItemName("__array__"), offset, elementTypeArray, true);
             break;
         default:
             state=valueStart;
             ptElementType type = readValue(value.clear());
             readNext();
-            beginNode(arrayItemName(), offset, type, true);
+            beginNode(arrayItemName("__item__"), offset, type, true);
             stateInfo->tagText.swapWith(value);
             break;
         }

system/jlib/jutil.cpp

@@ -27,6 +27,7 @@
 #include "jmutex.hpp"
 #include "jfile.hpp"
 #include "jprop.hpp"
+#include "jerror.hpp"
 #ifdef _WIN32
 #include <mmsystem.h> // for timeGetTime 
 #include <float.h> //for _isnan and _fpclass
@@ -1571,6 +1572,13 @@ unsigned __int64 greatestCommonDivisor(unsigned __int64 left, unsigned __int64 r
     }
 }
 
+//The whole point of this function is to force memory to be accessed on the stack to avoid page faults.
+//Therefore disable the gcc warning.
+#ifdef __GNUC__
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wuninitialized"
+#endif
+
 //In a separate module to stop optimizer removing the surrounding catch.
 void doStackProbe()
 {
@@ -1579,6 +1587,10 @@ void doStackProbe()
     byte forceload = x[-4096];
 }
 
+#ifdef __GNUC__
+#pragma GCC diagnostic pop
+#endif
+
 #ifdef _WIN32
 
 DWORD dwTlsIndex = -1;
@@ -1914,14 +1926,18 @@ public:
     {
         saveuid = geteuid();
         savegid = getegid();
-        setegid(gid);
-        seteuid(uid);
+        if (setegid(gid) == -1)
+            throw makeOsException(errno, "Failed to set effective group id");
+        if (seteuid(uid) == -1)
+            throw makeOsException(errno, "Failed to set effective user id");
     }
 
     void revert()
     {
-        seteuid(saveuid);
-        setegid(savegid);
+        if (seteuid(saveuid) == -1)
+            throw makeOsException(errno, "Failed to restore effective group id");
+        if (setegid(savegid) == -1)
+            throw makeOsException(errno, "Failed to restore effective user id");
     }
 
     const char *username()

system/xmllib/libxslt_processor.cpp

@@ -823,7 +823,7 @@ void globalLibXsltExtensionHandler(xmlXPathParserContextPtr ctxt, int nargs)
             msg.append(" without any arguments\n");
         else
             msg.append(" with too many arguments\n");
-        xsltGenericError(xsltGenericErrorContext, msg.str());
+        xsltGenericError(xsltGenericErrorContext, "%s", msg.str());
         ctxt->error = XPATH_INVALID_ARITY;
         return;
     }

testing/regress/README.rst

@@ -594,6 +594,11 @@ The format of the output is the same as 'run', except there is a log, result and
     Optionally 'no<target>' exclusion info can add at the end of tag.
 //version <n1>=<v1>,<n2>=<v2>,...[,no<target>[,no<target>]]
 
+    This tag should use when a test case intentionally fails to handle it as pass.
+    If a test case intentionally fails then it should fail on all allowed platforms.
+//fail
+
+
 7. Key file handling:
 ---------------------
 

testing/regress/ecl-test

@@ -81,10 +81,10 @@ class RegressMain:
                 if len(eclfiles) :
                     #Execute multiple ECL files like RUN to generates summary results and diff report.
                     self.regress.bootstrap(cluster, self.args,  eclfiles)
-                    if  self.args.pq:
-                        self.regress.runSuiteP(cluster, self.regress.suites[cluster])
-                    else:
+                    if (self.args.pq in (0, 1)) or (len(eclfiles) == 1):
                         self.regress.runSuite(cluster, self.regress.suites[cluster])
+                    else:
+                        self.regress.runSuiteP(cluster, self.regress.suites[cluster])
                 else:
                     logging.error("%s. No ECL file match for cluster:'%s'!" % (1,  self.args.target))
                     raise Error("4001")

testing/regress/ecl/cppbody11.ecl

@@ -16,7 +16,7 @@
 ############################################################################## */
 
 
-real scoreFunc(real a, real b) := 1;
+real scoreFunc(real a, real b) := 1;  // defines the prototype for the function argument
 
 real scoreIt(scoreFunc func, real a, real b) := DEFINE
     
@@ -24,7 +24,7 @@ real scoreIt(scoreFunc func, real a, real b) := DEFINE
     
 real scoreIt2(scoreFunc func, real a, real b) := BEGINC++
     
-    return func(a-1.0,b-1.0) * func(a+1.0,b+1.0);
+    return func(ctx, a-1.0,b-1.0) * func(ctx, a+1.0,b+1.0);
     
 ENDC++;
     

testing/regress/ecl/cppbody12.ecl

@@ -0,0 +1,36 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2014 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+real storedval := 1 : stored('s');
+
+real scoreFunc(real a, real b) := 1;   // defines the prototype for the function argument
+
+real scoreIt(scoreFunc func, real a, real b) := BEGINC++
+    return func(ctx,a-1.0,b-1.0) * func(ctx,a+1.0,b+1.0);
+ENDC++;
+    
+real doSum(real a, real b) := DEFINE (a + b + storedval);
+
+ds := DATASET(100, transform({unsigned id}, SELF.id := COUNTER));
+s := SORT(ds, HASH(id));
+c := COUNT(NOFOLD(s));
+
+real doSum2(real a, real b) := DEFINE (a + b + c);
+
+output(scoreIt(doSum, 10, 20));
+output(scoreIt(doSum2, 100, 200));
+output(scoreIt(doSum2, 1, 3));

testing/regress/ecl/dbz2a.ecl

@@ -18,6 +18,7 @@
 //nothor
 //nothorlcr
 //noroxie
+//fail
 
 //Test division by zero - default action to return 0
 #option ('divideByZero', 'fail'); 

testing/regress/ecl/dbz2b.ecl

@@ -18,6 +18,7 @@
 //nothor
 //nothorlcr
 //noroxie
+//fail
 
 //Test division by zero - default action to return 0
 #option ('divideByZero', 'fail'); 

testing/regress/ecl/dbz2c.ecl

@@ -18,6 +18,7 @@
 //nothor
 //nothorlcr
 //noroxie
+//fail
 
 //Test division by zero - default action to return 0
 #option ('divideByZero', 'fail'); 

testing/regress/ecl/key/cppbody12.xml

@@ -0,0 +1,9 @@
+<Dataset name='Result 1'>
+ <Row><Result_1>957.0</Result_1></Row>
+</Dataset>
+<Dataset name='Result 2'>
+ <Row><Result_2>159996.0</Result_2></Row>
+</Dataset>
+<Dataset name='Result 3'>
+ <Row><Result_3>10812.0</Result_3></Row>
+</Dataset>

testing/regress/ecl/key/redissynctest.xml

@@ -0,0 +1,57 @@
+<Dataset name='Result 1'>
+ <Row><Result_1>true</Result_1></Row>
+</Dataset>
+<Dataset name='Result 2'>
+ <Row><Result_2>3.14159265359</Result_2></Row>
+</Dataset>
+<Dataset name='Result 3'>
+ <Row><Result_3>9.869604401090658</Result_3></Row>
+</Dataset>
+<Dataset name='Result 4'>
+ <Row><Result_4>123456789</Result_4></Row>
+</Dataset>
+<Dataset name='Result 5'>
+ <Row><Result_5>9.869604401090658</Result_5></Row>
+</Dataset>
+<Dataset name='Result 6'>
+ <Row><Result_6>123456789</Result_6></Row>
+</Dataset>
+<Dataset name='Result 7'>
+ <Row><Result_7>7</Result_7></Row>
+</Dataset>
+<Dataset name='Result 8'>
+ <Row><Result_8>supercalifragilisticexpialidocious</Result_8></Row>
+</Dataset>
+<Dataset name='Result 9'>
+ <Row><Result_9>אבגדהוזחטיךכלםמןנסעףפץצקרשת</Result_9></Row>
+</Dataset>
+<Dataset name='Result 10'>
+ <Row><Result_10>אבגדהוזחטיךכלםמןנסעףפץצקרשת</Result_10></Row>
+</Dataset>
+<Dataset name='Result 11'>
+ <Row><Result_11>D790D791D792D793D794D795D796D798D799D79AD79BD79CD79DD79DD79ED79FD7A0D7A1D7A2D7A3D7A4D7A5D7A6D7A7D7A8D7A9D7AA</Result_11></Row>
+</Dataset>
+<Dataset name='Result 12'>
+ <Row><Result_12>true</Result_12></Row>
+</Dataset>
+<Dataset name='Result 13'>
+ <Row><Result_13>false</Result_13></Row>
+</Dataset>
+<Dataset name='Result 14'>
+ <Row><Result_14>4614256656552046314</Result_14></Row>
+</Dataset>
+<Dataset name='Result 15'>
+ <Row><Result_15>6</Result_15></Row>
+</Dataset>
+<Dataset name='Result 16'>
+ <Row><Result_16>1</Result_16></Row>
+</Dataset>
+<Dataset name='Result 17'>
+ <Row><Result_17>0</Result_17></Row>
+</Dataset>
+<Dataset name='Result 18'>
+ <Row><Result_18>false</Result_18></Row>
+</Dataset>
+<Dataset name='Result 19'>
+ <Row><Result_19>300</Result_19></Row>
+</Dataset>

testing/regress/ecl/memcachedtest.ecl

@@ -20,42 +20,42 @@ IMPORT memcached FROM lib_memcached;
 STRING servers := '--SERVER=127.0.0.1:11211';
 memcached.Clear(servers);
 
-memcached.SetBoolean(servers, 'b', TRUE);
-memcached.GetBoolean(servers, 'b');
+memcached.SetBoolean('b', TRUE, servers);
+memcached.GetBoolean('b', servers);
 
 REAL pi := 3.14159265359;
-memcached.SetReal(servers, 'pi', pi);
-memcached.GetReal(servers, 'pi');
+memcached.SetReal('pi', pi, servers);
+memcached.GetReal('pi', servers);
 
 INTEGER i := 123456789;
-memcached.SetInteger(servers, 'i', i);
-memcached.GetInteger(servers, 'i');
+memcached.SetInteger('i', i, servers);
+memcached.GetInteger('i', servers);
 
 UNSIGNED u := 7;
-memcached.SetUnsigned(servers, 'u', u);
-memcached.GetUnsigned(servers, 'u');
+memcached.SetUnsigned('u', u, servers);
+memcached.GetUnsigned('u', servers);
 
 STRING str  := 'supercalifragilisticexpialidocious';
-memcached.SetString(servers, 'str', str);
-memcached.GetString(servers, 'str');
+memcached.SetString('str', str, servers);
+memcached.GetString('str', servers);
 
 UNICODE uni := U'אבגדהוזחטיךכלםמןנסעףפץצקרשת';
-memcached.SetUnicode(servers, 'uni', uni);
-memcached.GetUnicode(servers, 'uni');
+memcached.SetUnicode('uni', uni, servers);
+memcached.GetUnicode('uni', servers);
 
 UTF8 utf := U'אבגדהוזחטיךכלםמןנסעףפץצקרשת';
-memcached.SetUtf8(servers, 'utf8', utf);
-memcached.GetUtf8(servers, 'utf8');
+memcached.SetUtf8('utf8', utf, servers);
+memcached.GetUtf8('utf8', servers);
 
 DATA mydata := x'd790d791d792d793d794d795d796d798d799d79ad79bd79cd79dd79dd79ed79fd7a0d7a1d7a2d7a3d7a4d7a5d7a6d7a7d7a8d7a9d7aa';
-memcached.SetData(servers, 'data', mydata);
-memcached.GetData(servers,'data');
+memcached.SetData('data', mydata, servers);
+memcached.GetData('data', servers);
 
-memcached.Exists(servers, 'utf8');
-memcached.KeyType(servers,'utf8');
+memcached.Exists('utf8', servers);
+memcached.KeyType('utf8', servers);
 
 //The following test some exceptions
-memcached.GetInteger(servers, 'pi');
+memcached.GetInteger('pi', servers);
 memcached.Clear(servers);
-memcached.Exists(servers, 'utf8');
-memcached.KeyType(servers,'utf8');
+memcached.Exists('utf8', servers);
+memcached.KeyType('utf8', servers);

testing/regress/ecl/redissynctest.ecl

@@ -0,0 +1,100 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2015 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+IMPORT sync FROM lib_redis;
+
+STRING server := '--SERVER=127.0.0.1:6379';
+STRING password := 'foobared';
+sync.FlushDB(server, /*database*/, password);
+
+sync.SetBoolean('b', TRUE, server, /*database*/, /*expire*/, password);
+sync.GetBoolean('b', server, /*database*/, password);
+
+IMPORT redisSync FROM lib_redis;
+myRedis := redisSync(server, password);
+
+REAL pi := 3.14159265359;
+myRedis.SetReal('pi', pi);
+myRedis.GetReal('pi');
+
+REAL pi2 := pi*pi;
+myRedis.SetReal('pi', pi2, 1);
+myRedis.GetReal('pi', 1);
+
+INTEGER i := 123456789;
+myRedis.SetInteger('i', i);
+myRedis.GetInteger('i');
+
+myRedis2 := redisSync('--SERVER=127.0.0.1:6380', 'youarefoobared');
+
+myRedis2.SetReal('pi', pi2, 1);
+myRedis2.GetReal('pi', 1);
+
+myRedis2.SetInteger('i', i);
+myRedis2.GetInteger('i');
+
+UNSIGNED u := 7;
+myRedis.SetUnsigned('u', u);
+myRedis.GetUnsigned('u');
+
+STRING str  := 'supercalifragilisticexpialidocious';
+myRedis.SetString('str', str);
+myRedis.GetString('str');
+
+UNICODE uni := U'אבגדהוזחטיךכלםמןנסעףפץצקרשת';
+myRedis.setUnicode('uni', uni);
+myRedis.getUnicode('uni');
+
+UTF8 utf := U'אבגדהוזחטיךכלםמןנסעףפץצקרשת';
+myRedis.SetUtf8('utf8', utf);
+myRedis.GetUtf8('utf8');
+
+DATA mydata := x'd790d791d792d793d794d795d796d798d799d79ad79bd79cd79dd79dd79ed79fd7a0d7a1d7a2d7a3d7a4d7a5d7a6d7a7d7a8d7a9d7aa';
+myRedis.SetData('data', mydata);
+myRedis.GetData('data');
+
+SEQUENTIAL(
+    myRedis.Exists('utf8'),
+    myRedis.Del('utf8'),
+    myRedis.Exists('uft8')
+    );
+
+myRedis.Expire('str', 1); 
+myRedis.Persist('str');
+
+myRedis.GetInteger('pi');
+
+NOFOLD(myRedis.DBSize());
+NOFOLD(myRedis.DBSize(1));
+NOFOLD(myRedis.DBSize(2));
+
+SEQUENTIAL(
+    myRedis.FlushDB(),
+    NOFOLD(myRedis.Exists('str'))
+    );
+myRedis2.FlushDB();
+
+//The follwoing tests the multithreaded caching of the redis connections
+//SUM(NOFOLD(s1 + s2), a) uses two threads
+myRedis.FlushDB();
+INTEGER x := 2;
+INTEGER N := 100;
+myRedis.SetInteger('i', x);
+s1 :=DATASET(N, TRANSFORM({ integer a }, SELF.a := NOFOLD(myRedis.GetInteger('i'))));
+s2 :=DATASET(N, TRANSFORM({ integer a }, SELF.a := NOFOLD(myRedis.GetInteger('i'))/2));
+SUM(NOFOLD(s1 + s2), a);//answer = (x+x/2)*N, in this case 3N.
+myRedis.FlushDB();

testing/regress/hpcc/regression/regress.py

@@ -204,6 +204,7 @@ class Regression:
         self.taskParam = [{'taskId':0,  'jobName':'',  'timeoutValue':0,  'retryCount': 0} for i in range(self.maxthreads)]
         self.goodStates = ('compiling', 'blocked')
 
+        logging.debug("runSuiteP(name:'%s', suite:'%s')" %  (name,  suite.getSuiteName()))
         logging.warn("Suite: %s ",  name)
         logging.warn("Queries: %s" % repr(len(suite.getSuite())))
         logging.warn('%s','' , extra={'filebuffer':True,  'filesort':True})
@@ -232,9 +233,9 @@ class Regression:
                         else:
                             self.timeouts[threadId] = self.timeout
 
-                        self.taskParam[threadId]['timeoutValue'] = self.timeouts[threadId]
+                        self.taskParam[threadId]['timeoutValue'] = self.timeout
                         query = suiteItems[self.taskParam[threadId]['taskId']]
-                        query.setTimeout(self.timeouts[threadId])
+                        query.setTimeout(self.timeout)
                         #logging.debug("self.timeout[%d]:%d", threadId, self.timeouts[threadId])
                         sysThreadId = thread.start_new_thread(self.runQuery, (cluster, query, report, cnt, suite.testPublish(query.ecl),  threadId))
                         time.sleep(0.4)
@@ -265,7 +266,7 @@ class Regression:
                                         self.loggermutex.acquire()
                                         query = suiteItems[self.taskParam[threadId]['taskId']]
                                         query.setAborReason('Timeout and retry count exhausted!')
-                                        logging.info("%3d. Timeout occured and no more attempt. Force to abort... " % (self.taskParam[threadId]['taskId']),  extra={'taskId':self.taskParam[threadId]['taskId']+1})
+                                        logging.info("%3d. Timeout occured and no more attempt left. Force to abort... " % (self.taskParam[threadId]['taskId']),  extra={'taskId':self.taskParam[threadId]['taskId']+1})
                                         logging.debug("%3d. Task parameters: thread id:%d, wuid:'%s', state:'%s', ecl:'%s'." % (self.taskParam[threadId]['taskId']+1, threadId, wuid['wuid'], wuid['state'],  query.ecl),  extra={'taskId':self.taskParam[threadId]['taskId']+1})
                                         self.loggermutex.release()
                                     else:
@@ -273,7 +274,7 @@ class Regression:
                                         self.loggermutex.acquire()
                                         query = suiteItems[self.taskParam[threadId]['taskId']]
                                         query.setAborReason('Timeout (does not started yet and retry count exhausted)')
-                                        logging.info("%3d. Timeout occured and no more attempt. Force to abort... " % (self.taskParam[threadId]['taskId']),  extra={'taskId':self.taskParam[threadId]['taskId']+1})
+                                        logging.info("%3d. Timeout occured and no more attempt left. Force to abort... " % (self.taskParam[threadId]['taskId']),  extra={'taskId':self.taskParam[threadId]['taskId']+1})
                                         logging.debug("%3d. Task parameters: thread id:%d, wuid:'%s', state:'%s', ecl:'%s'." % (self.taskParam[threadId]['taskId']+1, threadId, wuid['wuid'], wuid['state'],  query.ecl),  extra={'taskId':self.taskParam[threadId]['taskId']+1})
                                         self.loggermutex.release()
 
@@ -301,8 +302,13 @@ class Regression:
                 # give some time to other threads
                 time.sleep(0.4)
 
-            for mutex in self.exitmutexes:
-                while mutex.locked(): pass
+            # All tasks are scheduled
+            #Some of them finished, others are not yet, but should check the still running tasks' timeout and retry state
+            for threadId in range(self.maxthreads):
+                    if self.exitmutexes[threadId].locked():
+                        query = suiteItems[self.taskParam[threadId]['taskId']]
+                        self.retryCount = int(self.config.maxAttemptCount)
+                        self.CheckTimeout(self.taskParam[threadId]['taskId']+1, threadId,  query)
 
             self.StopTimeoutThread()
             logging.warn('%s','' , extra={'filebuffer':True,  'filesort':True})
@@ -327,25 +333,35 @@ class Regression:
         self.timeoutHandlerEnabled=True
         self.timeoutThread.start()
 
-    def CheckTimeout(self, cnt,  thread,  query):
-        while  self.exitmutexes[thread].locked():
+    def CheckTimeout(self, cnt,  threadId,  query):
+        while  self.exitmutexes[threadId].locked():
             sleepTime = 1.0
-            if self.timeouts[thread] >= 0:
+            if self.timeouts[threadId] >= 0:
                 self.loggermutex.acquire()
-                logging.debug("%3d. timeout counter:%d" % (cnt, self.timeouts[thread]),  extra={'taskId':cnt})
+                logging.debug("%3d. timeout counter:%d" % (cnt, self.timeouts[threadId]),  extra={'taskId':cnt})
                 self.loggermutex.release()
                 sleepTime = 1.0
-            if self.timeouts[thread] == 0:
-                # time out, abort tast
+            if self.timeouts[threadId] == 0:
                 wuid =  queryWuid(query.getJobname(),  query.getTaskId())
-                logging.debug("%3d. Abort WUID:'%s'" % (cnt,  str(wuid)),  extra={'taskId':cnt})
-                abortWorkunit(wuid['wuid'])
-                query.setAborReason('Timeout')
-                self.loggermutex.acquire()
-                logging.debug("%3d. Waiting for abort..." % (cnt),  extra={'taskId':cnt})
-                self.loggermutex.release()
-                self.timeouts[thread] = -1
-                sleepTime = 1.0
+                self.retryCount -= 1;
+                if self.retryCount> 0:
+                    self.timeouts[threadId] =  self.timeout
+                    self.loggermutex.acquire()
+                    logging.warn("%3d. Does not started yet. Reset timeout to %d sec (%d retry attempt(s) remains)." % (cnt, self.timeouts[threadId],  self.retryCount),  extra={'taskId':cnt})
+                    logging.debug("%3d. Task parameters: thread id: %d, ecl:'%s',state:'%s', retry count:%d." % (cnt, threadId,  query.ecl,   wuid['state'],  self.retryCount),  extra={'taskId':cnt})
+                    self.loggermutex.release()
+                else:
+                    # retry counter exhausted, give up and abort this test case if exists
+                    logging.debug("%3d. Abort WUID:'%s'" % (cnt,  str(wuid)),  extra={'taskId':cnt})
+                    abortWorkunit(wuid['wuid'])
+                    query.setAborReason('Timeout and retry count exhausted!')
+                    self.loggermutex.acquire()
+                    logging.error("%3d. Timeout occured and no more attempt left. Force to abort... " % (cnt),  extra={'taskId':cnt})
+                    logging.debug("%3d. Task parameters: wuid:'%s', state:'%s', ecl:'%s'." % (cnt, wuid['wuid'], wuid['state'],  query.ecl),  extra={'taskId':cnt})
+                    logging.debug("%3d. Waiting for abort..." % (cnt),  extra={'taskId':cnt})
+                    self.loggermutex.release()
+                    self.timeouts[threadId] = -1
+                    sleepTime = 1.0
             time.sleep(sleepTime)
 
     def StopTimeoutThread(self):
@@ -366,6 +382,7 @@ class Regression:
 
         report = self.buildLogging(logName)
 
+        logging.debug("runSuite(name:'%s', suite:'%s')" %  (name,  suite.getSuiteName()))
         logging.warn("Suite: %s" % name)
         logging.warn("Queries: %s" % repr(len(suite.getSuite())))
         suite.setStarTime(time.time())
@@ -383,7 +400,7 @@ class Regression:
                    self.timeouts[th] = timeout
                 else:
                     self.timeouts[th] = self.timeout
-
+                self.retryCount = int(self.config.maxAttemptCount)
                 query.setTimeout(self.timeouts[th])
                 thread.start_new_thread(self.runQuery, (cluster, query, report, cnt, suite.testPublish(query.ecl),  th))
                 time.sleep(0.1)
@@ -431,6 +448,7 @@ class Regression:
                 self.timeouts[threadId] = timeout
             else:
                 self.timeouts[threadId] = self.timeout
+            self.retryCount = int(self.config.maxAttemptCount)
             sysThreadId = thread.start_new_thread(self.runQuery, (cluster, eclfile, report, cnt, eclfile.testPublish(),  threadId))
             time.sleep(0.1)
             self.CheckTimeout(cnt, threadId,  eclfile)

testing/regress/hpcc/util/ecl/command.py

@@ -114,8 +114,8 @@ class ECLcmd(Shell):
             logging.error("------" + err + "------")
             raise err
         finally:
+            res = queryWuid(eclfile.getJobname(), eclfile.getTaskId())
             if wuid ==  'N/A':
-                res = queryWuid(eclfile.getJobname(), eclfile.getTaskId())
                 logging.debug("%3d. in finally queryWuid() -> 'result':'%s', 'wuid':'%s', 'state':'%s'", eclfile.getTaskId(),  res['result'],  res['wuid'],  res['state'])
                 wuid = res['wuid']
                 if res['result'] != "OK":
@@ -130,13 +130,16 @@ class ECLcmd(Shell):
                     test = False
                     eclfile.diff = 'Error'
             else:
-                if queryWuid(eclfile.getJobname(), eclfile.getTaskId())['state'] == 'aborted':
+                if (res['state'] == 'aborted') or eclfile.isAborted():
                     eclfile.diff = ("%3d. Test: %s\n") % (eclfile.taskId, eclfile.getBaseEclRealName())
                     eclfile.diff += '\t'+'Aborted ( reason: '+eclfile.getAbortReason()+' )'
                     test = False
                 elif eclfile.getIgnoreResult():
                     logging.debug("%3d. Ignore result (ecl:'%s')", eclfile.getTaskId(),  eclfile.getBaseEcl())
                     test = True
+                elif eclfile.testFail():
+                    logging.debug("%3d. Fail is the expected result (ecl:'%s')", eclfile.getTaskId(),  eclfile.getBaseEcl())
+                    test = True
                 elif eclfile.testNoKey():
                     # keyfile comparaison disabled with //nokey tag
                     if eclfile.testNoOutput():

testing/regress/hpcc/util/ecl/file.py

@@ -41,6 +41,7 @@ class ECLFile:
     wuid = None
     elapsTime = 0
     jobname = ''
+    aborted = False
     abortReason = ''
     taskId = -1
     ignoreResult=False
@@ -69,6 +70,7 @@ class ECLFile:
         self.xml_a = 'archive_' + self.baseXml
         self.jobname = self.basename
         self.diff = ''
+        self.aborted = False
         self.abortReason =''
         self.tags={}
         self.tempFile=None
@@ -231,10 +233,10 @@ class ECLFile:
         return realName
 
     def getWuid(self):
-        return self.wuid
+        return self.wuid.strip()
 
     def setWuid(self,  wuid):
-        self.wuid = wuid
+        self.wuid = wuid.strip()
 
     def addResults(self, results, wuid):
         filename = self.getResults()
@@ -339,6 +341,15 @@ class ECLFile:
         logging.debug("%3d. testInClass() returns with: %s",  self.taskId,  retVal)
         return retVal
 
+    def testFail(self):
+        # Standard string has a problem with unicode characters
+        # use byte arrays and binary file open instead
+        tag = b'//fail'
+        logging.debug("%3d. testFail(ecl:'%s', tag:'%s')", self.taskId, self.ecl,  tag)
+        retVal = self.__checkTag(tag)
+        logging.debug("%3d. testFail() returns with: %s",  self.taskId,  retVal)
+        return retVal
+
     # Test (and read all) //version tag in the ECL file
     def testVesion(self):
         if self.isVersions == False and not self.args.noversion:
@@ -437,6 +448,10 @@ class ECLFile:
 
     def setAborReason(self,  reason):
         self.abortReason = reason
+        self.aborted = True
+
+    def isAborted(self):
+        return self.aborted
 
     def getAbortReason(self):
         return self.abortReason