Merge remote-tracking branch 'origin/candidate-5.2.0'

Signed-off-by: Gavin Halliday <gavin.halliday@lexisnexis.com>

Conflicts:
	ecl/hql/hqlexpr.cpp
	version.cmake

cmake_modules/FindMEMCACHED.cmake

@@ -0,0 +1,52 @@
+################################################################################
+#    HPCC SYSTEMS software Copyright (C) 2014 HPCC Systems.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+################################################################################
+
+# - Try to find the libmemcached library
+# Once done this will define
+#
+#  LIBMEMCACHED_FOUND - system has the libmemcached library
+#  LIBMEMCACHED_INCLUDE_DIR - the libmemcached include directory(s)
+#  LIBMEMCACHED_LIBRARIES - The libraries needed to use libmemcached
+
+IF (NOT LIBMEMCACHED_FOUND)
+  IF (WIN32)
+    SET (libmemcached_lib "libmemcached")
+    SET (libmemcachedUtil_lib "libmemcachedutil")
+  ELSE()
+    SET (libmemcached_lib "memcached")
+    SET (libmemcachedUtil_lib "memcachedutil")
+  ENDIF()
+
+  FIND_PATH(LIBMEMCACHED_INCLUDE_DIR memcached.hpp PATHS /usr/include /usr/share/include /usr/local/include PATH_SUFFIXES libmemcached)
+
+  FIND_LIBRARY (LIBMEMCACHEDCORE_LIBRARY NAMES ${libmemcached_lib} PATHS /usr/lib usr/lib/libmemcached /usr/share /usr/lib64 /usr/local/lib /usr/local/lib64)
+  FIND_LIBRARY (LIBMEMCACHEDUTIL_LIBRARY NAMES ${libmemcachedUtil_lib} PATHS /usr/lib /usr/share /usr/lib64 /usr/local/lib /usr/local/lib64)
+
+  #IF (LIBMEMCACHED_LIBRARY STREQUAL "LIBMEMCACHED_LIBRARY-NOTFOUND")
+  #  SET (LIBMEMCACHEDCORE_LIBRARY "")    # Newer versions of libmemcached are header-only, with no associated library.
+  #ENDIF()
+
+  SET (LIBMEMCACHED_LIBRARIES ${LIBMEMCACHEDCORE_LIBRARY} ${LIBMEMCACHEDUTIL_LIBRARY})
+
+  include(FindPackageHandleStandardArgs)
+  find_package_handle_standard_args(memcached DEFAULT_MSG
+    LIBMEMCACHED_LIBRARIES
+    LIBMEMCACHED_INCLUDE_DIR
+  )
+
+  MARK_AS_ADVANCED(LIBMEMCACHED_INCLUDE_DIRS LIBMEMCACHED_LIBRARIES)
+ENDIF()
+

cmake_modules/commonSetup.cmake

@@ -82,10 +82,11 @@ IF ("${COMMONSETUP_DONE}" STREQUAL "")
   option(USE_V8 "Enable V8 JavaScript support" ON)
   option(USE_JNI "Enable Java JNI support" ON)
   option(USE_RINSIDE "Enable R support" ON)
+  option(USE_MEMCACHED "Enable Memcached support" ON)
 
   option(USE_OPTIONAL "Automatically disable requested features with missing dependencies" ON)
 
-  if ( USE_PYTHON OR USE_V8 OR USE_JNI OR USE_RINSIDE OR USE_SQLITE3 OR USE_MYSQL OR USE_CASSANDRA)
+  if ( USE_PYTHON OR USE_V8 OR USE_JNI OR USE_RINSIDE OR USE_SQLITE3 OR USE_MYSQL OR USE_CASSANDRA OR USE_MEMCACHED)
       set( WITH_PLUGINS ON )
   endif()
 

dali/base/dasds.cpp

@@ -8173,7 +8173,8 @@ public:
         INIT_NAMEDCOUNT;
         SDSManager->querySubscriberTable().getSubscribers(subs);
     }
-    bool match(const char *head, const char *path, bool &sub)
+    enum SubCommitType { subCommitNone, subCommitExact, subCommitBelow, subCommitAbove };
+    SubCommitType match(const char *head, const char *path)
     {
         bool wild = false;
         loop
@@ -8194,30 +8195,23 @@ public:
                 wild = true;
             }
             else if (*head != *path)
-                return false;
+                return subCommitNone;
             else
                 path++;
 
             head++;
             if ('\0' == *path)
             {
-                if ('\0' == *head) // absolute match
-                {
-                    sub = false;
-                    return true;
-                }
+                if ('\0' == *head)
+                    return subCommitExact; // absolute match
                 else if (!wild && '/' != *head) // e.g. change=/a/bc, subscriber=/a/b
-                    return false;
-                sub = true; // e.g. change=/a/b/c, subscriber=/a/b
-                return true;
+                    return subCommitNone;
+                return subCommitBelow; // e.g. change=/a/b/c, subscriber=/a/b
             }
             else 
             {
-                if ('\0' == *head) // e.g. change=/a/b, subscriber=/a/b/c - not matched yet, but returning true keeps it from being pruned
-                {
-                    sub = false;
-                    return true;
-                }
+                if ('\0' == *head)
+                    return subCommitAbove; // e.g. change=/a/b, subscriber=/a/b/c - not matched yet, but returning true keeps it from being pruned
             }
         }
     }
@@ -8231,22 +8225,45 @@ public:
         scan(*rootChanges, stack, pruned);
     }
 
-    bool prune(const char *xpath, bool &sub, CSubscriberCopyArray *matches, CSubscriberArray &pruned)
+    bool prune(const char *xpath, CSubscriberCopyArray &candidates, CSubscriberArray &pruned)
     {
-        sub = false;
         ForEachItemInRev(s, subs)
         {
             CSubscriberContainer &subscriber = subs.item(s);
-            bool _sub; // false = (xpath NOT below subscriber), (true = xpath equals or is below subscriber)
-            if (subscriber.isUnsubscribed() || !match(xpath, subscriber.queryXPath(), _sub))
+            SubCommitType subCommit;
+            if (subscriber.isUnsubscribed())
+                subCommit = subCommitNone;
+            else
+                subCommit = match(xpath, subscriber.queryXPath());
+            switch (subCommit)
             {
-                pruned.append(*LINK(&subscriber));
-                subs.remove(s);
+                case subCommitNone:
+                {
+                    pruned.append(*LINK(&subscriber));
+                    subs.remove(s);
+                    break;
+                }
+                case subCommitExact:
+                {
+                    candidates.append(subscriber);
+                    break;
+                }
+                case subCommitBelow: // e.g. change=/a/b/c, subscriber=/a/b
+                {
+                    if (!subscriber.querySub())
+                    {
+                        pruned.append(*LINK(&subscriber));
+                        subs.remove(s);
+                    }
+                    else
+                        candidates.append(subscriber);
+                    break;
+                }
+                case subCommitAbove: // e.g. change=/a/b, subscriber=/a/b/c
+                    break; // keep in subs, deeper changes may match
+                default:
+                    throwUnexpected();
             }
-            else if (_sub)
-                sub = true;
-            else if (matches)
-                matches->append(subscriber);
         }
         return (subs.ordinality() > 0);
     }
@@ -8254,15 +8271,15 @@ public:
     // recurse down all matching subscription stubs while qualified
     void scanAll(PDState state, CBranchChange &changes, CPTStack &stack, CSubscriberArray &pruned)
     {
-        bool sub;
-        if (prune(xpath.str(), sub, NULL, pruned))
+        CSubscriberCopyArray candidates;
+        if (prune(xpath.str(), candidates, pruned))
         {
             MemoryBuffer notifyData;
-            if (sub)
+            if (candidates.ordinality())
             {
-                ForEachItemInRev(s, subs)
+                ForEachItemInRev(s, candidates)
                 {
-                    CSubscriberContainer &subscriber = subs.item(s);
+                    CSubscriberContainer &subscriber = candidates.item(s);
                     if (!subscriber.isUnsubscribed())
                     {
                         if (subscriber.qualify(stack, true))
@@ -8274,7 +8291,7 @@ public:
                         else
                             pruned.append(*LINK(&subscriber));
                     }
-                    subs.remove(s);
+                    subs.zap(subscriber);
                 }
             }
             else
@@ -8320,9 +8337,8 @@ public:
 
     void scan(CBranchChange &changes, CPTStack &stack, CSubscriberArray &pruned)
     {
-        CSubscriberCopyArray matches;
-        bool sub;
-        if (!prune(xpath.str(), sub, &matches, pruned))
+        CSubscriberCopyArray candidates;
+        if (!prune(xpath.str(), candidates, pruned))
             return;
 
         PushPop pp(stack, *changes.tree);
@@ -8331,7 +8347,7 @@ public:
             scanAll(changes.local, changes, stack, pruned);
             return;
         }
-        else if (matches.ordinality()) // xpath matched some subscribers, and/or below some, need to check for sub subscribers
+        else if (candidates.ordinality()) // xpath matched some subscribers, and/or below some, need to check for sub subscribers
         {
             bool ret = false;
             // avoid notifying on PDS_Structure only, which signifies changes deeper down only
@@ -8339,9 +8355,9 @@ public:
             if (changes.state && changes.local && (changes.local != PDS_Structure))
             {
                 int lastSendValue = -1;
-                ForEachItemInRev(s, matches)
+                ForEachItemInRev(s, candidates)
                 {
-                    CSubscriberContainer &subscriber = matches.item(s);
+                    CSubscriberContainer &subscriber = candidates.item(s);
                     if (!subscriber.isUnsubscribed())
                     {
                         if (subscriber.qualify(stack, false))
@@ -8372,23 +8388,7 @@ public:
                     subs.zap(subscriber);
                 }
             }
-            else
-            {
-                // remove non-sub subcribers at this level
-                ForEachItemInRev(s, subs)
-                {
-                    CSubscriberContainer &subscriber = subs.item(s);
-                    unsigned subDepth = subscriber.queryDepth();
-                    unsigned stackDepth = stack.ordinality();
-                    if ((!subscriber.querySub() && subDepth==stackDepth) || 0 == changes.children.ordinality())
-                    {
-                        pruned.append(*LINK(&subscriber));
-                        subs.remove(s);
-                    }
-                }
-            }
         }
-
         ForEachItemIn(c, changes.children)
         {
             CBranchChange &childChanges = changes.children.item(c);

ecl/eclcc/eclcc.cpp

@@ -2193,6 +2193,7 @@ const char * const helpText[] = {
     "! -fmaxCompileThreads     Number of compiler instances to compile the c++",
     "! -fnoteRecordSizeInGraph Add estimates of record sizes to the graph",
     "! -fpickBestEngine        Allow simple thor queries to be passed to thor",
+    "! -freportCppWarnings     Report warnings from c++ compilation",
     "! -fsaveCppTempFiles      Retain the generated c++ files",
     "! -fshowActivitySizeInGraph Show estimates of generated c++ size in the graph",
     "! -fshowMetaInGraph       Add distribution/sort orders to the graph",

ecl/hql/hqlerror.cpp

@@ -79,6 +79,8 @@ WarnErrorCategory getCategory(const char * category)
         return CategoryUnusual;
     if (strieq(category, "unexpected"))
         return CategoryUnexpected;
+    if (strieq(category, "cpp"))
+        return CategoryCpp;
     return CategoryUnknown;
 }
 
@@ -89,89 +91,24 @@ ErrorSeverity getCheckSeverity(IAtom * name)
     return severity;
 }
 
-
-//---------------------------------------------------------------------------------------------------------------------
-
-class HQL_API CECLError : public CInterfaceOf<IECLError>
-{
-public:
-    CECLError(WarnErrorCategory _category,ErrorSeverity _severity, int _no, const char* _msg, const char* _filename, int _lineno, int _column, int _position);
-
-    virtual int             errorCode() const { return no; }
-    virtual StringBuffer &  errorMessage(StringBuffer & ret) const { return ret.append(msg); }
-    virtual MessageAudience errorAudience() const { return MSGAUD_user; }
-    virtual const char* getFilename() const { return filename; }
-    virtual WarnErrorCategory getCategory() const { return category; }
-    virtual int getLine() const { return lineno; }
-    virtual int getColumn() const { return column; }
-    virtual int getPosition() const { return position; }
-    virtual StringBuffer& toString(StringBuffer&) const;
-    virtual ErrorSeverity getSeverity() const { return severity; }
-    virtual IECLError * cloneSetSeverity(ErrorSeverity _severity) const;
-
-protected:
-    ErrorSeverity severity;
-    WarnErrorCategory category;
-    int no;
-    StringAttr msg;
-    StringAttr filename;
-    int lineno;
-    int column;
-    int position;
-};
-
-CECLError::CECLError(WarnErrorCategory _category, ErrorSeverity _severity, int _no, const char* _msg, const char* _filename, int _lineno, int _column, int _position):
-  category(_category),severity(_severity), msg(_msg), filename(_filename)
-{
-    no = _no;
-    lineno = _lineno;
-    column = _column;
-    position = _position;
-}
-
-
-StringBuffer& CECLError::toString(StringBuffer& buf) const
-{
-    buf.append(filename);
-
-    if(lineno && column)
-        buf.append('(').append(lineno).append(',').append(column).append(')');
-    buf.append(" : ");
-
-    buf.append(no).append(": ").append(msg);
-    return buf;
-}
-
-IECLError * CECLError::cloneSetSeverity(ErrorSeverity newSeverity) const
-{
-    return new CECLError(category, newSeverity,
-                         errorCode(), msg, filename,
-                         getLine(), getColumn(), getPosition());
-}
-
-extern HQL_API IECLError *createECLError(WarnErrorCategory category, ErrorSeverity severity, int errNo, const char *msg, const char * filename, int lineno, int column, int pos)
-{
-    return new CECLError(category,severity,errNo,msg,filename,lineno,column,pos);
-}
-
 //---------------------------------------------------------------------------------------------------------------------
 
 void IErrorReceiver::reportError(int errNo, const char *msg, const char *filename, int lineno, int column, int position)
 {
-    Owned<IECLError> err = createECLError(errNo,msg,filename,lineno,column,position);
+    Owned<IError> err = createError(errNo,msg,filename,lineno,column,position);
     report(err);
 }
 
 void IErrorReceiver::reportWarning(WarnErrorCategory category, int warnNo, const char *msg, const char *filename, int lineno, int column, int position)
 {
     ErrorSeverity severity = queryDefaultSeverity(category);
-    Owned<IECLError> warn = createECLError(category, severity,warnNo,msg,filename,lineno,column,position);
+    Owned<IError> warn = createError(category, severity,warnNo,msg,filename,lineno,column,position);
     report(warn);
 }
 
 //---------------------------------------------------------------------------------------------------------------------
 
-void ErrorReceiverSink::report(IECLError* error)
+void ErrorReceiverSink::report(IError* error)
 {
     switch (error->getSeverity())
     {
@@ -189,7 +126,7 @@ void ErrorReceiverSink::report(IECLError* error)
 
 
 
-void MultiErrorReceiver::report(IECLError* error)
+void MultiErrorReceiver::report(IError* error)
 {
     ErrorReceiverSink::report(error);
 
@@ -203,25 +140,25 @@ StringBuffer& MultiErrorReceiver::toString(StringBuffer& buf)
 {
     ForEachItemIn(i, msgs)
     {
-        IECLError* error=item(i);
+        IError* error=item(i);
         error->toString(buf);
         buf.append('\n');
     }
     return buf;
 }
 
-IECLError *MultiErrorReceiver::firstError()
+IError *MultiErrorReceiver::firstError()
 {
     ForEachItemIn(i, msgs)
     {
-        IECLError* error = item(i);
+        IError* error = item(i);
         if (isError(error->getSeverity()))
             return error;
     }
     return NULL;
 }
 
-extern HQL_API void reportErrors(IErrorReceiver & receiver, IECLErrorArray & errors)
+extern HQL_API void reportErrors(IErrorReceiver & receiver, IErrorArray & errors)
 {
     ForEachItemIn(i, errors)
         receiver.report(&errors.item(i));
@@ -237,7 +174,7 @@ public:
         f = _f;
     }
 
-    virtual void report(IECLError* error)
+    virtual void report(IError* error)
     {
         ErrorReceiverSink::report(error);
 
@@ -286,10 +223,10 @@ extern HQL_API IErrorReceiver *createFileErrorReceiver(FILE *f)
 
 class HQL_API ThrowingErrorReceiver : public ErrorReceiverSink
 {
-    virtual void report(IECLError* error);
+    virtual void report(IError* error);
 };
 
-void ThrowingErrorReceiver::report(IECLError* error)
+void ThrowingErrorReceiver::report(IError* error)
 {
     throw error;
 }
@@ -320,7 +257,7 @@ void reportErrorVa(IErrorReceiver * errors, int errNo, const ECLlocation & loc,
     if (errors)
         errors->reportError(errNo, msg.str(), loc.sourcePath->str(), loc.lineno, loc.column, loc.position);
     else
-        throw createECLError(errNo, msg.str(), loc.sourcePath->str(), loc.lineno, loc.column, loc.position);
+        throw createError(errNo, msg.str(), loc.sourcePath->str(), loc.lineno, loc.column, loc.position);
 }
 
 void reportError(IErrorReceiver * errors, int errNo, const ECLlocation & loc, const char * format, ...)
@@ -335,9 +272,9 @@ void reportError(IErrorReceiver * errors, int errNo, const ECLlocation & loc, co
 class ErrorInserter : public IndirectErrorReceiver
 {
 public:
-    ErrorInserter(IErrorReceiver & _prev, IECLError * _error) : IndirectErrorReceiver(_prev), error(_error) {}
+    ErrorInserter(IErrorReceiver & _prev, IError * _error) : IndirectErrorReceiver(_prev), error(_error) {}
 
-    virtual void report(IECLError* error)
+    virtual void report(IError* error)
     {
         if (isError(error->getSeverity()))
             flush();
@@ -355,7 +292,7 @@ protected:
     }
 
 protected:
-    Linked<IECLError> error;
+    Linked<IError> error;
 };
 
 //---------------------------------------------------------------------------------------------------------------------
@@ -365,9 +302,9 @@ class AbortingErrorReceiver : public IndirectErrorReceiver
 public:
     AbortingErrorReceiver(IErrorReceiver & _prev) : IndirectErrorReceiver(_prev) {}
 
-    virtual void report(IECLError* error)
+    virtual void report(IError* error)
     {
-        Owned<IECLError> mappedError = prevErrorProcessor->mapError(error);
+        Owned<IError> mappedError = prevErrorProcessor->mapError(error);
         prevErrorProcessor->report(mappedError);
         if (isError(mappedError->getSeverity()))
             throw MakeStringExceptionDirect(HQLERR_ErrorAlreadyReported, "");
@@ -386,7 +323,7 @@ class DedupingErrorReceiver : public IndirectErrorReceiver
 public:
     DedupingErrorReceiver(IErrorReceiver & _prev) : IndirectErrorReceiver(_prev) {}
 
-    virtual void report(IECLError* error)
+    virtual void report(IError* error)
     {
         if (errors.contains(*error))
             return;
@@ -420,14 +357,14 @@ void checkEclVersionCompatible(Shared<IErrorReceiver> & errors, const char * ecl
             else if (minor != LANGUAGE_VERSION_MINOR)
             {
                 VStringBuffer msg("Mismatch in minor version number (%s v %s)", eclVersion, LANGUAGE_VERSION);
-                Owned<IECLError> warning = createECLError(CategoryUnexpected, SeverityInfo, HQLERR_VersionMismatch, msg.str(), NULL, 0, 0);
+                Owned<IError> warning = createError(CategoryUnexpected, SeverityInfo, HQLERR_VersionMismatch, msg.str(), NULL, 0, 0);
                 errors.setown(new ErrorInserter(*errors, warning));
             }
             else if (subminor != LANGUAGE_VERSION_SUB)
             {
                 //This adds the warning if any other warnings occur.
                 VStringBuffer msg("Mismatch in subminor version number (%s v %s)", eclVersion, LANGUAGE_VERSION);
-                Owned<IECLError> warning = createECLError(CategoryUnexpected, SeverityInfo, HQLERR_VersionMismatch, msg.str(), NULL, 0, 0);
+                Owned<IError> warning = createError(CategoryUnexpected, SeverityInfo, HQLERR_VersionMismatch, msg.str(), NULL, 0, 0);
                 errors.setown(new ErrorInserter(*errors, warning));
             }
         }

ecl/hql/hqlerror.hpp

@@ -23,63 +23,10 @@
 
 #define HQLERR_ErrorAlreadyReported             4799            // special case...
 
-enum ErrorSeverity
-{
-    SeverityIgnore,
-    SeverityInfo,
-    SeverityWarning,
-    SeverityError,    // a warning treated as an error
-    SeverityFatal,      // a fatal error - can't be mapped to anything else
-    SeverityUnknown,
-};
-
-inline bool isError(ErrorSeverity severity) { return severity >= SeverityError; }
-inline bool isFatal(ErrorSeverity severity) { return severity == SeverityFatal; }
-
-//TBD in a separate commit - add support for warnings to be associated with different categories
-enum WarnErrorCategory
-{
-    CategoryInformation,// Some kind of information [default severity information]
-
-    CategoryCast,       // Suspicious casts between types or out of range values
-    CategoryConfuse,    // Likely to cause confusion
-    CategoryDeprecated, // deprecated features or syntax
-    CategoryEfficiency, // Something that is likely to be inefficient
-    CategoryFolding,    // Unusual results from constant folding
-    CategoryFuture,     // Likely to cause problems in future versions
-    CategoryIgnored,    // Something that has no effect, or is ignored
-    CategoryIndex,      // Unusual indexing of datasets or strings
-    CategoryMistake,    // Almost certainly a mistake
-    CategoryLimit,      // An operation that should really have some limits to protect data runaway
-    CategorySyntax,     // Invalid syntax which is painless to recover from
-    CategoryUnusual,    // Not strictly speaking an error, but highly unusual and likely to be a mistake
-    CategoryUnexpected, // Code that could be correct, but has the potential for unexpected behaviour
-
-    CategoryError,      // Typically severity fatal
-    CategoryAll,
-    CategoryUnknown,
-    CategoryMax,
-};
-
-interface HQL_API IECLError: public IException
-{
-public:
-    virtual const char* getFilename() const = 0;
-    virtual WarnErrorCategory getCategory() const = 0;
-    virtual int getLine() const = 0;
-    virtual int getColumn() const = 0;
-    virtual int getPosition() const = 0;
-    virtual StringBuffer& toString(StringBuffer&) const = 0;
-    virtual ErrorSeverity getSeverity() const = 0;
-    virtual IECLError * cloneSetSeverity(ErrorSeverity _severity) const = 0;
-};
-inline bool isError(IECLError * error) { return isError(error->getSeverity()); }
-inline bool isFatal(IECLError * error) { return isFatal(error->getSeverity()); }
-
 interface HQL_API IErrorReceiver : public IInterface
 {
-    virtual void report(IECLError* error) = 0;
-    virtual IECLError * mapError(IECLError * error) = 0;
+    virtual void report(IError* error) = 0;
+    virtual IError * mapError(IError * error) = 0;
     virtual size32_t errCount() = 0;
     virtual size32_t warnCount() = 0;
 
@@ -88,7 +35,7 @@ interface HQL_API IErrorReceiver : public IInterface
     void reportWarning(WarnErrorCategory category, int warnNo, const char *msg, const char *filename, int lineno, int column, int pos);
 };
 
-typedef IArrayOf<IECLError> IECLErrorArray;
+typedef IArrayOf<IError> IErrorArray;
 
 
 //---------------------------------------------------------------------------------------------------------------------
@@ -98,8 +45,8 @@ class HQL_API ErrorReceiverSink : public CInterfaceOf<IErrorReceiver>
 public:
     ErrorReceiverSink() { errs = warns = 0; }
 
-    virtual IECLError * mapError(IECLError * error) { return LINK(error); }
-    virtual void report(IECLError* err);
+    virtual IError * mapError(IError * error) { return LINK(error); }
+    virtual void report(IError* err);
     virtual size32_t errCount() { return errs; }
     virtual size32_t warnCount() { return warns; }
 
@@ -115,11 +62,11 @@ class IndirectErrorReceiver : public CInterfaceOf<IErrorReceiver>
 public:
     IndirectErrorReceiver(IErrorReceiver & _prev) : prevErrorProcessor(&_prev) {}
 
-    virtual void report(IECLError* error)
+    virtual void report(IError* error)
     {
         prevErrorProcessor->report(error);
     }
-    virtual IECLError * mapError(IECLError * error)
+    virtual IError * mapError(IError * error)
     {
         return prevErrorProcessor->mapError(error);
     }
@@ -143,16 +90,16 @@ class HQL_API MultiErrorReceiver : public ErrorReceiverSink
 public:
     MultiErrorReceiver() {}
 
-    virtual void report(IECLError* err);
+    virtual void report(IError* err);
 
     size32_t length() { return errCount() + warnCount();}
-    IECLError* item(size32_t index) { return &msgs.item(index); }
-    IECLError* firstError();
+    IError* item(size32_t index) { return &msgs.item(index); }
+    IError* firstError();
     StringBuffer& toString(StringBuffer& out);
     void clear() { msgs.kill(); }
 
 private:
-    IECLErrorArray msgs;
+    IErrorArray msgs;
 };
 
 //---------------------------------------------------------------------------------------------------------------------
@@ -164,12 +111,11 @@ extern HQL_API ErrorSeverity getCheckSeverity(IAtom * name);
 
 //---------------------------------------------------------------------------------------------------------------------
 
-extern HQL_API IECLError *createECLError(WarnErrorCategory category, ErrorSeverity severity, int errNo, const char *msg, const char *filename, int lineno=0, int column=0, int pos=0);
-inline IECLError * createECLError(int errNo, const char *msg, const char *filename, int lineno=0, int column=0, int pos=0)
+inline IError * createError(int errNo, const char *msg, const char *filename, int lineno=0, int column=0, int pos=0)
 {
-    return createECLError(CategoryError, SeverityFatal, errNo, msg, filename, lineno, column, pos);
+    return createError(CategoryError, SeverityFatal, errNo, msg, filename, lineno, column, pos);
 }
-extern HQL_API void reportErrors(IErrorReceiver & receiver, IECLErrorArray & errors);
+extern HQL_API void reportErrors(IErrorReceiver & receiver, IErrorArray & errors);
 void HQL_API reportErrorVa(IErrorReceiver * errors, int errNo, const ECLlocation & loc, const char* format, va_list args);
 void HQL_API reportError(IErrorReceiver * errors, int errNo, const ECLlocation & loc, const char * format, ...) __attribute__((format(printf, 4, 5)));
 

ecl/hql/hqlerrors.hpp

@@ -429,6 +429,8 @@
 #define ERR_INDEX_DEPEND_DATASET    2396
 #define ERR_DUBIOUS_NAME            2397
 
+#define ERR_CPP_COMPILE_ERROR       2999
+
 #define ERR_ASSERTION_FAILS         100000
 
 /* general error types */

ecl/hql/hqlexpr.cpp

@@ -7194,7 +7194,7 @@ bool isImport(IHqlExpression * expr)
     return symbol && ((symbol->getSymbolFlags() & ob_import) != 0);
 }
 
-IECLError * queryAnnotatedWarning(const IHqlExpression * expr)
+IError * queryAnnotatedWarning(const IHqlExpression * expr)
 {
     assertex(expr->getAnnotationKind() == annotate_warning);
     const CHqlWarningAnnotation * cast = static_cast<const CHqlWarningAnnotation *>(expr);
@@ -7424,7 +7424,7 @@ bool CHqlAnnotationExtraBase::equals(const IHqlExpression & other) const
 //  if (!areMatchingPTrees(doc, cast->doc))
 //      return false;
 
-CHqlWarningAnnotation::CHqlWarningAnnotation(IHqlExpression *_body, IECLError * _ownedWarning)
+CHqlWarningAnnotation::CHqlWarningAnnotation(IHqlExpression *_body, IError * _ownedWarning)
 : CHqlAnnotationExtraBase(_body, _ownedWarning)
 {
 }
@@ -7436,12 +7436,12 @@ IHqlExpression * CHqlWarningAnnotation::cloneAnnotation(IHqlExpression * newbody
     return createWarningAnnotation(LINK(newbody), LINK(queryWarning()));
 }
 
-IHqlExpression * CHqlWarningAnnotation::createWarningAnnotation(IHqlExpression * _ownedBody, IECLError * _ownedWarning)
+IHqlExpression * CHqlWarningAnnotation::createWarningAnnotation(IHqlExpression * _ownedBody, IError * _ownedWarning)
 {
     return (new CHqlWarningAnnotation(_ownedBody, _ownedWarning))->closeExpr();
 }
 
-IHqlExpression * createWarningAnnotation(IHqlExpression * _ownedBody, IECLError * _ownedWarning)
+IHqlExpression * createWarningAnnotation(IHqlExpression * _ownedBody, IError * _ownedWarning)
 {
     return CHqlWarningAnnotation::createWarningAnnotation(_ownedBody, _ownedWarning);
 }
@@ -7919,7 +7919,7 @@ void CHqlScope::throwRecursiveError(IIdAtom * searchName)
 
     StringBuffer msg("Definition of ");
     msg.append(*searchName).append(" contains a recursive dependency");
-    throw createECLError(ERR_RECURSIVE_DEPENDENCY, msg.str(), filename, 0, 0, 1);
+    throw createError(ERR_RECURSIVE_DEPENDENCY, msg.str(), filename, 0, 0, 1);
 }
 
 inline bool namesMatch(const char * lName, const char * rName)
@@ -8114,7 +8114,7 @@ IHqlExpression *CHqlRemoteScope::lookupSymbol(IIdAtom * searchName, unsigned loo
     {
         StringBuffer msg("Definition for ");
         msg.append(*searchName).append(" contains no text");
-        throw createECLError(ERR_EXPORT_OR_SHARE, msg.str(), filename, 0, 0, 1);
+        throw createError(ERR_EXPORT_OR_SHARE, msg.str(), filename, 0, 0, 1);
     }
 
     OwnedHqlExpr recursionGuard = createSymbol(searchName, LINK(processingMarker), ob_exported);
@@ -8140,7 +8140,7 @@ IHqlExpression *CHqlRemoteScope::lookupSymbol(IIdAtom * searchName, unsigned loo
             resolved->removeSymbol(searchName);
         StringBuffer msg("Definition must contain EXPORT or SHARED value for ");
         msg.append(*searchName);
-        throw createECLError(ERR_EXPORT_OR_SHARE, msg.str(), filename, 0, 0, 1);
+        throw createError(ERR_EXPORT_OR_SHARE, msg.str(), filename, 0, 0, 1);
     }
 
     //Preserve ob_sandbox etc. annotated on the original definition, but not on the parsed code.
@@ -9313,7 +9313,7 @@ IHqlExpression *CHqlForwardScope::lookupSymbol(IIdAtom * searchName, unsigned lo
         const char * filename = parentCtx->sourcePath->str();
         StringBuffer msg("Definition must contain EXPORT or SHARED value for ");
         msg.append(*searchName);
-        throw createECLError(ERR_EXPORT_OR_SHARE, msg.str(), filename, oldSymbol->getStartLine(), oldSymbol->getStartColumn(), 1);
+        throw createError(ERR_EXPORT_OR_SHARE, msg.str(), filename, oldSymbol->getStartLine(), oldSymbol->getStartColumn(), 1);
     }
 
     if (!(newSymbol->isExported() || (lookupFlags & LSFsharedOK)))

ecl/hql/hqlexpr.hpp

@@ -910,7 +910,7 @@ public:
     Owned<IPropertyTree> nestedDependTree;
     Owned<IPropertyTree> globalDependTree;
     Owned<IPropertyTree> metaTree;
-    IECLErrorArray orphanedWarnings;
+    IErrorArray orphanedWarnings;
     HqlExprArray defaultFunctionCache;
     CIArrayOf<ForwardScopeItem> forwardLinks;
     bool expandCallsWhenBound;
@@ -1310,7 +1310,7 @@ extern HQL_API IHqlExpression * createLocationAnnotation(IHqlExpression * _owned
 extern HQL_API IHqlExpression * createLocationAnnotation(IHqlExpression * ownedBody, ISourcePath * sourcePath, int lineno, int column);
 extern HQL_API IHqlExpression * createMetaAnnotation(IHqlExpression * _ownedBody, HqlExprArray & _args);
 extern HQL_API IHqlExpression * createParseMetaAnnotation(IHqlExpression * _ownedBody, HqlExprArray & _args);
-extern HQL_API IHqlExpression * createWarningAnnotation(IHqlExpression * _ownedBody, IECLError * _ownedWarning);
+extern HQL_API IHqlExpression * createWarningAnnotation(IHqlExpression * _ownedBody, IError * _ownedWarning);
 extern HQL_API IHqlExpression * createJavadocAnnotation(IHqlExpression * _ownedBody, IPropertyTree * _doc);
 
 extern HQL_API IHqlExpression * createCompound(IHqlExpression * expr1, IHqlExpression * expr2);
@@ -1604,7 +1604,7 @@ extern HQL_API bool isShared(IHqlExpression * expr);
 extern HQL_API bool isImport(IHqlExpression * expr);
 extern HQL_API bool isVirtualSymbol(IHqlExpression * expr);
 
-extern HQL_API IECLError * queryAnnotatedWarning(const IHqlExpression * expr);
+extern HQL_API IError * queryAnnotatedWarning(const IHqlExpression * expr);
 
 extern HQL_API bool isPublicSymbol(IHqlExpression * expr);
 extern HQL_API ITypeInfo * getSumAggType(IHqlExpression * arg);

ecl/hql/hqlexpr.ipp

@@ -745,14 +745,14 @@ protected:
 class HQL_API CHqlWarningAnnotation: public CHqlAnnotationExtraBase
 {
 public:
-    static IHqlExpression * createWarningAnnotation(IHqlExpression * _ownedBody, IECLError * _ownedWarning);
+    static IHqlExpression * createWarningAnnotation(IHqlExpression * _ownedBody, IError * _ownedWarning);
     virtual annotate_kind getAnnotationKind() const { return annotate_warning; }
     virtual IHqlExpression * cloneAnnotation(IHqlExpression * body);
 
-    inline IECLError * queryWarning() const { return static_cast<IECLError *>(extra.get()); }
+    inline IError * queryWarning() const { return static_cast<IError *>(extra.get()); }
 
 protected:
-    CHqlWarningAnnotation(IHqlExpression * _expr, IECLError * _ownedWarning);
+    CHqlWarningAnnotation(IHqlExpression * _expr, IError * _ownedWarning);
 };
 
 class HQL_API CHqlJavadocAnnotation: public CHqlAnnotationExtraBase

ecl/hql/hqlgram.hpp

@@ -574,7 +574,7 @@ public:
     void doReportWarning(WarnErrorCategory category, int warnNo, const char *msg, const char *filename, int lineno, int column, int pos);
     void reportError(int errNo, const attribute& a, const char* format, ...) __attribute__((format(printf, 4, 5)));
     void reportError(int errNo, const ECLlocation & pos, const char* format, ...) __attribute__((format(printf, 4, 5)));
-    void reportMacroExpansionPosition(IECLError * warning, HqlLex * lexer);
+    void reportMacroExpansionPosition(IError * warning, HqlLex * lexer);
     void reportErrorUnexpectedX(const attribute & errpos, IAtom * unexpected);
 
     // Don't use overloading: va_list is the same as char*!!
@@ -588,8 +588,8 @@ public:
 
     // interface IErrorReceiver
     virtual void reportError(int errNo, const char *msg, const char *filename=NULL, int lineno=0, int column=0, int pos=0);
-    virtual void report(IECLError * error);
-    virtual IECLError * mapError(IECLError * error);
+    virtual void report(IError * error);
+    virtual IError * mapError(IError * error);
     void reportWarning(WarnErrorCategory category, int warnNo, const char *msg, const char *filename=NULL, int lineno=0, int column=0, int pos=0);
     virtual size32_t errCount();
     virtual size32_t warnCount();
@@ -846,7 +846,7 @@ protected:
     bool expandingMacroPosition;
     unsigned m_maxErrorsAllowed;
 
-    IECLErrorArray pendingWarnings;
+    IErrorArray pendingWarnings;
     Linked<ISourcePath> sourcePath;
     IIdAtom * moduleName;
     IIdAtom * current_id;

ecl/hql/hqlgram.y

@@ -8014,7 +8014,7 @@ simpleDataSet
                         {
                             IHqlExpression * left = $3.getExpr();
                             IHqlExpression * right = $5.getExpr();
-                            IHqlExpression * transform = parser->createDefJoinTransform(parser->queryLeftScope(),parser->queryRightScope(),$1, $7.queryExpr(),NULL);
+                            IHqlExpression * transform = parser->createDefJoinTransform(left,right,$1, $7.queryExpr(),NULL);
                             IHqlExpression * combine = createDataset(no_combine, left, createComma(right, transform, $7.getExpr()));
                             $$.setExpr(combine);
                             $$.setPosition($1);

ecl/hql/hqlgram2.cpp

@@ -262,11 +262,11 @@ bool HqlGramCtx::hasAnyActiveParameters()
 }
 
 
-static IECLError * createErrorVA(WarnErrorCategory category, ErrorSeverity severity, int errNo, const ECLlocation & pos, const char* format, va_list args)
+static IError * createErrorVA(WarnErrorCategory category, ErrorSeverity severity, int errNo, const ECLlocation & pos, const char* format, va_list args)
 {
     StringBuffer msg;
     msg.valist_appendf(format, args);
-    return createECLError(category, severity, errNo, msg.str(), pos.sourcePath->str(), pos.lineno, pos.column, pos.position);
+    return createError(category, severity, errNo, msg.str(), pos.sourcePath->str(), pos.lineno, pos.column, pos.position);
 }
 
 IHqlExpression * HqlGram::createSetRange(attribute & array, attribute & range)
@@ -3484,7 +3484,7 @@ IHqlExpression *HqlGram::lookupSymbol(IIdAtom * searchName, const attribute& err
         }
         return NULL;
     }
-    catch(IECLError* error)
+    catch(IError* error)
     {
         if(errorHandler && !errorDisabled)
             errorHandler->report(error);
@@ -5846,11 +5846,11 @@ void HqlGram::reportErrorUnexpectedX(const attribute& errpos, IAtom * unexpected
 
 void HqlGram::doReportWarning(WarnErrorCategory category, int warnNo, const char *msg, const char *filename, int lineno, int column, int pos)
 {
-    Owned<IECLError> error = createECLError(category, queryDefaultSeverity(category), warnNo, msg, filename, lineno, column, pos);
+    Owned<IError> error = createError(category, queryDefaultSeverity(category), warnNo, msg, filename, lineno, column, pos);
     report(error);
 }
 
-void HqlGram::reportMacroExpansionPosition(IECLError * warning, HqlLex * lexer)
+void HqlGram::reportMacroExpansionPosition(IError * warning, HqlLex * lexer)
 {
     if (expandingMacroPosition)
         return;
@@ -5873,7 +5873,7 @@ void HqlGram::reportMacroExpansionPosition(IECLError * warning, HqlLex * lexer)
 
 void HqlGram::reportErrorVa(int errNo, const ECLlocation & pos, const char* format, va_list args)
 {
-    Owned<IECLError> error = createErrorVA(CategoryError, SeverityFatal, errNo, pos, format, args);
+    Owned<IError> error = createErrorVA(CategoryError, SeverityFatal, errNo, pos, format, args);
     report(error);
 }
 
@@ -5881,7 +5881,7 @@ void HqlGram::reportError(int errNo, const char *msg, int lineno, int column, in
 {
     if (errorHandler && !errorDisabled)
     {
-        Owned<IECLError> error = createECLError(errNo, msg, lexObject->queryActualSourcePath()->str(), lineno, column, position);
+        Owned<IError> error = createError(errNo, msg, lexObject->queryActualSourcePath()->str(), lineno, column, position);
         report(error);
     }
 }
@@ -5892,7 +5892,7 @@ void HqlGram::reportWarning(WarnErrorCategory category, int warnNo, const ECLloc
     {
         va_list args;
         va_start(args, format);
-        Owned<IECLError> error = createErrorVA(category, queryDefaultSeverity(category), warnNo, pos, format, args);
+        Owned<IError> error = createErrorVA(category, queryDefaultSeverity(category), warnNo, pos, format, args);
         va_end(args);
 
         report(error);
@@ -5906,7 +5906,7 @@ void HqlGram::reportWarning(WarnErrorCategory category, ErrorSeverity severity,
         StringBuffer msg;
         va_list args;
         va_start(args, format);
-        Owned<IECLError> error = createErrorVA(category, severity, warnNo, pos, format, args);
+        Owned<IError> error = createErrorVA(category, severity, warnNo, pos, format, args);
         va_end(args);
 
         report(error);
@@ -5918,7 +5918,7 @@ void HqlGram::reportWarningVa(WarnErrorCategory category, int warnNo, const attr
     const ECLlocation & pos = a.pos;
     if (errorHandler && !errorDisabled)
     {
-        Owned<IECLError> error = createErrorVA(category, queryDefaultSeverity(category), warnNo, pos, format, args);
+        Owned<IError> error = createErrorVA(category, queryDefaultSeverity(category), warnNo, pos, format, args);
         report(error);
     }
 }
@@ -5933,16 +5933,16 @@ void HqlGram::reportWarning(WarnErrorCategory category, int warnNo, const char *
 //interface IErrorReceiver
 void HqlGram::reportError(int errNo, const char *msg, const char *filename, int lineno, int column, int position)
 {
-    Owned<IECLError> err = createECLError(errNo,msg,filename,lineno,column,position);
+    Owned<IError> err = createError(errNo,msg,filename,lineno,column,position);
     report(err);
 }
 
-IECLError * HqlGram::mapError(IECLError * error)
+IError * HqlGram::mapError(IError * error)
 {
     return errorHandler->mapError(error);
 }
 
-void HqlGram::report(IECLError* error)
+void HqlGram::report(IError* error)
 {
     if (errorHandler && !errorDisabled)
     {
@@ -9020,7 +9020,7 @@ IHqlExpression * HqlGram::associateSideEffects(IHqlExpression * expr, const ECLl
             {
                 if (expr->isScope())
                 {
-                    Owned<IECLError> error = createECLError(CategorySyntax, SeverityError, ERR_RESULT_IGNORED_SCOPE, "Cannot associate a side effect with a module - action will be lost", errpos.sourcePath->str(), errpos.lineno, errpos.column, errpos.position);
+                    Owned<IError> error = createError(CategorySyntax, SeverityError, ERR_RESULT_IGNORED_SCOPE, "Cannot associate a side effect with a module - action will be lost", errpos.sourcePath->str(), errpos.lineno, errpos.column, errpos.position);
                     //Unusual processing.  Create a warning and save it in the parse context
                     //The reason is that this error is reporting "the associated side-effects will be lost" - but
                     //the same will apply to the warning, and if it's lost there will be no way to report it later...

ecl/hql/hqlir.cpp

@@ -842,7 +842,7 @@ public:
 public:
     exprid_t expr;
     const char * name;
-    IECLError * warning;
+    IError * warning;
     IPropertyTree * tree;
     unsigned value;
     unsigned line;
@@ -1327,7 +1327,7 @@ public:
             }
         case annotate_warning:
             {
-                IECLError * warning = info.warning;
+                IError * warning = info.warning;
                 StringBuffer msg;
                 warning->errorMessage(msg);
                 const char * filename = warning->getFilename();

ecl/hql/hqlutil.cpp

@@ -5795,7 +5795,7 @@ void TempTableTransformer::reportError(IHqlExpression * location, int code,const
     va_start(args, format);
     errorMsg.valist_appendf(format, args);
     va_end(args);
-    Owned<IECLError> err = createECLError(code, errorMsg.str(), where->sourcePath->str(), where->lineno, where->column, where->position);
+    Owned<IError> err = createError(code, errorMsg.str(), where->sourcePath->str(), where->lineno, where->column, where->position);
     errorProcessor.report(err);
 }
 
@@ -6657,10 +6657,10 @@ void ErrorSeverityMapper::restoreState(const ErrorSeverityMapperState & saved)
     activeSymbol = saved.symbol;
 }
 
-IECLError * ErrorSeverityMapper::mapError(IECLError * error)
+IError * ErrorSeverityMapper::mapError(IError * error)
 {
     //An error that is fatal cannot be mapped.
-    Owned<IECLError> mappedError = IndirectErrorReceiver::mapError(error);
+    Owned<IError> mappedError = IndirectErrorReceiver::mapError(error);
     if (!isFatal(mappedError))
     {
         //This takes precedence over mappings in the parent
@@ -6736,8 +6736,8 @@ public:
             }
         case annotate_warning:
             {
-                IECLError * error = static_cast<CHqlWarningAnnotation *>(expr)->queryWarning();
-                Owned<IECLError> mappedError = mapper.mapError(error);
+                IError * error = static_cast<CHqlWarningAnnotation *>(expr)->queryWarning();
+                Owned<IError> mappedError = mapper.mapError(error);
                 mapper.report(mappedError);
                 break;
             }
@@ -6756,7 +6756,7 @@ protected:
 };
 
 
-void gatherParseWarnings(IErrorReceiver * errs, IHqlExpression * expr, IECLErrorArray & orphanedWarnings)
+void gatherParseWarnings(IErrorReceiver * errs, IHqlExpression * expr, IErrorArray & orphanedWarnings)
 {
     if (!errs || !expr)
         return;
@@ -6774,7 +6774,7 @@ void gatherParseWarnings(IErrorReceiver * errs, IHqlExpression * expr, IECLError
 
     ForEachItemIn(i, orphanedWarnings)
     {
-        Owned<IECLError> mappedError = globalOnWarning->mapError(&orphanedWarnings.item(i));
+        Owned<IError> mappedError = globalOnWarning->mapError(&orphanedWarnings.item(i));
         globalOnWarning->report(mappedError);
     }
 }
@@ -8296,12 +8296,12 @@ IHqlExpression * ensureOwned(IHqlExpression * expr)
 }
 
 
-IECLError * annotateExceptionWithLocation(IException * e, IHqlExpression * location)
+IError * annotateExceptionWithLocation(IException * e, IHqlExpression * location)
 {
     StringBuffer errorMsg;
     e->errorMessage(errorMsg);
     unsigned code = e->errorCode();
-    return createECLError(code, errorMsg.str(), location->querySourcePath()->str(), location->getStartLine(), location->getStartColumn(), 0);
+    return createError(code, errorMsg.str(), location->querySourcePath()->str(), location->getStartLine(), location->getStartColumn(), 0);
 }
 
 StringBuffer & appendLocation(StringBuffer & s, IHqlExpression * location, const char * suffix)

ecl/hql/hqlutil.hpp

@@ -554,7 +554,7 @@ public:
 public:
     ErrorSeverityMapper(IErrorReceiver & errorProcessor);
 
-    virtual IECLError * mapError(IECLError * error);
+    virtual IError * mapError(IError * error);
 
     bool addCommandLineMapping(const char * mapping);
     bool addMapping(const char * category, const char * value);
@@ -668,7 +668,7 @@ extern HQL_API IPropertyTree * queryEnsureArchiveModule(IPropertyTree * archive,
 extern HQL_API IPropertyTree * queryArchiveAttribute(IPropertyTree * module, const char * name);
 extern HQL_API IPropertyTree * createArchiveAttribute(IPropertyTree * module, const char * name);
 
-extern HQL_API IECLError * annotateExceptionWithLocation(IException * e, IHqlExpression * location);
+extern HQL_API IError * annotateExceptionWithLocation(IException * e, IHqlExpression * location);
 extern HQL_API IHqlExpression * expandMacroDefinition(IHqlExpression * expr, HqlLookupContext & ctx, bool reportError);
 extern HQL_API IHqlExpression * convertAttributeToQuery(IHqlExpression * expr, HqlLookupContext & ctx);
 extern HQL_API StringBuffer & appendLocation(StringBuffer & s, IHqlExpression * location, const char * suffix = NULL);
@@ -745,6 +745,6 @@ protected:
 };
 
 extern HQL_API bool joinHasRightOnlyHardMatch(IHqlExpression * expr, bool allowSlidingMatch);
-extern HQL_API void gatherParseWarnings(IErrorReceiver * errs, IHqlExpression * expr, IECLErrorArray & warnings);
+extern HQL_API void gatherParseWarnings(IErrorReceiver * errs, IHqlExpression * expr, IErrorArray & warnings);
 
 #endif

ecl/hql/hqlwuerr.cpp

@@ -43,12 +43,12 @@ void WorkUnitErrorReceiver::initializeError(IWUException * exception, int errNo,
     exception->setTimeStamp(NULL);
 }
 
-IECLError * WorkUnitErrorReceiver::mapError(IECLError * error)
+IError * WorkUnitErrorReceiver::mapError(IError * error)
 {
     return LINK(error);
 }
 
-void WorkUnitErrorReceiver::report(IECLError* eclError)
+void WorkUnitErrorReceiver::report(IError* eclError)
 {
     WUExceptionSeverity wuSeverity = ExceptionSeverityInformation;
     ErrorSeverity severity = eclError->getSeverity();
@@ -103,13 +103,13 @@ public:
     CompoundErrorReceiver(IErrorReceiver * _primary, IErrorReceiver * _secondary) { primary.set(_primary); secondary.set(_secondary); }
     IMPLEMENT_IINTERFACE;
 
-    virtual IECLError * mapError(IECLError * error)
+    virtual IError * mapError(IError * error)
     {
-        Owned<IECLError> mappedError = primary->mapError(error);
+        Owned<IError> mappedError = primary->mapError(error);
         assertex(mappedError == error); // should not expect any mapping below a compound.
         return mappedError.getClear();
     }
-    virtual void report(IECLError* err)
+    virtual void report(IError* err)
     {
         primary->report(err);
         secondary->report(err);

ecl/hql/hqlwuerr.hpp

@@ -26,8 +26,8 @@ public:
     WorkUnitErrorReceiver(IWorkUnit * _wu, const char * _component) { wu.set(_wu); component.set(_component); }
     IMPLEMENT_IINTERFACE;
 
-    virtual IECLError * mapError(IECLError * error);
-    virtual void report(IECLError*);
+    virtual IError * mapError(IError * error);
+    virtual void report(IError*);
     virtual size32_t errCount();
     virtual size32_t warnCount();
 

ecl/hqlcpp/hqlcpp.cpp

@@ -1371,7 +1371,7 @@ HqlCppTranslator::HqlCppTranslator(IErrorReceiver * _errors, const char * _soNam
             if (errs.errCount())
             {
                 StringBuffer errtext;
-                IECLError *first = errs.firstError();
+                IError *first = errs.firstError();
                 first->toString(errtext);
                 throw MakeStringException(HQLERR_FailedToLoadSystemModule, "%s @ %d:%d", errtext.str(), first->getColumn(), first->getLine());
             } 
@@ -2038,14 +2038,14 @@ bool HqlCppTranslator::getDebugFlag(const char * name, bool defValue)
 
 void HqlCppTranslator::doReportWarning(WarnErrorCategory category, ErrorSeverity explicitSeverity, IHqlExpression * location, unsigned id, const char * msg)
 {
-    Owned<IECLError> warnError;
+    Owned<IError> warnError;
     if (!location)
         location = queryActiveActivityLocation();
     ErrorSeverity severity = (explicitSeverity == SeverityUnknown) ? queryDefaultSeverity(category) : explicitSeverity;
     if (location)
-        warnError.setown(createECLError(category, severity, id, msg, location->querySourcePath()->str(), location->getStartLine(), location->getStartColumn(), 0));
+        warnError.setown(createError(category, severity, id, msg, location->querySourcePath()->str(), location->getStartLine(), location->getStartColumn(), 0));
     else
-        warnError.setown(createECLError(category, severity, id, msg, NULL, 0, 0, 0));
+        warnError.setown(createError(category, severity, id, msg, NULL, 0, 0, 0));
 
     errorProcessor->report(warnError);
 }
@@ -2137,7 +2137,7 @@ void HqlCppTranslator::ThrowStringException(int code,const char *format, ...)
         va_start(args, format);
         errorMsg.valist_appendf(format, args);
         va_end(args);
-        throw createECLError(code, errorMsg.str(), location->querySourcePath()->str(), location->getStartLine(), location->getStartColumn(), 0);
+        throw createError(code, errorMsg.str(), location->querySourcePath()->str(), location->getStartLine(), location->getStartColumn(), 0);
     }
 
     va_list args;
@@ -2154,7 +2154,7 @@ void HqlCppTranslator::reportErrorDirect(IHqlExpression * location, int code,con
         ECLlocation loc;
         loc.extractLocationAttr(location);
         if (alwaysAbort)
-            throw createECLError(code, msg, loc.sourcePath->str(), loc.lineno, loc.column, loc.position);
+            throw createError(code, msg, loc.sourcePath->str(), loc.lineno, loc.column, loc.position);
         errorProcessor->reportError(code, msg, loc.sourcePath->str(), loc.lineno, loc.column, loc.position);
     }
     else

ecl/hqlcpp/hqlecl.cpp

@@ -393,7 +393,7 @@ bool HqlDllGenerator::generateCode(HqlQueryContext & query)
             translator.finalizeResources();
             translator.expandFunctions(true);
         }
-        catch (IECLError * e)
+        catch (IError * e)
         {
             if (e->errorCode() != HQLERR_ErrorAlreadyReported)
             {
@@ -554,6 +554,16 @@ bool HqlDllGenerator::doCompile(ICppCompiler * compiler)
     else
         PrintLog("Failed to compile %s", wuname);
 
+    bool reportCppWarnings = wu->getDebugValueBool("reportCppWarnings", false);
+    IArrayOf<IError> errors;
+    compiler->extractErrors(errors);
+    ForEachItemIn(iErr, errors)
+    {
+        IError & cur = errors.item(iErr);
+        if (isError(&cur) || reportCppWarnings)
+            errs->report(&cur);
+    }
+
     cycle_t elapsedCycles = get_cycles_now() - startCycles;
     //For eclcc the workunit has been written to the resource - so any further timings will not be preserved -> need to report differently
     queryActiveTimer()->addTiming("compile:compile c++", elapsedCycles);

ecl/hqlcpp/hqlhtcpp.cpp

@@ -6793,12 +6793,12 @@ ABoundActivity * HqlCppTranslator::buildActivity(BuildCtx & ctx, IHqlExpression
     }
     catch (IException * e)
     {
-        if (dynamic_cast<IECLError *>(e))
+        if (dynamic_cast<IError *>(e))
             throw;
         IHqlExpression * location = queryActiveActivityLocation();
         if (location)
         {
-            IECLError * error = annotateExceptionWithLocation(e, location);
+            IError * error = annotateExceptionWithLocation(e, location);
             e->Release();
             throw error;
         }

ecl/hqlcpp/hqlttcpp.cpp

@@ -9558,7 +9558,7 @@ void HqlScopeTagger::reportError(WarnErrorCategory category, const char * msg)
     int startColumn = location ? location->getStartColumn() : 0;
     ISourcePath * sourcePath = location ? location->querySourcePath() : NULL;
     ErrorSeverity severity = queryDefaultSeverity(category);
-    Owned<IECLError> err = createECLError(category, severity, ERR_ASSERT_WRONGSCOPING, msg, sourcePath->str(), startLine, startColumn, 0);
+    Owned<IError> err = createError(category, severity, ERR_ASSERT_WRONGSCOPING, msg, sourcePath->str(), startLine, startColumn, 0);
     errors.report(err);        // will throw immediately if it is an error.
 }
 
@@ -11576,12 +11576,12 @@ IHqlExpression * HqlTreeNormalizer::createTransformed(IHqlExpression * expr)
         }
         catch (IException * e)
         {
-            if (dynamic_cast<IECLError *>(e))
+            if (dynamic_cast<IError *>(e))
                 throw;
             IHqlExpression * location = queryLocation(expr);
             if (location)
             {
-                IECLError * error = annotateExceptionWithLocation(e, location);
+                IError * error = annotateExceptionWithLocation(e, location);
                 e->Release();
                 throw error;
             }

ecl/regress/cpperr.ecl

@@ -0,0 +1,14 @@
+
+boolean MyFunc() := BEGINC++
+
+    char * temp = "Invalid const string assignment";
+    
+    char * x = strdup(temp);
+    iff (syntaxError);
+    
+    return false;
+
+ENDC++;
+
+output(MyFunc());
+

ecl/regress/cpperr2.ecl

@@ -0,0 +1,11 @@
+
+boolean MyFunc() := BEGINC++
+
+extern bool doesNotExist();
+#body
+    return doesNotExist();
+
+ENDC++;
+
+output(MyFunc());
+

ecl/regress/issue12491.ecl

@@ -0,0 +1,25 @@
+#option ('targetClusterType', 'roxie');
+
+inrec := RECORD
+  UNSIGNED6 did;
+END;
+outrec := RECORD(inrec)
+  STRING20  name;
+  STRING10  ssn;
+  UNSIGNED8  dob;
+END;
+ds := DATASET([1,2,3,4,5,6], inrec);
+i1 := DATASET([{1, 'Kevin'}, {2, 'Richard'}, {5,'Nigel'}],
+    { UNSIGNED6 did, STRING10 name });
+i2 := DATASET([{3, '123462'}, {5, '1287234'}, {6,'007001002'}],
+    { UNSIGNED6 did, STRING10 ssn });
+i3 := DATASET([{1, 19700117}, {4, 19831212}, {6,20000101}],
+    { UNSIGNED6 did, UNSIGNED8 dob}); 
+j1 := JOIN(ds, i1, LEFT.did = RIGHT.did, LEFT OUTER, LOOKUP);
+j2 := JOIN(ds, i2, LEFT.did = RIGHT.did, LEFT OUTER, LOOKUP);
+j3 := JOIN(ds, i3, LEFT.did = RIGHT.did, LEFT OUTER, LOOKUP);
+combined1 := COMBINE(j1, j2, TRANSFORM(outRec, SELF := LEFT; SELF := RIGHT; SELF := []));
+// combined2 := COMBINE(combined1, j3, TRANSFORM(outRec, SELF.dob := RIGHT.dob; SELF := LEFT));
+combined2 := COMBINE(combined1, j3);
+
+output(combined2);

ecllibrary/teststd/Date/TestDate.ecl

@@ -6,33 +6,208 @@ IMPORT Std.Date;
 
 EXPORT TestDate := MODULE
 
+  SHARED vCreateDateTimeFromSeconds := ROW(Date.CreateDateTimeFromSeconds(917872496)); // Feb 1, 1999 @ 12:34:56
+  SHARED vCreateDateFromSeconds := ROW(Date.CreateDateFromSeconds(917872496)); // Feb 1, 1999
+  SHARED vCreateTimeFromSeconds := ROW(Date.CreateTimeFromSeconds(917872496)); // 12:34:56
+  SHARED vCreateTime := ROW(Date.CreateTime(12,34,56)); // 12:34:56
+  SHARED vCreateDateTime := ROW(Date.CreateDateTime(1999,2,1,12,34,56)); // Feb 1, 1999 @ 12:34:56
+  SHARED vDate := Date.CurrentDate(); // UTC
+  SHARED vToday := Date.Today(); // Local
+  SHARED vTime := Date.CurrentTime(); // UTC
+  SHARED vTimeLocal := Date.CurrentTime(TRUE); // Local
+  SHARED vSeconds := Date.CurrentSeconds(); // UTC
+  SHARED vSecondsLocal := Date.CurrentSeconds(TRUE); // Local
+  SHARED vIndTimestamp := Date.CurrentTimestamp() : INDEPENDENT; // UTC, evaluated before all others
+  SHARED vTimestamp := Date.CurrentTimestamp(); // UTC
+  SHARED vTimestampLocal := Date.CurrentTimestamp(TRUE); // Local
+  SHARED vLocalTimeZoneOffset := Date.LocalTimeZoneOffset();
+
   EXPORT TestConstant := [
-    ASSERT(NOT date.IsLeapYear(1900), CONST);
-    ASSERT(date.IsLeapYear(1904), CONST);
-    ASSERT(NOT date.IsLeapYear(2100), CONST);
-    ASSERT(date.IsLeapYear(2000), CONST);
-    ASSERT(NOT date.IsLeapYear(1901), CONST);
-    ASSERT(date.FromDaysSince1900(0) = 19000101, CONST);
-    ASSERT(date.ToGregorianDate(1) = 00010101, CONST);
-    ASSERT(date.DaysSince1900(1900,1,1)=0, CONST);
-    ASSERT(date.FromGregorianYMD(1,1,1)=1, CONST);
-    ASSERT(date.ToJulianDate(1) = 00010101, CONST);
-    ASSERT(date.FromJulianYMD(1,1,1)=1, CONST);
-    ASSERT(date.MonthsBetween(19700101,19701231)=11, CONST);
-    ASSERT(date.MonthsBetween(19701231,19710101)=0, CONST);
-    ASSERT(date.MonthsBetween(19701231,19711231)=12, CONST);
-    ASSERT(date.MonthsBetween(19711231,19701231)=-12, CONST);
-    ASSERT(date.MonthsBetween(19700606,19700706)=1, CONST);
-    ASSERT(date.MonthsBetween(19700606,19700705)=0, CONST);
-    ASSERT(date.MonthsBetween(19700606,19700607)=0, CONST);
+    ASSERT(Date.FromDaysSince1900(0) = 19000101, CONST);
+    ASSERT(Date.ToGregorianDate(1) = 00010101, CONST);
+    ASSERT(Date.DaysSince1900(1900,1,1)=0, CONST);
+    ASSERT(Date.FromGregorianYMD(1,1,1)=1, CONST);
+    ASSERT(Date.ToJulianDate(1) = 00010101, CONST);
+    ASSERT(Date.FromJulianYMD(1,1,1)=1, CONST);
+
+    ASSERT(Date.Year(19990201) = 1999, CONST);
+    ASSERT(Date.Month(19990201) = 2, CONST);
+    ASSERT(Date.Day(19990201) = 1, CONST);
+
+    ASSERT(Date.Hour(123456) = 12, CONST);
+    ASSERT(Date.Minute(123456) = 34, CONST);
+    ASSERT(Date.Second(123456) = 56, CONST);
+
+    ASSERT(Date.DateFromParts(1999,2,1) = 19990201, CONST);
+    ASSERT(Date.TimeFromParts(12,34,56) = 123456, CONST);
+
+    ASSERT(NOT Date.IsLeapYear(1900), CONST);
+    ASSERT(Date.IsLeapYear(1904), CONST);
+    ASSERT(NOT Date.IsLeapYear(2100), CONST);
+    ASSERT(Date.IsLeapYear(2000), CONST);
+    ASSERT(NOT Date.IsLeapYear(1901), CONST);
+
+    ASSERT(Date.IsDateLeapYear(20000201) = TRUE, CONST);
+    ASSERT(Date.IsDateLeapYear(20010201) = FALSE, CONST);
+    ASSERT(Date.IsDateLeapYear(21000201) = FALSE, CONST);
+
+    ASSERT(Date.IsJulianLeapYear(2000) = TRUE, CONST);
+    ASSERT(Date.IsJulianLeapYear(2001) = FALSE, CONST);
+
+    ASSERT(Date.YearsBetween(20010615,20020615) = 1, CONST);
+    ASSERT(Date.YearsBetween(20010615,20020614) = 0, CONST);
+    ASSERT(Date.YearsBetween(20020615,20010615) = -1, CONST);
+
+    ASSERT(Date.MonthsBetween(19700101,19701231)=11, CONST);
+    ASSERT(Date.MonthsBetween(19701231,19710101)=0, CONST);
+    ASSERT(Date.MonthsBetween(19701231,19711231)=12, CONST);
+    ASSERT(Date.MonthsBetween(19711231,19701231)=-12, CONST);
+    ASSERT(Date.MonthsBetween(19700606,19700706)=1, CONST);
+    ASSERT(Date.MonthsBetween(19700606,19700705)=0, CONST);
+    ASSERT(Date.MonthsBetween(19700606,19700607)=0, CONST);
+
+    ASSERT(Date.DaysBetween(20010615,20020615) = 365, CONST);
+    ASSERT(Date.DaysBetween(20010615,20020614) = 364, CONST);
+    ASSERT(Date.DaysBetween(20020615,20010615) = -365, CONST);
+
     ASSERT(TRUE, CONST)
   ];
 
+  EXPORT TestDynamicFunctions := [
+    ASSERT(Date.SecondsFromParts(1999,2,1,12,34,56,FALSE) = 917872496);     // UTC
+    ASSERT(Date.SecondsFromParts(1965,2,17,0,0,0,FALSE) = -153705600);      // UTC
+
+    ASSERT(Date.SecondsToParts(917872496).year = 1999);
+    ASSERT(Date.SecondsToParts(917872496).month = 2);
+    ASSERT(Date.SecondsToParts(917872496).day = 1);
+    ASSERT(Date.SecondsToParts(917872496).hour = 12);
+    ASSERT(Date.SecondsToParts(917872496).minute = 34);
+    ASSERT(Date.SecondsToParts(917872496).second = 56);
+
+    ASSERT(Date.SecondsToParts(-153705600).year = 1965);
+    ASSERT(Date.SecondsToParts(-153705600).month = 2);
+    ASSERT(Date.SecondsToParts(-153705600).day = 17);
+    ASSERT(Date.SecondsToParts(-153705600).hour = 0);
+    ASSERT(Date.SecondsToParts(-153705600).minute = 0);
+    ASSERT(Date.SecondsToParts(-153705600).second = 0);
+
+    ASSERT(Date.SecondsToParts(0).year = 1970);     // Epoch test
+    ASSERT(Date.SecondsToParts(0).month = 1);       // Epoch test
+    ASSERT(Date.SecondsToParts(0).day = 1);         // Epoch test
+    ASSERT(Date.SecondsToParts(0).hour = 0);        // Epoch test
+    ASSERT(Date.SecondsToParts(0).minute = 0);      // Epoch test
+    ASSERT(Date.SecondsToParts(0).second = 0);      // Epoch test
+
+    ASSERT(Date.DayOfYear(20000101) = 1);
+    ASSERT(Date.DayOfYear(20001231) = 366);
+    ASSERT(Date.DayOfYear(20011231) = 365);
+
+    ASSERT(Date.DayOfWeek(20140130) = 5);   // 5=Thursday
+    ASSERT(Date.DayOfWeek(19650217) = 4);   // 4=Wednesday
+    ASSERT(Date.DayOfWeek(20530213) = 5);   // 5=Thursday
+
+    ASSERT(Date.AdjustDate(20000130, month_delta:=1) = 20000301);
+    ASSERT(Date.AdjustDate(20000130, month_delta:=1, day_delta:=-1) = 20000229);
+    ASSERT(Date.AdjustDate(20000229, year_delta:=1) = 20010301);
+    ASSERT(Date.AdjustDate(20000229, year_delta:=-1) = 19990301);
+    ASSERT(Date.AdjustDate(19650217, year_delta:=49) = 20140217);
+
+    ASSERT(Date.AdjustDateBySeconds(20140130, 172800) = 20140201);
+
+    ASSERT(Date.AdjustTime(180000, hour_delta:=7) = 010000);
+    ASSERT(Date.AdjustTime(180000, minute_delta:=420) = 010000);
+    ASSERT(Date.AdjustTime(180000, second_delta:=-86400) = 180000);
+
+    ASSERT(Date.AdjustTimeBySeconds(180000, 86400) = 180000);
+    ASSERT(Date.AdjustTimeBySeconds(180000, -86400) = 180000);
+
+    ASSERT(Date.AdjustSeconds(917872496, hour_delta:=1) = 917876096);
+
+    ASSERT(Date.AdjustCalendar(20140130, month_delta:=1) = 20140228);
+    ASSERT(Date.AdjustCalendar(20000229, year_delta:=1) = 20010228);
+    ASSERT(Date.AdjustCalendar(20000229, year_delta:=4) = 20040229);
+
+    ASSERT(vSeconds + vLocalTimeZoneOffset = vSecondsLocal);
+
+    ASSERT(vTimestamp != vIndTimestamp); // Test for non-pure calls to C++ code
+
+    ASSERT(Date.TimestampToSeconds(vTimestamp) = vSeconds);
+    ASSERT(Date.TimestampToSeconds(vTimestampLocal) = vSecondsLocal);
+
+    // IsLocalDaylightSavingsInEffect() -- not possible to check without pinning both cluster location and date
+
+    ASSERT(Date.DatesForMonth(20141215).startDate = 20141201);
+    ASSERT(Date.DatesForMonth(20141215).endDate = 20141231);
+
+    ASSERT(Date.DatesForWeek(20141030).startDate = 20141026);
+    ASSERT(Date.DatesForWeek(20141030).endDate = 20141101);
+
+    ASSERT(Date.IsValidDate(vDate, 2014, 2050));
+    ASSERT(NOT Date.IsValidDate(vDate, 2000, 2010));
+    ASSERT(Date.IsValidDate(vToday, 2014, 2050));
+    ASSERT(NOT Date.IsValidDate(vToday, 2000, 2010));
+    ASSERT(Date.IsValidDate(20141030) = TRUE);
+    ASSERT(Date.IsValidDate(20000229) = TRUE);
+    ASSERT(Date.IsValidDate(20010229) = FALSE); // Invalid date (leap year check)
+    ASSERT(Date.IsValidDate(20141000) = FALSE); // Invalid day
+    ASSERT(Date.IsValidDate(20141032) = FALSE); // Invalid day
+    ASSERT(Date.IsValidDate(20000001) = FALSE); // Invalid month
+    ASSERT(Date.IsValidDate(20001301) = FALSE); // Invalid month
+
+    ASSERT(Date.IsValidTime(vTime));
+    ASSERT(Date.IsValidTime(vTimeLocal));
+    ASSERT(Date.IsValidTime(123456) = TRUE);
+    ASSERT(Date.IsValidTime(123465) = FALSE); // Invalid seconds
+    ASSERT(Date.IsValidTime(127456) = FALSE); // Invalid minutes
+    ASSERT(Date.IsValidTime(243456) = FALSE); // Invalid hours
+
+    ASSERT(NOT Date.IsValidGregorianDate(16001231));
+    ASSERT(Date.IsValidGregorianDate(16010101));
+    ASSERT(Date.IsValidGregorianDate(308271231));
+    ASSERT(NOT Date.IsValidGregorianDate(308280101));
+
+    ASSERT(vCreateDateTimeFromSeconds.year = 1999);
+    ASSERT(vCreateDateTimeFromSeconds.month = 2);
+    ASSERT(vCreateDateTimeFromSeconds.day = 1);
+    ASSERT(vCreateDateTimeFromSeconds.hour = 12);
+    ASSERT(vCreateDateTimeFromSeconds.minute = 34);
+    ASSERT(vCreateDateTimeFromSeconds.second = 56);
+
+    ASSERT(vCreateDateFromSeconds.year = 1999);
+    ASSERT(vCreateDateFromSeconds.month = 2);
+    ASSERT(vCreateDateFromSeconds.day = 1);
+
+    ASSERT(vCreateTimeFromSeconds.hour = 12);
+    ASSERT(vCreateTimeFromSeconds.minute = 34);
+    ASSERT(vCreateTimeFromSeconds.second = 56);
+
+    ASSERT(vCreateTime.hour = 12);
+    ASSERT(vCreateTime.minute = 34);
+    ASSERT(vCreateTime.second = 56);
+
+    ASSERT(vCreateDateTime.year = 1999);
+    ASSERT(vCreateDateTime.month = 2);
+    ASSERT(vCreateDateTime.day = 1);
+    ASSERT(vCreateDateTime.hour = 12);
+    ASSERT(vCreateDateTime.minute = 34);
+    ASSERT(vCreateDateTime.second = 56);
+
+    ASSERT(Date.TimeFromTimeRec(vCreateTime) = 123456);
+
+    ASSERT(Date.DateFromDateTimeRec(vCreateDateTime) = 19990201);
+
+    ASSERT(Date.TimeFromDateTimeRec(vCreateDateTime) = 123456);
+
+    ASSERT(Date.SecondsFromDateTimeRec(vCreateDateTime) = 917872496);
+
+    ASSERT(TRUE)
+  ];
+
   EXPORT TestDynamic := MODULE
     //Iterate through all lots of dates, incrementing the day and the date to check they convert correctly.
     Date_rec := Date.Date_rec;
     test_rec := { Date.Days_t day, Date_rec gregorian, Date_rec julian };
-    firstDate := DATASET([{ 1, ROW(Date.createDate(1,1,1)), ROW(Date.createDate(1,1,1))}], test_rec);
+    firstDate := DATASET([{ 1, ROW(Date.CreateDate(1,1,1)), ROW(Date.CreateDate(1,1,1))}], test_rec);
     daysInLeapYearMonth := [31,29,31,30,31,30,31,31,30,31,30,31];
     daysInNonLeapYearMonth := [31,28,31,30,31,30,31,31,30,31,30,31];
 
@@ -59,10 +234,10 @@ EXPORT TestDate := MODULE
 
     processNextDate(DATASET(test_rec) in) := FUNCTION
        next := PROJECT(in, nextRecord(LEFT));
-       result1 := ASSERT(next, Date.ToGregorianDate(next.day) = Date.DateFromRec(next.gregorian));
-       result2 := ASSERT(result1, next.day = Date.FromGregorianDate(Date.DateFromRec(next.gregorian)));
-       result3 := ASSERT(result2, Date.ToJulianDate(next.day) = Date.DateFromRec(next.julian));
-       result4 := ASSERT(result3, next.day = Date.FromJulianDate(Date.DateFromRec(next.julian)));
+       result1 := ASSERT(next, Date.ToGregorianDate(next.day) = Date.DateFromDateRec(next.gregorian));
+       result2 := ASSERT(result1, next.day = Date.FromGregorianDate(Date.DateFromDateRec(next.gregorian)));
+       result3 := ASSERT(result2, Date.ToJulianDate(next.day) = Date.DateFromDateRec(next.julian));
+       result4 := ASSERT(result3, next.day = Date.FromJulianDate(Date.DateFromDateRec(next.julian)));
        RETURN result4;
     END;
 
@@ -71,6 +246,6 @@ EXPORT TestDate := MODULE
     EXPORT Test01 := OUTPUT(x);
   END;
 
-  EXPORT Main := [EVALUATE(TestConstant), EVALUATE(TestDynamic)];
+  EXPORT Main := [EVALUATE(TestConstant), EVALUATE(TestDynamicFunctions), EVALUATE(TestDynamic)];
 
 END;

ecllibrary/teststd/Date/TestFormat.ecl

@@ -6,36 +6,69 @@ IMPORT Std.Date;
 
 EXPORT TestFormat := MODULE
 
-  SHARED DateFormats := ['%d %b %Y', '%Y %b %d', '%Y%m%d', '%Y-%m-%d', '%d/%m/%Y'];
+  SHARED DateFormats := ['%d %b %Y', '%Y %b %d', '%Y%m%d', '%Y-%m-%d', '%d/%m/%Y', '%m/%d/%Y'];
+  SHARED TimeFormats := ['%H%M%S', '%T', '%R'];
 
   EXPORT TestConstant := [
-    ASSERT(date.ToString(19700101, '%Y-%m-%d') = '1970-01-01', CONST);
-    ASSERT(date.ToString(19700101, '%d/%m/%y') = '01/01/70', CONST);
-    ASSERT(date.ToString(20110302, '%d %b %Y') = '02 Mar 2011', CONST);
-    ASSERT(date.ToString(20111202, '%d %B %Y') = '02 December 2011', CONST);
-
-    ASSERT(date.FromString('19700001', '%Y%m%d') = 0, CONST);
-    ASSERT(date.FromString('19701000', '%Y%m%d') = 0, CONST);
-    ASSERT(date.FromString('19700101', '%Y%m%d') = 19700101, CONST);
-    ASSERT(date.FromString('68011', '%y%m%d') = 20680101, CONST);
-    ASSERT(date.FromString('69011', '%y%m%d') = 19690101, CONST);
-    ASSERT(date.FromString('1 \t Dec   2056', '%d %b %Y') = 20561201, CONST);
-    ASSERT(date.FromString('1 \t December  1862', '%d %b %Y') = 18621201, CONST);
-    ASSERT(date.FromString('31 \t jAN 12', '%d %b %Y') = 120131, CONST);
-    ASSERT(date.FromString('1 \t De   2056', '%d %b %Y') = 0, CONST);
-    ASSERT(date.FromString('1December1', '%d%b%Y') = 00011201, CONST);
-//    ASSERT(date.MatchDateString('1dec2011',DateFormats) = 20111201, CONST);
+    ASSERT(Date.FromStringToDate('19700001', '%Y%m%d') = 0, CONST);
+    ASSERT(Date.FromStringToDate('19701000', '%Y%m%d') = 0, CONST);
+    ASSERT(Date.FromStringToDate('19700101', '%Y%m%d') = 19700101, CONST);
+    ASSERT(Date.FromStringToDate('68011', '%y%m%d') = 20680101, CONST);
+    ASSERT(Date.FromStringToDate('69011', '%y%m%d') = 19690101, CONST);
+    ASSERT(Date.FromStringToDate('1 \t Dec   2056', '%d %b %Y') = 20561201, CONST);
+    ASSERT(Date.FromStringToDate('1 \t December  1862', '%d %b %Y') = 18621201, CONST);
+    ASSERT(Date.FromStringToDate('31 \t jAN 12', '%d %b %Y') = 120131, CONST);
+    ASSERT(Date.FromStringToDate('1 \t De   2056', '%d %b %Y') = 0, CONST);
+    ASSERT(Date.FromStringToDate('1December1', '%d%b%Y') = 00011201, CONST);
+    ASSERT(Date.FromStringToDate('1970-02-01', '%F') = 19700201, CONST);
+
+    ASSERT(Date.FromStringToTime('12:34:56', '%H:%M:%S') = 123456, CONST);
+    ASSERT(Date.FromStringToTime('12:34:56', '%T') = 123456, CONST);
+    ASSERT(Date.FromStringToTime('12:34', '%R') = 123400, CONST);
+
     ASSERT(TRUE)
   ];
 
   EXPORT TestDynamic := [
-    ASSERT(date.MatchDateString('1dec2011',DateFormats) = 20111201);
-    ASSERT(date.MatchDateString('2011dec1',DateFormats) = 20111201);
-    ASSERT(date.MatchDateString('1 december 2011',DateFormats) = 20111201);
-    ASSERT(date.MatchDateString('2011\tdecem\t01',DateFormats) = 20111201);
-    ASSERT(date.MatchDateString('20111201',DateFormats) = 20111201);
-    ASSERT(date.MatchDateString('2011-12-01',DateFormats) = 20111201);
-    ASSERT(date.MatchDateString('1/12/2011',DateFormats) = 20111201);
+    ASSERT(Date.MatchDateString('1dec2011',DateFormats) = 20111201);
+    ASSERT(Date.MatchDateString('2011dec1',DateFormats) = 20111201);
+    ASSERT(Date.MatchDateString('1 december 2011',DateFormats) = 20111201);
+    ASSERT(Date.MatchDateString('2011\tdecem\t01',DateFormats) = 20111201);
+    ASSERT(Date.MatchDateString('20111201',DateFormats) = 20111201);
+    ASSERT(Date.MatchDateString('2011-12-01',DateFormats) = 20111201);
+    ASSERT(Date.MatchDateString('1/12/2011',DateFormats) = 20111201);
+
+    ASSERT(Date.DateToString(19700101, '%Y-%m-%d') = '1970-01-01');
+    ASSERT(Date.DateToString(19700101, '%d/%m/%y') = '01/01/70');
+    ASSERT(Date.DateToString(20110302, '%d %b %Y') = '02 Mar 2011');
+    ASSERT(Date.DateToString(20111202, '%d %B %Y') = '02 December 2011');
+
+    ASSERT(Date.MatchDateString('1dec2011',DateFormats) = 20111201);
+    ASSERT(Date.MatchDateString('2011dec1',DateFormats) = 20111201);
+    ASSERT(Date.MatchDateString('1 december 2011',DateFormats) = 20111201);
+    ASSERT(Date.MatchDateString('2011\tdecem\t01',DateFormats) = 20111201);
+    ASSERT(Date.MatchDateString('20111201',DateFormats) = 20111201);
+    ASSERT(Date.MatchDateString('2011-12-01',DateFormats) = 20111201);
+    ASSERT(Date.MatchDateString('1/12/2011',DateFormats) = 20111201);
+
+    ASSERT(Date.MatchTimeString('123456',TimeFormats) = 123456);
+    ASSERT(Date.MatchTimeString('12:34:56',TimeFormats) = 123456);
+    ASSERT(Date.MatchTimeString('12:34',TimeFormats) = 123400);
+
+    ASSERT(Date.DateToString(19990201,'%F') = '1999-02-01');
+
+    ASSERT(Date.TimeToString(123456,'%T') = '12:34:56');
+
+    ASSERT(Date.SecondsToString(917872496,'%FT%T') = '1999-02-01T12:34:56');
+
+    ASSERT(Date.ConvertDateFormat('1/12/2011','%m/%d/%Y','%F') = '2011-01-12');
+
+    ASSERT(Date.ConvertTimeFormat('123456','%H%M%S','%T') = '12:34:56');
+
+    ASSERT(Date.ConvertDateFormatMultiple('1/31/2011',DateFormats,'%F') = '2011-01-31');
+
+    ASSERT(Date.ConvertTimeFormatMultiple('123456',TimeFormats,'%T') = '12:34:56');
+
     ASSERT(TRUE)
   ];
 

esp/services/ws_dfu/ws_dfuService.cpp

@@ -1496,7 +1496,7 @@ void CWsDfuEx::getDefFile(IUserDescriptor* udesc, const char* FileName,StringBuf
     if (errs.errCount())
     {
         StringBuffer errtext;
-        IECLError *first = errs.firstError();
+        IError *first = errs.firstError();
         first->toString(errtext);
         throw MakeStringException(ECLWATCH_CANNOT_PARSE_ECL_QUERY, "Failed in parsing ECL query: %s @ %d:%d.", errtext.str(), first->getColumn(), first->getLine());
     } 

esp/src/eclwatch/DelayLoadWidget.js

@@ -57,7 +57,7 @@ define([
             this.deferred = new Deferred();
             this.startLoading();
             var context = this;
-            require(["hpcc/" + this.delayWidget], function (widget) {
+            require([(this.delayFolder ? "plugins/" + this.delayFolder + "/" : "hpcc/") + this.delayWidget], function (widget) {
                 if (widget.fixCircularDependency) {
                     widget = widget.fixCircularDependency;
                 }

esp/src/eclwatch/HPCCPlatformServicesPluginWidget.js

@@ -0,0 +1,91 @@
+/*##############################################################################
+#	HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+#
+#	Licensed under the Apache License, Version 2.0 (the "License");
+#	you may not use this file except in compliance with the License.
+#	You may obtain a copy of the License at
+#
+#	   http://www.apache.org/licenses/LICENSE-2.0
+#
+#	Unless required by applicable law or agreed to in writing, software
+#	distributed under the License is distributed on an "AS IS" BASIS,
+#	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#	See the License for the specific language governing permissions and
+#	limitations under the License.
+############################################################################## */
+define([
+    "dojo/_base/declare",
+    "dojo/_base/lang",
+    "dojo/i18n",
+    "dojo/i18n!./nls/hpcc",
+    "dojo/_base/array",
+
+    "hpcc/_TabContainerWidget",
+    "hpcc/DelayLoadWidget",
+    "hpcc/WsTopology",
+
+    "dojo/text!../templates/HPCCPlatformServicesPluginWidget.html",
+
+    "dijit/layout/BorderContainer",
+    "dijit/layout/TabContainer",
+    "dijit/layout/ContentPane"
+
+], function (declare, lang, i18n, nlsHPCC, arrayUtil,
+                _TabContainerWidget, DelayLoadWidget, WsTopology,
+                template) {
+    return declare("HPCCPlatformServicesPluginWidget", [_TabContainerWidget], {
+        templateString: template,
+        baseClass: "HPCCPlatformServicesPluginWidget",
+        i18n: nlsHPCC,
+
+        postCreate: function (args) {
+            this.inherited(arguments);
+        },
+
+        startup: function (args) {
+            this.inherited(arguments);
+        },
+
+        getTitle: function () {
+            return this.i18n.title_HPCCPlatformServicesPlugin;
+        },
+
+        //  Implementation  ---
+        init: function (params) {
+            if (this.inherited(arguments))
+                return;
+
+            var context = this;
+            WsTopology.TpGetServicePlugins({
+                request: {
+                }
+            }).then(function (response) {
+                if (lang.exists("TpGetServicePluginsResponse.Plugins.Plugin", response) && response.TpGetServicePluginsResponse.Plugins.Plugin.length) {
+                    arrayUtil.forEach(response.TpGetServicePluginsResponse.Plugins.Plugin, function (item) {
+                        var pluginWidget = new DelayLoadWidget({
+                            id: context.createChildTabID(item.ShortName),
+                            title: item.ShortName,
+                            delayFolder: item.FolderName,
+                            delayWidget: item.WidgetName,
+                            hpcc: {
+                                params: {
+                                }
+                            }
+                        });
+                        context.addChild(pluginWidget);
+                        context.resize();
+                    });
+                }
+            });
+        },
+
+        initTab: function () {
+            var currSel = this.getSelectedChild();
+            if (currSel && !currSel.initalized) {
+                if (currSel.init) {
+                    currSel.init({});
+                }
+            }
+        }
+    });
+});

esp/src/eclwatch/HPCCPlatformWidget.js

@@ -37,6 +37,7 @@ define([
     "hpcc/ws_access",
     "hpcc/WsDfu",
     "hpcc/WsSMC",
+    "hpcc/WsTopology",
     "hpcc/GraphWidget",
     "hpcc/DelayLoadWidget",
 
@@ -63,7 +64,7 @@ define([
 ], function (declare, lang, i18n, nlsHPCC, arrayUtil, dom, domForm, domStyle, domGeo, cookie,
                 registry, Tooltip,
                 UpgradeBar,
-                _TabContainerWidget, ESPRequest, ESPActivity, WsAccount, WsAccess, WsDfu, WsSMC, GraphWidget, DelayLoadWidget,
+                _TabContainerWidget, ESPRequest, ESPActivity, WsAccount, WsAccess, WsDfu, WsSMC, WsTopology, GraphWidget, DelayLoadWidget,
                 template) {
     return declare("HPCCPlatformWidget", [_TabContainerWidget], {
         templateString: template,
@@ -81,6 +82,7 @@ define([
             this.stackContainer = registry.byId(this.id + "TabContainer");
             this.mainPage = registry.byId(this.id + "_Main");
             this.errWarnPage = registry.byId(this.id + "_ErrWarn");
+            this.pluginsPage = registry.byId(this.id + "_Plugins");
             registry.byId(this.id + "SetBanner").set("disabled", true);
 
             this.upgradeBar = new UpgradeBar({
@@ -91,6 +93,9 @@ define([
 
         startup: function (args) {
             this.inherited(arguments);
+            domStyle.set(dom.byId(this.id + "StackController_stub_Plugins").parentNode.parentNode, {
+                visibility: "hidden"
+            });
             domStyle.set(dom.byId(this.id + "StackController_stub_ErrWarn").parentNode.parentNode, {
                 visibility: "hidden"
             });
@@ -213,6 +218,17 @@ define([
                 }
             });
 
+            WsTopology.TpGetServicePlugins({
+                request: {
+                }
+            }).then(function (response) {
+                if (lang.exists("TpGetServicePluginsResponse.Plugins.Plugin", response) && response.TpGetServicePluginsResponse.Plugins.Plugin.length) {
+                    domStyle.set(dom.byId(context.id + "StackController_stub_Plugins").parentNode.parentNode, {
+                        visibility: "visible"
+                    });
+                }
+            });
+
             this.activity = ESPActivity.Get();
             this.activity.watch("Build", function (name, oldValue, newValue) {
                 context.parseBuildString(newValue);
@@ -226,6 +242,7 @@ define([
             this.createStackControllerTooltip(this.id + "_Files", this.i18n.Files);
             this.createStackControllerTooltip(this.id + "_RoxieQueries", this.i18n.PublishedQueries);
             this.createStackControllerTooltip(this.id + "_OPS", this.i18n.Operations);
+            this.createStackControllerTooltip(this.id + "_Plugins", this.i18n.Plugins);
             this.initTab();
         },
 

esp/src/eclwatch/WsTopology.js

@@ -121,6 +121,9 @@ define([
         },
         TpThorStatus: function (params) {
             return ESPRequest.send("WsTopology", "TpThorStatus", params);
+        },
+        TpGetServicePlugins: function (params) {
+            return ESPRequest.send("WsTopology", "TpGetServicePlugins", params);
         }
     };
 });

esp/src/eclwatch/css/hpcc.css

@@ -363,6 +363,13 @@ hr.dashedLine {
     cursor: pointer;
 }
 
+.iconPlugins{
+    background-image: url("../img/Plugins.png");
+    width: 32px;
+    height: 32px;
+    cursor: pointer;
+}
+
 .iconLandingZone{
     background-image: url("../img/LandingZone.png");
     width: 32px;

esp/src/eclwatch/dojoConfig.js

@@ -14,6 +14,7 @@ var dojoConfig = (function () {
             params: searchNodes.length >= 2 ? searchNodes[1] : "",
             baseHost: baseHost,
             basePath: baseHost + "/esp/files",
+            pluginsPath: baseHost + "/esp/files",
             resourcePath: baseHost + "/esp/files/eclwatch",
             scriptsPath: baseHost + "/esp/files/eclwatch",
             thisPath: pathnodes.join("/")
@@ -54,6 +55,9 @@ var dojoConfig = (function () {
             name: "ecl",
             location: urlInfo.resourcePath + "/ecl"
         }, {
+            name: "plugins",
+            location: urlInfo.pluginsPath
+        }, {
             name: "this",
             location: urlInfo.thisPath
         }],

esp/src/eclwatch/nls/hpcc.js

@@ -327,6 +327,7 @@ define({root:
     PlaceholderFirstName: "John",
     PlaceholderLastName: "Smith",
     Playground: "Playground",
+    Plugins: "Plugins",
     Port: "Port",
     Prefix: "Prefix",
     PrefixPlaceholder: "filename{:length}, filesize{:[B|L][1-8]}",
@@ -486,6 +487,7 @@ define({root:
     title_HPCCPlatformMain: "ECL Watch - Home",
     title_HPCCPlatformOps: "ECL Watch - Operations",
     title_HPCCPlatformRoxie: "ECL Watch - Roxie",
+    title_HPCCPlatformServicesPlugin: "ECL Watch - Plugins",
     title_Inputs: "Inputs",
     title_LFDetails: "Logical File Details",
     title_LZBrowse: "Landing Zones",

esp/src/eclwatch/templates/HPCCPlatformServicesPluginWidget.html

@@ -0,0 +1,9 @@
+<div class="${baseClass}">
+    <div id="${id}BorderContainer" class="${baseClass}BorderContainer" style="width: 100%; height: 100%; margin:0; padding:0" data-dojo-props="gutters:false, liveSplitters:false" data-dojo-type="dijit.layout.BorderContainer">
+        <div id="${id}TitlebarMini" class="miniTitlebar" data-dojo-props="region: 'top'" data-dojo-type="dijit.layout.ContentPane">
+            <div id="${id}StackController" style="width: 100%" data-dojo-props="containerId:'${id}TabContainer'" data-dojo-type="dijit.layout.StackController"></div>
+        </div>
+        <div id="${id}TabContainer" data-dojo-props="region: 'center', tabPosition: 'top'" style="width: 100%; height: 100%" data-dojo-type="dijit.layout.StackContainer">
+        </div>
+    </div>
+</div>

esp/src/eclwatch/templates/HPCCPlatformWidget.html

@@ -49,6 +49,8 @@
             </div>
             <div id="${id}_OPS" data-dojo-props="iconClass: 'iconOperations', showLabel: false, delayWidget: 'HPCCPlatformOpsWidget'" data-dojo-type="DelayLoadWidget">
             </div>
+            <div id="${id}_Plugins" data-dojo-props="iconClass: 'iconPlugins', showLabel: false, delayWidget: 'HPCCPlatformServicesPluginWidget'" data-dojo-type="DelayLoadWidget">
+            </div>
             <div id="${id}_ErrWarn" data-dojo-props="showLabel: false" data-dojo-type="dijit.layout.TabContainer">
                 <div id="${id}ErrWarnGrid" data-dojo-props="title: 'Error/Warning(s)', errWarn: true, showToolbar: true" data-dojo-type="InfoGridWidget">
                 </div>

plugins/CMakeLists.txt

@@ -19,6 +19,7 @@ add_subdirectory (fileservices)
 add_subdirectory (logging)
 add_subdirectory (parselib)
 add_subdirectory (stringlib)
+add_subdirectory (timelib)
 add_subdirectory (unicodelib)
 add_subdirectory (workunitservices)
 add_subdirectory (proxies)
@@ -29,3 +30,4 @@ add_subdirectory (pyembed)
 add_subdirectory (javaembed)
 add_subdirectory (Rembed)
 add_subdirectory (cassandra)
+add_subdirectory (memcached)

plugins/memcached/CMakeLists.txt

@@ -0,0 +1,67 @@
+################################################################################
+#    HPCC SYSTEMS software Copyright (C) 2014 HPCC Systems.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+################################################################################
+
+# Component: memcached
+
+#####################################################
+# Description:
+# ------------
+#    Cmake Input File for memcached
+#####################################################
+
+project( memcached )
+
+if (USE_MEMCACHED)
+  ADD_PLUGIN(memcached PACKAGES MEMCACHED OPTION MAKE_MEMCACHED)
+  if ( MAKE_MEMCACHED )
+    set (    SRCS
+             memcachedplugin.hpp
+             memcachedplugin.cpp
+        )
+
+    include_directories (
+             ./../../system/include
+             ./../../rtl/eclrtl
+             ./../../rtl/include
+             ./../../common/deftype
+             ./../../common/workunit
+             ./../../system/jlib
+             ${LIBMEMCACHED_INCLUDE_DIR}
+        )
+
+    ADD_DEFINITIONS( -D_USRDLL -DECL_MEMCACHED_EXPORTS)
+    SET (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}  -std=gnu++0x")
+
+    HPCC_ADD_LIBRARY( memcached SHARED ${SRCS} )
+    if (${CMAKE_VERSION} VERSION_LESS "2.8.9")
+      message("WARNING: Cannot set NO_SONAME. shlibdeps will give warnings when package is installed")
+    else()
+      set_target_properties( memcached PROPERTIES NO_SONAME 1 )
+    endif()
+
+    install ( TARGETS memcached DESTINATION plugins)
+
+    target_link_libraries ( memcached
+        eclrtl
+        jlib
+        workunit
+        ${LIBMEMCACHED_LIBRARIES}
+        )
+  endif()
+endif()
+
+#Even if not making the memcached plugin, we want to install the header
+install ( FILES ${CMAKE_CURRENT_SOURCE_DIR}/lib_memcached.ecllib DESTINATION plugins COMPONENT Runtime)

plugins/memcached/lib_memcached.ecllib

@@ -0,0 +1,40 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2014 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the License);
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an AS IS BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+export memcached := SERVICE : plugin('memcached')
+  BOOLEAN SetUnicode(CONST VARSTRING servers, CONST VARSTRING key, CONST UNICODE value, CONST VARSTRING partitionKey = '', UNSIGNED4 expire = 0) : cpp,action,context,entrypoint='MSet';
+  BOOLEAN SetString(CONST VARSTRING servers, CONST VARSTRING key, CONST STRING value, CONST VARSTRING partitionKey = '', UNSIGNED4 expire = 0) : cpp,action,context,entrypoint='MSet';
+  BOOLEAN SetUtf8(CONST VARSTRING servers, CONST VARSTRING key, CONST UTF8 value, CONST VARSTRING partitionKey = '', UNSIGNED4 expire = 0) : cpp,action,context,entrypoint='MSetUtf8';
+  BOOLEAN SetBoolean(CONST VARSTRING servers, CONST VARSTRING key, BOOLEAN value, CONST VARSTRING partitionKey = '', UNSIGNED4 expire = 0) : cpp,action,context,entrypoint='MSet';
+  BOOLEAN SetReal(CONST VARSTRING servers, CONST VARSTRING key, REAL value, CONST VARSTRING partitionKey = '', UNSIGNED4 expire = 0) : cpp,action,context,entrypoint='MSet';
+  BOOLEAN SetInteger(CONST VARSTRING servers, CONST VARSTRING key, INTEGER value, CONST VARSTRING partitionKey = '', UNSIGNED4 expire = 0) : cpp,action,context,entrypoint='MSet';
+  BOOLEAN SetUnsigned(CONST VARSTRING servers, CONST VARSTRING key, UNSIGNED value,CONST VARSTRING partitionKey = '',  UNSIGNED4 expire = 0) : cpp,action,context,entrypoint='MSet';
+  BOOLEAN SetData(CONST VARSTRING servers, CONST VARSTRING key, CONST DATA value, CONST VARSTRING partitionKey = '', UNSIGNED4 expire = 0) : cpp,action,context,entrypoint='MSetData';
+
+  INTEGER8 GetInteger(CONST VARSTRING servers, CONST VARSTRING key, CONST VARSTRING partitionKey = '') : cpp,action,context,entrypoint='MGetInt8';
+  UNSIGNED8 GetUnsigned(CONST VARSTRING servers, CONST VARSTRING key, CONST VARSTRING partitionKey = '') : cpp,action,context,entrypoint='MGetUint8';
+  STRING GetString(CONST VARSTRING servers, CONST VARSTRING key, CONST VARSTRING partitionKey = '') : cpp,action,context,entrypoint='MGetStr';
+  UNICODE GetUnicode(CONST VARSTRING servers, CONST VARSTRING key, CONST VARSTRING partitionKey = '') : cpp,action,context,entrypoint='MGetUChar';
+  UTF8 GetUtf8(CONST VARSTRING servers, CONST VARSTRING key, CONST VARSTRING partitionKey = '') : cpp,action,context,entrypoint='MGetUtf8';
+  BOOLEAN GetBoolean(CONST VARSTRING servers, CONST VARSTRING key, CONST VARSTRING partitionKey = '') : cpp,action,context,entrypoint='MGetBool';
+  REAL GetReal(CONST VARSTRING servers, CONST VARSTRING key, CONST VARSTRING partitionKey = '') : cpp,action,context,entrypoint='MGetDouble';
+  DATA GetData(CONST VARSTRING servers, CONST VARSTRING key, CONST VARSTRING partitionKey = '') : cpp,action,context,entrypoint='MGetData';
+ 
+  BOOLEAN Exist(CONST VARSTRING servers, CONST VARSTRING key, CONST VARSTRING partitionKey = '') : cpp,action,context,entrypoint='MExist';
+  CONST VARSTRING KeyType(CONST VARSTRING servers, CONST VARSTRING key, CONST VARSTRING partitionKey = '') : cpp,action,context,entrypoint='MKeyType'; //NOTE: calls get
+  BOOLEAN Clear(CONST VARSTRING servers) : cpp,action,context,entrypoint='MClear';
+END;

plugins/memcached/memcachedplugin.cpp

@@ -0,0 +1,624 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2014 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#include "platform.h"
+#include "memcachedplugin.hpp"
+#include "eclrtl.hpp"
+#include "jexcept.hpp"
+#include "jstring.hpp"
+#include "workunit.hpp"
+#include <libmemcached/util.h>
+
+#define MEMCACHED_VERSION "memcached plugin 1.0.0"
+
+ECL_MEMCACHED_API bool getECLPluginDefinition(ECLPluginDefinitionBlock *pb)
+{
+    if (pb->size != sizeof(ECLPluginDefinitionBlock))
+        return false;
+
+    pb->magicVersion = PLUGIN_VERSION;
+    pb->version = MEMCACHED_VERSION;
+    pb->moduleName = "lib_memcached";
+    pb->ECL = NULL;
+    pb->flags = PLUGIN_IMPLICIT_MODULE;
+    pb->description = "ECL plugin library for the C/C++ API libmemcached (http://libmemcached.org/)\n";
+    return true;
+}
+
+namespace MemCachedPlugin {
+IPluginContext * parentCtx = NULL;
+static const unsigned unitExpire = 86400;//1 day (secs)
+
+enum eclDataType {
+    ECL_BOOLEAN,
+    ECL_DATA,
+    ECL_INTEGER,
+    ECL_REAL,
+    ECL_STRING,
+    ECL_UTF8,
+    ECL_UNICODE,
+    ECL_UNSIGNED,
+    ECL_NONE
+};
+
+const char * enumToStr(eclDataType type)
+{
+    switch(type)
+    {
+    case ECL_BOOLEAN:
+        return "BOOLEAN";
+    case ECL_INTEGER:
+        return "INTEGER";
+    case ECL_UNSIGNED:
+        return "UNSIGNED";
+    case ECL_REAL:
+        return "REAL";
+    case ECL_STRING:
+        return "STRING";
+    case ECL_UTF8:
+        return "UTF8";
+    case ECL_UNICODE:
+        return "UNICODE";
+    case ECL_DATA:
+        return "DATA";
+    case ECL_NONE:
+        return "Nonexistent";
+    default:
+        return "UNKNOWN";
+    }
+}
+
+class MCached : public CInterface
+{
+public :
+    MCached(ICodeContext * ctx, const char * servers);
+    ~MCached();
+
+    //set
+    template <class type> bool set(ICodeContext * ctx, const char * partitionKey, const char * key, type value, unsigned expire, eclDataType eclType);
+    template <class type> bool set(ICodeContext * ctx, const char * partitionKey, const char * key, size32_t valueLength, const type * value, unsigned expire, eclDataType eclType);
+    //get
+    template <class type> void get(ICodeContext * ctx, const char * partitionKey, const char * key, type & value, eclDataType eclType);
+    template <class type> void get(ICodeContext * ctx, const char * partitionKey, const char * key, size_t & valueLength, type * & value, eclDataType eclType);
+    void getVoidPtrLenPair(ICodeContext * ctx, const char * partitionKey, const char * key, size_t & valueLength, void * & value, eclDataType eclType);
+
+    bool clear(ICodeContext * ctx, unsigned when);
+    bool exist(ICodeContext * ctx, const char * key, const char * partitionKey);
+    eclDataType getKeyType(const char * key, const char * partitionKey);
+
+    bool isSameConnection(const char * _servers) const;
+
+private :
+    void checkServersUp(ICodeContext * ctx);
+    void assertOnError(memcached_return_t error, const char * msgSuffix = "");
+    const char * keyNotFoundMsg(memcached_return_t error, const char * key, StringBuffer & target) const;
+    void connect(ICodeContext * ctx);
+    bool reportErrorOnFail(ICodeContext * ctx, memcached_return_t error);
+    void reportKeyTypeMismatch(ICodeContext * ctx, const char * key, uint32_t flag, eclDataType eclType);
+    void * cpy(const char * src, size_t length);
+    void logServerStats(ICodeContext * ctx);
+    void init(ICodeContext * ctx);
+    void invokePoolSecurity(ICodeContext * ctx);
+    void invokeConnectionSecurity(ICodeContext * ctx);
+    void setPoolSettings();
+    void assertPool();//For internal purposes to insure correct order of the above processes and instantiation.
+
+private :
+    memcached_st * connection;
+    memcached_pool_st * pool;
+    StringAttr servers;
+    bool alreadyInitialized;
+    unsigned typeMismatchCount;
+};
+
+#define OwnedMCached Owned<MemCachedPlugin::MCached>
+
+#define MAX_TYPEMISMATCHCOUNT 10
+
+static CriticalSection crit;
+static OwnedMCached cachedConnection;
+
+MCached * createConnection(ICodeContext * ctx, const char * servers)
+{
+    CriticalBlock block(crit);
+    if (!cachedConnection)
+    {
+        cachedConnection.setown(new MemCachedPlugin::MCached(ctx, servers));
+        return LINK(cachedConnection);
+    }
+
+    if (cachedConnection->isSameConnection(servers))
+        return LINK(cachedConnection);
+
+    cachedConnection.setown(new MemCachedPlugin::MCached(ctx, servers));
+    return LINK(cachedConnection);
+}
+
+//-------------------------------------------SET-----------------------------------------
+template<class type> bool MSet(ICodeContext * ctx, const char * _servers, const char * partitionKey, const char * key, type value, unsigned expire, eclDataType eclType)
+{
+    OwnedMCached serverPool = createConnection(ctx, _servers);
+    bool success = serverPool->set(ctx, partitionKey, key, value, expire, eclType);
+    return success;
+}
+//Set pointer types
+template<class type> bool MSet(ICodeContext * ctx, const char * _servers, const char * partitionKey, const char * key, size32_t valueLength, const type * value, unsigned expire, eclDataType eclType)
+{
+    OwnedMCached serverPool = createConnection(ctx, _servers);
+    bool success = serverPool->set(ctx, partitionKey, key, valueLength, value, expire, eclType);
+    return success;
+}
+//-------------------------------------------GET-----------------------------------------
+template<class type> void MGet(ICodeContext * ctx, const char * servers, const char * partitionKey, const char * key, type & returnValue, eclDataType eclType)
+{
+    OwnedMCached serverPool = createConnection(ctx, servers);
+    serverPool->get(ctx, partitionKey, key, returnValue, eclType);
+}
+template<class type> void MGet(ICodeContext * ctx, const char * servers, const char * partitionKey, const char * key, size_t & returnLength, type * & returnValue, eclDataType eclType)
+{
+    OwnedMCached serverPool = createConnection(ctx, servers);
+    serverPool->get(ctx, partitionKey, key, returnLength, returnValue, eclType);
+}
+void MGetVoidPtrLenPair(ICodeContext * ctx, const char * servers, const char * partitionKey, const char * key, size_t & returnLength, void * & returnValue, eclDataType eclType)
+{
+    OwnedMCached serverPool = createConnection(ctx, servers);
+    serverPool->getVoidPtrLenPair(ctx, partitionKey, key, returnLength, returnValue, eclType);
+}
+}//close namespace
+
+//----------------------------------SET----------------------------------------
+template<class type> bool MemCachedPlugin::MCached::set(ICodeContext * ctx, const char * partitionKey, const char * key, type value, unsigned expire, eclDataType eclType)
+{
+    const char * _value = reinterpret_cast<const char *>(&value);//Do this even for char * to prevent compiler complaining
+    size_t partitionKeyLength = strlen(partitionKey);
+    if (partitionKeyLength)
+        return !reportErrorOnFail(ctx, memcached_set_by_key(connection, partitionKey, partitionKeyLength, key, strlen(key), _value, sizeof(value), (time_t)(expire*unitExpire), (uint32_t)eclType));
+    else
+        return !reportErrorOnFail(ctx, memcached_set(connection, key, strlen(key), _value, sizeof(value), (time_t)(expire*unitExpire), (uint32_t)eclType));
+}
+template<class type> bool MemCachedPlugin::MCached::set(ICodeContext * ctx, const char * partitionKey, const char * key, size32_t valueLength, const type * value, unsigned expire, eclDataType eclType)
+{
+    const char * _value = reinterpret_cast<const char *>(value);//Do this even for char * to prevent compiler complaining
+    size_t partitionKeyLength = strlen(partitionKey);
+    if (partitionKeyLength)
+        return !reportErrorOnFail(ctx, memcached_set_by_key(connection, partitionKey, partitionKeyLength, key, strlen(key), _value, (size_t)(valueLength), (time_t)(expire*unitExpire), (uint32_t)eclType));
+    else
+        return !reportErrorOnFail(ctx, memcached_set(connection, key, strlen(key), _value, (size_t)(valueLength), (time_t)(expire*unitExpire), (uint32_t)eclType));
+}
+//----------------------------------GET----------------------------------------
+template<class type> void MemCachedPlugin::MCached::get(ICodeContext * ctx, const char * partitionKey, const char * key, type & returnValue, eclDataType eclType)
+{
+    uint32_t flag = 0;
+    size_t returnLength = 0;
+    memcached_return_t error;
+
+    OwnedMalloc<char> value;
+    size_t partitionKeyLength = strlen(partitionKey);
+    if (partitionKeyLength)
+        value.setown(memcached_get_by_key(connection, partitionKey, partitionKeyLength, key, strlen(key), &returnLength, &flag, &error));
+    else
+        value.setown(memcached_get(connection, key, strlen(key), &returnLength, &flag, &error));
+
+    StringBuffer keyMsg;
+    assertOnError(error, keyNotFoundMsg(error, key, keyMsg));
+    reportKeyTypeMismatch(ctx, key, flag, eclType);
+
+    if (sizeof(type)!=returnLength)
+    {
+        VStringBuffer msg("MemCachedPlugin: ERROR - Requested type of different size (%uB) from that stored (%uB). Check logs for more information.", (unsigned)sizeof(type), (unsigned)returnLength);
+        rtlFail(0, msg.str());
+    }
+    memcpy(&returnValue, value, returnLength);
+}
+template<class type> void MemCachedPlugin::MCached::get(ICodeContext * ctx, const char * partitionKey, const char * key, size_t & returnLength, type * & returnValue, eclDataType eclType)
+{
+    uint32_t flag = 0;
+    memcached_return_t error;
+
+    OwnedMalloc<char> value;
+    size_t partitionKeyLength = strlen(partitionKey);
+    if (partitionKeyLength)
+        value.setown(memcached_get_by_key(connection, partitionKey, partitionKeyLength, key, strlen(key), &returnLength, &flag, &error));
+    else
+        value.setown(memcached_get(connection, key, strlen(key), &returnLength, &flag, &error));
+
+    StringBuffer keyMsg;
+    assertOnError(error, keyNotFoundMsg(error, key, keyMsg));
+    reportKeyTypeMismatch(ctx, key, flag, eclType);
+
+    returnValue = reinterpret_cast<type*>(cpy(value, returnLength));
+}
+void MemCachedPlugin::MCached::getVoidPtrLenPair(ICodeContext * ctx, const char * partitionKey, const char * key, size_t & returnLength, void * & returnValue, eclDataType eclType)
+{
+    uint32_t flag = 0;
+    size_t returnValueLength = 0;
+    memcached_return_t error;
+
+    OwnedMalloc<char> value;
+    size_t partitionKeyLength = strlen(partitionKey);
+    if (partitionKeyLength)
+        value.setown(memcached_get_by_key(connection, partitionKey, partitionKeyLength, key, strlen(key), &returnValueLength, &flag, &error));
+    else
+        value.setown(memcached_get(connection, key, strlen(key), &returnValueLength, &flag, &error));
+
+    StringBuffer keyMsg;
+    assertOnError(error, keyNotFoundMsg(error, key, keyMsg));
+    reportKeyTypeMismatch(ctx, key, flag, eclType);
+
+    returnLength = (size32_t)(returnValueLength);
+    returnValue = reinterpret_cast<void*>(cpy(value, returnLength));
+}
+
+ECL_MEMCACHED_API void setPluginContext(IPluginContext * ctx) { MemCachedPlugin::parentCtx = ctx; }
+
+MemCachedPlugin::MCached::MCached(ICodeContext * ctx, const char * _servers)
+{
+    alreadyInitialized = false;
+    connection = NULL;
+    pool = NULL;
+    servers.set(_servers);
+    typeMismatchCount = 0;
+
+    pool = memcached_pool(_servers, strlen(_servers));
+    assertPool();
+
+    setPoolSettings();
+    invokePoolSecurity(ctx);
+    connect(ctx);
+    checkServersUp(ctx);
+}
+//-----------------------------------------------------------------------------
+MemCachedPlugin::MCached::~MCached()
+{
+    if (pool)
+    {
+        memcached_pool_release(pool, connection);
+        connection = NULL;//For safety (from changing this destructor) as not implicit in either the above or below.
+        memcached_pool_destroy(pool);
+    }
+    else if (connection)//This should never be needed but just in case.
+    {
+        memcached_free(connection);
+    }
+}
+
+bool MemCachedPlugin::MCached::isSameConnection(const char * _servers) const
+{
+    if (!_servers)
+        return false;
+
+    return stricmp(servers.get(), _servers) == 0;
+}
+
+void MemCachedPlugin::MCached::assertPool()
+{
+    if (!pool)
+    {
+        StringBuffer msg = "Memcached Plugin: Failed to instantiate server pool with:";
+        msg.newline().append(servers);
+        rtlFail(0, msg.str());
+    }
+}
+
+void * MemCachedPlugin::MCached::cpy(const char * src, size_t length)
+{
+    void * value = rtlMalloc(length);
+    return memcpy(value, src, length);
+}
+
+void MemCachedPlugin::MCached::checkServersUp(ICodeContext * ctx)
+{
+    memcached_return_t error;
+    char * args = NULL;
+
+    OwnedMalloc<memcached_stat_st> stats;
+    stats.setown(memcached_stat(connection, args, &error));
+    assertex(stats);
+
+    unsigned int numberOfServers = memcached_server_count(connection);
+    unsigned int numberOfServersDown = 0;
+    for (unsigned i = 0; i < numberOfServers; ++i)
+    {
+        if (stats[i].pid == -1)//perhaps not the best test?
+        {
+            numberOfServersDown++;
+            VStringBuffer msg("Memcached Plugin: Failed connecting to entry %u\nwithin the server list: %s", i+1, servers.str());
+            ctx->addWuException(msg.str(), WRN_FROM_PLUGIN, ExceptionSeverityWarning, "");
+        }
+    }
+    if (numberOfServersDown == numberOfServers)
+        rtlFail(0,"Memcached Plugin: Failed connecting to ALL servers. Check memcached on all servers and \"memcached -B ascii\" not used.");
+
+    //check memcached version homogeneity
+    for (unsigned i = 0; i < numberOfServers-1; ++i)
+    {
+        if (strcmp(stats[i].version, stats[i+1].version) != 0)
+            ctx->addWuException("Memcached Plugin: Inhomogeneous versions of memcached across servers.", WRN_FROM_PLUGIN, ExceptionSeverityInformation, "");
+    }
+}
+
+bool MemCachedPlugin::MCached::reportErrorOnFail(ICodeContext * ctx, memcached_return_t error)
+{
+    if (error == MEMCACHED_SUCCESS)
+        return false;
+
+    VStringBuffer msg("Memcached Plugin: %s", memcached_strerror(connection, error));
+    ctx->addWuException(msg.str(), ERR_FROM_PLUGIN, ExceptionSeverityInformation, "");
+    return true;
+}
+
+void MemCachedPlugin::MCached::assertOnError(memcached_return_t error, const char * msgSuffix)
+{
+    if (error != MEMCACHED_SUCCESS)
+    {
+        VStringBuffer msg("Memcached Plugin: %s%s", memcached_strerror(connection, error), msgSuffix);
+        rtlFail(0, msg.str());
+    }
+}
+
+const char * MemCachedPlugin::MCached::keyNotFoundMsg(memcached_return_t error, const char * key, StringBuffer & target) const
+{
+    target.clear();
+    if (error == MEMCACHED_NOTFOUND)
+    {
+        target = " (key: '";
+        target.append(key).append("') ");
+    }
+    return target.str();
+}
+
+bool MemCachedPlugin::MCached::clear(ICodeContext * ctx, unsigned when)
+{
+    //NOTE: memcached_flush is the actual cache flush/clear/delete and not an io buffer flush.
+    return !reportErrorOnFail(ctx, memcached_flush(connection, (time_t)(when)));
+}
+
+bool MemCachedPlugin::MCached::exist(ICodeContext * ctx, const char * key, const char * partitionKey)
+{
+    memcached_return_t error;
+    size_t partitionKeyLength = strlen(partitionKey);
+    if (partitionKeyLength)
+        error = memcached_exist_by_key(connection, partitionKey, partitionKeyLength, key, strlen(key));
+    else
+        error = memcached_exist(connection, key, strlen(key));
+
+    if (error == MEMCACHED_SUCCESS)
+        return true;
+    else if (error == MEMCACHED_NOTFOUND)
+        return false;
+
+    reportErrorOnFail(ctx, error);
+    return false;
+}
+
+MemCachedPlugin::eclDataType MemCachedPlugin::MCached::getKeyType(const char * key, const char * partitionKey)
+{
+    size_t returnValueLength;
+    uint32_t flag;
+    memcached_return_t error;
+
+    size_t partitionKeyLength = strlen(partitionKey);
+    if (partitionKeyLength)
+        memcached_get_by_key(connection, partitionKey, partitionKeyLength, key, strlen(key), &returnValueLength, &flag, &error);
+    else
+        memcached_get(connection, key, strlen(key), &returnValueLength, &flag, &error);
+
+    if (error == MEMCACHED_SUCCESS)
+        return (MemCachedPlugin::eclDataType)(flag);
+    else if (error == MEMCACHED_NOTFOUND)
+        return ECL_NONE;
+    else
+    {
+        StringBuffer msg = "Memcached Plugin: ";
+        rtlFail(0, msg.append(memcached_strerror(connection, error)).str());
+    }
+}
+
+void MemCachedPlugin::MCached::reportKeyTypeMismatch(ICodeContext * ctx, const char * key, uint32_t flag, eclDataType eclType)
+{
+    if (flag && eclType != ECL_DATA && flag != eclType)
+    {
+        VStringBuffer msg("Memcached Plugin: The requested key '%s' is of type %s, not %s as requested.", key, enumToStr((eclDataType)(flag)), enumToStr(eclType));
+        if (++typeMismatchCount <= MAX_TYPEMISMATCHCOUNT)
+            ctx->logString(msg.str());//NOTE: logging locally, rather than calling ctx->addWuException, to prevent flooding the WU if this is called multiple times by every node
+    }
+}
+
+void MemCachedPlugin::MCached::logServerStats(ICodeContext * ctx)
+{
+    //NOTE: errors are ignored here so that at least some info is reported, such as non-connection related libmemcached version numbers
+    memcached_return_t error;
+    char * args = NULL;
+
+    OwnedMalloc<memcached_stat_st> stats;
+    stats.setown(memcached_stat(connection, args, &error));
+
+    OwnedMalloc<char*> keys;
+    keys.setown(memcached_stat_get_keys(connection, stats, &error));
+
+    unsigned int numberOfServers = memcached_server_count(connection);
+    for (unsigned int i = 0; i < numberOfServers; ++i)
+    {
+        StringBuffer statsStr;
+        unsigned j = 0;
+        do
+        {
+            OwnedMalloc<char> value;
+            value.setown(memcached_stat_get_value(connection, &stats[i], keys[j], &error));
+            statsStr.newline().append("libmemcached server stat - ").append(keys[j]).append(":").append(value);
+        } while (keys[++j]);
+        statsStr.newline().append("libmemcached client stat - libmemcached version:").append(memcached_lib_version());
+        ctx->logString(statsStr.str());
+    }
+}
+
+void MemCachedPlugin::MCached::init(ICodeContext * ctx)
+{
+    logServerStats(ctx);
+}
+
+void MemCachedPlugin::MCached::setPoolSettings()
+{
+    assertPool();
+    assertOnError(memcached_pool_behavior_set(pool, MEMCACHED_BEHAVIOR_HASH_WITH_PREFIX_KEY, 1));//key set in invokeConnectionSecurity. Only hashed with keys and not partitionKeys
+    assertOnError(memcached_pool_behavior_set(pool, MEMCACHED_BEHAVIOR_KETAMA, 1));//NOTE: alias of MEMCACHED_DISTRIBUTION_CONSISTENT_KETAMA amongst others.
+    assertOnError(memcached_pool_behavior_set(pool, MEMCACHED_BEHAVIOR_USE_UDP, 0));
+    assertOnError(memcached_pool_behavior_set(pool, MEMCACHED_BEHAVIOR_SERVER_FAILURE_LIMIT, 1));
+    assertOnError(memcached_pool_behavior_set(pool, MEMCACHED_BEHAVIOR_REMOVE_FAILED_SERVERS, 1));
+    assertOnError(memcached_pool_behavior_set(pool, MEMCACHED_BEHAVIOR_NO_BLOCK, 0));
+    assertOnError(memcached_pool_behavior_set(pool, MEMCACHED_BEHAVIOR_CONNECT_TIMEOUT, 100));//units of ms MORE: What should I set this to or get from?
+    assertOnError(memcached_pool_behavior_set(pool, MEMCACHED_BEHAVIOR_BUFFER_REQUESTS, 0));// Buffering does not work with the ecl runtime paradigm
+}
+
+void MemCachedPlugin::MCached::invokePoolSecurity(ICodeContext * ctx)
+{
+    assertPool();
+    assertOnError(memcached_pool_behavior_set(pool, MEMCACHED_BEHAVIOR_BINARY_PROTOCOL, 1));
+}
+
+void MemCachedPlugin::MCached::invokeConnectionSecurity(ICodeContext * ctx)
+{
+    //NOTE: Whether to assert or just report? This depends on when this is called. If before checkServersUp() and
+    //a server is down, it will cause the following to fail if asserted with only a 'poor' libmemcached error message.
+    //Reporting means that these 'security' measures may not be carried out. Moving checkServersUp() to here is probably the best
+    //soln. however, this comes with extra overhead.
+    reportErrorOnFail(ctx, memcached_verbosity(connection, (uint32_t)(0)));
+    //reportErrorOnFail(ctx, memcached_callback_set(connection, MEMCACHED_CALLBACK_PREFIX_KEY, "ecl"));//NOTE: MEMCACHED_CALLBACK_PREFIX_KEY is an alias of MEMCACHED_CALLBACK_NAMESPACE
+}
+
+void MemCachedPlugin::MCached::connect(ICodeContext * ctx)
+{
+    assertPool();
+    if (connection)
+        memcached_pool_release(pool, connection);
+
+    memcached_return_t error;
+    connection = memcached_pool_fetch(pool, (struct timespec *)0 , &error);
+    invokeConnectionSecurity(ctx);
+
+    if (!alreadyInitialized)//Do this now rather than after assert. Better to have something even if it could be jiberish.
+    {
+        init(ctx);//doesn't necessarily initialize anything, instead outputs specs etc for debugging
+        alreadyInitialized = true;
+    }
+    assertOnError(error);
+}
+
+//--------------------------------------------------------------------------------
+//                           ECL SERVICE ENTRYPOINTS
+//--------------------------------------------------------------------------------
+ECL_MEMCACHED_API bool ECL_MEMCACHED_CALL MClear(ICodeContext * ctx, const char * servers)
+{
+    OwnedMCached serverPool = MemCachedPlugin::createConnection(ctx, servers);
+    bool returnValue = serverPool->clear(ctx, 0);
+    return returnValue;
+}
+ECL_MEMCACHED_API bool ECL_MEMCACHED_CALL MExist(ICodeContext * ctx, const char * servers, const char * key, const char * partitionKey)
+{
+    OwnedMCached serverPool = MemCachedPlugin::createConnection(ctx, servers);
+    bool returnValue = serverPool->exist(ctx, key, partitionKey);
+    return returnValue;
+}
+ECL_MEMCACHED_API const char * ECL_MEMCACHED_CALL MKeyType(ICodeContext * ctx, const char * servers, const char * key, const char * partitionKey)
+{
+    OwnedMCached serverPool = MemCachedPlugin::createConnection(ctx, servers);
+    const char * keyType = enumToStr(serverPool->getKeyType(key, partitionKey));
+    return keyType;
+}
+//-----------------------------------SET------------------------------------------
+//NOTE: These were all overloaded by 'value' type, however; this caused problems since ecl implicitly casts and doesn't type check.
+ECL_MEMCACHED_API bool ECL_MEMCACHED_CALL MSet(ICodeContext * ctx, const char * servers, const char * key, size32_t valueLength, const char * value, const char * partitionKey, unsigned expire /* = 0 (ECL default)*/)
+{
+    return MemCachedPlugin::MSet(ctx, servers, partitionKey, key, valueLength, value, expire, MemCachedPlugin::ECL_STRING);
+}
+ECL_MEMCACHED_API bool ECL_MEMCACHED_CALL MSet(ICodeContext * ctx, const char * servers, const char * key, size32_t valueLength, const UChar * value, const char * partitionKey, unsigned expire /* = 0 (ECL default)*/)
+{
+    return MemCachedPlugin::MSet(ctx, servers, partitionKey, key, (valueLength)*sizeof(UChar), value, expire, MemCachedPlugin::ECL_UNICODE);
+}
+ECL_MEMCACHED_API bool ECL_MEMCACHED_CALL MSet(ICodeContext * ctx, const char * servers, const char * key, signed __int64 value, const char * partitionKey, unsigned expire /* = 0 (ECL default)*/)
+{
+    return MemCachedPlugin::MSet(ctx, servers, partitionKey, key, value, expire, MemCachedPlugin::ECL_INTEGER);
+}
+ECL_MEMCACHED_API bool ECL_MEMCACHED_CALL MSet(ICodeContext * ctx, const char * servers, const char * key, unsigned __int64 value, const char * partitionKey, unsigned expire /* = 0 (ECL default)*/)
+{
+    return MemCachedPlugin::MSet(ctx, servers, partitionKey, key, value, expire, MemCachedPlugin::ECL_UNSIGNED);
+}
+ECL_MEMCACHED_API bool ECL_MEMCACHED_CALL MSet(ICodeContext * ctx, const char * servers, const char * key, double value, const char * partitionKey, unsigned expire /* = 0 (ECL default)*/)
+{
+    return MemCachedPlugin::MSet(ctx, servers, partitionKey, key, value, expire, MemCachedPlugin::ECL_REAL);
+}
+ECL_MEMCACHED_API bool ECL_MEMCACHED_CALL MSet(ICodeContext * ctx, const char * servers, const char * key, bool value, const char * partitionKey, unsigned expire)
+{
+    return MemCachedPlugin::MSet(ctx, servers, partitionKey, key, value, expire, MemCachedPlugin::ECL_BOOLEAN);
+}
+ECL_MEMCACHED_API bool ECL_MEMCACHED_CALL MSetData(ICodeContext * ctx, const char * servers, const char * key, size32_t valueLength, const void * value, const char * partitionKey, unsigned expire)
+{
+    return MemCachedPlugin::MSet(ctx, servers, partitionKey, key, valueLength, value, expire, MemCachedPlugin::ECL_DATA);
+}
+ECL_MEMCACHED_API bool ECL_MEMCACHED_CALL MSetUtf8(ICodeContext * ctx, const char * servers, const char * key, size32_t valueLength, const char * value, const char * partitionKey, unsigned expire /* = 0 (ECL default)*/)
+{
+    return MemCachedPlugin::MSet(ctx, servers, partitionKey, key, rtlUtf8Size(valueLength, value), value, expire, MemCachedPlugin::ECL_UTF8);
+}
+//-------------------------------------GET----------------------------------------
+ECL_MEMCACHED_API bool ECL_MEMCACHED_CALL MGetBool(ICodeContext * ctx, const char * servers, const char * key, const char * partitionKey)
+{
+    bool value;
+    MemCachedPlugin::MGet(ctx, servers, partitionKey, key, value, MemCachedPlugin::ECL_BOOLEAN);
+    return value;
+}
+ECL_MEMCACHED_API double ECL_MEMCACHED_CALL MGetDouble(ICodeContext * ctx, const char * servers, const char * key, const char * partitionKey)
+{
+    double value;
+    MemCachedPlugin::MGet(ctx, servers, partitionKey, key, value, MemCachedPlugin::ECL_REAL);
+    return value;
+}
+ECL_MEMCACHED_API signed __int64 ECL_MEMCACHED_CALL MGetInt8(ICodeContext * ctx, const char * servers, const char * key, const char * partitionKey)
+{
+    signed __int64 value;
+    MemCachedPlugin::MGet(ctx, servers, partitionKey, key, value, MemCachedPlugin::ECL_INTEGER);
+    return value;
+}
+ECL_MEMCACHED_API unsigned __int64 ECL_MEMCACHED_CALL MGetUint8(ICodeContext * ctx, const char * servers, const char * key, const char * partitionKey)
+{
+    unsigned __int64 value;
+    MemCachedPlugin::MGet(ctx, servers, partitionKey, key, value, MemCachedPlugin::ECL_UNSIGNED);
+    return value;
+}
+ECL_MEMCACHED_API void ECL_MEMCACHED_CALL MGetStr(ICodeContext * ctx, size32_t & returnLength, char * & returnValue, const char * servers, const char * key, const char * partitionKey)
+{
+    size_t _returnLength;
+    MemCachedPlugin::MGet(ctx, servers, partitionKey, key, _returnLength, returnValue, MemCachedPlugin::ECL_STRING);
+    returnLength = static_cast<size32_t>(_returnLength);
+}
+ECL_MEMCACHED_API void ECL_MEMCACHED_CALL MGetUChar(ICodeContext * ctx, size32_t & returnLength, UChar * & returnValue,  const char * servers, const char * key, const char * partitionKey)
+{
+    size_t _returnSize;
+    MemCachedPlugin::MGet(ctx, servers, partitionKey, key, _returnSize, returnValue, MemCachedPlugin::ECL_UNICODE);
+    returnLength = static_cast<size32_t>(_returnSize/sizeof(UChar));
+}
+ECL_MEMCACHED_API void ECL_MEMCACHED_CALL MGetUtf8(ICodeContext * ctx, size32_t & returnLength, char * & returnValue, const char * servers, const char * key, const char * partitionKey)
+{
+    size_t returnSize;
+    MemCachedPlugin::MGet(ctx, servers, partitionKey, key, returnSize, returnValue, MemCachedPlugin::ECL_UTF8);
+    returnLength = static_cast<size32_t>(rtlUtf8Length(returnSize, returnValue));
+}
+ECL_MEMCACHED_API void ECL_MEMCACHED_CALL MGetData(ICodeContext * ctx, size32_t & returnLength, void * & returnValue, const char * servers, const char * key, const char * partitionKey)
+{
+    size_t _returnLength;
+    MemCachedPlugin::MGetVoidPtrLenPair(ctx, servers, partitionKey, key, _returnLength, returnValue, MemCachedPlugin::ECL_DATA);
+    returnLength = static_cast<size32_t>(_returnLength);
+}

plugins/memcached/memcachedplugin.hpp

@@ -0,0 +1,71 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2014 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#ifndef ECL_MEMCACHED_INCL
+#define ECL_MEMCACHED_INCL
+
+#ifdef _WIN32
+#define ECL_MEMCACHED_CALL _cdecl
+#ifdef ECL_MEMCACHED_EXPORTS
+#define ECL_MEMCACHED_API __declspec(dllexport)
+#else
+#define ECL_MEMCACHED_API __declspec(dllimport)
+#endif
+#else
+#define ECL_MEMCACHED_CALL
+#define ECL_MEMCACHED_API
+#endif
+
+#include "hqlplugins.hpp"
+#include "eclhelper.hpp"
+
+#define WRN_FROM_PLUGIN 10001
+#define ERR_FROM_PLUGIN 10002
+
+extern "C"
+{
+    ECL_MEMCACHED_API bool getECLPluginDefinition(ECLPluginDefinitionBlock *pb);
+    ECL_MEMCACHED_API void setPluginContext(IPluginContext * _ctx);
+}
+
+//NB: LIBMEMCACHED_API already used by libmemcached
+extern "C++"
+{
+    //--------------------------SET----------------------------------------
+    ECL_MEMCACHED_API bool ECL_MEMCACHED_CALL MSet    (ICodeContext * _ctx, const char * servers, const char * key, bool value, const char * partitionKey, unsigned expire);
+    ECL_MEMCACHED_API bool ECL_MEMCACHED_CALL MSet    (ICodeContext * _ctx, const char * servers, const char * key, signed __int64 value, const char * partitionKey, unsigned expire);
+    ECL_MEMCACHED_API bool ECL_MEMCACHED_CALL MSet    (ICodeContext * _ctx, const char * servers, const char * key, unsigned __int64 value, const char * partitionKey, unsigned expire);
+    ECL_MEMCACHED_API bool ECL_MEMCACHED_CALL MSet    (ICodeContext * _ctx, const char * servers, const char * key, double value, const char * partitionKey, unsigned expire);
+    ECL_MEMCACHED_API bool ECL_MEMCACHED_CALL MSetUtf8(ICodeContext * _ctx, const char * servers, const char * key, size32_t valueLength, const char * value, const char * partitionKey, unsigned expire);
+    ECL_MEMCACHED_API bool ECL_MEMCACHED_CALL MSet    (ICodeContext * _ctx, const char * servers, const char * key, size32_t valueLength, const char * value, const char * partitionKey, unsigned expire);
+    ECL_MEMCACHED_API bool ECL_MEMCACHED_CALL MSet    (ICodeContext * _ctx, const char * servers, const char * key, size32_t valueLength, const UChar * value, const char * partitionKey, unsigned expire);
+    ECL_MEMCACHED_API bool ECL_MEMCACHED_CALL MSetData(ICodeContext * _ctx, const char * servers, const char * key, size32_t valueLength, const void * value, const char * partitionKey, unsigned expire);
+    //--------------------------GET----------------------------------------
+    ECL_MEMCACHED_API bool             ECL_MEMCACHED_CALL MGetBool  (ICodeContext * _ctx, const char * servers, const char * key, const char * partitionKey);
+    ECL_MEMCACHED_API signed __int64   ECL_MEMCACHED_CALL MGetInt8  (ICodeContext * _ctx, const char * servers, const char * key, const char * partitionKey);
+    ECL_MEMCACHED_API unsigned __int64 ECL_MEMCACHED_CALL MGetUint8 (ICodeContext * _ctx, const char * servers, const char * key, const char * partitionKey);
+    ECL_MEMCACHED_API double           ECL_MEMCACHED_CALL MGetDouble(ICodeContext * _ctx, const char * servers, const char * key, const char * partitionKey);
+    ECL_MEMCACHED_API void             ECL_MEMCACHED_CALL MGetUtf8  (ICodeContext * _ctx, size32_t & valueLength, char * & returnValue, const char * servers, const char * key, const char * partitionKey);
+    ECL_MEMCACHED_API void             ECL_MEMCACHED_CALL MGetStr   (ICodeContext * _ctx, size32_t & valueLength, UChar * & returnValue, const char * servers, const char * key, const char * partitionKey);
+    ECL_MEMCACHED_API void             ECL_MEMCACHED_CALL MGetUChar (ICodeContext * _ctx, size32_t & valueLength, char * & returnValue, const char * servers, const char * key, const char * partitionKey);
+    ECL_MEMCACHED_API void             ECL_MEMCACHED_CALL MGetData  (ICodeContext * _ctx,size32_t & returnLength, void * & returnValue, const char * servers, const char * key, const char * partitionKey);
+    //--------------------------------AUXILLARIES---------------------------
+    ECL_MEMCACHED_API bool             ECL_MEMCACHED_CALL MExist  (ICodeContext * _ctx, const char * servers, const char * key, const char * partitionKey);
+    ECL_MEMCACHED_API const char *     ECL_MEMCACHED_CALL MKeyType(ICodeContext * _ctx, const char * servers, const char * key, const char * partitionKey);
+    ECL_MEMCACHED_API bool             ECL_MEMCACHED_CALL MClear  (ICodeContext * _ctx, const char * servers);
+}
+#endif

plugins/stringlib/stringlib.cpp

@@ -83,7 +83,9 @@ static const char * EclDefinition =
 "  SET OF STRING SplitWords(const string src, const string _separator, BOOLEAN allow_blanks) : c, pure,entrypoint='slSplitWords'; \n"
 "  STRING CombineWords(set of string src, const string _separator) : c, pure,entrypoint='slCombineWords'; \n"
 "  UNSIGNED4 StringToDate(const string src, const varstring format) : c, pure,entrypoint='slStringToDate'; \n"
+"  UNSIGNED4 StringToTimeOfDay(const string src, const varstring format) : c, pure,entrypoint='slStringToTimeOfDay'; \n"
 "  UNSIGNED4 MatchDate(const string src, set of varstring formats) : c, pure,entrypoint='slMatchDate'; \n"
+"  UNSIGNED4 MatchTimeOfDay(const string src, set of varstring formats) : c, pure,entrypoint='slMatchTimeOfDay'; \n"
 "  STRING FormatDate(UNSIGNED4 date, const varstring format) : c, pure,entrypoint='slFormatDate'; \n"
 "  STRING StringRepeat(const string src, unsigned4 n) : c, pure,entrypoint='slStringRepeat'; \n"
 "END;";
@@ -1320,7 +1322,7 @@ STRINGLIB_API void STRINGLIB_CALL slCombineWords(size32_t & __lenResult, void *
 
 //--------------------------------------------------------------------------------------------------------------------
 
-inline bool readValue(unsigned & value, size32_t & _offset, size32_t lenStr, const char * str, unsigned max)
+inline bool readValue(unsigned & value, size32_t & _offset, size32_t lenStr, const char * str, unsigned max, bool spaceIsZero = false)
 {
     unsigned total = 0;
     unsigned offset = _offset;
@@ -1332,6 +1334,8 @@ inline bool readValue(unsigned & value, size32_t & _offset, size32_t lenStr, con
         char next = str[offset+i];
         if (next >= '0' && next <= '9')
             total = total * 10 + (next - '0');
+    	else if (next == ' ' && spaceIsZero)
+            total = total * 10;
         else
             break;
     }
@@ -1384,6 +1388,44 @@ static const char * simple_strptime(size32_t lenStr, const char * str, const cha
         {
             switch (*curFormat++)
             {
+            // Recursive cases
+            case 'F':
+            	{
+            		const char*	newPtr = simple_strptime(lenStr-offset, str+offset, "%Y-%m-%d", tm);
+            		
+            		if (!newPtr)
+            			return NULL;
+            		offset = newPtr - str;
+            	}
+            	break;
+            case 'D':
+            	{
+            		const char*	newPtr = simple_strptime(lenStr-offset, str+offset, "%m/%d/%y", tm);
+            		
+            		if (!newPtr)
+            			return NULL;
+            		offset = newPtr - str;
+            	}
+            	break;
+            case 'R':
+            	{
+            		const char*	newPtr = simple_strptime(lenStr-offset, str+offset, "%H:%M", tm);
+            		
+            		if (!newPtr)
+            			return NULL;
+            		offset = newPtr - str;
+            	}
+            	break;
+            case 'T':
+            	{
+            		const char*	newPtr = simple_strptime(lenStr-offset, str+offset, "%H:%M:%S", tm);
+            		
+            		if (!newPtr)
+            			return NULL;
+            		offset = newPtr - str;
+            	}
+            	break;
+            // Non-recursive cases
             case 't':
                 while ((offset < lenStr) && isspace(src[offset]))
                     offset++;
@@ -1408,6 +1450,11 @@ static const char * simple_strptime(size32_t lenStr, const char * str, const cha
                     return NULL;
                 tm->tm_mday = value;
                 break;
+            case 'e':
+                if (!readValue(value, offset, lenStr, str, 2, true) || (value < 1) || (value > 31))
+                    return NULL;
+                tm->tm_mday = value;
+                break;
             case 'b':
             case 'B':
             case 'h':
@@ -1420,6 +1467,11 @@ static const char * simple_strptime(size32_t lenStr, const char * str, const cha
                     return NULL;
                 tm->tm_hour = value;
                 break;
+            case 'k':
+                if (!readValue(value, offset, lenStr, str, 2, true)|| (value > 24))
+                    return NULL;
+                tm->tm_hour = value;
+                break;
             case 'M':
                 if (!readValue(value, offset, lenStr, str, 2)|| (value > 59))
                     return NULL;
@@ -1457,6 +1509,12 @@ inline unsigned makeDate(const tm & tm)
     return (tm.tm_year + 1900) * 10000 + (tm.tm_mon + 1) * 100 + tm.tm_mday;
 }
 
+
+inline unsigned makeTimeOfDay(const tm & tm)
+{
+    return (tm.tm_hour * 10000) + (tm.tm_min * 100) + tm.tm_sec;
+}
+
 inline void extractDate(tm & tm, unsigned date)
 {
     tm.tm_year = (date / 10000) - 1900;
@@ -1475,16 +1533,26 @@ STRINGLIB_API unsigned STRINGLIB_CALL slStringToDate(size32_t lenS, const char *
     return 0;
 }
 
+STRINGLIB_API unsigned STRINGLIB_CALL slStringToTimeOfDay(size32_t lenS, const char * s, const char * fmtin)
+{
+    struct tm tm;
+    memset(&tm, 0, sizeof(tm));
+    if (simple_strptime(lenS, s, fmtin, &tm))
+        return makeTimeOfDay(tm);
+    return 0;
+}
+
 
 STRINGLIB_API unsigned STRINGLIB_CALL slMatchDate(size32_t lenS, const char * s, bool isAllFormats, unsigned lenFormats, const void * _formats)
 {
     struct tm tm;
-    memset(&tm, 0, sizeof(tm));
 
     const char * formats = (const char *)_formats;
     for (unsigned off=0; off < lenFormats; )
     {
         const char * curFormat = formats+off;
+        
+        memset(&tm, 0, sizeof(tm));
         if (simple_strptime(lenS, s, curFormat, &tm))
             return makeDate(tm);
         off += strlen(curFormat) + 1;
@@ -1492,6 +1560,24 @@ STRINGLIB_API unsigned STRINGLIB_CALL slMatchDate(size32_t lenS, const char * s,
     return 0;
 }
 
+
+STRINGLIB_API unsigned STRINGLIB_CALL slMatchTimeOfDay(size32_t lenS, const char * s, bool isAllFormats, unsigned lenFormats, const void * _formats)
+{
+    struct tm tm;
+
+    const char * formats = (const char *)_formats;
+    for (unsigned off=0; off < lenFormats; )
+    {
+        const char * curFormat = formats+off;
+        
+        memset(&tm, 0, sizeof(tm));
+        if (simple_strptime(lenS, s, curFormat, &tm))
+            return makeTimeOfDay(tm);
+        off += strlen(curFormat) + 1;
+    }
+    return 0;
+}
+
 STRINGLIB_API void STRINGLIB_CALL slFormatDate(size32_t & __lenResult, char * & __result, unsigned date, const char * format)
 {
     size32_t len = 0;

plugins/stringlib/stringlib.hpp

@@ -85,7 +85,9 @@ STRINGLIB_API unsigned STRINGLIB_CALL slCountWords(size32_t lenSrc, const char *
 STRINGLIB_API void STRINGLIB_CALL slSplitWords(bool & __isAllResult, size32_t & __lenResult, void * & __result, size32_t lenSrc, const char * src, size32_t lenSeparator, const char * separator, bool allowBlankItems);
 STRINGLIB_API void STRINGLIB_CALL slCombineWords(size32_t & __lenResult, void * & __result, bool isAllSrc, size32_t lenSrc, const char * src, size32_t lenSeparator, const char * separator, bool allowBlankItems);
 STRINGLIB_API unsigned STRINGLIB_CALL slStringToDate(size32_t lenS, const char * s, const char * fmtin);
+STRINGLIB_API unsigned STRINGLIB_CALL slStringToTimeOfDay(size32_t lenS, const char * s, const char * fmtin);
 STRINGLIB_API unsigned STRINGLIB_CALL slMatchDate(size32_t lenS, const char * s, bool isAllFormats, unsigned lenFormats, const void * _formats);
+STRINGLIB_API unsigned STRINGLIB_CALL slMatchTimeOfDay(size32_t lenS, const char * s, bool isAllFormats, unsigned lenFormats, const void * _formats);
 STRINGLIB_API void STRINGLIB_CALL slFormatDate(size32_t & __lenResult, char * & __result, unsigned date, const char * format);
 STRINGLIB_API void STRINGLIB_CALL slStringRepeat(unsigned & tgtLen, char * & tgt, unsigned srcLen, const char * src, unsigned n);
 }

plugins/timelib/CMakeLists.txt

@@ -0,0 +1,48 @@
+################################################################################
+#    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+################################################################################
+
+
+# Component: timelib 
+
+#####################################################
+# Description:
+# ------------
+#    Cmake Input File for timelib
+#####################################################
+
+set ( toolsdir "${HPCC_SOURCE_DIR}/tools" )
+
+
+project( timelib ) 
+
+set (    SRCS 
+         timelib.cpp 
+    )
+
+include_directories ( 
+         ./../../system/include 
+         ./../../system/jlib
+         ./../../rtl/include
+         ./../../rtl/eclrtl
+    )
+
+ADD_DEFINITIONS( -D_USRDLL -DTIMELIB_EXPORTS )
+
+HPCC_ADD_LIBRARY( timelib SHARED ${SRCS} )
+install ( TARGETS timelib DESTINATION plugins )
+target_link_libraries ( timelib
+         eclrtl
+    )

plugins/timelib/sourcedoc.xml

@@ -0,0 +1,26 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!--
+################################################################################
+#    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+################################################################################
+-->
+<!DOCTYPE section PUBLIC "-//OASIS//DTD DocBook XML V4.3//EN" "http://www.oasis-open.org/docbook/xml/4.3/docbookx.dtd">
+<section>
+    <title>plugins/timelib</title>
+
+    <para>
+        The plugins/timelib directory contains the sources for the plugins/timelib library.
+    </para>
+</section>

plugins/timelib/timelib.cpp

@@ -0,0 +1,955 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#include <platform.h>
+#include <time.h>
+#include <stdlib.h>
+#include <string.h>
+#include <ctype.h>
+#include <assert.h>
+#include <eclrtl.hpp>
+
+#ifdef _WIN32
+#include <sys/timeb.h>
+#endif
+
+#include "timelib.hpp"
+
+static const char * compatibleVersions[] = {
+    NULL };
+
+#define TIMELIB_VERSION "TIMELIB 1.0.0"
+
+static const char * EclDefinition =
+"EXPORT TMPartsRec := RECORD \n"
+"  INTEGER4 sec; \n"
+"  INTEGER4 min; \n"
+"  INTEGER4 hour; \n"
+"  INTEGER4 mday; \n"
+"  INTEGER4 mon; \n"
+"  INTEGER4 year; \n"
+"  INTEGER4 wday; \n"
+"END;"
+"EXPORT TMDateRangeRec := RECORD \n"
+"  UNSIGNED4 startDate; \n"
+"  UNSIGNED4 endDate; \n"
+"END;"
+"EXPORT TimeLib := SERVICE\n"
+"  INTEGER8 SecondsFromParts(integer2 year, unsigned1 month, unsigned1 day, unsigned1 hour, unsigned1 minute, unsigned1 second, boolean is_local_time) : c,pure,entrypoint='tlSecondsFromParts'; \n"
+"  TRANSFORM(TMPartsRec) SecondsToParts(INTEGER8 seconds) : c,pure,entrypoint='tlSecondsToParts'; \n"
+"  UNSIGNED2 GetDayOfYear(INTEGER2 year, UNSIGNED1 month, UNSIGNED1 day) : c,pure,entrypoint='tlGetDayOfYear'; \n"
+"  UNSIGNED1 GetDayOfWeek(INTEGER2 year, UNSIGNED1 month, UNSIGNED1 day) : c,pure,entrypoint='tlGetDayOfWeek'; \n"
+"  STRING DateToString(UNSIGNED4 date, CONST VARSTRING format) : c,pure,entrypoint='tlDateToString'; \n"
+"  STRING TimeToString(UNSIGNED3 time, CONST VARSTRING format) : c,pure,entrypoint='tlTimeToString'; \n"
+"  STRING SecondsToString(INTEGER8 seconds, CONST VARSTRING format) : c,pure,entrypoint='tlSecondsToString'; \n"
+"  UNSIGNED4 AdjustDate(UNSIGNED4 date, INTEGER2 year_delta, INTEGER4 month_delta, INTEGER4 day_delta) : c,pure,entrypoint='tlAdjustDate'; \n"
+"  UNSIGNED4 AdjustDateBySeconds(UNSIGNED4 date, INTEGER4 seconds_delta) : c,pure,entrypoint='tlAdjustDateBySeconds'; \n"
+"  UNSIGNED4 AdjustTime(UNSIGNED3 time, INTEGER2 hour_delta, INTEGER4 minute_delta, INTEGER4 second_delta) : c,pure,entrypoint='tlAdjustTime'; \n"
+"  UNSIGNED4 AdjustTimeBySeconds(UNSIGNED3 time, INTEGER4 seconds_delta) : c,pure,entrypoint='tlAdjustTimeBySeconds'; \n"
+"  INTEGER4 AdjustSeconds(INTEGER8 seconds, INTEGER2 year_delta, INTEGER4 month_delta, INTEGER4 day_delta, INTEGER2 hour_delta, INTEGER4 minute_delta, INTEGER4 second_delta) : c,pure,entrypoint='tlAdjustSeconds'; \n"
+"  UNSIGNED4 AdjustCalendar(UNSIGNED4 date, INTEGER2 year_delta, INTEGER4 month_delta, INTEGER4 day_delta) : c,pure,entrypoint='tlAdjustCalendar'; \n"
+"  BOOLEAN IsLocalDaylightSavingsInEffect() : c,pure,entrypoint='tlIsLocalDaylightSavingsInEffect'; \n"
+"  INTEGER4 LocalTimeZoneOffset() : c,pure,entrypoint='tlLocalTimeZoneOffset'; \n"
+"  UNSIGNED4 CurrentDate(BOOLEAN in_local_time) : c,once,entrypoint='tlCurrentDate'; \n"
+"  UNSIGNED4 CurrentTime(BOOLEAN in_local_time) : c,entrypoint='tlCurrentTime'; \n"
+"  INTEGER8 CurrentSeconds(BOOLEAN in_local_time) : c,entrypoint='tlCurrentSeconds'; \n"
+"  INTEGER8 CurrentTimestamp(BOOLEAN in_local_time) : c,entrypoint='tlCurrentTimestamp'; \n"
+"  UNSIGNED4 GetLastDayOfMonth(UNSIGNED4 date) : c,pure,entrypoint='tlGetLastDayOfMonth'; \n"
+"  TRANSFORM(TMDateRangeRec) DatesForWeek(UNSIGNED4 date) : c,pure,entrypoint='tlDatesForWeek'; \n"
+"END;";
+
+TIMELIB_API bool getECLPluginDefinition(ECLPluginDefinitionBlock *pb)
+{
+    if (pb->size == sizeof(ECLPluginDefinitionBlockEx))
+    {
+        ECLPluginDefinitionBlockEx * pbx = (ECLPluginDefinitionBlockEx *) pb;
+        pbx->compatibleVersions = compatibleVersions;
+    }
+    else if (pb->size != sizeof(ECLPluginDefinitionBlock))
+        return false;
+    pb->magicVersion = PLUGIN_VERSION;
+    pb->version = TIMELIB_VERSION;
+    pb->moduleName = "lib_timelib";
+    pb->ECL = EclDefinition;
+    pb->flags = PLUGIN_IMPLICIT_MODULE | PLUGIN_MULTIPLE_VERSIONS;
+    pb->description = "TimeLib time manipulation library";
+    return true;
+}
+
+IPluginContext * parentCtx = NULL;
+
+TIMELIB_API void setPluginContext(IPluginContext * _ctx) { parentCtx = _ctx; }
+
+//------------------------------------------------------------------------------
+
+#ifdef _WIN32
+const __int64 _onesec_in100ns = (__int64)10000000;
+
+static __int64 tlFileTimeToInt64(FILETIME f)
+{
+    __int64     seconds;
+
+    seconds = f.dwHighDateTime;
+    seconds <<= 32;
+    seconds |= f.dwLowDateTime;
+
+    return seconds;
+}
+
+static FILETIME tlInt64ToFileTime(__int64 seconds)
+{
+    FILETIME    f;
+
+    f.dwHighDateTime = (DWORD)((seconds >> 32) & 0x00000000FFFFFFFF);
+    f.dwLowDateTime = (DWORD)(seconds & 0x00000000FFFFFFFF);
+
+    return f;
+}
+
+static FILETIME tlFileTimeFromYear(WORD year)
+{
+    SYSTEMTIME  s;
+    FILETIME    f;
+
+    memset(&s, 0, sizeof(s));
+
+    s.wYear = year;
+    s.wMonth = 1;
+    s.wDayOfWeek = 1;
+    s.wDay = 1;
+
+    SystemTimeToFileTime(&s, &f);
+
+    return f;
+}
+
+static unsigned int tlYearDayFromSystemTime(const SYSTEMTIME* s)
+{
+    __int64     seconds;
+    FILETIME    f1;
+    FILETIME    f2;
+
+    f1 = tlFileTimeFromYear(s->wYear);
+    SystemTimeToFileTime(s, &f2);
+
+    seconds = tlFileTimeToInt64(f2) - tlFileTimeToInt64(f1);
+
+    return static_cast<unsigned int>((seconds / _onesec_in100ns) / (60 * 60 * 24));
+}
+
+static SYSTEMTIME tlTimeStructToSystemTime(const struct tm* timeInfoPtr)
+{
+    SYSTEMTIME s;
+
+    s.wYear = timeInfoPtr->tm_year + 1900;
+    s.wMonth = timeInfoPtr->tm_mon + 1;
+    s.wDayOfWeek = timeInfoPtr->tm_wday;
+    s.wDay = timeInfoPtr->tm_mday;
+    s.wHour = timeInfoPtr->tm_hour;
+    s.wMinute = timeInfoPtr->tm_min;
+    s.wSecond = timeInfoPtr->tm_sec;
+    s.wMilliseconds = 0;
+
+    return s;
+}
+
+static void tlSystemTimeToTimeStruct_r(const SYSTEMTIME* s, struct tm* timeInfoPtr)
+{
+    memset(timeInfoPtr, 0, sizeof(struct tm));
+
+    timeInfoPtr->tm_year = s->wYear - 1900;
+    timeInfoPtr->tm_mon = s->wMonth - 1;
+    timeInfoPtr->tm_wday = s->wDayOfWeek;
+    timeInfoPtr->tm_mday = s->wDay;
+    timeInfoPtr->tm_yday = tlYearDayFromSystemTime(s);
+    timeInfoPtr->tm_hour = s->wHour;
+    timeInfoPtr->tm_min = s->wMinute;
+    timeInfoPtr->tm_sec = s->wSecond;
+    timeInfoPtr->tm_isdst = 0;
+}
+
+static time_t tlFileTimeToSeconds(const FILETIME* f)
+{
+    const __int64   offset = I64C(11644473600); // Number of seconds between 1601 and 1970 (Jan 1 of each)
+
+    return static_cast<time_t>((tlFileTimeToInt64(*f) / _onesec_in100ns) - offset);
+}
+
+static FILETIME tlSecondsToFileTime(const time_t seconds)
+{
+    FILETIME    f1970 = tlFileTimeFromYear(1970);
+    FILETIME    f;
+    __int64     time;
+
+    time = (seconds * _onesec_in100ns) + tlFileTimeToInt64(f1970);
+
+    f = tlInt64ToFileTime(time);
+
+    return f;
+}
+
+static __int64 tlLocalTimeZoneDiffIn100nsIntervals()
+{
+    SYSTEMTIME  systemUTC;
+    SYSTEMTIME  systemLocal;
+    FILETIME    fileUTC;
+    FILETIME    fileLocal;
+
+    GetSystemTime(&systemUTC);
+    GetLocalTime(&systemLocal);
+
+    SystemTimeToFileTime(&systemUTC, &fileUTC);
+    SystemTimeToFileTime(&systemLocal, &fileLocal);
+
+    return tlFileTimeToInt64(fileLocal) - tlFileTimeToInt64(fileUTC);
+}
+
+static void tlBoundaryMod(int* tensPtr, int* unitsPtr, int base)
+{
+    if (*unitsPtr >= base)
+    {
+        *tensPtr += *unitsPtr / base;
+        *unitsPtr %= base;
+    }
+    else if (*unitsPtr < 0)
+    {
+        --*tensPtr;
+        *unitsPtr += base;
+        if (*unitsPtr < 0)
+        {
+            *tensPtr -= 1 + (-*unitsPtr) / base;
+            *unitsPtr = base - (-*unitsPtr) % base;
+
+            if (*unitsPtr == base)
+            {
+                *tensPtr += 1;
+                *unitsPtr = 0;
+            }
+        }
+    }
+}
+
+static void tlNormalizeTimeStruct(struct tm* timeInfoPtr)
+{
+    // Normalize incoming struct tm
+    const int           secondsPerMinute = 60;
+    const int           minutesPerHour = 60;
+    const int           hoursPerDay = 24;
+    const int           daysPerWeek = 7;
+    const int           daysPerNYear = 365;
+    const int           daysPerLYear = 366;
+    const int           yearLengths[2] = { daysPerNYear, daysPerLYear };
+    const int           secondsPerHour = secondsPerMinute * minutesPerHour;
+    const long          secondsPerDay = secondsPerHour * hoursPerDay;
+    const int           monthsPerYear = 12;
+    const int           yearBase = 1900;
+    const int           monthLengths[2][monthsPerYear] = { { 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31 }, { 31, 29, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31 } };
+    int                 leapYearIndex = 0;
+
+    tlBoundaryMod(&timeInfoPtr->tm_min, &timeInfoPtr->tm_sec, secondsPerMinute);
+    tlBoundaryMod(&timeInfoPtr->tm_hour, &timeInfoPtr->tm_min, minutesPerHour);
+    tlBoundaryMod(&timeInfoPtr->tm_mday, &timeInfoPtr->tm_hour, hoursPerDay);
+    tlBoundaryMod(&timeInfoPtr->tm_year, &timeInfoPtr->tm_mon, monthsPerYear);
+
+    leapYearIndex = ((((timeInfoPtr->tm_year + yearBase) % 4) == 0 && ((timeInfoPtr->tm_year + yearBase) % 100) != 0) || ((timeInfoPtr->tm_year + yearBase) % 400) == 0);
+    while (timeInfoPtr->tm_mday <= 0)
+    {
+        --timeInfoPtr->tm_mon;
+
+        if (timeInfoPtr->tm_mon < 0)
+        {
+            timeInfoPtr->tm_mon = 11;
+            --timeInfoPtr->tm_year;
+            leapYearIndex = ((((timeInfoPtr->tm_year + yearBase) % 4) == 0 && ((timeInfoPtr->tm_year + yearBase) % 100) != 0) || ((timeInfoPtr->tm_year + yearBase) % 400) == 0);
+        }
+
+        timeInfoPtr->tm_mday += monthLengths[leapYearIndex][timeInfoPtr->tm_mon];
+    }
+
+    while (timeInfoPtr->tm_mday > monthLengths[leapYearIndex][timeInfoPtr->tm_mon])
+    {
+        timeInfoPtr->tm_mday -= monthLengths[leapYearIndex][timeInfoPtr->tm_mon];
+        ++timeInfoPtr->tm_mon;
+
+        if (timeInfoPtr->tm_mon >= 12)
+        {
+            timeInfoPtr->tm_mon = 0;
+            ++timeInfoPtr->tm_year;
+            leapYearIndex = ((((timeInfoPtr->tm_year + yearBase) % 4) == 0 && ((timeInfoPtr->tm_year + yearBase) % 100) != 0) || ((timeInfoPtr->tm_year + yearBase) % 400) == 0);
+        }
+    }
+}
+
+//---------------------------
+
+static void tlWinLocalTime_r(const time_t* clock, struct tm* timeInfoPtr)
+{
+    SYSTEMTIME  s;
+    FILETIME    f;
+    __int64     time;
+
+    f = tlSecondsToFileTime(*clock);
+    time = tlFileTimeToInt64(f) + tlLocalTimeZoneDiffIn100nsIntervals();
+    f = tlInt64ToFileTime(time);
+
+    FileTimeToSystemTime(&f, &s);
+
+    tlSystemTimeToTimeStruct_r(&s, timeInfoPtr);
+}
+
+static void tlWinGMTime_r(const time_t* clock, struct tm* timeInfo)
+{
+    FILETIME    f;
+    SYSTEMTIME  s;
+
+    f = tlSecondsToFileTime(*clock);
+    FileTimeToSystemTime(&f, &s);
+    tlSystemTimeToTimeStruct_r(&s, timeInfo);
+}
+
+static time_t tlWinMKTime(struct tm* timeInfoPtr)
+{
+    SYSTEMTIME  s;
+    FILETIME    f;
+    time_t      diff;
+
+    // Windows apparently doesn't normalize/fix bogus date values before
+    // doing conversions, so we need to normalize them first
+    tlNormalizeTimeStruct(timeInfoPtr);
+
+    s = tlTimeStructToSystemTime(timeInfoPtr);
+    SystemTimeToFileTime(&s, &f);
+
+    // Reset day of week
+    FileTimeToSystemTime(&f, &s);
+    timeInfoPtr->tm_wday = s.wDayOfWeek;
+
+    // The above assumes UTC but Linux's mktime() assumes a local
+    // time zone, so we need to offset the result into the local time zone
+    diff = tlLocalTimeZoneDiffIn100nsIntervals() / _onesec_in100ns;
+
+    return tlFileTimeToSeconds(&f) - diff;
+}
+#endif
+
+//------------------------------------------------------------------------------
+
+void tlLocalTime_r(const time_t* clock, struct tm* timeInfoPtr)
+{
+    #ifdef _WIN32
+    tlWinLocalTime_r(clock, timeInfoPtr);
+    #else
+    localtime_r(clock, timeInfoPtr);
+    #endif
+}
+
+void tlGMTime_r(const time_t* clock, struct tm* timeInfoPtr)
+{
+    #ifdef _WIN32
+    tlWinGMTime_r(clock, timeInfoPtr);
+    #else
+    gmtime_r(clock, timeInfoPtr);
+    #endif
+}
+
+time_t tlMKTime(struct tm* timeInfoPtr, bool inLocalTimeZone)
+{
+    time_t      the_time = 0;
+
+    #ifdef _WIN32
+    the_time = tlWinMKTime(timeInfoPtr);
+
+    if (!inLocalTimeZone)
+    {
+        // Adjust for time zone offset
+        the_time += (tlLocalTimeZoneDiffIn100nsIntervals() / _onesec_in100ns);
+    }
+    #else
+    // Get the initial time components; note that mktime assumes local time
+    the_time = mktime(timeInfoPtr);
+
+    if (!inLocalTimeZone)
+    {
+        // Adjust for time zone offset
+        the_time += timeInfoPtr->tm_gmtoff;
+    }
+    #endif
+
+    return the_time;
+}
+
+//------------------------------------------------------------------------------
+
+void tlMakeTimeStructFromUTCSeconds(time_t seconds, struct tm* timeInfo)
+{
+    tlGMTime_r(&seconds, timeInfo);
+}
+
+void tlInsertDateIntoTimeStruct(struct tm* timeInfo, unsigned int date)
+{
+    unsigned int    year = date / 10000;
+    unsigned int    month = (date - (year * 10000)) / 100;
+    unsigned int    day = date - (year * 10000) - (month * 100);
+
+    timeInfo->tm_year = year - 1900;
+    timeInfo->tm_mon = month - 1;
+    timeInfo->tm_mday = day;
+}
+
+unsigned int tlExtractDateFromTimeStruct(const struct tm* timeInfo)
+{
+    unsigned int    result = 0;
+
+    result = (timeInfo->tm_year + 1900) * 10000;
+    result += (timeInfo->tm_mon + 1) * 100;
+    result += timeInfo->tm_mday;
+
+    return result;
+}
+
+void tlInsertTimeIntoTimeStruct(struct tm* timeInfo, unsigned int time)
+{
+    unsigned int    hour = time / 10000;
+    unsigned int    minute = (time - (hour * 10000)) / 100;
+    unsigned int    second = time - (hour * 10000) - (minute * 100);
+
+    timeInfo->tm_hour = hour;
+    timeInfo->tm_min = minute;
+    timeInfo->tm_sec = second;
+}
+
+unsigned int tlExtractTimeFromTimeStruct(const struct tm* timeInfo)
+{
+    unsigned int    result = 0;
+
+    result = timeInfo->tm_hour * 10000;
+    result += timeInfo->tm_min * 100;
+    result += timeInfo->tm_sec;
+
+    return result;
+}
+
+//------------------------------------------------------------------------------
+
+TIMELIB_API __int64 TIMELIB_CALL tlSecondsFromParts(int year, unsigned int month, unsigned int day, unsigned int hour, unsigned int minute, unsigned int second, bool is_local_time)
+{
+    struct tm       timeInfo;
+    time_t          the_time = 0;
+
+    memset(&timeInfo, 0, sizeof(timeInfo));
+
+    // Push each time part value into the tm struct
+    timeInfo.tm_sec = second;
+    timeInfo.tm_min = minute;
+    timeInfo.tm_hour = hour;
+    timeInfo.tm_mday = day;
+    timeInfo.tm_mon = month - 1;
+    timeInfo.tm_year = year - 1900;
+
+    the_time = tlMKTime(&timeInfo, is_local_time);
+
+    return static_cast<__int64>(the_time);
+}
+
+//------------------------------------------------------------------------------
+
+TIMELIB_API size32_t TIMELIB_CALL tlSecondsToParts(ARowBuilder& __self, __int64 seconds)
+{
+    struct tm       timeInfo;
+
+    struct TMParts
+    {
+        __int32 sec;
+        __int32 min;
+        __int32 hour;
+        __int32 mday;
+        __int32 mon;
+        __int32 year;
+        __int32 wday;
+    };
+
+    tlMakeTimeStructFromUTCSeconds(seconds, &timeInfo);
+
+    TMParts* result = reinterpret_cast<TMParts*>(__self.getSelf());
+
+    result->sec = timeInfo.tm_sec;
+    result->min = timeInfo.tm_min;
+    result->hour = timeInfo.tm_hour;
+    result->mday = timeInfo.tm_mday;
+    result->mon = timeInfo.tm_mon;
+    result->year = timeInfo.tm_year;
+    result->wday = timeInfo.tm_wday;
+
+    return static_cast<size32_t>(sizeof(TMParts));
+}
+
+//------------------------------------------------------------------------------
+
+TIMELIB_API unsigned int TIMELIB_CALL tlGetDayOfYear(short year, unsigned short month, unsigned short day)
+{
+    unsigned int    dayOfYear = 0;
+
+    #ifdef _WIN32
+    SYSTEMTIME  s;
+
+    memset(&s, 0, sizeof(s));
+
+    s.wYear = year;
+    s.wMonth = month;
+    s.wDay = day;
+
+    dayOfYear = tlYearDayFromSystemTime(&s);
+    #else
+    struct tm       timeInfo;
+
+    memset(&timeInfo, 0, sizeof(timeInfo));
+
+    // Push each time part value into the tm struct
+    timeInfo.tm_mday = day;
+    timeInfo.tm_mon = month - 1;
+    timeInfo.tm_year = year - 1900;
+
+    tlMKTime(&timeInfo);
+
+    dayOfYear = timeInfo.tm_yday;
+    #endif
+
+    return dayOfYear;
+}
+
+//------------------------------------------------------------------------------
+
+TIMELIB_API unsigned int TIMELIB_CALL tlGetDayOfWeek(short year, unsigned short month, unsigned short day)
+{
+    struct tm       timeInfo;
+
+    memset(&timeInfo, 0, sizeof(timeInfo));
+
+    // Push each time part value into the tm struct
+    timeInfo.tm_mday = day;
+    timeInfo.tm_mon = month - 1;
+    timeInfo.tm_year = year - 1900;
+
+    tlMKTime(&timeInfo);
+
+    return timeInfo.tm_wday;
+}
+
+//------------------------------------------------------------------------------
+
+TIMELIB_API void TIMELIB_CALL tlDateToString(size32_t &__lenResult, char* &__result, unsigned int date, const char* format)
+{
+    struct tm       timeInfo;
+    const size_t    kBufferSize = 256;
+    char            buffer[kBufferSize];
+
+    memset(&timeInfo, 0, sizeof(timeInfo));
+    tlInsertDateIntoTimeStruct(&timeInfo, date);
+    tlMKTime(&timeInfo);
+
+    __lenResult = strftime(buffer, kBufferSize, format, &timeInfo);
+    __result = NULL;
+
+    if (__lenResult > 0)
+    {
+        __result = reinterpret_cast<char*>(CTXMALLOC(parentCtx, __lenResult));
+        memcpy(__result, buffer, __lenResult);
+    }
+}
+
+//------------------------------------------------------------------------------
+
+TIMELIB_API void TIMELIB_CALL tlTimeToString(size32_t &__lenResult, char* &__result, unsigned int time, const char* format)
+{
+    struct tm       timeInfo;
+    const size_t    kBufferSize = 256;
+    char            buffer[kBufferSize];
+
+    memset(&timeInfo, 0, sizeof(timeInfo));
+    tlInsertTimeIntoTimeStruct(&timeInfo, time);
+    tlMKTime(&timeInfo);
+
+    __lenResult = strftime(buffer, kBufferSize, format, &timeInfo);
+    __result = NULL;
+
+    if (__lenResult > 0)
+    {
+        __result = reinterpret_cast<char*>(rtlMalloc(__lenResult));
+        memcpy(__result, buffer, __lenResult);
+    }
+}
+
+//------------------------------------------------------------------------------
+
+TIMELIB_API void TIMELIB_CALL tlSecondsToString(size32_t &__lenResult, char* &__result, __int64 seconds, const char* format)
+{
+    struct tm       timeInfo;
+    time_t          theTime = seconds;
+    const size_t    kBufferSize = 256;
+    char            buffer[kBufferSize];
+
+    memset(buffer, 0, kBufferSize);
+
+    tlGMTime_r(&theTime, &timeInfo);
+
+    __lenResult = strftime(buffer, kBufferSize, format, &timeInfo);
+    __result = NULL;
+
+    if (__lenResult > 0)
+    {
+        __result = reinterpret_cast<char*>(rtlMalloc(__lenResult));
+        memcpy(__result, buffer, __lenResult);
+    }
+}
+
+//------------------------------------------------------------------------------
+
+TIMELIB_API unsigned int TIMELIB_CALL tlAdjustDate(unsigned int date, short year_delta, int month_delta, int day_delta)
+{
+    struct tm       timeInfo;
+    unsigned int    result = 0;
+
+    memset(&timeInfo, 0, sizeof(timeInfo));
+
+    tlInsertDateIntoTimeStruct(&timeInfo, date);
+
+    timeInfo.tm_year += year_delta;
+    timeInfo.tm_mon += month_delta;
+    timeInfo.tm_mday += day_delta;
+
+    tlMKTime(&timeInfo);
+
+    result = tlExtractDateFromTimeStruct(&timeInfo);
+
+    return result;
+}
+
+//------------------------------------------------------------------------------
+
+TIMELIB_API unsigned int TIMELIB_CALL tlAdjustDateBySeconds(unsigned int date, int seconds_delta)
+{
+    struct tm       timeInfo;
+    unsigned int    result = 0;
+
+    memset(&timeInfo, 0, sizeof(timeInfo));
+
+    tlInsertDateIntoTimeStruct(&timeInfo, date);
+    timeInfo.tm_sec = seconds_delta;
+
+    tlMKTime(&timeInfo);
+
+    result = tlExtractDateFromTimeStruct(&timeInfo);
+
+    return result;
+}
+
+//------------------------------------------------------------------------------
+
+TIMELIB_API unsigned int TIMELIB_CALL tlAdjustTime(unsigned int time, short hour_delta, int minute_delta, int second_delta)
+{
+    struct tm       timeInfo;
+    unsigned int    result = 0;
+
+    memset(&timeInfo, 0, sizeof(timeInfo));
+
+    tlInsertTimeIntoTimeStruct(&timeInfo, time);
+
+    timeInfo.tm_hour += hour_delta;
+    timeInfo.tm_min += minute_delta;
+    timeInfo.tm_sec += second_delta;
+
+    tlMKTime(&timeInfo);
+
+    result = tlExtractTimeFromTimeStruct(&timeInfo);
+
+    return result;
+}
+
+//------------------------------------------------------------------------------
+
+TIMELIB_API unsigned int TIMELIB_CALL tlAdjustTimeBySeconds(unsigned int time, int seconds_delta)
+{
+    struct tm       timeInfo;
+    unsigned int    result = 0;
+
+    memset(&timeInfo, 0, sizeof(timeInfo));
+
+    tlInsertTimeIntoTimeStruct(&timeInfo, time);
+    timeInfo.tm_sec += seconds_delta;
+
+    tlMKTime(&timeInfo);
+
+    result = tlExtractTimeFromTimeStruct(&timeInfo);
+
+    return result;
+}
+
+//------------------------------------------------------------------------------
+
+TIMELIB_API __int64 TIMELIB_CALL tlAdjustSeconds(__int64 seconds, short year_delta, int month_delta, int day_delta, short hour_delta, int minute_delta, int second_delta)
+{
+    struct tm       timeInfo;
+    time_t          theTime = seconds;
+    time_t          result = 0;
+
+    tlLocalTime_r(&theTime, &timeInfo);
+
+    timeInfo.tm_year += year_delta;
+    timeInfo.tm_mon += month_delta;
+    timeInfo.tm_mday += day_delta;
+    timeInfo.tm_hour += hour_delta;
+    timeInfo.tm_min += minute_delta;
+    timeInfo.tm_sec += second_delta;
+
+    result = tlMKTime(&timeInfo);
+
+    return static_cast<__int64>(result);
+}
+
+//------------------------------------------------------------------------------
+
+TIMELIB_API unsigned int TIMELIB_CALL tlAdjustCalendar(unsigned int date, short year_delta, int month_delta, int day_delta)
+{
+    struct tm       timeInfo;
+    unsigned int    year = date / 10000;
+    unsigned int    month = (date - (year * 10000)) / 100;
+    unsigned int    day = date - (year * 10000) - (month * 100);
+    int             expectedMonthVal = month + month_delta - 1;
+    time_t          seconds;
+    unsigned int    result = 0;
+
+    // Normalize the expected month value
+    if (expectedMonthVal >= 0)
+    {
+        expectedMonthVal = expectedMonthVal % 12;
+    }
+    else
+    {
+        expectedMonthVal = 12 - (abs(expectedMonthVal) % 12);
+    }
+
+    memset(&timeInfo, 0, sizeof(timeInfo));
+
+    timeInfo.tm_year = year - 1900;
+    timeInfo.tm_mon = month - 1;
+    timeInfo.tm_mday = day;
+
+    timeInfo.tm_year += year_delta;
+    timeInfo.tm_mon += month_delta;
+
+    seconds = tlMKTime(&timeInfo);
+
+    if (timeInfo.tm_mon != expectedMonthVal)
+    {
+        // If the returned month doesn't match the expected month, we need to
+        // go back to the last day of the previous month
+        timeInfo.tm_mday = 0;
+        tlMKTime(&timeInfo);
+    }
+
+    if (day_delta != 0)
+    {
+        // Now apply the day delta
+        timeInfo.tm_mday += day_delta;
+        tlMKTime(&timeInfo);
+    }
+
+    result = tlExtractDateFromTimeStruct(&timeInfo);
+
+    return result;
+}
+
+//------------------------------------------------------------------------------
+
+TIMELIB_API bool TIMELIB_CALL tlIsLocalDaylightSavingsInEffect()
+{
+    struct tm       timeInfo;
+    time_t          theTime = time(NULL);
+
+    tlLocalTime_r(&theTime, &timeInfo);
+
+    return (timeInfo.tm_isdst == 1);
+}
+
+//------------------------------------------------------------------------------
+
+TIMELIB_API int TIMELIB_CALL tlLocalTimeZoneOffset()
+{
+    int     offset = 0;
+
+    #ifdef _WIN32
+    offset = static_cast<int>(tlLocalTimeZoneDiffIn100nsIntervals() / _onesec_in100ns);
+    #else
+    struct tm       timeInfo;
+    time_t          theTime = time(NULL);
+
+    tlLocalTime_r(&theTime, &timeInfo);
+
+    offset = timeInfo.tm_gmtoff;
+    #endif
+
+    return offset;
+}
+
+//------------------------------------------------------------------------------
+
+TIMELIB_API unsigned int TIMELIB_CALL tlCurrentDate(bool in_local_time)
+{
+    struct tm       timeInfo;
+    time_t          theTime = time(NULL);
+    unsigned int    result = 0;
+
+    // Create time parts differently depending on whether you need
+    // UTC or local time
+    if (in_local_time)
+    {
+        tlLocalTime_r(&theTime, &timeInfo);
+    }
+    else
+    {
+        tlGMTime_r(&theTime, &timeInfo);
+    }
+
+    result = tlExtractDateFromTimeStruct(&timeInfo);
+
+    return result;
+}
+
+//------------------------------------------------------------------------------
+
+TIMELIB_API unsigned int TIMELIB_CALL tlCurrentTime(bool in_local_time)
+{
+    struct tm       timeInfo;
+    time_t          theTime = time(NULL);
+    unsigned int    result = 0;
+
+    // Create time parts differently depending on whether you need
+    // UTC or local time
+    if (in_local_time)
+    {
+        tlLocalTime_r(&theTime, &timeInfo);
+    }
+    else
+    {
+        tlGMTime_r(&theTime, &timeInfo);
+    }
+
+    result = tlExtractTimeFromTimeStruct(&timeInfo);
+
+    return result;
+}
+
+//------------------------------------------------------------------------------
+
+TIMELIB_API __int64 TIMELIB_CALL tlCurrentSeconds(bool in_local_time)
+{
+    time_t    result = time(NULL);
+
+    if (in_local_time)
+    {
+        result += tlLocalTimeZoneOffset();
+    }
+
+    return static_cast<__int64>(result);
+}
+
+//------------------------------------------------------------------------------
+
+TIMELIB_API __int64 TIMELIB_CALL tlCurrentTimestamp(bool in_local_time)
+{
+    __int64     result = 0;
+
+    #ifdef _WIN32
+    struct _timeb   now;
+
+    _ftime_s(&now);
+
+    result = (now.time * I64C(1000000)) + (now.millitm * 1000);
+    #else
+    struct timeval  tv;
+
+    if (gettimeofday(&tv, NULL) == 0)
+    {
+        result = (tv.tv_sec * I64C(1000000)) + tv.tv_usec;
+    }
+    #endif
+
+    if (in_local_time)
+    {
+        result += (static_cast<__int64>(tlLocalTimeZoneOffset()) * I64C(1000000));
+    }
+
+    return result;
+}
+
+//------------------------------------------------------------------------------
+
+TIMELIB_API unsigned int TIMELIB_CALL tlGetLastDayOfMonth(unsigned int date)
+{
+    struct tm       timeInfo;
+    unsigned int    result = 0;
+
+    memset(&timeInfo, 0, sizeof(timeInfo));
+    tlInsertDateIntoTimeStruct(&timeInfo, date);
+
+    // Call mktime once to fix up any bogus data
+    tlMKTime(&timeInfo);
+
+    // Adjust and call again
+    timeInfo.tm_mon += 1;
+    timeInfo.tm_mday = 0;
+    tlMKTime(&timeInfo);
+
+    result = tlExtractDateFromTimeStruct(&timeInfo);
+
+    return result;
+}
+
+//------------------------------------------------------------------------------
+
+TIMELIB_API size32_t TIMELIB_CALL tlDatesForWeek(ARowBuilder& __self, unsigned int date)
+{
+    struct tm       timeInfo;
+
+    struct TMDateRange
+    {
+        unsigned int    startDate;
+        unsigned int    endDate;
+    };
+
+    TMDateRange* result = reinterpret_cast<TMDateRange*>(__self.getSelf());
+
+    memset(&timeInfo, 0, sizeof(timeInfo));
+    tlInsertDateIntoTimeStruct(&timeInfo, date);
+
+    // Call mktime once to fix up any bogus data
+    tlMKTime(&timeInfo);
+
+    // Adjust and call again
+    timeInfo.tm_mday -= timeInfo.tm_wday;
+    tlMKTime(&timeInfo);
+
+    result->startDate = tlExtractDateFromTimeStruct(&timeInfo);
+
+    // Adjust to the beginning of the week
+    timeInfo.tm_mday += 6;
+    tlMKTime(&timeInfo);
+
+    result->endDate = tlExtractDateFromTimeStruct(&timeInfo);
+
+    return static_cast<size32_t>(sizeof(TMDateRange));
+}

plugins/timelib/timelib.hpp

@@ -0,0 +1,80 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#ifndef TIMELIB_INCL
+#define TIMELIB_INCL
+
+#ifdef _WIN32
+#define TIMELIB_CALL _cdecl
+#ifdef TIMELIB_EXPORTS
+#define TIMELIB_API __declspec(dllexport)
+#else
+#define TIMELIB_API __declspec(dllimport)
+#endif
+#else
+#define TIMELIB_CALL
+#define TIMELIB_API
+#endif
+
+#include <time.h>
+
+#include "platform.h"
+#include "hqlplugins.hpp"
+#include "eclinclude4.hpp"
+#include "eclrtl.hpp"
+
+extern "C" {
+
+#ifdef TIMELIB_EXPORTS
+TIMELIB_API bool getECLPluginDefinition(ECLPluginDefinitionBlock *pb);
+TIMELIB_API void setPluginContext(IPluginContext * _ctx);
+#endif
+
+void tlMakeTimeStructFromUTCSeconds(time_t seconds, struct tm* timeInfo);
+void tlInsertDateIntoTimeStruct(struct tm* timeInfo, unsigned int date);
+unsigned int tlExtractDateFromTimeStruct(const struct tm* timeInfo);
+void tlInsertTimeIntoTimeStruct(struct tm* timeInfo, unsigned int time);
+unsigned int tlExtractTimeFromTimeStruct(const struct tm* timeInfo);
+
+void tlLocalTime_r(const time_t* clock, struct tm* timeInfoPtr);
+void tlGMTime_r(const time_t* clock, struct tm* timeInfoPtr);
+time_t tlMKTime(struct tm* timeInfoPtr, bool inLocalTimeZone = true);
+
+TIMELIB_API __int64 TIMELIB_CALL tlSecondsFromParts(int year, unsigned int month, unsigned int day, unsigned int hour, unsigned int minute, unsigned int second, bool is_local_time = false);
+TIMELIB_API size32_t TIMELIB_CALL tlSecondsToParts(ARowBuilder & __self, __int64 seconds);
+TIMELIB_API unsigned int TIMELIB_CALL tlGetDayOfYear(short year, unsigned short month, unsigned short day);
+TIMELIB_API unsigned int TIMELIB_CALL tlGetDayOfWeek(short year, unsigned short month, unsigned short day);
+TIMELIB_API void TIMELIB_CALL tlDateToString(size32_t &__lenResult, char* &__result, unsigned int date, const char* format);
+TIMELIB_API void TIMELIB_CALL tlTimeToString(size32_t &__lenResult, char* &__result, unsigned int time, const char* format);
+TIMELIB_API unsigned int TIMELIB_CALL tlAdjustDate(unsigned int date, short year_delta, int month_delta, int day_delta);
+TIMELIB_API void TIMELIB_CALL tlSecondsToString(size32_t &__lenResult, char* &__result, __int64 seconds, const char* format);
+TIMELIB_API unsigned int TIMELIB_CALL tlAdjustDateBySeconds(unsigned int date, int seconds_delta);
+TIMELIB_API unsigned int TIMELIB_CALL tlAdjustTime(unsigned int time, short hour_delta, int minute_delta, int second_delta);
+TIMELIB_API unsigned int TIMELIB_CALL tlAdjustTimeBySeconds(unsigned int time, int seconds_delta);
+TIMELIB_API __int64 TIMELIB_CALL tlAdjustSeconds(__int64 seconds, short year_delta, int month_delta, int day_delta, short hour_delta, int minute_delta, int second_delta);
+TIMELIB_API unsigned int TIMELIB_CALL tlAdjustCalendar(unsigned int date, short year_delta, int month_delta, int day_delta);
+TIMELIB_API bool TIMELIB_CALL tlIsLocalDaylightSavingsInEffect();
+TIMELIB_API int TIMELIB_CALL tlLocalTimeZoneOffset();
+TIMELIB_API unsigned int TIMELIB_CALL tlCurrentDate(bool in_local_time);
+TIMELIB_API unsigned int TIMELIB_CALL tlCurrentTime(bool in_local_time);
+TIMELIB_API __int64 TIMELIB_CALL tlCurrentSeconds(bool in_local_time);
+TIMELIB_API __int64 TIMELIB_CALL tlCurrentTimestamp(bool in_local_time);
+TIMELIB_API unsigned int TIMELIB_CALL tlGetLastDayOfMonth(unsigned int date);
+TIMELIB_API size32_t TIMELIB_CALL tlDatesForWeek(ARowBuilder & __self, unsigned int date);
+
+}
+#endif

system/include/portlist.h

@@ -78,6 +78,7 @@
 #define WS_JABBER_DEFAULT_PORT          8080
 #define WS_REF_TRACKER_DEFAULT_PORT     8080
 #define WS_ATTRIBUTES_DEFAULT_PORT      8145
+#define WS_LOGGING_DEFAULT_PORT         8146
 #define WS_ITS_DEFAULT_PORT             8888
 #define WS_FACTS_DEFAULT_PORT           8999
 #define WS_MOXIE_DEFAULT_PORT           8999

system/jlib/jcomp.cpp

@@ -22,6 +22,7 @@
 #include "jcomp.hpp"
 #include "jsem.hpp"
 #include "jexcept.hpp"
+#include "jregexp.hpp"
 
 #ifdef _WIN32
 #include <windows.h>
@@ -507,6 +508,100 @@ bool CppCompiler::compileFile(IThreadPool * pool, const char * filename, Semapho
     return true;
 }
 
+void CppCompiler::extractErrors(IArrayOf<IError> & errors)
+{
+    const char* cclog = ccLogPath.get();
+    if(!cclog||!*cclog)
+        cclog = queryCcLogName();
+    Owned <IFile> logfile = createIFile(cclog);
+    if (!logfile->exists())
+        return;
+
+    try
+    {
+        StringBuffer file;
+        file.loadFile(logfile);
+
+        RegExpr vsErrorPattern("^{.+}({[0-9]+}) : error {.*$}");
+        RegExpr vsLinkErrorPattern("^{.+} : error {.*$}");
+
+        //cpperr.ecl:7:10: error: ‘syntaxError’ was not declared in this scope
+        RegExpr gccErrorPattern("^{.+}:{[0-9]+}:{[0-9]+}: {[a-z]+}: {.*$}");
+        RegExpr gccLinkErrorPattern("^{.+}:[^:]+: {.*$}"); // undefined reference
+        RegExpr gccLinkErrorPattern2("^.+ld: {.*$}"); // fail to find library etc.
+        RegExpr gccExitStatusPattern("^.*exit status$"); // collect2: error: ld returned 1 exit status
+        const char * cur = file.str();
+        do
+        {
+            const char * newline = strchr(cur, '\n');
+            StringAttr next;
+            if (newline)
+            {
+                next.set(cur, newline-cur);
+                cur = newline+1;
+                if (*cur == '\r')
+                    cur++;
+            }
+            else
+            {
+                next.set(cur);
+                cur = NULL;
+            }
+
+            if (gccExitStatusPattern.find(next))
+            {
+                //ignore
+            }
+            else if (gccErrorPattern.find(next))
+            {
+                StringBuffer filename, line, column, kind, msg;
+                gccErrorPattern.findstr(filename, 1);
+                gccErrorPattern.findstr(line, 2);
+                gccErrorPattern.findstr(column, 3);
+                gccErrorPattern.findstr(kind, 4);
+                gccErrorPattern.findstr(msg, 5);
+
+                if (streq(kind, "warning"))
+                    errors.append(*createError(CategoryCpp, SeverityWarning, 2999, msg.str(), filename.str(), atoi(line), atoi(column), 0));
+                else
+                    errors.append(*createError(CategoryError, SeverityError, 2999, msg.str(), filename.str(), atoi(line), atoi(column), 0));
+            }
+            else if (gccLinkErrorPattern.find(next))
+            {
+                StringBuffer filename, msg;
+                gccLinkErrorPattern.findstr(filename, 1);
+                gccLinkErrorPattern.findstr(msg, 2);
+                errors.append(*createError(CategoryError, SeverityError, 2999, msg.str(), filename.str(), 0, 0, 0));
+            }
+            else if (gccLinkErrorPattern2.find(next))
+            {
+                StringBuffer msg("C++ link error: ");
+                gccLinkErrorPattern2.findstr(msg, 1);
+                errors.append(*createError(CategoryError, SeverityError, 2999, msg.str(), NULL, 0, 0, 0));
+            }
+            else if (vsErrorPattern.find(next))
+            {
+                StringBuffer filename, line, msg("C++ compiler error: ");
+                vsErrorPattern.findstr(filename, 1);
+                vsErrorPattern.findstr(line, 2);
+                vsErrorPattern.findstr(msg, 3);
+                errors.append(*createError(CategoryError, SeverityError, 2999, msg.str(), filename.str(), atoi(line), 0, 0));
+            }
+            else if (vsLinkErrorPattern.find(next))
+            {
+                StringBuffer filename, msg("C++ link error: ");
+                vsLinkErrorPattern.findstr(filename, 1);
+                vsLinkErrorPattern.findstr(msg, 2);
+                errors.append(*createError(CategoryError, SeverityError, 2999, msg.str(), filename.str(), 0, 0, 0));
+            }
+        } while (cur);
+    }
+    catch (IException * e)
+    {
+        e->Release();
+    }
+}
+
 void CppCompiler::expandCompileOptions(StringBuffer & target)
 {
     target.append(" ").append(CC_OPTION_CORE[targetCompiler]).append(" ");

system/jlib/jcomp.hpp

@@ -52,6 +52,7 @@ public:
     virtual void addLinkOption(const char * option) = 0;
     virtual void addSourceFile(const char * filename) = 0;
     virtual bool compile() = 0;
+    virtual void extractErrors(IArrayOf<IError> & errors) = 0;
     virtual void setDebug(bool _debug) = 0;
     virtual void setDebugLibrary(bool _debug) = 0;
     virtual void setOnlyCompile(bool _onlyCompile) = 0;

system/jlib/jcomp.ipp

@@ -36,6 +36,7 @@ public:
     virtual void addInclude(const char * includePath);
     virtual void addSourceFile(const char * filename);
     virtual bool compile();
+    virtual void extractErrors(IArrayOf<IError> & errors);
     virtual void setDebug(bool _debug);
     virtual void setDebugLibrary(bool _debug);
     virtual void setOnlyCompile(bool _onlyCompile) { onlyCompile = _onlyCompile; }

system/jlib/jexcept.cpp

@@ -1356,3 +1356,68 @@ void printStackReport()
 #endif
     queryLogMsgManager()->flushQueue(10*1000);
 }
+
+//---------------------------------------------------------------------------------------------------------------------
+
+class jlib_decl CError : public CInterfaceOf<IError>
+{
+public:
+    CError(WarnErrorCategory _category,ErrorSeverity _severity, int _no, const char* _msg, const char* _filename, int _lineno, int _column, int _position);
+
+    virtual int             errorCode() const { return no; }
+    virtual StringBuffer &  errorMessage(StringBuffer & ret) const { return ret.append(msg); }
+    virtual MessageAudience errorAudience() const { return MSGAUD_user; }
+    virtual const char* getFilename() const { return filename; }
+    virtual WarnErrorCategory getCategory() const { return category; }
+    virtual int getLine() const { return lineno; }
+    virtual int getColumn() const { return column; }
+    virtual int getPosition() const { return position; }
+    virtual StringBuffer& toString(StringBuffer&) const;
+    virtual ErrorSeverity getSeverity() const { return severity; }
+    virtual IError * cloneSetSeverity(ErrorSeverity _severity) const;
+
+protected:
+    ErrorSeverity severity;
+    WarnErrorCategory category;
+    int no;
+    StringAttr msg;
+    StringAttr filename;
+    int lineno;
+    int column;
+    int position;
+};
+
+CError::CError(WarnErrorCategory _category, ErrorSeverity _severity, int _no, const char* _msg, const char* _filename, int _lineno, int _column, int _position):
+  category(_category),severity(_severity), msg(_msg), filename(_filename)
+{
+    no = _no;
+    lineno = _lineno;
+    column = _column;
+    position = _position;
+}
+
+
+StringBuffer& CError::toString(StringBuffer& buf) const
+{
+    buf.append(filename);
+
+    if(lineno && column)
+        buf.append('(').append(lineno).append(',').append(column).append(')');
+    buf.append(" : ");
+
+    buf.append(no).append(": ").append(msg);
+    return buf;
+}
+
+IError * CError::cloneSetSeverity(ErrorSeverity newSeverity) const
+{
+    return new CError(category, newSeverity,
+                         errorCode(), msg, filename,
+                         getLine(), getColumn(), getPosition());
+}
+
+IError *createError(WarnErrorCategory category, ErrorSeverity severity, int errNo, const char *msg, const char * filename, int lineno, int column, int pos)
+{
+    return new CError(category,severity,errNo,msg,filename,lineno,column,pos);
+}
+

system/jlib/jexcept.hpp

@@ -160,5 +160,64 @@ void  jlib_decl printStackReport();
 #define throwError3(x,a,b,c)                    throwStringExceptionV(x, (x ## _Text), a, b, c)
 #define throwError4(x,a,b,c,d)                  throwStringExceptionV(x, (x ## _Text), a, b, c, d)
 
+//---------------------------------------------------------------------------------------------------------------------
+
+enum ErrorSeverity
+{
+    SeverityIgnore,
+    SeverityInfo,
+    SeverityWarning,
+    SeverityError,    // a warning treated as an error
+    SeverityFatal,      // a fatal error - can't be mapped to anything else
+    SeverityUnknown,
+};
+
+inline bool isError(ErrorSeverity severity) { return severity >= SeverityError; }
+inline bool isFatal(ErrorSeverity severity) { return severity == SeverityFatal; }
+
+//TBD in a separate commit - add support for warnings to be associated with different categories
+enum WarnErrorCategory
+{
+    CategoryInformation,// Some kind of information [default severity information]
+
+    CategoryCast,       // Suspicious casts between types or out of range values
+    CategoryConfuse,    // Likely to cause confusion
+    CategoryDeprecated, // deprecated features or syntax
+    CategoryEfficiency, // Something that is likely to be inefficient
+    CategoryFolding,    // Unusual results from constant folding
+    CategoryFuture,     // Likely to cause problems in future versions
+    CategoryIgnored,    // Something that has no effect, or is ignored
+    CategoryIndex,      // Unusual indexing of datasets or strings
+    CategoryMistake,    // Almost certainly a mistake
+    CategoryLimit,      // An operation that should really have some limits to protect data runaway
+    CategorySyntax,     // Invalid syntax which is painless to recover from
+    CategoryUnusual,    // Not strictly speaking an error, but highly unusual and likely to be a mistake
+    CategoryUnexpected, // Code that could be correct, but has the potential for unexpected behaviour
+    CategoryCpp,        // Warning passed through from C++ compiler
+
+    CategoryError,      // Typically severity fatal
+    CategoryAll,
+    CategoryUnknown,
+    CategoryMax,
+};
+
+interface jlib_thrown_decl IError : public IException
+{
+public:
+    virtual const char* getFilename() const = 0;
+    virtual WarnErrorCategory getCategory() const = 0;
+    virtual int getLine() const = 0;
+    virtual int getColumn() const = 0;
+    virtual int getPosition() const = 0;
+    virtual StringBuffer& toString(StringBuffer&) const = 0;
+    virtual ErrorSeverity getSeverity() const = 0;
+    virtual IError * cloneSetSeverity(ErrorSeverity _severity) const = 0;
+};
+
+inline bool isError(IError * error) { return isError(error->getSeverity()); }
+inline bool isFatal(IError * error) { return isFatal(error->getSeverity()); }
+
+extern jlib_decl IError *createError(WarnErrorCategory category, ErrorSeverity severity, int errNo, const char *msg, const char *filename, int lineno=0, int column=0, int pos=0);
+
 #endif
 

system/jlib/jstats.h

@@ -121,7 +121,7 @@ enum StatisticKind
 
     StSizeGeneratedCpp,
     StSizePeakMemory,
-    StSizeMaxRowSize,                   // Is measurement in K appropriate?
+    StSizeMaxRowSize,
 
     StNumRowsProcessed,                 // on edge
     StNumSlaves,                        // on edge

testing/regress/ecl-test.json

@@ -22,7 +22,7 @@
             "thor",
             "roxie"
         ],
-        "timeout":"72000000",
+        "timeout":"720",
         "maxAttemptCount":"3",
         "defaultSetupClusters": [
             "all"

testing/regress/ecl/date2str.ecl

@@ -19,51 +19,51 @@ import std;
 
 // Test the abbreviated weekday and full weekday name format 
 // for every day a week
-ASSERT(std.date.tostring(20140825, '%a %A') = 'Mon Monday', CONST);
-ASSERT(std.date.tostring(20140826, '%a %A') = 'Tue Tuesday', CONST);
-ASSERT(std.date.tostring(20140827, '%a %A') = 'Wed Wednesday', CONST);
-ASSERT(std.date.tostring(20140828, '%a %A') = 'Thu Thursday', CONST);
-ASSERT(std.date.tostring(20140829, '%a %A') = 'Fri Friday', CONST);
-ASSERT(std.date.tostring(20140830, '%a %A') = 'Sat Saturday', CONST);
-ASSERT(std.date.tostring(20140824, '%a %A') = 'Sun Sunday', CONST);
+ASSERT(std.date.datetostring(20140825, '%a %A') = 'Mon Monday', CONST);
+ASSERT(std.date.datetostring(20140826, '%a %A') = 'Tue Tuesday', CONST);
+ASSERT(std.date.datetostring(20140827, '%a %A') = 'Wed Wednesday', CONST);
+ASSERT(std.date.datetostring(20140828, '%a %A') = 'Thu Thursday', CONST);
+ASSERT(std.date.datetostring(20140829, '%a %A') = 'Fri Friday', CONST);
+ASSERT(std.date.datetostring(20140830, '%a %A') = 'Sat Saturday', CONST);
+ASSERT(std.date.datetostring(20140824, '%a %A') = 'Sun Sunday', CONST);
 
 // Test the abbreviated and full Month name 
-ASSERT(std.date.tostring(20140824, '%b %B') = 'Aug August', CONST);
+ASSERT(std.date.datetostring(20140824, '%b %B') = 'Aug August', CONST);
 
 // Test the century number (year/100) as a 2-digit integer 
-ASSERT(std.date.tostring(20140824, '%C') = '20', CONST);
+ASSERT(std.date.datetostring(20140824, '%C') = '20', CONST);
 
 // Test the day of the month as a decimal number (range 01 to 31). 
-ASSERT(std.date.tostring(20140824, '%d') = '24', CONST);
+ASSERT(std.date.datetostring(20140824, '%d') = '24', CONST);
 
 // Test the MM/DD/YY format. 
-ASSERT(std.date.tostring(20140824, '%D') = '08/24/14', CONST);
+ASSERT(std.date.datetostring(20140824, '%D') = '08/24/14', CONST);
 
 // Test the ISO 8601 week-based year
-ASSERT(std.date.tostring(20140824, '%G') = '2014', CONST);
+ASSERT(std.date.datetostring(20140824, '%G') = '2014', CONST);
 
 // Test the ISO 8601 week-based year without century
-ASSERT(std.date.tostring(20140824, '%g') = '14', CONST);
+ASSERT(std.date.datetostring(20140824, '%g') = '14', CONST);
 
 // Test the %Y-%m-%d (the ISO 8601 date format)
-ASSERT(std.date.tostring(20140824, '%F') = '2014-08-24', CONST);
+ASSERT(std.date.datetostring(20140824, '%F') = '2014-08-24', CONST);
 
 // Test the day of the year as a decimal number (range 001 to 366)
-ASSERT(std.date.tostring(20140824, '%j') = '236', CONST);
+ASSERT(std.date.datetostring(20140824, '%j') = '236', CONST);
 
 // Test the day of the week as a decimal, range 1 to 7, Monday being  1.
-ASSERT(std.date.tostring(20140824, '%u') = '7', CONST);
+ASSERT(std.date.datetostring(20140824, '%u') = '7', CONST);
 
 // Test the day of the week as a decimal, range 0 to 6, Sunday being  0.
-ASSERT(std.date.tostring(20140824, '%w') = '0', CONST);
+ASSERT(std.date.datetostring(20140824, '%w') = '0', CONST);
 
 // Test the  week  number of the current year as a decimal number, range
 // 00 to 53, starting with the first Sunday as  the  first  day  of week 01.
-ASSERT(std.date.tostring(20140824, '%U') = '34', CONST);
+ASSERT(std.date.datetostring(20140824, '%U') = '34', CONST);
 
 // Test the  week  number of the current year as a decimal number, range
 // 00 to 53, starting with the first Monday as  the  first  day  of week 01
-ASSERT(std.date.tostring(20140824, '%W') = '33', CONST);
+ASSERT(std.date.datetostring(20140824, '%W') = '33', CONST);
 
 
 // The time formatting works well, but the result depends on 

testing/regress/ecl/key/memcachedtest.xml

@@ -0,0 +1,71 @@
+ <Info><Code>10001</Code><Source></Source><Message>Memcached Plugin: The key &apos;pi&apos; to fetch is of type REAL, not INTEGER as requested.
+</Message></Info>
+<Dataset name='Result 1'>
+ <Row><Result_1>true</Result_1></Row>
+</Dataset>
+<Dataset name='Result 2'>
+ <Row><Result_2>true</Result_2></Row>
+</Dataset>
+<Dataset name='Result 3'>
+ <Row><Result_3>true</Result_3></Row>
+</Dataset>
+<Dataset name='Result 4'>
+ <Row><Result_4>true</Result_4></Row>
+</Dataset>
+<Dataset name='Result 5'>
+ <Row><Result_5>3.14159265359</Result_5></Row>
+</Dataset>
+<Dataset name='Result 6'>
+ <Row><Result_6>true</Result_6></Row>
+</Dataset>
+<Dataset name='Result 7'>
+ <Row><Result_7>12345689</Result_7></Row>
+</Dataset>
+<Dataset name='Result 8'>
+ <Row><Result_8>true</Result_8></Row>
+</Dataset>
+<Dataset name='Result 9'>
+ <Row><Result_9>7</Result_9></Row>
+</Dataset>
+<Dataset name='Result 10'>
+ <Row><Result_10>true</Result_10></Row>
+</Dataset>
+<Dataset name='Result 11'>
+ <Row><Result_11>supercalifragilisticexpialidocious</Result_11></Row>
+</Dataset>
+<Dataset name='Result 12'>
+ <Row><Result_12>true</Result_12></Row>
+</Dataset>
+<Dataset name='Result 13'>
+ <Row><Result_13>אבגדהוזחטיךכלםמןנסעףפץצקרשת</Result_13></Row>
+</Dataset>
+<Dataset name='Result 14'>
+ <Row><Result_14>true</Result_14></Row>
+</Dataset>
+<Dataset name='Result 15'>
+ <Row><Result_15>אבגדהוזחטיךכלםמןנסעףפץצקרשת</Result_15></Row>
+</Dataset>
+<Dataset name='Result 16'>
+ <Row><Result_16>true</Result_16></Row>
+</Dataset>
+<Dataset name='Result 17'>
+ <Row><Result_17>D790D791D792D793D794D795D796D798D799D79AD79BD79CD79DD79DD79ED79FD7A0D7A1D7A2D7A3D7A4D7A5D7A6D7A7D7A8D7A9D7AA</Result_17></Row>
+</Dataset>
+<Dataset name='Result 18'>
+ <Row><Result_18>true</Result_18></Row>
+</Dataset>
+<Dataset name='Result 19'>
+ <Row><Result_19>UTF8</Result_19></Row>
+</Dataset>
+<Dataset name='Result 20'>
+ <Row><Result_20>4614256656552046314</Result_20></Row>
+</Dataset>
+<Dataset name='Result 21'>
+ <Row><Result_21>true</Result_21></Row>
+</Dataset>
+<Dataset name='Result 22'>
+ <Row><Result_22>false</Result_22></Row>
+</Dataset>
+<Dataset name='Result 23'>
+ <Row><Result_23>Nonexistent</Result_23></Row>
+</Dataset>

testing/regress/ecl/memcachedtest.ecl

@@ -0,0 +1,61 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2014 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+IMPORT * FROM lib_memcached.memcached;
+
+STRING servers := '--SERVER=127.0.0.1:11211';
+Clear(servers);
+
+SetBoolean(servers, 'b', TRUE);
+GetBoolean(servers, 'b');
+
+REAL pi := 3.14159265359;
+SetReal(servers, 'pi', pi);
+GetReal(servers, 'pi');
+
+INTEGER i := 12345689;
+SetInteger(servers, 'i', i);
+GetInteger(servers, 'i');
+
+UNSIGNED u := 7;
+SetUnsigned(servers, 'u', u);
+GetUnsigned(servers, 'u');
+
+STRING str  := 'supercalifragilisticexpialidocious';
+SetString(servers, 'str', str);
+GetString(servers, 'str');
+
+UNICODE uni := U'אבגדהוזחטיךכלםמןנסעףפץצקרשת';
+SetUnicode(servers, 'uni', uni);
+GetUnicode(servers, 'uni');
+
+UTF8 utf := U'אבגדהוזחטיךכלםמןנסעףפץצקרשת';
+SetUtf8(servers, 'utf8', utf);
+GetUtf8(servers, 'utf8');
+
+DATA mydata := x'd790d791d792d793d794d795d796d798d799d79ad79bd79cd79dd79dd79ed79fd7a0d7a1d7a2d7a3d7a4d7a5d7a6d7a7d7a8d7a9d7aa';
+SetData(servers, 'data', mydata);
+GetData(servers,'data');
+
+Exist(servers, 'utf8');
+KeyType(servers,'utf8');
+
+//The following test some exceptions
+GetInteger(servers, 'pi');
+Clear(servers);
+Exist(servers, 'utf8');
+KeyType(servers,'utf8');

testing/unittests/dalitests.cpp

@@ -446,6 +446,7 @@ class CDaliTests : public CppUnit::TestFixture
 //        CPPUNIT_TEST(testReadAllSDS); // Ignoring this test; See comments below
         CPPUNIT_TEST(testSDSRW);
         CPPUNIT_TEST(testSDSSubs);
+        CPPUNIT_TEST(testSDSSubs2);
         CPPUNIT_TEST(testFiles);
         CPPUNIT_TEST(testGroups);
         CPPUNIT_TEST(testMultiCluster);
@@ -2247,6 +2248,117 @@ public:
         pool->joinAll();
         PROGLOG("Hammer test took: %d ms", tm.elapsed());
     }
+
+    void testSDSSubs2()
+    {
+        class CSubscriber : public CSimpleInterfaceOf<ISDSSubscription>
+        {
+            StringAttr xpath;
+            bool sub;
+            StringBuffer &result;
+            SubscriptionId id;
+        public:
+            CSubscriber(StringBuffer &_result, const char *_xpath, bool _sub) : result(_result), xpath(_xpath), sub(_sub)
+            {
+                id = querySDS().subscribe(xpath, *this, sub, !sub);
+                PROGLOG("Subscribed to %s", xpath.get());
+            }
+            ~CSubscriber()
+            {
+                querySDS().unsubscribe(id);
+            }
+            virtual void notify(SubscriptionId id, const char *_xpath, SDSNotifyFlags flags, unsigned valueLen, const void *valueData)
+            {
+                PROGLOG("CSubscriber notified path=%s for subscriber=%s, sub=%s", _xpath, xpath.get(), sub?"true":"false");
+                if (result.length())
+                    result.append("|");
+                result.append(xpath);
+                if (!sub && valueLen)
+                    result.append(",").append(valueLen, (const char *)valueData);
+            }
+        };
+        Owned<IRemoteConnection> conn = querySDS().connect("/DAREGRESS/TestSub2", myProcessSession(), RTM_CREATE, INFINITE);
+        Owned<IPropertyTree> tree = createPTreeFromXMLString("<a><b1><c/></b1><b2/><b3><d><e/></d></b3></a>");
+        IPropertyTree *root = conn->queryRoot();
+        root->setPropTree("a", tree.getClear());
+        conn->commit();
+
+        StringBuffer result;
+        Owned<ISDSSubscription> sub1 = new CSubscriber(result, "/DAREGRESS/TestSub2/a", true);
+        Owned<ISDSSubscription> sub2 = new CSubscriber(result, "/DAREGRESS/TestSub2/a/b1", false);
+        Owned<ISDSSubscription> sub3 = new CSubscriber(result, "/DAREGRESS/TestSub2/a/b2", false);
+        Owned<ISDSSubscription> sub4 = new CSubscriber(result, "/DAREGRESS/TestSub2/a/b1/c", false);
+        Owned<ISDSSubscription> sub5 = new CSubscriber(result, "/DAREGRESS/TestSub2/a/b3", true);
+
+        MilliSleep(1000);
+
+        StringArray expectedResults;
+        expectedResults.append("/DAREGRESS/TestSub2/a");
+        expectedResults.append("/DAREGRESS/TestSub2/a|/DAREGRESS/TestSub2/a/b1,testv");
+        expectedResults.append("/DAREGRESS/TestSub2/a|/DAREGRESS/TestSub2/a/b2,testv");
+        expectedResults.append("/DAREGRESS/TestSub2/a|/DAREGRESS/TestSub2/a/b1/c,testv");
+        expectedResults.append("/DAREGRESS/TestSub2/a|/DAREGRESS/TestSub2/a/b1,testv");
+        expectedResults.append("/DAREGRESS/TestSub2/a|/DAREGRESS/TestSub2/a/b2,testv");
+        expectedResults.append("/DAREGRESS/TestSub2/a");
+        expectedResults.append("/DAREGRESS/TestSub2/a|/DAREGRESS/TestSub2/a/b3");
+        expectedResults.append("/DAREGRESS/TestSub2/a|/DAREGRESS/TestSub2/a/b3");
+
+        StringArray props;
+        props.appendList("S:a,S:a/b1,S:a/b2,S:a/b1/c,S:a/b1/d,S:a/b2/e,S:a/b2/e/f,D:a/b3/d/e,D:a/b3/d", ",");
+
+        assertex(expectedResults.ordinality() == props.ordinality());
+
+        ForEachItemIn(p, props)
+        {
+            result.clear(); // filled by subscriber notifications
+            const char *cmd = props.item(p);
+            const char *propPath=cmd+2;
+            switch (*cmd)
+            {
+                case 'S':
+                {
+                    PROGLOG("Changing %s", propPath);
+                    root->setProp(propPath, "testv");
+                    break;
+                }
+                case 'D':
+                {
+                    PROGLOG("Deleting %s", propPath);
+                    root->removeProp(propPath);
+                    break;
+                }
+                default:
+                    throwUnexpected();
+            }
+            conn->commit();
+
+            MilliSleep(100); // time for notifications to come through
+
+            PROGLOG("Checking results");
+            StringArray resultArray;
+            resultArray.appendList(result, "|");
+            result.clear();
+            resultArray.sortAscii();
+            ForEachItemIn(r, resultArray)
+            {
+                if (result.length())
+                    result.append("|");
+                result.append(resultArray.item(r));
+            }
+            const char *expectedResult = expectedResults.item(p);
+            if (0 == strcmp(expectedResult, result))
+                PROGLOG("testSDSSubs2 [ %s ]: MATCH", cmd);
+            else
+            {
+                VStringBuffer errMsg("testSDSSubs2 [ %s ]: MISMATCH", cmd);
+                errMsg.newline().append("Expected: ").append(expectedResult);
+                errMsg.newline().append("Got: ").append(result);
+                PROGLOG("%s", errMsg.str());
+                CPPUNIT_ASSERT_MESSAGE(errMsg.str(), 0);
+            }
+        }
+    }
+
 };
 
 CPPUNIT_TEST_SUITE_REGISTRATION( CDaliTests );

thorlcr/activities/lookupjoin/thlookupjoin.cpp

@@ -22,7 +22,7 @@
 
 class CLookupJoinActivityMaster : public CMasterActivity
 {
-    mptag_t broadcast2MpTag, lhsDistributeTag, rhsDistributeTag;
+    mptag_t broadcast2MpTag, broadcast3MpTag, lhsDistributeTag, rhsDistributeTag;
 
     bool isAll() const
     {
@@ -39,13 +39,14 @@ public:
     CLookupJoinActivityMaster(CMasterGraphElement * info) : CMasterActivity(info)
     {
         if (container.queryLocal())
-            broadcast2MpTag = lhsDistributeTag = rhsDistributeTag = TAG_NULL;
+            broadcast2MpTag = broadcast3MpTag, lhsDistributeTag = rhsDistributeTag = TAG_NULL;
         else
         {
             mpTag = container.queryJob().allocateMPTag(); // NB: base takes ownership and free's
             if (!isAll())
             {
                 broadcast2MpTag = container.queryJob().allocateMPTag();
+                broadcast3MpTag = container.queryJob().allocateMPTag();
                 lhsDistributeTag = container.queryJob().allocateMPTag();
                 rhsDistributeTag = container.queryJob().allocateMPTag();
             }
@@ -56,6 +57,7 @@ public:
         if (!container.queryLocal() && !isAll())
         {
             container.queryJob().freeMPTag(broadcast2MpTag);
+            container.queryJob().freeMPTag(broadcast3MpTag);
             container.queryJob().freeMPTag(lhsDistributeTag);
             container.queryJob().freeMPTag(rhsDistributeTag);
             // NB: if mpTag is allocated, the activity base class frees
@@ -69,6 +71,7 @@ public:
             if (!isAll())
             {
                 serializeMPtag(dst, broadcast2MpTag);
+                serializeMPtag(dst, broadcast3MpTag);
                 serializeMPtag(dst, lhsDistributeTag);
                 serializeMPtag(dst, rhsDistributeTag);
             }

thorlcr/activities/lookupjoin/thlookupjoinslave.cpp

@@ -39,7 +39,7 @@ enum join_t { JT_Undefined, JT_Inner, JT_LeftOuter, JT_RightOuter, JT_LeftOnly,
 #define MAX_QUEUE_BLOCKS 5
 
 enum broadcast_code { bcast_none, bcast_send, bcast_sendStopping, bcast_stop };
-enum broadcast_flags { bcastflag_spilt=0x100 };
+enum broadcast_flags { bcastflag_null=0, bcastflag_spilt=0x100, bcastflag_standardjoin=0x200 };
 #define BROADCAST_CODE_MASK 0x00FF
 #define BROADCAST_FLAG_MASK 0xFF00
 class CSendItem : public CSimpleInterface
@@ -331,7 +331,7 @@ class CBroadcaster : public CSimpleInterface
                     CriticalBlock b(allDoneLock);
                     if (slaveStop(sendItem->queryOrigin()-1) || allDone)
                     {
-                        recvInterface->bCastReceive(NULL); // signal last
+                        recvInterface->bCastReceive(sendItem.getClear());
                         ActPrintLog(&activity, "recvLoop, received last slaveStop");
                         // NB: this slave has nothing more to receive.
                         // However the sender will still be re-broadcasting some packets, including these stop packets
@@ -372,7 +372,7 @@ public:
     {
         stopping = _stopping;
         if (stopping)
-            slavesStopping->set(myNode-1, true);
+            setStopping();
         recvInterface = _recvInterface;
         stopRecv = false;
         mpTag = _mpTag;
@@ -431,6 +431,14 @@ public:
         broadcastToOthers(sendItem);
         return !allRequestStop;
     }
+    bool isStopping()
+    {
+        return slavesStopping->test(myNode-1);
+    }
+    void setStopping()
+    {
+        slavesStopping->set(myNode-1, true);
+    }
 };
 
 /* CMarker processes a sorted set of rows, comparing every adjacent row.
@@ -663,6 +671,7 @@ class CInMemJoinBase : public CSlaveActivity, public CThorDataLink, public CAllO
     Owned<IException> leftexception;
 
     bool eos, eog, someSinceEog;
+    SpinLock rHSRowSpinLock;
 
 protected:
     typedef CAllOrLookupHelper<HELPER> HELPERBASE;
@@ -711,7 +720,12 @@ protected:
                 blockQueue.enqueue(NULL);
             }
         }
-        void wait() { threaded.join(); }
+        void wait()
+        {
+            threaded.join();
+            if (exception)
+                throw exception.getClear();
+        }
         void addBlock(CSendItem *sendItem)
         {
             if (exception)
@@ -835,6 +849,10 @@ protected:
     }
     void processRHSRows(unsigned slave, MemoryBuffer &mb)
     {
+        /* JCSMORE - I wonder if this should be done asynchronously on a few threads (<=1 per target)
+         * It also might be better to use hash the rows now and assign to rhsNodeRows arrays, it's a waste of hash() calls
+         * if it never spills, but will make flushing non-locals simpler if spilling occurs.
+         */
         CThorSpillableRowArray &rows = *rhsNodeRows.item(slave-1);
         RtlDynamicRowBuilder rowBuilder(rightAllocator);
         CThorStreamDeserializerSource memDeserializer(mb.length(), mb.toByteArray());
@@ -842,13 +860,13 @@ protected:
         {
             size32_t sz = rightDeserializer->deserialize(rowBuilder, memDeserializer);
             OwnedConstThorRow fRow = rowBuilder.finalizeRowClear(sz);
-            // NB: If spilt, addLocalRHSRow will filter out non-locals
-            addLocalRHSRow(rows, fRow);
+            // NB: If spilt, addRHSRow will filter out non-locals
+            if (!addRHSRow(rows, fRow)) // NB: in SMART case, must succeed
+                throw MakeActivityException(this, 0, "Out of memory: Unable to add any more rows to RHS");
         }
     }
     void broadcastRHS() // broadcasting local rhs
     {
-        bool stopRHSBroadcast = false;
         Owned<CSendItem> sendItem = broadcaster.newSendItem(bcast_send);
         MemoryBuffer mb;
         try
@@ -863,21 +881,21 @@ protected:
                     if (!row)
                         break;
 
-                    if (!addLocalRHSRow(localRhsRows, row))
-                        stopRHSBroadcast = true;
+                    if (!addRHSRow(localRhsRows, row)) // may cause broadcaster to be told to stop (for isStopping() to become true)
+                        throw MakeActivityException(this, 0, "Out of memory: Unable to add any more rows to RHS");
 
                     rightSerializer->serialize(mbser, (const byte *)row.get());
-                    if (mb.length() >= MAX_SEND_SIZE || stopRHSBroadcast)
+                    if (mb.length() >= MAX_SEND_SIZE || broadcaster.isStopping())
                         break;
                 }
                 if (0 == mb.length())
                     break;
-                if (stopRHSBroadcast)
+                if (broadcaster.isStopping())
                     sendItem->setFlag(bcastflag_spilt);
                 ThorCompress(mb, sendItem->queryMsg());
                 if (!broadcaster.send(sendItem))
                     break;
-                if (stopRHSBroadcast)
+                if (broadcaster.isStopping())
                     break;
                 mb.clear();
                 broadcaster.resetSendItem(sendItem);
@@ -890,7 +908,7 @@ protected:
         }
 
         sendItem.setown(broadcaster.newSendItem(bcast_stop));
-        if (stopRHSBroadcast)
+        if (broadcaster.isStopping())
             sendItem->setFlag(bcastflag_spilt);
         ActPrintLog("Sending final RHS broadcast packet");
         broadcaster.send(sendItem); // signals stop to others
@@ -1124,6 +1142,8 @@ public:
         allocator.set(queryRowAllocator());
         leftAllocator.set(::queryRowAllocator(leftITDL));
         outputMeta.set(leftITDL->queryFromActivity()->queryContainer().queryHelper()->queryOutputMeta());
+        for (unsigned s=0; s<container.queryJob().querySlaves(); s++)
+            rhsNodeRows.item(s)->setup(queryRowInterfaces(rightITDL));
 
         eos = eog = someSinceEog = false;
 
@@ -1198,6 +1218,15 @@ public:
         broadcaster.end();
         rowProcessor.wait();
     }
+    void doBroadcastStop(mptag_t tag, broadcast_flags flag)
+    {
+        broadcaster.start(this, tag, false);
+        Owned<CSendItem> sendItem = broadcaster.newSendItem(bcast_stop);
+        if (flag)
+            sendItem->setFlag(flag);
+        broadcaster.send(sendItem); // signals stop to others
+        broadcaster.end();
+    }
     rowidx_t getGlobalRHSTotal()
     {
         rowcount_t rhsRows = 0;
@@ -1211,12 +1240,16 @@ public:
         }
         return (rowidx_t)rhsRows;
     }
-    virtual bool addLocalRHSRow(CThorSpillableRowArray &localRhsRows, const void *row)
+    virtual bool addRHSRow(CThorSpillableRowArray &rhsRows, const void *row)
     {
         LinkThorRow(row);
-        if (!localRhsRows.append(row))
-            throw MakeActivityException(this, 0, "Out of memory: Cannot append local rhs row");
-        return true;
+        {
+            SpinBlock b(rHSRowSpinLock);
+            if (rhsRows.append(row))
+                return true;
+        }
+        ReleaseThorRow(row);
+        return false;
     }
 // ISmartBufferNotify
     virtual void onInputStarted(IException *except)
@@ -1289,6 +1322,7 @@ protected:
     using PARENT::fuzzyMatch;
     using PARENT::keepLimit;
     using PARENT::doBroadcastRHS;
+    using PARENT::doBroadcastStop;
     using PARENT::getGlobalRHSTotal;
     using PARENT::getOptBool;
     using PARENT::broadcaster;
@@ -1310,32 +1344,27 @@ protected:
     unsigned abortLimit, atMost;
     bool dedup, stable;
 
-    mptag_t lhsDistributeTag, rhsDistributeTag, broadcast2MpTag;
+    mptag_t lhsDistributeTag, rhsDistributeTag, broadcast2MpTag, broadcast3MpTag;
 
     // Handling failover to a) hashed local lookupjoin b) hash distributed standard join
+    bool smart;
     bool rhsCollated;
-    bool failoverToLocalLookupJoin, failoverToStdJoin;
     Owned<IHashDistributor> lhsDistributor, rhsDistributor;
     ICompare *compareLeft;
     UnsignedArray flushedRowMarkers;
-
-    atomic_t performLocalLookup;
+    CriticalSection overflowCrit;
+    Owned<CFileOwner> overflowWriteFile;
+    Owned<IRowWriter> overflowWriteStream;
+    rowcount_t overflowWriteCount;
+    atomic_t localLookup, standardJoin;
     CriticalSection broadcastSpillingLock;
     Owned<IJoinHelper> joinHelper;
 
-    inline bool isSmart() const
-    {
-        switch (container.getKind())
-        {
-            case TAKsmartjoin:
-            case TAKsmartdenormalize:
-            case TAKsmartdenormalizegroup:
-                return true;
-        }
-        return false;
-    }
-    inline bool doPerformLocalLookup() const { return 0 != atomic_read(&performLocalLookup); }
-    inline void setPerformLocalLookup(bool tf) { atomic_set(&performLocalLookup, (int)tf); }
+    inline bool isSmart() const { return smart; }
+    inline bool isLocalLookup() const { return 0 != atomic_read(&localLookup); }
+    inline void setLocalLookup(bool tf) { atomic_set(&localLookup, (int)tf); }
+    inline bool isStandardJoin() const { return 0 != atomic_read(&standardJoin); }
+    inline void setStandardJoin(bool tf) { atomic_set(&standardJoin, (int)tf); }
     rowidx_t clearNonLocalRows(CThorSpillableRowArray &rows, rowidx_t startPos)
     {
         CThorArrayLockBlock block(rows);
@@ -1373,11 +1402,13 @@ protected:
         // This is likely to free memory, so block others (threads) until done
         // NB: This will not block appends
         CriticalBlock b(broadcastSpillingLock);
-        if (doPerformLocalLookup())
+        if (isLocalLookup())
             return false;
-        setPerformLocalLookup(true);
+        setLocalLookup(true);
         ActPrintLog("Clearing non-local rows - cause: %s", msg);
 
+        broadcaster.setStopping(); // signals to broadcast to start stopping
+
         rowidx_t clearedRows = 0;
         if (rhsCollated)
         {
@@ -1403,16 +1434,6 @@ protected:
         ActPrintLog("handleLowMem: clearedRows = %"RIPF"d", clearedRows);
         return 0 != clearedRows;
     }
-    void setupDistributors()
-    {
-        if (!rhsDistributor)
-        {
-            rhsDistributor.setown(createHashDistributor(this, queryJob().queryJobComm(), rhsDistributeTag, false, NULL, "RHS"));
-            right.setown(rhsDistributor->connect(queryRowInterfaces(rightITDL), right.getClear(), rightHash, NULL));
-            lhsDistributor.setown(createHashDistributor(this, queryJob().queryJobComm(), lhsDistributeTag, false, NULL, "LHS"));
-            left.setown(lhsDistributor->connect(queryRowInterfaces(leftITDL), left.getClear(), leftHash, NULL));
-        }
-    }
     bool setupHT(rowidx_t size)
     {
         if (size < 10)
@@ -1469,23 +1490,27 @@ protected:
  * 10) The LHS side is loaded and spilt and sorted if necessary
  * 11) A regular join helper is created to perform a local join against the two hash distributed sorted sides.
  */
-        bool rhsAlreadySorted = helper->isRightAlreadyLocallySorted();
+        bool rhsAlreadySorted = false;
         CMarker marker(*this);
         if (needGlobal)
         {
+            rhsAlreadySorted = helper->isRightAlreadySorted();
             doBroadcastRHS(stopping);
             rowidx_t rhsRows;
             {
                 CriticalBlock b(broadcastSpillingLock);
-                rhsRows = getGlobalRHSTotal();
+                rhsRows = getGlobalRHSTotal(); // total count of flushed rows
             }
-            if (!doPerformLocalLookup())
+            overflowWriteStream.clear(); // if created, would imply already localLookup=true, there cannot be anymore after broadcast complete
+            if (!isLocalLookup())
             {
                 if (stable && !rhsAlreadySorted)
                     rhs.setup(NULL, false, stableSort_earlyAlloc);
+                bool success=false;
                 try
                 {
-                    rhs.ensure(rhsRows, SPILL_PRIORITY_LOW); // NB: Could OOM, handled by exception handler
+                    // NB: If this ensure returns false, it will have called the MM callbacks and have setup isLocalLookup() already
+                    success = rhs.ensure(rhsRows, SPILL_PRIORITY_LOW); // NB: Could OOM, handled by exception handler
                 }
                 catch (IException *e)
                 {
@@ -1500,9 +1525,16 @@ protected:
                     default:
                         throw;
                     }
+                    success = false;
+                }
+                if (!success)
+                {
                     ActPrintLog("Out of memory trying to size the global RHS row table for a SMART join, will now attempt a distributed local lookup join");
-                    clearAllNonLocalRows("OOM on sizing global row table"); // NB: someone else could have provoked callback already
-                    dbgassertex(doPerformLocalLookup());
+                    if (!isLocalLookup())
+                    {
+                        clearAllNonLocalRows("OOM on sizing global row table"); // NB: someone else could have provoked callback already
+                        dbgassertex(isLocalLookup());
+                    }
                 }
             }
 
@@ -1518,13 +1550,13 @@ protected:
                  * and need to protect rhsNodeRows access
                  */
                 CriticalBlock b(broadcastSpillingLock);
-                if (!doPerformLocalLookup())
+                if (!isLocalLookup())
                 {
                     // If spilt, don't size ht table now, if local rhs fits, ht will be sized later
                     ForEachItemIn(a, rhsNodeRows)
                     {
                         CThorSpillableRowArray &rows = *rhsNodeRows.item(a);
-                        rhs.appendRows(rows, true);
+                        rhs.appendRows(rows, true); // NB: This should not cause spilling, rhs is already sized and we are only copying ptrs in
                         rows.kill(); // free up ptr table asap
                     }
                     // Have to keep broadcastSpillingLock locked until sort and calculate are done
@@ -1533,16 +1565,16 @@ protected:
                     ActPrintLog("Collated all RHS rows");
                 }
             }
-            if (!doPerformLocalLookup()) // check again after processing above
+            if (!isLocalLookup()) // check again after processing above
             {
-                if (!setupHT(uniqueKeys)) // NB: Sizing could cause spilling callback to be triggered and/or OOM
+                if (!setupHT(uniqueKeys)) // NB: Sizing can cause spilling callback to be triggered or OOM in case of !smart
                 {
                     ActPrintLog("Out of memory trying to size the global hash table for a SMART join, will now attempt a distributed local lookup join");
-                    clearAllNonLocalRows("OOM on sizing global hash table"); // NB: someone else could have provoked callback already
-                    dbgassertex(doPerformLocalLookup());
+                    clearAllNonLocalRows("OOM on sizing global hash table"); // NB: setupHT should have provoked callback already
+                    dbgassertex(isLocalLookup());
                 }
             }
-            if (failoverToLocalLookupJoin)
+            if (isSmart())
             {
                 /* NB: Potentially one of the slaves spilt late after broadcast and rowprocessor finished
                  * NB2: This is also to let others know of this slaves spill state.
@@ -1553,24 +1585,25 @@ protected:
 
                 ActPrintLog("Broadcasting final split status");
                 broadcaster.reset();
-                // NB: using a different tag from 1st broadcast, as 2nd on other nodes can start sending before 1st on this has quit receiving
-                broadcaster.start(this, broadcast2MpTag, false);
-                Owned<CSendItem> sendItem = broadcaster.newSendItem(bcast_stop);
-                if (doPerformLocalLookup())
-                    sendItem->setFlag(bcastflag_spilt);
-                broadcaster.send(sendItem); // signals stop to others
-                broadcaster.end();
+                doBroadcastStop(broadcast2MpTag, isLocalLookup() ? bcastflag_spilt : bcastflag_null);
             }
 
             /* All slaves now know whether any one spilt or not, i.e. whether to perform local hash join or not
              * If any have, still need to distribute rest of RHS..
              */
 
-            // flush spillable row arrays, and clear any non-locals if performLocalLookup and compact
-            if (doPerformLocalLookup())
+            if (!isLocalLookup())
+            {
+                if (stopping) // broadcast done and no-one spilt, this node can now stop
+                    return;
+            }
+            else
             {
                 ActPrintLog("Spilt whilst broadcasting, will attempt distribute local lookup join");
 
+                // Either it has collated already and remaining rows are sorted, or if didn't get that far, can't rely on previous state of rhsAlreadySorted
+                rhsAlreadySorted = rhsCollated;
+
                 // NB: lhs ordering and grouping lost from here on..
                 if (grouped)
                     throw MakeActivityException(this, 0, "Degraded to distributed lookup join, LHS order cannot be preserved");
@@ -1578,8 +1611,6 @@ protected:
                 // If HT sized already and now spilt, too big clear and size when local size known
                 clearHT();
 
-                setupDistributors();
-
                 if (stopping)
                 {
                     ActPrintLog("getRHS stopped");
@@ -1589,103 +1620,169 @@ protected:
                      */
                     return;
                 }
-
-                /* NB: The collected broadcast rows thus far (in rhsNodeRows or rhs) were ordered/deterministic.
-                 * However, the rest of the rows received via the distributor are non-deterministic.
-                 * Therefore the order overall is non-deterministic from this point on.
-                 * For that reason, the rest of the RHS (distributed) rows will be processed ahead of the
-                 * collected [broadcast] rows in the code below for efficiency reasons.
-                 */
-                IArrayOf<IRowStream> streams;
-                streams.append(*right.getLink()); // what remains of 'right' will be read through distributor
-
-                if (rhsCollated)
-                {
-                    /* JCSMORE - I think 'right' must be empty if here (if rhsCollated=true)
-                     * so above streams.append(*right.getLink()); should go in else block below only AFAICS
-                     */
-
-                    // NB: If spilt after rhsCollated, callback will have cleared and compacted
-                    streams.append(*rhs.createRowStream()); // NB: will kill array when stream exhausted
-                }
-                else
-                {
-                    // NB: If cleared before rhsCollated, then need to clear non-locals that were added after spill
-                    ForEachItemIn(a, rhsNodeRows)
-                    {
-                        CThorSpillableRowArray &sRowArray = *rhsNodeRows.item(a);
-                        CThorExpandingRowArray rowArray(*this, NULL);
-                        rowArray.transferFrom(sRowArray);
-                        clearNonLocalRows(rowArray, flushedRowMarkers.item(a));
-                        rowArray.compact();
-                        streams.append(*rowArray.createRowStream()); // NB: will kill array when stream exhausted
-                    }
-                }
-                right.setown(createConcatRowStream(streams.ordinality(), streams.getArray()));
-            }
-            else
-            {
-                if (stopping) // broadcast done and no-one spilt, this node can now stop
-                    return;
             }
         }
         else
         {
+            rhsAlreadySorted = helper->isRightAlreadyLocallySorted();
             if (stopping) // if local can stop now
                 return;
-            setPerformLocalLookup(true);
+            setLocalLookup(true);
         }
 
-        if (doPerformLocalLookup())
+        if (isLocalLookup())
         {
             Owned<IThorRowLoader> rowLoader;
-            ICompare *cmp = rhsAlreadySorted ? NULL : compareRight;
-            if (failoverToStdJoin)
+            if (!needGlobal || !rhsCollated) // If global && rhsCollated, then no need for rowLoader
             {
-                dbgassertex(!stable);
-                if (getOptBool(THOROPT_LKJOIN_HASHJOINFAILOVER)) // for testing only (force to disk, as if spilt)
-                    rowLoader.setown(createThorRowLoader(*this, queryRowInterfaces(rightITDL), cmp, stableSort_none, rc_allDisk, SPILL_PRIORITY_LOOKUPJOIN));
+                ICompare *cmp = rhsAlreadySorted ? NULL : compareRight;
+                if (isSmart())
+                {
+                    dbgassertex(!stable);
+                    if (getOptBool(THOROPT_LKJOIN_HASHJOINFAILOVER)) // for testing only (force to disk, as if spilt)
+                        rowLoader.setown(createThorRowLoader(*this, queryRowInterfaces(rightITDL), cmp, stableSort_none, rc_allDisk, SPILL_PRIORITY_LOOKUPJOIN));
+                    else
+                        rowLoader.setown(createThorRowLoader(*this, queryRowInterfaces(rightITDL), cmp, stableSort_none, rc_mixed, SPILL_PRIORITY_LOOKUPJOIN));
+                }
                 else
-                    rowLoader.setown(createThorRowLoader(*this, queryRowInterfaces(rightITDL), cmp, stableSort_none, rc_mixed, SPILL_PRIORITY_LOOKUPJOIN));
+                {
+                    // i.e. will fire OOM if runs out of memory loading local right
+                    rowLoader.setown(createThorRowLoader(*this, queryRowInterfaces(rightITDL), cmp, stable ? stableSort_lateAlloc : stableSort_none, rc_allMem, SPILL_PRIORITY_DISABLE));
+                }
             }
-            else
+            Owned<IRowStream> rightStream;
+            if (needGlobal)
             {
-                // i.e. will fire OOM if runs out of memory loading local right
-                rowLoader.setown(createThorRowLoader(*this, queryRowInterfaces(rightITDL), cmp, stable ? stableSort_lateAlloc : stableSort_none, rc_allMem, SPILL_PRIORITY_DISABLE));
+                if (!rhsCollated) // NB: If spilt after rhsCollated, callback will have cleared and compacted, rows will still be sorted
+                {
+                    /* NB: If cleared before rhsCollated, then need to clear non-locals that were added after spill
+                     * There should not be many, as broadcast starts to stop as soon as a slave notifies it is spilling
+                     * and ignores all non-locals.
+                     */
+
+                    // First identify largest row set from nodes, clear (non-locals) and compact.
+                    rowidx_t largestRowCount = 0;
+                    CThorSpillableRowArray *largestRhsNodeRows = NULL;
+                    ForEachItemIn(a, rhsNodeRows)
+                    {
+                        CThorSpillableRowArray &rows = *rhsNodeRows.item(a);
+                        clearNonLocalRows(rows, flushedRowMarkers.item(a));
+
+                        rows.compact(); // JCS->GH - really we want to resize rhsNodeRows now to free up as much space as possible (see HPCC-12511)
+
+                        rowidx_t c = rows.numCommitted();
+                        if (c > largestRowCount)
+                        {
+                            largestRowCount = c;
+                            largestRhsNodeRows = &rows;
+                        }
+                    }
+                    /* NB: The collected broadcast rows thus far (in rhsNodeRows or rhs) were ordered/deterministic.
+                     * However, the rest of the rows received via the distributor are non-deterministic.
+                     * Therefore the order overall is non-deterministic from this point on.
+                     * For that reason, the rest of the RHS (distributed) rows will be processed ahead of the
+                     * collected [broadcast] rows in the code below for efficiency reasons.
+                     */
+                    IArrayOf<IRowStream> rightStreams;
+                    if (largestRhsNodeRows) // unlikely, but could happen if 0 local rhs rows left
+                    {
+                        // add largest directly into loader, so it can alleviate some memory immediately. NB: doesn't allocate
+                        rowLoader->transferRowsIn(*largestRhsNodeRows);
+
+                        /* the others + the rest of the RHS (distributed) are streamed into loader
+                         * the streams have a lower spill priority than the loader, IOW, they will spill before the loader spills.
+                         * If the loader has to spill, then we have to failover to standard join
+                         */
+                        ForEachItemIn(a, rhsNodeRows)
+                        {
+                            CThorSpillableRowArray &rows = *rhsNodeRows.item(a);
+                            if (&rows == largestRhsNodeRows)
+                                continue;
+                            /* NB: will kill array when stream exhausted or if spilt
+                             * Ensure spill priority of these spillable streams is lower than the stream in the loader in the next stage
+                             */
+                            rightStreams.append(*rows.createRowStream()); // NB: default SPILL_PRIORITY_SPILLABLE_STREAM is lower than SPILL_PRIORITY_LOOKUPJOIN
+                        }
+                    }
+                    // NB: 'right' deliberately added after rhsNodeRow streams, so that rhsNodeRow can be consumed into loader 1st
+                    right.setown(rhsDistributor->connect(queryRowInterfaces(rightITDL), right.getClear(), rightHash, NULL));
+                    rightStreams.append(*right.getLink()); // what remains of 'right' will be read through distributor
+                    if (overflowWriteFile)
+                    {
+                        unsigned rwFlags = DEFAULT_RWFLAGS;
+                        if (getOptBool(THOROPT_COMPRESS_SPILLS, true))
+                            rwFlags |= rw_compress;
+                        ActPrintLog("Reading overflow RHS broadcast rows : %"RCPF"d", overflowWriteCount);
+                        Owned<IRowStream> overflowStream = createRowStream(&overflowWriteFile->queryIFile(), queryRowInterfaces(rightITDL), rwFlags);
+                        rightStreams.append(* overflowStream.getClear());
+                    }
+                    if (rightStreams.ordinality()>1)
+                        right.setown(createConcatRowStream(rightStreams.ordinality(), rightStreams.getArray()));
+                    else
+                        right.set(&rightStreams.item(0));
+                    rightStream.setown(rowLoader->load(right, abortSoon, false, &rhs, NULL, false));
+                }
             }
+            else
+                rightStream.setown(rowLoader->load(right, abortSoon, false, &rhs, NULL, false));
 
-            Owned<IRowStream> rightStream = rowLoader->load(right, abortSoon, false, &rhs);
             if (!rightStream)
             {
                 ActPrintLog("RHS local rows fitted in memory, count: %"RIPF"d", rhs.ordinality());
                 // all fitted in memory, rows were transferred out back into 'rhs' and sorted
 
                 /* Now need to size HT.
-                 * transfer rows back into a spillable container
-                 * If HT sizing DOESN'T cause spill, then, row will be transferred back into 'rhs'
+                 * If HT sizing DOESN'T cause spill, then rows will be transferred back into 'rhs'
                  * If HT sizing DOES cause spill, sorted rightStream will be created.
+                 * transfer rows back into a spillable container
                  */
 
                 rowLoader.clear();
 
-                // If stable (and sort needed), already sorted by rowLoader
-                rowidx_t uniqueKeys = marker.calculate(rhs, compareRight, !rhsAlreadySorted && !stable);
-
-                Owned<IThorRowCollector> collector = createThorRowCollector(*this, queryRowInterfaces(rightITDL), cmp, stableSort_none, rc_mixed, SPILL_PRIORITY_LOOKUPJOIN);
-                collector->setOptions(rcflag_noAllInMemSort); // If fits into memory, don't want it resorted
-                collector->transferRowsIn(rhs); // can spill after this
+                // Either was already sorted, or rowLoader->load() sorted on transfer out to rhs
+                rowidx_t uniqueKeys = marker.calculate(rhs, compareRight, false);
 
-                if (!setupHT(uniqueKeys))
+                /* Although HT is allocated with a low spill priority, it can still cause callbacks
+                 * so try to allocate before rhs is transferred to spillable collector
+                 */
+                bool htAllocated = setupHT(uniqueKeys);
+                if (!htAllocated)
                 {
-                    ActPrintLog("Out of memory trying to allocate the [LOCAL] hash table for a SMART join, will now failover to a std hash join");
+                    ActPrintLog("Out of memory trying to allocate the [LOCAL] hash table for a SMART join (%"RIPF"d rows), will now failover to a std hash join", uniqueKeys);
+                    Owned<IThorRowCollector> collector = createThorRowCollector(*this, queryRowInterfaces(rightITDL), NULL, stableSort_none, rc_mixed, SPILL_PRIORITY_LOOKUPJOIN);
+                    collector->setOptions(rcflag_noAllInMemSort); // If fits into memory, don't want it resorted
+                    collector->transferRowsIn(rhs); // can spill after this
                     rightStream.setown(collector->getStream());
                 }
-                else
-                    rightStream.setown(collector->getStream(false, &rhs));
             }
-            if (rightStream) // NB: returned stream, implies (spilt or setupHT OOM'd) AND sorted, if not, 'rhs' is filled
+            if (rightStream)
             {
-                ActPrintLog("RHS spilt to disk. Standard Join will be used");
+                ActPrintLog("Local RHS spilt to disk. Standard Join will be used");
+                setStandardJoin(true);
+            }
+
+            // Barrier point, did all slaves succeed in building local RHS HT?
+            if (needGlobal && isSmart())
+            {
+                bool localRequiredStandardJoin = isStandardJoin();
+                ActPrintLog("Checking other slaves for local RHS lookup status, this slaves RHS %s", localRequiredStandardJoin?"did NOT fit" : "DID fit");
+                broadcaster.reset();
+                broadcast_flags flag = localRequiredStandardJoin ? bcastflag_standardjoin : bcastflag_null;
+                doBroadcastStop(broadcast3MpTag, flag);
+                if (!localRequiredStandardJoin && isStandardJoin())
+                {
+                    ActPrintLog("Other slaves did NOT fit, consequently this slave must fail over to standard join as well");
+                    dbgassertex(!rightStream);
+                    rightStream.setown(rhs.createRowStream());
+                }
+
+                // start LHS distributor, needed either way, by local lookup or full join
+                left.setown(lhsDistributor->connect(queryRowInterfaces(leftITDL), left.getClear(), leftHash, NULL));
+            }
+
+            if (isStandardJoin()) // NB: returned stream, implies (spilt or setupHT OOM'd) AND sorted, if not, 'rhs' is filled
+            {
+                ActPrintLog("Performing standard join");
 
                 // NB: lhs ordering and grouping lost from here on.. (will have been caught earlier if global)
                 if (grouped)
@@ -1751,8 +1848,9 @@ public:
     CLookupJoinActivityBase(CGraphElementBase *_container) : PARENT(_container)
     {
         rhsCollated = false;
-        broadcast2MpTag = lhsDistributeTag = rhsDistributeTag = TAG_NULL;
-        setPerformLocalLookup(false);
+        broadcast2MpTag = broadcast3MpTag = lhsDistributeTag = rhsDistributeTag = TAG_NULL;
+        setLocalLookup(false);
+        setStandardJoin(false);
 
         leftHash = helper->queryHashLeft();
         rightHash = helper->queryHashRight();
@@ -1770,9 +1868,19 @@ public:
         if (abortLimit < atMost)
             atMost = abortLimit;
 
-        failoverToLocalLookupJoin = failoverToStdJoin = isSmart();
-        ActPrintLog("failoverToLocalLookupJoin=%s, failoverToStdJoin=%s",
-                failoverToLocalLookupJoin?"true":"false", failoverToStdJoin?"true":"false");
+        switch (container.getKind())
+        {
+            case TAKsmartjoin:
+            case TAKsmartdenormalize:
+            case TAKsmartdenormalizegroup:
+                smart = true;
+                break;
+            default:
+                smart = false;
+                break;
+        }
+        overflowWriteCount = 0;
+        ActPrintLog("Smart join = %s", smart?"true":"false");
     }
     bool exceedsLimit(rowidx_t count, const void *left, const void *right, const void *&failRow)
     {
@@ -1821,30 +1929,38 @@ public:
         if (!container.queryLocal())
         {
             broadcast2MpTag = container.queryJob().deserializeMPTag(data);
+            broadcast3MpTag = container.queryJob().deserializeMPTag(data);
             lhsDistributeTag = container.queryJob().deserializeMPTag(data);
             rhsDistributeTag = container.queryJob().deserializeMPTag(data);
+            rhsDistributor.setown(createHashDistributor(this, queryJob().queryJobComm(), rhsDistributeTag, false, NULL, "RHS"));
+            lhsDistributor.setown(createHashDistributor(this, queryJob().queryJobComm(), lhsDistributeTag, false, NULL, "LHS"));
         }
     }
     virtual void start()
     {
         ActivityTimer s(totalCycles, timeActivities);
         PARENT::start();
+        if (isSmart())
+        {
+            setLocalLookup(false);
+            setStandardJoin(false);
+            rhsCollated = false;
+            flushedRowMarkers.kill();
 
-        if (!isSmart())
+            if (needGlobal)
+            {
+                overflowWriteCount = 0;
+                overflowWriteFile.clear();
+                overflowWriteStream.clear();
+                queryJob().queryRowManager()->addRowBuffer(this);
+            }
+        }
+        else
         {
             bool inputGrouped = leftITDL->isGrouped();
             dbgassertex(inputGrouped == grouped); // std. lookup join expects these to match
         }
 
-        setPerformLocalLookup(false);
-        rhsCollated = false;
-        flushedRowMarkers.kill();
-
-        if (failoverToLocalLookupJoin)
-        {
-            if (needGlobal)
-                queryJob().queryRowManager()->addRowBuffer(this);
-        }
     }
     CATCH_NEXTROW()
     {
@@ -1856,9 +1972,9 @@ public:
             if (isSmart())
             {
                 msg.append("SmartJoin - ");
-                if (joinHelper)
+                if (isStandardJoin())
                     msg.append("Failed over to standard join");
-                else if (needGlobal && doPerformLocalLookup())
+                else if (needGlobal && isLocalLookup())
                     msg.append("Failed over to hash distributed local lookup join");
                 else
                     msg.append("All in memory lookup join");
@@ -1912,15 +2028,22 @@ public:
 // IBCastReceive
     virtual void bCastReceive(CSendItem *sendItem)
     {
-        if (sendItem)
+        if (0 != (sendItem->queryFlags() & bcastflag_spilt))
         {
-            if (0 != (sendItem->queryFlags() & bcastflag_spilt))
-            {
-                VStringBuffer msg("Notification that slave %d spilt", sendItem->queryOrigin());
-                clearAllNonLocalRows(msg.str());
-            }
+            VStringBuffer msg("Notification that slave %d spilt", sendItem->queryOrigin());
+            clearAllNonLocalRows(msg.str());
+        }
+        else if (0 != (sendItem->queryFlags() & bcastflag_standardjoin))
+        {
+            VStringBuffer msg("Notification that slave %d required standard join", sendItem->queryOrigin());
+            setStandardJoin(true);
         }
-        rowProcessor.addBlock(sendItem); // NB: NULL indicates end
+        if (bcast_stop == sendItem->queryCode())
+        {
+            sendItem->Release();
+            sendItem = NULL; // NB: NULL indicates end
+        }
+        rowProcessor.addBlock(sendItem);
     }
 // IBufferedRowCallback
     virtual unsigned getSpillCost() const
@@ -1932,27 +2055,47 @@ public:
         // NB: only installed if lookup join and global
         return clearAllNonLocalRows("Out of memory callback");
     }
-    virtual bool addLocalRHSRow(CThorSpillableRowArray &localRhsRows, const void *row)
+    virtual bool addRHSRow(CThorSpillableRowArray &rhsRows, const void *row)
     {
-        if (doPerformLocalLookup())
+        /* NB: If PARENT::addRHSRow fails, it will cause clearAllNonLocalRows() to have been triggered and localLookup to be set
+         * When all is done, a last pass is needed to clear out non-locals
+         */
+        if (!overflowWriteFile)
         {
+            if (!isLocalLookup() && PARENT::addRHSRow(rhsRows, row))
+                return true;
+            dbgassertex(isLocalLookup());
             // keep it only if it hashes to my node
             unsigned hv = rightHash->hash(row);
-            if ((myNode-1) == (hv % numNodes))
-                PARENT::addLocalRHSRow(localRhsRows, row);
-            // ok so switch tactics.
-            // clearAllNonLocalRows() will have cleared out non-locals by now
-            // Returning false here, will stop the broadcaster
+            if ((myNode-1) != (hv % numNodes))
+                return true; // throw away non-local row
+            if (PARENT::addRHSRow(rhsRows, row))
+                return true;
 
-            return false;
-        }
-        else
-        {
-            /* NB: It could still spill here, i.e. before appending a non-local row
-             * When all is done, a last pass is needed to clear out non-locals
+            /* Could OOM whilst still failing over to local lookup again, dealing with last row, or trailing
+             * few rows being received. Unlikely since all local rows will have been cleared, but possible,
+             * particularly if last rows end up causing row ptr table expansion here.
+             *
+             * Need to stash away somewhere to allow it continue.
              */
-            PARENT::addLocalRHSRow(localRhsRows, row);
+            CriticalBlock b(overflowCrit); // could be coming from broadcaster or receiver
+            if (!overflowWriteFile)
+            {
+                unsigned rwFlags = DEFAULT_RWFLAGS;
+                if (getOptBool(THOROPT_COMPRESS_SPILLS, true))
+                    rwFlags |= rw_compress;
+                StringBuffer tempFilename;
+                GetTempName(tempFilename, "lookup_local", true);
+                ActPrintLog("Overflowing RHS broadcast rows to spill file: %s", tempFilename.str());
+                OwnedIFile iFile = createIFile(tempFilename.str());
+                overflowWriteFile.setown(new CFileOwner(iFile.getLink()));
+                overflowWriteStream.setown(createRowWriter(iFile, queryRowInterfaces(rightITDL), rwFlags));
+            }
         }
+        ++overflowWriteCount;
+        LinkThorRow(row);
+        CriticalBlock b(overflowCrit); // could be coming from broadcaster or receiver
+        overflowWriteStream->putRow(row);
         return true;
     }
 };
@@ -2341,7 +2484,12 @@ public:
 // IBCastReceive
     virtual void bCastReceive(CSendItem *sendItem)
     {
-        rowProcessor.addBlock(sendItem); // NB: NULL indicates end
+        if (bcast_stop == sendItem->queryCode())
+        {
+            sendItem->Release();
+            sendItem = NULL; // NB: NULL indicates end
+        }
+        rowProcessor.addBlock(sendItem);
     }
 };
 

thorlcr/thorutil/thmem.cpp

@@ -173,15 +173,15 @@ StringBuffer &getRecordString(const void *key, IOutputRowSerializer *serializer,
 
 //====
 
+// NB: rows are transferred into derivatives of CSpillableStreamBase and read or spilt, but are never written to
 class CSpillableStreamBase : public CSimpleInterface, implements roxiemem::IBufferedRowCallback
 {
 protected:
     CActivityBase &activity;
     IRowInterfaces *rowIf;
-    bool preserveNulls, ownsRows, useCompression;
+    bool preserveNulls, ownsRows, useCompression, spillPriority;
     CThorSpillableRowArray rows;
     OwnedIFile spillFile;
-    Owned<IRowStream> spillStream;
 
     bool spillRows()
     {
@@ -197,22 +197,26 @@ protected:
 
         VStringBuffer spillPrefixStr("SpillableStream(%d)", SPILL_PRIORITY_SPILLABLE_STREAM); // const for now
         rows.save(*spillFile, useCompression, spillPrefixStr.str()); // saves committed rows
+        rows.kill(); // no longer needed, readers will pull from spillFile. NB: ok to kill array as rows is never written to or expanded
         return true;
     }
-
+    void clearSpillingCallback()
+    {
+        activity.queryJob().queryRowManager()->removeRowBuffer(this);
+    }
 public:
     IMPLEMENT_IINTERFACE_USING(CSimpleInterface);
 
-    CSpillableStreamBase(CActivityBase &_activity, CThorSpillableRowArray &inRows, IRowInterfaces *_rowIf, bool _preserveNulls)
-        : activity(_activity), rowIf(_rowIf), rows(_activity, _rowIf, _preserveNulls), preserveNulls(_preserveNulls)
+    CSpillableStreamBase(CActivityBase &_activity, CThorSpillableRowArray &inRows, IRowInterfaces *_rowIf, bool _preserveNulls, unsigned _spillPriorirty)
+        : activity(_activity), rowIf(_rowIf), rows(_activity, _rowIf, _preserveNulls), preserveNulls(_preserveNulls), spillPriority(_spillPriorirty)
     {
+    	assertex(inRows.isFlushed());
         rows.swap(inRows);
         useCompression = false;
     }
     ~CSpillableStreamBase()
     {
-        activity.queryJob().queryRowManager()->removeRowBuffer(this);
-        spillStream.clear();
+        clearSpillingCallback();
         if (spillFile)
             spillFile->remove();
     }
@@ -220,7 +224,7 @@ public:
 // IBufferedRowCallback
     virtual unsigned getSpillCost() const
     {
-        return SPILL_PRIORITY_SPILLABLE_STREAM;
+        return spillPriority;
     }
     virtual bool freeBufferedRows(bool critical)
     {
@@ -262,8 +266,8 @@ class CSharedSpillableRowSet : public CSpillableStreamBase, implements IInterfac
             CThorArrayLockBlock block(owner->rows);
             if (owner->spillFile) // i.e. has spilt
             {
+                owner->clearSpillingCallback();
                 assertex(((offset_t)-1) != outputOffset);
-                owner->rows.kill(); // no longer needed, frees pointer array
                 unsigned rwFlags = DEFAULT_RWFLAGS;
                 if (owner->preserveNulls)
                     rwFlags |= rw_grouped;
@@ -272,7 +276,10 @@ class CSharedSpillableRowSet : public CSpillableStreamBase, implements IInterfac
                 return spillStream->nextRow();
             }
             else if (pos == owner->rows.numCommitted())
+            {
+                owner->clearSpillingCallback();
                 return NULL;
+            }
             return owner->rows.get(pos++);
         }
         virtual void stop() { }
@@ -291,8 +298,8 @@ class CSharedSpillableRowSet : public CSpillableStreamBase, implements IInterfac
 public:
     IMPLEMENT_IINTERFACE_USING(CSimpleInterface);
 
-    CSharedSpillableRowSet(CActivityBase &_activity, CThorSpillableRowArray &inRows, IRowInterfaces *_rowIf, bool _preserveNulls)
-        : CSpillableStreamBase(_activity, inRows, _rowIf, _preserveNulls)
+    CSharedSpillableRowSet(CActivityBase &_activity, CThorSpillableRowArray &inRows, IRowInterfaces *_rowIf, bool _preserveNulls, unsigned _spillPriority)
+        : CSpillableStreamBase(_activity, inRows, _rowIf, _preserveNulls, _spillPriority)
     {
         activity.queryJob().queryRowManager()->addRowBuffer(this);
     }
@@ -303,6 +310,7 @@ public:
             CThorArrayLockBlock block(rows);
             if (spillFile)
             {
+                clearSpillingCallback();
                 unsigned rwFlags = DEFAULT_RWFLAGS;
                 if (preserveNulls)
                     rwFlags |= rw_grouped;
@@ -318,12 +326,13 @@ class CSpillableStream : public CSpillableStreamBase, implements IRowStream
 {
     rowidx_t pos, numReadRows, granularity;
     const void **readRows;
+    Owned<IRowStream> spillStream;
 
 public:
     IMPLEMENT_IINTERFACE_USING(CSimpleInterface);
 
-    CSpillableStream(CActivityBase &_activity, CThorSpillableRowArray &inRows, IRowInterfaces *_rowIf, bool _preserveNulls)
-        : CSpillableStreamBase(_activity, inRows, _rowIf, _preserveNulls)
+    CSpillableStream(CActivityBase &_activity, CThorSpillableRowArray &inRows, IRowInterfaces *_rowIf, bool _preserveNulls, unsigned _spillPriority)
+        : CSpillableStreamBase(_activity, inRows, _rowIf, _preserveNulls, _spillPriority)
     {
         useCompression = activity.getOptBool(THOROPT_COMPRESS_SPILLS, true);
         pos = numReadRows = 0;
@@ -336,6 +345,7 @@ public:
     }
     ~CSpillableStream()
     {
+        spillStream.clear();
         while (pos < numReadRows)
         {
             ReleaseThorRow(readRows[pos++]);
@@ -353,7 +363,7 @@ public:
             CThorArrayLockBlock block(rows);
             if (spillFile)
             {
-                rows.kill(); // no longer needed, frees pointer array
+                clearSpillingCallback();
                 unsigned rwFlags = DEFAULT_RWFLAGS;
                 if (preserveNulls)
                     rwFlags |= rw_grouped;
@@ -362,13 +372,17 @@ public:
                 spillStream.setown(createRowStream(spillFile, rowIf, rwFlags));
                 return spillStream->nextRow();
             }
-            rowidx_t fetch = rows.numCommitted();
-            if (0 == fetch)
+            rowidx_t available = rows.numCommitted();
+            if (0 == available)
                 return NULL;
-            if (fetch >= granularity)
-                fetch = granularity;
+            rowidx_t fetch = (available >= granularity) ? granularity : available;
             // consume 'fetch' rows
             rows.readBlock(readRows, fetch);
+            if (available == fetch)
+            {
+                clearSpillingCallback();
+                rows.kill();
+            }
             numReadRows = fetch;
             pos = 0;
         }
@@ -632,6 +646,7 @@ void CThorExpandingRowArray::compact()
         }
     }
     numRows = freeFinger-rows;
+    // JCSMORE - this may be a good time to call IRowManager::compactRows
 }
 
 void CThorExpandingRowArray::kill()
@@ -1321,6 +1336,13 @@ void CThorSpillableRowArray::transferFrom(CThorExpandingRowArray &src)
     commitRows = numRows;
 }
 
+void CThorSpillableRowArray::transferFrom(CThorSpillableRowArray &src)
+{
+    CThorArrayLockBlock block(*this);
+    CThorExpandingRowArray::transferFrom(src);
+    commitRows = numRows;
+}
+
 void CThorSpillableRowArray::swap(CThorSpillableRowArray &other)
 {
     CThorArrayLockBlock block(*this);
@@ -1365,9 +1387,10 @@ void CThorSpillableRowArray::transferRowsCopy(const void **outRows, bool takeOwn
     }
 }
 
-IRowStream *CThorSpillableRowArray::createRowStream()
+IRowStream *CThorSpillableRowArray::createRowStream(unsigned spillPriority)
 {
-    return new CSpillableStream(activity, *this, rowIf, allowNulls);
+    assertex(rowIf);
+    return new CSpillableStream(activity, *this, rowIf, allowNulls, spillPriority);
 }
 
 
@@ -1534,10 +1557,10 @@ protected:
                     return NULL;
                 }
                 if (!shared)
-                    instrms.append(*spillableRows.createRowStream()); // NB: stream will take ownership of rows in spillableRows
+                    instrms.append(*spillableRows.createRowStream(spillPriority)); // NB: stream will take ownership of rows in spillableRows
                 else
                 {
-                    spillableRowSet.setown(new CSharedSpillableRowSet(activity, spillableRows, rowIf, preserveGrouping));
+                    spillableRowSet.setown(new CSharedSpillableRowSet(activity, spillableRows, rowIf, preserveGrouping, spillPriority));
                     instrms.append(*spillableRowSet->createRowStream());
                 }
             }
@@ -1638,6 +1661,12 @@ public:
         spillableRows.transferFrom(src);
         enableSpillingCallback();
     }
+    virtual void transferRowsIn(CThorSpillableRowArray &src)
+    {
+        reset();
+        spillableRows.transferFrom(src);
+        enableSpillingCallback();
+    }
     virtual void setup(ICompare *_iCompare, StableSortFlag _stableSort, RowCollectorSpillFlags _diskMemMix, unsigned _spillPriority)
     {
         iCompare = _iCompare;
@@ -1678,9 +1707,10 @@ public:
 enum TRLGroupFlag { trl_ungroup, trl_preserveGrouping, trl_stopAtEog };
 class CThorRowLoader : public CThorRowCollectorBase, implements IThorRowLoader
 {
-    IRowStream *load(IRowStream *in, const bool &abort, TRLGroupFlag grouping, CThorExpandingRowArray *allMemRows, memsize_t *memUsage)
+    IRowStream *load(IRowStream *in, const bool &abort, TRLGroupFlag grouping, CThorExpandingRowArray *allMemRows, memsize_t *memUsage, bool doReset)
     {
-        reset();
+        if (doReset)
+            reset();
         enableSpillingCallback();
         setPreserveGrouping(trl_preserveGrouping == grouping);
         while (!abort)
@@ -1717,6 +1747,7 @@ public:
     virtual unsigned overflowScale() const { return CThorRowCollectorBase::overflowScale(); }
     virtual void transferRowsOut(CThorExpandingRowArray &dst, bool sort) { CThorRowCollectorBase::transferRowsOut(dst, sort); }
     virtual void transferRowsIn(CThorExpandingRowArray &src) { CThorRowCollectorBase::transferRowsIn(src); }
+    virtual void transferRowsIn(CThorSpillableRowArray &src) { CThorRowCollectorBase::transferRowsIn(src); }
     virtual void setup(ICompare *iCompare, StableSortFlag stableSort, RowCollectorSpillFlags diskMemMix=rc_mixed, unsigned spillPriority=50)
     {
         CThorRowCollectorBase::setup(iCompare, stableSort, diskMemMix, spillPriority);
@@ -1724,14 +1755,14 @@ public:
     virtual void ensure(rowidx_t max) { CThorRowCollectorBase::ensure(max); }
     virtual void setOptions(unsigned options)  { CThorRowCollectorBase::setOptions(options); }
 // IThorRowLoader
-    virtual IRowStream *load(IRowStream *in, const bool &abort, bool preserveGrouping, CThorExpandingRowArray *allMemRows, memsize_t *memUsage)
+    virtual IRowStream *load(IRowStream *in, const bool &abort, bool preserveGrouping, CThorExpandingRowArray *allMemRows, memsize_t *memUsage, bool doReset)
     {
         assertex(!iCompare || !preserveGrouping); // can't sort if group preserving
-        return load(in, abort, preserveGrouping?trl_preserveGrouping:trl_ungroup, allMemRows, memUsage);
+        return load(in, abort, preserveGrouping?trl_preserveGrouping:trl_ungroup, allMemRows, memUsage, doReset);
     }
-    virtual IRowStream *loadGroup(IRowStream *in, const bool &abort, CThorExpandingRowArray *allMemRows, memsize_t *memUsage)
+    virtual IRowStream *loadGroup(IRowStream *in, const bool &abort, CThorExpandingRowArray *allMemRows, memsize_t *memUsage, bool doReset)
     {
-        return load(in, abort, trl_stopAtEog, allMemRows, memUsage);
+        return load(in, abort, trl_stopAtEog, allMemRows, memUsage, doReset);
     }
 };
 
@@ -1767,6 +1798,7 @@ public:
     virtual unsigned overflowScale() const { return CThorRowCollectorBase::overflowScale(); }
     virtual void transferRowsOut(CThorExpandingRowArray &dst, bool sort) { CThorRowCollectorBase::transferRowsOut(dst, sort); }
     virtual void transferRowsIn(CThorExpandingRowArray &src) { CThorRowCollectorBase::transferRowsIn(src); }
+    virtual void transferRowsIn(CThorSpillableRowArray &src) { CThorRowCollectorBase::transferRowsIn(src); }
     virtual void setup(ICompare *iCompare, StableSortFlag stableSort, RowCollectorSpillFlags diskMemMix=rc_mixed, unsigned spillPriority=50)
     {
         CThorRowCollectorBase::setup(iCompare, stableSort, diskMemMix, spillPriority);

thorlcr/thorutil/thmem.hpp

@@ -422,6 +422,7 @@ public:
     void kill();
     void compact();
     void flush();
+    inline bool isFlushed() const { return numRows == numCommitted(); }
     inline bool append(const void *row) __attribute__((warn_unused_result))
     {
         //GH->JCS Should this really be inline?
@@ -471,8 +472,9 @@ public:
         swap(from);
     }
     void transferFrom(CThorExpandingRowArray &src);
+    void transferFrom(CThorSpillableRowArray &src);
 
-    IRowStream *createRowStream();
+    IRowStream *createRowStream(unsigned spillPriority=SPILL_PRIORITY_SPILLABLE_STREAM);
 
     offset_t serializedSize()
     {
@@ -513,6 +515,7 @@ interface IThorRowCollectorCommon : extends IInterface
     virtual unsigned overflowScale() const = 0;
     virtual void transferRowsOut(CThorExpandingRowArray &dst, bool sort=true) = 0;
     virtual void transferRowsIn(CThorExpandingRowArray &src) = 0;
+    virtual void transferRowsIn(CThorSpillableRowArray &src) = 0;
     virtual void setup(ICompare *iCompare, StableSortFlag stableSort=stableSort_none, RowCollectorSpillFlags diskMemMix=rc_mixed, unsigned spillPriority=50) = 0;
     virtual void ensure(rowidx_t max) = 0;
     virtual void setOptions(unsigned options) = 0;
@@ -520,8 +523,8 @@ interface IThorRowCollectorCommon : extends IInterface
 
 interface IThorRowLoader : extends IThorRowCollectorCommon
 {
-    virtual IRowStream *load(IRowStream *in, const bool &abort, bool preserveGrouping=false, CThorExpandingRowArray *allMemRows=NULL, memsize_t *memUsage=NULL) = 0;
-    virtual IRowStream *loadGroup(IRowStream *in, const bool &abort, CThorExpandingRowArray *allMemRows=NULL, memsize_t *memUsage=NULL) = 0;
+    virtual IRowStream *load(IRowStream *in, const bool &abort, bool preserveGrouping=false, CThorExpandingRowArray *allMemRows=NULL, memsize_t *memUsage=NULL, bool doReset=true) = 0;
+    virtual IRowStream *loadGroup(IRowStream *in, const bool &abort, CThorExpandingRowArray *allMemRows=NULL, memsize_t *memUsage=NULL, bool doReset=true) = 0;
 };
 
 interface IThorRowCollector : extends IThorRowCollectorCommon