Merge remote-tracking branch 'origin/closedown-5.0.x'

Conflicts:
	dali/base/dasds.cpp
	roxie/roxiemem/roxiemem.cpp
	testing/regress/README.rst
	testing/regress/ecl-test
	testing/regress/hpcc/util/ecl/file.py

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

cmake_modules/dependencies/el5.cmake

@@ -1 +1 @@
-set ( CPACK_RPM_PACKAGE_REQUIRES "boost141-regex, openldap, libicu, m4, libtool, xalan-c, xerces-c, gcc-c++, openssh-server, openssh-clients, expect, libarchive, rsync")
+set ( CPACK_RPM_PACKAGE_REQUIRES "boost141-regex, openldap, libicu, m4, libtool, xalan-c, xerces-c, gcc-c++, openssh-server, openssh-clients, expect, libarchive, rsync, apr, apr-util")

cmake_modules/dependencies/el6.cmake

@@ -1 +1 @@
-set ( CPACK_RPM_PACKAGE_REQUIRES "boost-regex, openldap, libicu, m4, libtool, libxslt, libxml2, gcc-c++, openssh-server, openssh-clients, expect, libarchive, rsync")
+set ( CPACK_RPM_PACKAGE_REQUIRES "boost-regex, openldap, libicu, m4, libtool, libxslt, libxml2, gcc-c++, openssh-server, openssh-clients, expect, libarchive, rsync, apr, apr-util")

cmake_modules/dependencies/lenny.cmake

@@ -1 +1 @@
-set ( CPACK_DEBIAN_PACKAGE_DEPENDS "libboost-regex1.34.1, libicu38, libxalan110, libxerces-c28, binutils, libldap-2.4-2, openssl, zlib1g, g++, sudo, openssh-client, openssh-server, expect, libarchive, rsync")
+set ( CPACK_DEBIAN_PACKAGE_DEPENDS "libboost-regex1.34.1, libicu38, libxalan110, libxerces-c28, binutils, libldap-2.4-2, openssl, zlib1g, g++, sudo, openssh-client, openssh-server, expect, libarchive, rsync, libapr1, libaprutil1")

cmake_modules/dependencies/oneiric.cmake

@@ -1 +1 @@
-set ( CPACK_DEBIAN_PACKAGE_DEPENDS "libboost-regex1.46.1, libicu44, libxalan110, libxerces-c28, binutils, libldap-2.4-2, openssl, zlib1g, g++, openssh-client, openssh-server, expect, libarchive1, rsync")
+set ( CPACK_DEBIAN_PACKAGE_DEPENDS "libboost-regex1.46.1, libicu44, libxalan110, libxerces-c28, binutils, libldap-2.4-2, openssl, zlib1g, g++, openssh-client, openssh-server, expect, libarchive1, rsync, libapr1, libaprutil1")

cmake_modules/dependencies/precise.cmake

@@ -1 +1 @@
-set ( CPACK_DEBIAN_PACKAGE_DEPENDS "libboost-regex1.46.1, libicu48, libxalan110, libxerces-c28, binutils, libldap-2.4-2, openssl, zlib1g, g++, openssh-client, openssh-server, expect, libarchive12, rsync")
+set ( CPACK_DEBIAN_PACKAGE_DEPENDS "libboost-regex1.46.1, libicu48, libxalan110, libxerces-c28, binutils, libldap-2.4-2, openssl, zlib1g, g++, openssh-client, openssh-server, expect, libarchive12, rsync, libapr1, libaprutil1")

cmake_modules/dependencies/quantal.cmake

@@ -1 +1 @@
-set ( CPACK_DEBIAN_PACKAGE_DEPENDS "libboost-regex1.49.0, libicu48, libxalan110, libxerces-c28, binutils, libldap-2.4-2, openssl, zlib1g, g++, openssh-client, openssh-server, expect, libarchive12, rsync")
+set ( CPACK_DEBIAN_PACKAGE_DEPENDS "libboost-regex1.49.0, libicu48, libxalan110, libxerces-c28, binutils, libldap-2.4-2, openssl, zlib1g, g++, openssh-client, openssh-server, expect, libarchive12, rsync, libapr1, libaprutil1")

cmake_modules/dependencies/raring.cmake

@@ -1 +1 @@
-set ( CPACK_DEBIAN_PACKAGE_DEPENDS "libboost-regex1.49.0, libicu48, libxalan110, libxerces-c28, binutils, libldap-2.4-2, openssl, zlib1g, g++, openssh-client, openssh-server, expect, libarchive13, rsync")
+set ( CPACK_DEBIAN_PACKAGE_DEPENDS "libboost-regex1.49.0, libicu48, libxalan110, libxerces-c28, binutils, libldap-2.4-2, openssl, zlib1g, g++, openssh-client, openssh-server, expect, libarchive13, rsync, libapr1, libaprutil1")

cmake_modules/dependencies/saucy.cmake

@@ -1 +1 @@
-set ( CPACK_DEBIAN_PACKAGE_DEPENDS "libboost-regex1.53.0, libicu48, libxalan-c111, libxerces-c3.1, binutils, libldap-2.4-2, openssl, zlib1g, g++, openssh-client, openssh-server, expect, libarchive13, rsync")
+set ( CPACK_DEBIAN_PACKAGE_DEPENDS "libboost-regex1.53.0, libicu48, libxalan-c111, libxerces-c3.1, binutils, libldap-2.4-2, openssl, zlib1g, g++, openssh-client, openssh-server, expect, libarchive13, rsync, libapr1, libaprutil1")

cmake_modules/dependencies/squeeze.cmake

@@ -1 +1 @@
-set ( CPACK_DEBIAN_PACKAGE_DEPENDS "libboost-regex1.42.0, libicu44, libxalan110, libxerces-c28, binutils, libldap-2.4-2, openssl, zlib1g, g++, openssh-client, openssh-server, expect, libarchive1, rsync")
+set ( CPACK_DEBIAN_PACKAGE_DEPENDS "libboost-regex1.42.0, libicu44, libxalan110, libxerces-c28, binutils, libldap-2.4-2, openssl, zlib1g, g++, openssh-client, openssh-server, expect, libarchive1, rsync, libapr1, libaprutil1")

cmake_modules/dependencies/suse11.3.cmake

@@ -1 +1 @@
-set ( CPACK_RPM_PACKAGE_REQUIRES  "binutils, gcc-c++, openssh, libldap-2_4-2, libicu, libboost_regex1_42_0, libxerces-c-3_0, libxalan-c110, expect, libarchive2, rsync" )
+set ( CPACK_RPM_PACKAGE_REQUIRES  "binutils, gcc-c++, openssh, libldap-2_4-2, libicu, libboost_regex1_42_0, libxerces-c-3_0, libxalan-c110, expect, libarchive2, rsync, apr, apr-util" )

cmake_modules/dependencies/suse11.4.cmake

@@ -1 +1 @@
-set ( CPACK_RPM_PACKAGE_REQUIRES  "binutils, gcc-c++, openssh, libldap-2_4-2, libicu, libboost_regex1_44_0, libxerces-c-3_0, libxalan-c110, expect, libarchive2, rsync" )
+set ( CPACK_RPM_PACKAGE_REQUIRES  "binutils, gcc-c++, openssh, libldap-2_4-2, libicu, libboost_regex1_44_0, libxerces-c-3_0, libxalan-c110, expect, libarchive2, rsync, apr, apr-util" )

cmake_modules/dependencies/trusty.cmake

@@ -1 +1 @@
-set ( CPACK_DEBIAN_PACKAGE_DEPENDS "libboost-regex1.54.0, libicu52, libxalan-c111, libxerces-c3.1, binutils, libldap-2.4-2, openssl, zlib1g, g++, openssh-client, openssh-server, expect, libarchive13, rsync")
+set ( CPACK_DEBIAN_PACKAGE_DEPENDS "libboost-regex1.54.0, libicu52, libxalan-c111, libxerces-c3.1, binutils, libldap-2.4-2, openssl, zlib1g, g++, openssh-client, openssh-server, expect, libarchive13, rsync, libapr1, libaprutil1")

common/workunit/workunit.cpp

@@ -10197,7 +10197,7 @@ extern WORKUNIT_API IPropertyTree * getPackageSetRegistry(const char * wsEclId,
     return conn->getRoot();
 }
 
-void addQueryToQuerySet(IWorkUnit *workunit, const char *querySetName, const char *queryName, IPropertyTree *packageInfo, WUQueryActivationOptions activateOption, StringBuffer &newQueryId, const char *userid)
+void addQueryToQuerySet(IWorkUnit *workunit, IPropertyTree *queryRegistry, const char *queryName, WUQueryActivationOptions activateOption, StringBuffer &newQueryId, const char *userid)
 {
     StringBuffer cleanQueryName;
     appendUtf8XmlName(cleanQueryName, strlen(queryName), queryName);
@@ -10211,8 +10211,6 @@ void addQueryToQuerySet(IWorkUnit *workunit, const char *querySetName, const cha
     SCMStringBuffer wuid;
     workunit->getWuid(wuid);
 
-    Owned<IPropertyTree> queryRegistry = getQueryRegistry(querySetName, false);
-
     StringBuffer currentTargetClusterType;
     queryRegistry->getProp("@targetclustertype", currentTargetClusterType); 
 
@@ -10239,11 +10237,19 @@ void addQueryToQuerySet(IWorkUnit *workunit, const char *querySetName, const cha
     workunit->setIsQueryService(true); //will check querysets before delete
     workunit->commit();
 
+    activateQuery(queryRegistry, activateOption, queryName, newQueryId, userid);
+}
+
+void activateQuery(IPropertyTree *queryRegistry, WUQueryActivationOptions activateOption, const char *queryName, const char *queryId, const char *userid)
+{
+    StringBuffer cleanQueryName;
+    appendUtf8XmlName(cleanQueryName, strlen(queryName), queryName);
+
     if (activateOption == ACTIVATE_SUSPEND_PREVIOUS|| activateOption == ACTIVATE_DELETE_PREVIOUS)
     {
         Owned<IPropertyTree> prevQuery = resolveQueryAlias(queryRegistry, cleanQueryName);
-        setQueryAlias(queryRegistry, cleanQueryName, newQueryId);
-        if (prevQuery)
+        setQueryAlias(queryRegistry, cleanQueryName, queryId);
+        if (prevQuery && !streq(queryId, prevQuery->queryProp("@id")))
         {
             if (activateOption == ACTIVATE_SUSPEND_PREVIOUS)
                 setQuerySuspendedState(queryRegistry, prevQuery->queryProp("@id"), true, userid);
@@ -10252,7 +10258,13 @@ void addQueryToQuerySet(IWorkUnit *workunit, const char *querySetName, const cha
         }
     }
     else if (activateOption == MAKE_ACTIVATE || activateOption == MAKE_ACTIVATE_LOAD_DATA_ONLY)
-        setQueryAlias(queryRegistry, cleanQueryName, newQueryId.str());
+        setQueryAlias(queryRegistry, cleanQueryName, queryId);
+}
+
+void addQueryToQuerySet(IWorkUnit *workunit, const char *querySetName, const char *queryName, WUQueryActivationOptions activateOption, StringBuffer &newQueryId, const char *userid)
+{
+    Owned<IPropertyTree> queryRegistry = getQueryRegistry(querySetName, false);
+    addQueryToQuerySet(workunit, queryRegistry, queryName, activateOption, newQueryId, userid);
 }
 
 bool removeQuerySetAlias(const char *querySetName, const char *alias)
@@ -10294,9 +10306,10 @@ void setQueryCommentForNamedQuery(const char *querySetName, const char *id, cons
     setQueryCommentForNamedQuery(queryRegistry, id, queryComment);
 }
 
-const char *queryIdFromQuerySetWuid(const char *querySetName, const char *wuid, IStringVal &id)
+const char *queryIdFromQuerySetWuid(IPropertyTree *queryRegistry, const char *wuid, IStringVal &id)
 {
-    Owned<IPropertyTree> queryRegistry = getQueryRegistry(querySetName, false);
+    if (!queryRegistry)
+        return NULL;
     StringBuffer xpath;
     xpath.appendf("Query[@wuid='%s']", wuid);
     IPropertyTree *q = queryRegistry->queryPropTree(xpath.str());
@@ -10307,6 +10320,12 @@ const char *queryIdFromQuerySetWuid(const char *querySetName, const char *wuid,
     return id.str();
 }
 
+const char *queryIdFromQuerySetWuid(const char *querySetName, const char *wuid, IStringVal &id)
+{
+    Owned<IPropertyTree> queryRegistry = getQueryRegistry(querySetName, true);
+    return queryIdFromQuerySetWuid(queryRegistry, wuid, id);
+}
+
 extern WORKUNIT_API void gatherLibraryNames(StringArray &names, StringArray &unresolved, IWorkUnitFactory &workunitFactory, IConstWorkUnit &cw, IPropertyTree *queryset)
 {
     IConstWULibraryIterator &wulibraries = cw.getLibraries();

common/workunit/workunit.hpp

@@ -1336,11 +1336,14 @@ extern WORKUNIT_API IPropertyTree * addNamedPackageSet(IPropertyTree * packageRe
 extern WORKUNIT_API void removeNamedPackage(IPropertyTree * packageRegistry, const char * id);
 extern WORKUNIT_API IPropertyTree * getPackageSetRegistry(const char * wsEclId, bool readonly);
 
-extern WORKUNIT_API void addQueryToQuerySet(IWorkUnit *workunit, const char *querySetName, const char *queryName, IPropertyTree *packageInfo, WUQueryActivationOptions activateOption, StringBuffer &newQueryId, const char *userid);
+extern WORKUNIT_API void addQueryToQuerySet(IWorkUnit *workunit, IPropertyTree *queryRegistry, const char *queryName, WUQueryActivationOptions activateOption, StringBuffer &newQueryId, const char *userid);
+extern WORKUNIT_API void addQueryToQuerySet(IWorkUnit *workunit, const char *querySetName, const char *queryName, WUQueryActivationOptions activateOption, StringBuffer &newQueryId, const char *userid);
+extern WORKUNIT_API void activateQuery(IPropertyTree *queryRegistry, WUQueryActivationOptions activateOption, const char *queryName, const char *queryId, const char *userid);
 extern WORKUNIT_API bool removeQuerySetAlias(const char *querySetName, const char *alias);
 extern WORKUNIT_API void addQuerySetAlias(const char *querySetName, const char *alias, const char *id);
 extern WORKUNIT_API void setSuspendQuerySetQuery(const char *querySetName, const char *id, bool suspend, const char *userid);
 extern WORKUNIT_API void deleteQuerySetQuery(const char *querySetName, const char *id);
+extern WORKUNIT_API const char *queryIdFromQuerySetWuid(IPropertyTree *queryRegistry, const char *wuid, IStringVal &id);
 extern WORKUNIT_API const char *queryIdFromQuerySetWuid(const char *querySetName, const char *wuid, IStringVal &id);
 extern WORKUNIT_API void removeQuerySetAliasesFromNamedQuery(const char *querySetName, const char * id);
 extern WORKUNIT_API void setQueryCommentForNamedQuery(const char *querySetName, const char *id, const char *comment);

dali/base/dacoven.cpp

@@ -37,12 +37,12 @@ extern void closedownDFS();
 // base is saved in store whenever block exhausted, so replacement coven servers can restart 
 
 // server side versioning.
-#define ServerVersion    "3.11"
+#define ServerVersion    "3.12"
 #define MinClientVersion "1.5"
 
 
 // client side default versioning.
-static StringAttr ClientVersion("3.5");
+static StringAttr ClientVersion("3.6");
 static StringAttr MinServerVersion("3.1");      // when this upped check initClientProcess instances
 static CDaliVersion _ServerVersion;
 

dali/base/dacsds.cpp

@@ -41,6 +41,7 @@ static unsigned clientThrottleDelay;
 #define MIN_GETXPATHS_CONNECT_SVER "3.2"
 #define MIN_APPEND_OPT_SVER "3.3"
 #define MIN_GETIDS_SVER "3.5"
+#define MIN_NODESUBSCRIBE_SVER "3.12"
 
 
 static ISDSManager *SDSManager=NULL;
@@ -1817,11 +1818,35 @@ SubscriptionId CClientSDSManager::subscribe(const char *xpath, ISDSSubscription
     return subscriber->getId();
 }
 
+
+SubscriptionId CClientSDSManager::subscribeExact(const char *xpath, ISDSNodeSubscription &notify, bool sendValue)
+{
+    if (queryDaliServerVersion().compare(MIN_NODESUBSCRIBE_SVER) < 0)
+        throw MakeSDSException(SDSExcpt_VersionMismatch, "Requires dali server version >= " MIN_NODESUBSCRIBE_SVER " for subscribeExact");
+    assertex(xpath);
+    StringBuffer s;
+    if ('/' != *xpath)
+    {
+        s.append('/').append(xpath);
+        xpath = s.str();
+    }
+    CSDSNodeSubscriberProxy *subscriber = new CSDSNodeSubscriberProxy(xpath, sendValue, notify);
+    querySubscriptionManager(SDSNODE_PUBLISHER)->add(subscriber, subscriber->getId());
+    return subscriber->getId();
+}
+
 void CClientSDSManager::unsubscribe(SubscriptionId id)
 {
     querySubscriptionManager(SDS_PUBLISHER)->remove(id);
 }
 
+void CClientSDSManager::unsubscribeExact(SubscriptionId id)
+{
+    if (queryDaliServerVersion().compare(MIN_NODESUBSCRIBE_SVER) < 0)
+        throw MakeSDSException(SDSExcpt_VersionMismatch, "Requires dali server version >= " MIN_NODESUBSCRIBE_SVER " for unsubscribeExact");
+    querySubscriptionManager(SDSNODE_PUBLISHER)->remove(id);
+}
+
 StringBuffer &CClientSDSManager::getInfo(SdsDiagCommand cmd, StringBuffer &out)
 {
     CMessageBuffer mb;

dali/base/dacsds.ipp

@@ -28,52 +28,6 @@
 
 #include "dasds.ipp"
   
-
-class CSubscriberContainerBase : public CInterface, implements IInterface
-{
-    DECL_NAMEDCOUNT;
-public:
-    IMPLEMENT_IINTERFACE;
-
-    CSubscriberContainerBase(ISubscription *_subscriber, SubscriptionId _id) : 
-      subscriber(_subscriber), id(_id)
-    {
-        INIT_NAMEDCOUNT;
-        unsubscribed = false;
-    }
-
-    bool notify(MemoryBuffer &mb)
-    {
-        try { 
-            subscriber->notify(mb); 
-            return true;
-        }
-        catch (IException *e)
-        {
-            LOG(MCuserWarning, e, "SDS: Error notifying subscriber");
-            e->Release();
-            
-        }
-        return false; // unsubscribe 
-    }
-
-    const SubscriptionId &queryId() const { return id; }
-    const void *queryFindParam() const
-    {
-        return (const void *) &id;
-    }
-
-    bool isUnsubscribed() { return unsubscribed || subscriber->aborted(); }
-    void setUnsubscribed() { unsubscribed = true; }
-
-protected:
-    Owned<ISubscription> subscriber;
-    SubscriptionId id;
-    bool unsubscribed;
-};
-
-/////////////////
-
 class CClientSDSManager;
 class CClientRemoteTree;
 class CRemoteConnection : public CConnectionBase, public CTrackChanges, implements IRemoteConnection
@@ -158,12 +112,10 @@ public:
     ~CConnectionLock() { conn.lockCrit.leave(); }
 };
 //////////////////
-class CSDSConnectionSubscriberProxy : public CInterface, implements ISubscription
+class CSDSConnectionSubscriberProxy : public CInterfaceOf<ISubscription>
 {
     DECL_NAMEDCOUNT;
 public:
-    IMPLEMENT_IINTERFACE;
-
     CSDSConnectionSubscriberProxy(ISDSConnectionSubscription &_sdsNotify, ConnectionId connId) : sdsNotify(&_sdsNotify)
     {
         INIT_NAMEDCOUNT;
@@ -199,40 +151,46 @@ private:
     Linked<ISDSConnectionSubscription> sdsNotify;
 };
 
+static void checkValidSubscriptionPath(const char *xpath)
+{
+    bool quote=false, sep=false;
+    const char *_xpath = xpath;
+    loop
+    {
+        char next = *_xpath;
+        if ('\0' == next)
+            break;
+        if ('\"' == next)
+        {
+            sep = false;
+            quote = !quote;
+        }
+        else if ('/' == next && !quote)
+        {
+            if (sep)
+                throw MakeStringException(0, "UNSUPPORTED: '//' syntax unsupported in subscriber xpath (path=\"%s\")", xpath); // JCSMORE - TBD?
+            sep = true;
+        }
+        else
+            sep = false;
+        ++_xpath;
+    }
+}
+
 //////////////////
-class CSDSSubscriberProxy : public CInterface, implements ISubscription
+class CSDSSubscriberProxyBase : public CInterfaceOf<ISubscription>
 {
     DECL_NAMEDCOUNT;
+protected:
+    SubscriptionId id;
+    StringAttr xpath;
+    MemoryAttr data;
 public:
-    IMPLEMENT_IINTERFACE;
-
-    CSDSSubscriberProxy(const char *xpath, bool sub, bool sendValue, ISDSSubscription &_sdsNotify) : sdsNotify(&_sdsNotify)
+    CSDSSubscriberProxyBase(const char *_xpath, bool sendValue)
     {
         INIT_NAMEDCOUNT;
-        bool quote=false, sep=false;
-        const char *_xpath = xpath;
-        const char *end = _xpath+strlen(_xpath);
-        while (_xpath != end)
-        {
-            if ('\"' == *_xpath)
-            {
-                sep = false;
-                if (quote) quote = false;
-                else quote = true;
-            }
-            else if ('/' == *_xpath && !quote)
-            {
-                if (sep)
-                    throw MakeStringException(0, "UNSUPPORTED: '//' syntax unsupported in subscriber xpath (path=\"%s\")", xpath); // JCSMORE - TBD?
-                sep = true;
-            }
-            else
-                sep = false;
-            ++_xpath;
-        }
-        MemoryBuffer _data;
-        _data.append(xpath).append(sub).append(sendValue);
-        data.set(_data.length(), _data.toByteArray());
+        checkValidSubscriptionPath(_xpath);
+        xpath.set(_xpath);
         id = queryCoven().getUniqueId();
     }
     SubscriptionId getId() const { return id; }
@@ -242,12 +200,34 @@ public:
     {
         return data;
     }
+    virtual void abort() // called when server closes
+    { 
+        // JCS TBD?
+    }
+    virtual bool aborted() // called when server closes
+    { 
+        return false;
+    }
+};
+
+class CSDSSubscriberProxy : public CSDSSubscriberProxyBase
+{
+    Linked<ISDSSubscription> sdsNotify;
+public:
+    CSDSSubscriberProxy(const char *_xpath, bool sub, bool sendValue, ISDSSubscription &_sdsNotify)
+        : CSDSSubscriberProxyBase(_xpath, sendValue), sdsNotify(&_sdsNotify)
+    {
+        MemoryBuffer _data;
+        _data.append(xpath).append(sub).append(sendValue);
+        data.set(_data.length(), _data.toByteArray());
+    }
+// ISubscription impl.
     virtual void notify(MemoryBuffer &returnData)
     {
         StringAttr xpath;
-        SDSNotifyFlags flags;
+        int flags;
         returnData.read(xpath);
-        returnData.read((int &) flags);
+        returnData.read(flags);
         bool valueData;
         if (returnData.length()-returnData.getPos()) // remaining
         {
@@ -256,29 +236,43 @@ public:
             {
                 unsigned l;
                 returnData.read(l);
-                sdsNotify->notify(id, xpath, flags, l, returnData.readDirect(l));
+                sdsNotify->notify(id, xpath, (SDSNotifyFlags)flags, l, returnData.readDirect(l));
             }
             else
-                sdsNotify->notify(id, xpath, flags);
+                sdsNotify->notify(id, xpath, (SDSNotifyFlags)flags);
         }
         else
-            sdsNotify->notify(id, xpath, flags);
+            sdsNotify->notify(id, xpath, (SDSNotifyFlags)flags);
     }
+};
 
-    virtual void abort() // called when server closes
-    { 
-        // JCS TBD?
+class CSDSNodeSubscriberProxy : public CSDSSubscriberProxyBase
+{
+    Linked<ISDSNodeSubscription> sdsNotify;
+public:
+    CSDSNodeSubscriberProxy(const char *_xpath, bool sendValue, ISDSNodeSubscription &_sdsNotify)
+        : CSDSSubscriberProxyBase(_xpath, sendValue) , sdsNotify(&_sdsNotify)
+    {
+        MemoryBuffer _data;
+        _data.append(xpath).append(sendValue);
+        data.set(_data.length(), _data.toByteArray());
     }
-
-    virtual bool aborted() // called when server closes
-    { 
-        return false;
+// ISubscription impl.
+    virtual void notify(MemoryBuffer &returnData)
+    {
+        int flags;
+        returnData.read(flags);
+        unsigned valueLen = 0;
+        const void *valueData = NULL;
+        bool isValueData;
+        returnData.read(isValueData);
+        if (isValueData)
+        {
+            returnData.read(valueLen);
+            valueData = returnData.readDirect(valueLen);
+        }
+        sdsNotify->notify(id, (SDSNotifyFlags)flags, valueLen, valueData);
     }
-
-private:
-    SubscriptionId id;
-    MemoryAttr data, valueData;
-    Linked<ISDSSubscription> sdsNotify;
 };
 
 ////////////////////
@@ -399,7 +393,9 @@ public:
     virtual IRemoteConnections *connect(IMultipleConnector *mConnect, SessionId id, unsigned timeout);
     virtual IRemoteConnection *connect(const char *xpath, SessionId id, unsigned mode, unsigned timeout);
     virtual SubscriptionId subscribe(const char *xpath, ISDSSubscription &notify, bool sub=true, bool sendValue=false);
+    virtual SubscriptionId subscribeExact(const char *xpath, ISDSNodeSubscription &notify, bool sendValue=false);
     virtual void unsubscribe(SubscriptionId id);
+    virtual void unsubscribeExact(SubscriptionId id);
     virtual StringBuffer &getLocks(StringBuffer &out);
     virtual StringBuffer &getUsageStats(StringBuffer &out);
     virtual StringBuffer &getConnections(StringBuffer &out);

dali/base/dasds.cpp

@@ -486,6 +486,46 @@ private:
 };
 
 //////////////
+
+class CSubscriberContainerBase : public CInterfaceOf<IInterface>
+{
+    DECL_NAMEDCOUNT;
+protected:
+    bool unsubscribed;
+    Owned<ISubscription> subscriber;
+    SubscriptionId id;
+public:
+    CSubscriberContainerBase(ISubscription *_subscriber, SubscriptionId _id) :
+      subscriber(_subscriber), id(_id)
+    {
+        INIT_NAMEDCOUNT;
+        unsubscribed = false;
+    }
+    bool notify(MemoryBuffer &mb) const
+    {
+        try
+        {
+            subscriber->notify(mb);
+            return true;
+        }
+        catch (IException *e)
+        {
+            LOG(MCuserWarning, e, "SDS: Error notifying subscriber");
+            e->Release();
+        }
+        return false; // unsubscribe
+    }
+    const SubscriptionId &queryId() const { return id; }
+    const void *queryFindParam() const
+    {
+        return (const void *) &id;
+    }
+    bool isUnsubscribed() { return unsubscribed || subscriber->aborted(); }
+    void setUnsubscribed() { unsubscribed = true; }
+};
+
+/////////////////
+
 class CConnectionSubscriberContainer : public CSubscriberContainerBase
 {
 public:
@@ -651,7 +691,7 @@ public:
         MemoryBuffer mb(ma.length(), ma.get());
         mb.read(xpath);
         mb.read(sub);
-        if (mb.length()-mb.getPos()) // remaining
+        if (mb.remaining()) // remaining
             mb.read(sendValue);
         else
             sendValue = false;
@@ -1686,37 +1726,49 @@ SDSNotifyFlags translatePDState(PDState state)
     return (SDSNotifyFlags) state; // mirrored for now.
 }
 
-void buildNotifyData(CPTStack &stack, MemoryBuffer &mb)
+void buildNotifyData(MemoryBuffer &notifyData, PDState state, CPTStack *stack, MemoryBuffer *data)
 {
-    mb.append('/');
-    PTree *parent = &stack.item(0); // root
-
-    unsigned n = stack.ordinality();
-    if (n>1)
+    if (stack)
     {
-        unsigned s = 1;
-        loop
+        notifyData.append('/');
+        PTree *parent = &stack->item(0); // root
+
+        unsigned n = stack->ordinality();
+        if (n>1)
         {
-            PTree &child = stack.item(s);
-            const char *str = child.queryName();
-            mb.append(strlen(str), str);
-            if (child.queryParent())
+            unsigned s = 1;
+            loop
             {
-                char temp[12];
-                unsigned written = numtostr(temp, parent->findChild(&child)+1);
-                mb.append('[').append(written, temp).append(']');
+                PTree &child = stack->item(s);
+                const char *str = child.queryName();
+                notifyData.append(strlen(str), str);
+                if (child.queryParent())
+                {
+                    char temp[12];
+                    unsigned written = numtostr(temp, parent->findChild(&child)+1);
+                    notifyData.append('[').append(written, temp).append(']');
+                }
+                else
+                    notifyData.append(3, "[1]");
+                parent = &child;
+                s++;
+                if (s<n)
+                    notifyData.append('/');
+                else
+                    break;
             }
-            else
-                mb.append(3, "[1]");
-            parent = &child;
-            s++;
-            if (s<n)
-                mb.append('/');
-            else
-                break;
         }
+        notifyData.append('\0');
+    }
+    notifyData.append((int)translatePDState(state));
+    if (data)
+    {
+        notifyData.append(true);
+        notifyData.append(data->length());
+        notifyData.append(*data);
     }
-    mb.append('\0');
+    else
+        notifyData.append(false);
 }
 
 class CSubscriberNotifier;
@@ -1732,7 +1784,7 @@ class CSubscriberNotifier : public CInterface
         MemoryBuffer notifyData;
     };
 public:
-    CSubscriberNotifier(CSubscriberNotifierTable &_table, CSubscriberContainer &_subscriber, MemoryBuffer &notifyData)
+    CSubscriberNotifier(CSubscriberNotifierTable &_table, CSubscriberContainerBase &_subscriber, MemoryBuffer &notifyData)
         : table(_table), subscriber(_subscriber)
     {
         INIT_NAMEDCOUNT;
@@ -1787,7 +1839,7 @@ public:
 private:
     Linked<CChange> change;
     CIArrayOf<CChange> changeQueue;
-    CSubscriberContainer &subscriber;
+    CSubscriberContainerBase &subscriber;
     MemoryAttr notifyData;
     CSubscriberNotifierTable &table;
 };
@@ -1803,8 +1855,6 @@ public:
     CConnectionSubscriptionManager()
     {
     }
-
-
     unsigned querySubscribers()
     {
         CHECKEDCRITICALBLOCK(crit, fakeCritTimeout);
@@ -1855,6 +1905,18 @@ interface ICoalesce : extends IInterface
 
 //////////////////////
 
+class CNodeSubscriberContainer;
+interface INodeSubscriptionManager : extends ISubscriptionManager
+{
+    virtual void associateSubscriber(CNodeSubscriberContainer &subscriber) = 0;
+    virtual void removeSubscriberAssociation(SubscriptionId id) = 0;
+    virtual void notifyDelete(CServerRemoteTree &node) = 0;
+    virtual void notify(CServerRemoteTree &node, PDState state) = 0;
+    virtual MemoryBuffer &collectSubscribers(MemoryBuffer &out) const = 0;
+};
+
+//////////////////////
+
 enum LockStatus { LockFailed, LockHeld, LockTimedOut, LockSucceeded };
 
 class CCovenSDSManager : public CSDSManagerBase, implements ISDSManagerServer, implements ISubscriptionManager, implements IExceptionHandler
@@ -1879,7 +1941,7 @@ public:
     CLockInfo *queryLockInfo(__int64 id) { return lockTable.find(&id); }
     CSubscriberTable &querySubscriberTable() { return subscribers; }
     IExternalHandler *queryExternalHandler(const char *handler) { if (!handler) return NULL; CExternalHandlerMapping *mapping = externalHandlers.find(handler); return mapping ? &mapping->query() : NULL; }
-    void handleNotify(CSubscriberContainer &subscriber, PDState state, CPTStack &stack, MemoryBuffer *data=NULL);
+    void handleNotify(CSubscriberContainerBase &subscriber, MemoryBuffer &notifyData);
     void startNotification(IPropertyTree &changeTree, CPTStack &stack, CBranchChange &changes); // subscription notification
     MemoryBuffer &collectUsageStats(MemoryBuffer &out);
     MemoryBuffer &collectConnections(MemoryBuffer &out);
@@ -1909,6 +1971,10 @@ public:
     CSubscriberContainerList *getSubscribers(const char *xpath, CPTStack &stack);
     void getExternalValue(__int64 index, MemoryBuffer &mb);
     IPropertyTree *getXPathsSortLimitMatchTree(const char *baseXPath, const char *matchXPath, const char *sortby, bool caseinsensitive, bool ascending, unsigned from, unsigned limit);
+    void addNodeSubscriber(ISubscription *sub, SubscriptionId id);
+    void removeNodeSubscriber(SubscriptionId id);
+    void notifyNodeDelete(CServerRemoteTree &node);
+    void notifyNode(CServerRemoteTree &node, PDState state);
 
 // ISDSConnectionManager
     virtual CRemoteTreeBase *get(CRemoteConnection &connection, __int64 serverId);
@@ -1927,7 +1993,9 @@ public:
     virtual IRemoteConnections *connect(IMultipleConnector *mConnect, SessionId id, unsigned timeout);
     virtual IRemoteConnection *connect(const char *xpath, SessionId id, unsigned mode, unsigned timeout);
     virtual SubscriptionId subscribe(const char *xpath, ISDSSubscription &notify, bool sub=true, bool sendValue=false);
+    virtual SubscriptionId subscribeExact(const char *xpath, ISDSNodeSubscription &notify, bool sendValue=false);
     virtual void unsubscribe(SubscriptionId id);
+    virtual void unsubscribeExact(SubscriptionId id);
     virtual StringBuffer &getLocks(StringBuffer &out);
     virtual StringBuffer &getUsageStats(StringBuffer &out);
     virtual StringBuffer &getConnections(StringBuffer &out);
@@ -1940,7 +2008,6 @@ public:
     virtual unsigned countConnections();
     virtual bool setSDSDebug(StringArray &params, StringBuffer &reply);
     virtual unsigned countActiveLocks();
-    virtual unsigned countSubscribers() const;
     virtual unsigned queryExternalSizeThreshold() const { return externalSizeThreshold; }
     virtual void setExternalSizeThreshold(unsigned _size) { externalSizeThreshold = _size; }
     virtual bool queryRestartOnError() const { return restartOnError; }
@@ -2004,6 +2071,7 @@ private:
     CExternalHandlerTable externalHandlers;
     CSubscriberNotifierTable subscriberNotificationTable;
     Owned<CConnectionSubscriptionManager> connectionSubscriptionManager;
+    Owned<INodeSubscriptionManager> nodeSubscriptionManager;
     bool restartOnError, externalEnvironment;
     IStoreHelper *iStoreHelper;
     bool doTimeComparison;
@@ -2398,6 +2466,8 @@ public:
             }
         }
         if (SDSManager->queryStopped()) return; // don't bother building up free list that will never be used hence (could get v. big/slow)
+        if (isSubscribed())
+            SDSManager->notifyNodeDelete(*this);
         CHECKEDCRITICALBLOCK(SDSManager->treeRegCrit, fakeCritTimeout);
 
         SDSManager->queryAllNodes().freeElem(serverId);
@@ -2551,13 +2621,198 @@ public:
         ret.setBuffer(len, mem, len-1);
         return true;
     }
-
+    void setSubscribed(bool tf)
+    {
+        if (tf)
+            IptFlagSet(flags, ipt_ext4);
+        else
+            IptFlagClr(flags, ipt_ext4);
+    }
+    inline bool isSubscribed() const
+    {
+        return IptFlagTst(flags, ipt_ext4);
+    }
 private:
     PDState processData(IPropertyTree &changeTree, Owned<CBranchChange> &parentBranchChange, MemoryBuffer &newIds);
     PDState checkChange(IPropertyTree &tree, CBranchChange *parentBranchChange=NULL);
 friend class COrphanHandler;
 };
 
+class CNodeSubscriberContainer : public CSubscriberContainerBase
+{
+    StringAttr xpath;
+    bool sendValue;
+    ICopyArrayOf<CServerRemoteTree> nodes; // never linked, node must signal the unsubscription and removal of subscriber and these references
+public:
+    CNodeSubscriberContainer(ISubscription *subscriber, SubscriptionId id, bool _sendValue, const char *_xpath)
+        : CSubscriberContainerBase(subscriber, id), sendValue(_sendValue), xpath(_xpath)
+    {
+    }
+    bool querySendValue() const { return sendValue; }
+    void add(CServerRemoteTree &node) { nodes.append(node); }
+    ICopyArrayOf<CServerRemoteTree> &queryNodes() { return nodes; }
+    MemoryBuffer &getInfo(MemoryBuffer &out) const
+    {
+        out.append(id).append(xpath).append(nodes.ordinality());
+        return out;
+    }
+};
+
+class CNodeSubscriptionManager : public CSimpleInterface, implements INodeSubscriptionManager
+{
+public:
+    class CNodeSubscriberContainerList : public CSimpleInterfaceOf<IInterface>
+    {
+        CServerRemoteTree *node;
+        ICopyArrayOf<CNodeSubscriberContainer> subscribers;
+    public:
+        CNodeSubscriberContainerList(CServerRemoteTree *_node) : node(_node)
+        {
+        }
+        const void *queryFindParam() const { return &node; }
+        ICopyArrayOf<CNodeSubscriberContainer> &querySubscribers() { return subscribers; }
+        void add(CNodeSubscriberContainer &subscriber) { subscribers.append(subscriber); }
+    };
+
+    CCovenSDSManager &owner;
+    OwningSimpleHashTableOf<CNodeSubscriberContainer, SubscriptionId> subscribersById;
+    OwningSimpleHashTableOf<CNodeSubscriberContainerList, CServerRemoteTree *> subscriberListByNode;
+    CriticalSection lock;
+
+    void _notify(CServerRemoteTree *node, PDState state)
+    {
+        MemoryBuffer sendValueNotifyData;
+        CNodeSubscriberContainerList *subscriberList = subscriberListByNode.find(node);
+        assertex(subscriberList);
+        ICopyArrayOf<CNodeSubscriberContainer> &subscribers = subscriberList->querySubscribers();
+        int lastSendValue = -1;
+        ForEachItemIn(s, subscribers)
+        {
+            CNodeSubscriberContainer &subscriber = subscribers.item(s);
+            if (subscriber.querySendValue())
+            {
+                if (1 != lastSendValue) // overkill unless many subscribers to same node
+                {
+                    MemoryBuffer mb;
+                    node->getPropBin(NULL, mb);
+                    buildNotifyData(sendValueNotifyData.clear(), state, NULL, &mb);
+                    lastSendValue = 1;
+                }
+                SDSManager->handleNotify(subscriber, sendValueNotifyData);
+            }
+            else
+            {
+                if (0 != lastSendValue) // overkill unless many subscribers to same node
+                {
+                    buildNotifyData(sendValueNotifyData.clear(), state, NULL, NULL);
+                    lastSendValue = 0;
+                }
+                SDSManager->handleNotify(subscriber, sendValueNotifyData);
+            }
+        }
+    }
+    void _removeNode(CServerRemoteTree *node, SubscriptionId id)
+    {
+        CNodeSubscriberContainerList *subscriberList = subscriberListByNode.find(node);
+        assertex(subscriberList);
+        ICopyArrayOf<CNodeSubscriberContainer> &subscribers = subscriberList->querySubscribers();
+        ForEachItemInRev(s, subscribers)
+        {
+            CNodeSubscriberContainer &subscriber = subscribers.item(s);
+            if (0 == id) // remove all associated subscribers (node being deleted)
+            {
+                ICopyArrayOf<CServerRemoteTree> &nodes = subscriber.queryNodes();
+                verifyex(nodes.zap(*node));
+                SubscriptionId sid = subscriber.queryId();
+                subscribers.remove(s);
+                if (0 == nodes.ordinality()) // IOW this was the last node this subscriber was associated with
+                    subscribersById.remove(&sid);
+            }
+            else if (subscriber.queryId() == id)
+                subscribers.remove(s);
+        }
+        if (0 == subscribers.ordinality())
+        {
+            node->setSubscribed(false);
+            subscriberListByNode.removeExact(subscriberList);
+        }
+    }
+public:
+    IMPLEMENT_IINTERFACE_USING(CSimpleInterface);
+
+    CNodeSubscriptionManager(CCovenSDSManager &_owner) : owner(_owner) { }
+    void notify(CServerRemoteTree &node, PDState state)
+    {
+        // shouldn't be here, unless node is in subscribers table
+        CriticalBlock b(lock);
+        _notify(&node, state);
+    }
+    void notifyDelete(CServerRemoteTree &node)
+    {
+        // shouldn't be here, unless node is in subscribers table
+        CriticalBlock b(lock);
+        _notify(&node, PDS_Deleted);
+        _removeNode(&node, 0);
+    }
+    // ISubscriptionManager impl.
+    virtual void add(ISubscription *sub, SubscriptionId id)
+    {
+        CriticalBlock b(lock);
+        /* calls back out to owner to scan for match, so that SDSManager can protect root/treereg.
+         * It calls back (associateSubscriber) in this class to add subscribers based on matches.
+         */
+        owner.addNodeSubscriber(sub, id);
+    }
+    virtual void remove(SubscriptionId id)
+    {
+        CriticalBlock b(lock);
+        /* calls back out to owner to protect root/treereg.
+         * It calls back into removeSubscriberAssociation.
+         */
+        owner.removeNodeSubscriber(id);
+    }
+    virtual void removeSubscriberAssociation(SubscriptionId id) // Always called back from within remove() above.
+    {
+        CNodeSubscriberContainer *subscriber = subscribersById.find(id);
+        if (!subscriber)
+            return; // may not exist if removed already
+        ICopyArrayOf<CServerRemoteTree> &nodes = subscriber->queryNodes();
+        ForEachItemIn(n, nodes)
+        {
+            CServerRemoteTree &node = nodes.item(n);
+            _removeNode(&node, id);
+        }
+        verifyex(subscribersById.removeExact(subscriber));
+    }
+    void associateSubscriber(CNodeSubscriberContainer &subscriber) // Always called back from within add() above.
+    {
+        /* caller has established there are matches and added them to 'subscriber'
+         * add to HT's
+         */
+        verifyex(subscribersById.add(*LINK(&subscriber)));
+        ICopyArrayOf<CServerRemoteTree> &nodes = subscriber.queryNodes();
+        ForEachItemIn(n, nodes)
+        {
+            CServerRemoteTree *node = &nodes.item(n);
+            CNodeSubscriberContainerList *subscriberList = subscriberListByNode.find(node);
+            if (!subscriberList)
+            {
+                subscriberList = new CNodeSubscriberContainerList(node);
+                verifyex(subscriberListByNode.add(* subscriberList));
+            }
+            subscriberList->add(subscriber);
+        }
+    }
+    MemoryBuffer &collectSubscribers(MemoryBuffer &out) const
+    {
+        out.append(subscribersById.count());
+        SuperHashIteratorOf<CNodeSubscriberContainer> sdsNodeIter(subscribersById);
+        ForEach(sdsNodeIter)
+            sdsNodeIter.query().getInfo(out);
+        return out;
+    }
+};
+
 #if defined(_WIN32) && defined(__old_new)
 #define new __old_new
 #endif
@@ -2845,8 +3100,9 @@ PDState CServerRemoteTree::processData(IPropertyTree &changeTree, Owned<CBranchC
             if (childChanges.getPropBool("@new"))
             {
                 child = (CServerRemoteTree *)createChild(childChanges.getPropInt("@pos", -1), childChanges.queryProp("@name"));
-                mergePDState(res, PDS_Structure);
                 newIds.append(child->queryServerId());
+                mergePDState(localChange, PDS_Added);
+                mergePDState(res, PDS_Added);
             }
             else
             {
@@ -2876,6 +3132,8 @@ PDState CServerRemoteTree::processData(IPropertyTree &changeTree, Owned<CBranchC
     else
         branchChange->noteChange(localChange, res);
 
+    if ((localChange != PDS_None) && isSubscribed())
+        SDSManager->notifyNode(*this, localChange);
     if (!parentBranchChange.get())
         parentBranchChange.setown(branchChange.getClear());
     else
@@ -5511,7 +5769,9 @@ CCovenSDSManager::CCovenSDSManager(ICoven &_coven, IPropertyTree &_config, const
     }
     registerSubscriptionManager(SDS_PUBLISHER, this);
     connectionSubscriptionManager.setown(new CConnectionSubscriptionManager());
+    nodeSubscriptionManager.setown(new CNodeSubscriptionManager(*this));
     registerSubscriptionManager(SDSCONN_PUBLISHER, connectionSubscriptionManager.get());
+    registerSubscriptionManager(SDSNODE_PUBLISHER, nodeSubscriptionManager);
 
     // add external handlers
     Owned<CXMLFileExternal> xmlExternalHandler = new CXMLFileExternal(dataPath, backupHandler);
@@ -6407,11 +6667,30 @@ SubscriptionId CCovenSDSManager::subscribe(const char *xpath, ISDSSubscription &
     return subscriber->getId();
 }
 
+SubscriptionId CCovenSDSManager::subscribeExact(const char *xpath, ISDSNodeSubscription &notify, bool sendValue)
+{
+    assertex(xpath);
+    StringBuffer s;
+    if ('/' != *xpath)
+    {
+        s.append('/').append(xpath);
+        xpath = s.str();
+    }
+    CSDSNodeSubscriberProxy *subscriber = new CSDSNodeSubscriberProxy(xpath, sendValue, notify);
+    querySubscriptionManager(SDSNODE_PUBLISHER)->add(subscriber, subscriber->getId());
+    return subscriber->getId();
+}
+
 void CCovenSDSManager::unsubscribe(SubscriptionId id)
 {
     querySubscriptionManager(SDS_PUBLISHER)->remove(id);
 }
 
+void CCovenSDSManager::unsubscribeExact(SubscriptionId id)
+{
+    querySubscriptionManager(SDSNODE_PUBLISHER)->remove(id);
+}
+
 bool CCovenSDSManager::removeNotifyHandler(const char *handlerKey)
 {
     return nodeNotifyHandlers.remove(handlerKey);
@@ -7575,6 +7854,16 @@ StringBuffer &formatSubscriberInfo(MemoryBuffer &src, StringBuffer &out)
     return out;
 }
 
+StringBuffer &formatNodeSubscriberInfo(MemoryBuffer &src, StringBuffer &out)
+{
+    SubscriptionId subscriptionId;
+    StringAttr xpath;
+    unsigned nodeCount;
+    src.read(subscriptionId).read(xpath).read(nodeCount);
+    out.append("SubscriptionId=").appendf("%"I64F"x", subscriptionId).append(", xpath=").append(xpath).append(", nodes=").append(nodeCount);
+    return out;
+}
+
 StringBuffer &formatConnections(MemoryBuffer &src, StringBuffer &out)
 {
     unsigned count;
@@ -7594,18 +7883,25 @@ StringBuffer &formatConnections(MemoryBuffer &src, StringBuffer &out)
 
 StringBuffer &formatSubscribers(MemoryBuffer &src, StringBuffer &out)
 {
-    unsigned count;
-    src.read(count);
-    if (count)
+    unsigned sdsSubscribers, sdsNodeSubscribers=0;
+    src.read(sdsSubscribers);
+    unsigned s = sdsSubscribers;
+    while (s--)
     {
-        while (count--)
+        formatSubscriberInfo(src, out);
+        if (s) out.newline();
+    }
+    if (src.remaining())
+    {
+        src.read(sdsNodeSubscribers);
+        s = sdsNodeSubscribers;
+        while (s--)
         {
-            formatSubscriberInfo(src, out);
-            if (count) out.newline();
+            formatNodeSubscriberInfo(src, out);
+            if (s) out.newline();
         }
     }
-    else
-        out.append("No current subscriptions");
+    out.newline().appendf("%d xpath subscribers, %d node subscribers\n", sdsSubscribers, sdsNodeSubscribers);
     return out;
 }
 
@@ -7627,11 +7923,6 @@ unsigned CCovenSDSManager::countActiveLocks()
     return activeLocks;
 }
 
-unsigned CCovenSDSManager::countSubscribers() const
-{
-    return subscribers.count();
-}
-
 MemoryBuffer &CCovenSDSManager::collectUsageStats(MemoryBuffer &out)
 {
     { CHECKEDCRITICALBLOCK(cTableCrit, fakeCritTimeout);
@@ -7666,9 +7957,10 @@ MemoryBuffer &CCovenSDSManager::collectSubscribers(MemoryBuffer &out)
 {
     CHECKEDCRITICALBLOCK(sTableCrit, fakeCritTimeout);
     out.append(subscribers.count());
-    SuperHashIteratorOf<CSubscriberContainer> iter(subscribers.queryBaseTable());
-    ForEach(iter)
-        iter.query().getInfo(out);
+    SuperHashIteratorOf<CSubscriberContainer> sdsIter(subscribers.queryBaseTable());
+    ForEach(sdsIter)
+        sdsIter.query().getInfo(out);
+    nodeSubscriptionManager->collectSubscribers(out);
     return out;
 }
 
@@ -7765,7 +8057,7 @@ void CCovenSDSManager::handleNodeNotify(notifications n, CServerRemoteTree &tree
     }
 }
 
-void CCovenSDSManager::handleNotify(CSubscriberContainer &subscriber, PDState state, CPTStack &stack, MemoryBuffer *data)
+void CCovenSDSManager::handleNotify(CSubscriberContainerBase &subscriber, MemoryBuffer &notifyData)
 {
     class CNotifyPoolFactory : public CInterface, public IThreadFactory
     {
@@ -7809,18 +8101,6 @@ void CCovenSDSManager::handleNotify(CSubscriberContainer &subscriber, PDState st
         factory->Release();
     }
 
-    MemoryBuffer notifyData;
-    buildNotifyData(stack, notifyData);
-    notifyData.append(translatePDState(state));
-    if (data)
-    {
-        notifyData.append(true);
-        notifyData.append(data->length());
-        notifyData.append(*data);
-    }
-    else
-        notifyData.append(false);
-
     Owned<CSubscriberNotifier> _notifier;
     { CHECKEDCRITICALBLOCK(nfyTableCrit, fakeCritTimeout);
         SubscriptionId id = subscriber.queryId();
@@ -7936,6 +8216,7 @@ public:
         bool sub;
         if (prune(xpath.str(), sub, pruned))
         {
+            MemoryBuffer notifyData;
             if (sub)
             {
                 ForEachItemInRev(s, subs)
@@ -7944,7 +8225,11 @@ public:
                     if (!subscriber.isUnsubscribed())
                     {
                         if (subscriber.qualify(stack))
-                            SDSManager->handleNotify(subscriber, state, stack);
+                        {
+                            if (0 == notifyData.length())
+                                buildNotifyData(notifyData, state, &stack, NULL);
+                            SDSManager->handleNotify(subscriber, notifyData);
+                        }
                         else
                             pruned.append(*LINK(&subscriber));
                     }
@@ -7961,7 +8246,11 @@ public:
                         if (!subscriber.isUnsubscribed())
                         {
                             if (subscriber.qualify(stack))
-                                SDSManager->handleNotify(subscriber, state, stack);
+                            {
+                                if (0 == notifyData.length())
+                                    buildNotifyData(notifyData, state, &stack, NULL);
+                                SDSManager->handleNotify(subscriber, notifyData);
+                            }
                             else
                                 pruned.append(*LINK(&subscriber));
                         }
@@ -8005,8 +8294,10 @@ public:
         else if (sub) // xpath matched some subscribers, and/or below some, need to check for sub subscribers
         {
             bool ret = false;
+            MemoryBuffer notifyData;
             if (changes.state && changes.local)
             {
+                int lastSendValue = -1;
                 ForEachItemInRev(s, subs)
                 {
                     CSubscriberContainer &subscriber = subs.item(s);
@@ -8016,12 +8307,23 @@ public:
                         {
                             if (subscriber.querySendValue())
                             {
-                                MemoryBuffer mb;
-                                changes.tree->getPropBin(NULL, mb);
-                                SDSManager->handleNotify(subscriber, changes.state, stack, &mb);
+                                if (1 != lastSendValue)
+                                {
+                                    MemoryBuffer mb;
+                                    changes.tree->getPropBin(NULL, mb);
+                                    buildNotifyData(notifyData.clear(), changes.state, &stack, &mb);
+                                    lastSendValue = 1;
+                                }
                             }
                             else
-                                SDSManager->handleNotify(subscriber, changes.state, stack);
+                            {
+                                if (0 != lastSendValue)
+                                {
+                                    buildNotifyData(notifyData.clear(), changes.state, &stack, NULL);
+                                    lastSendValue = 0;
+                                }
+                            }
+                            SDSManager->handleNotify(subscriber, notifyData);
                         }
                         else
                             pruned.append(*LINK(&subscriber));
@@ -8241,6 +8543,51 @@ bool CCovenSDSManager::fireException(IException *e)
     return true;
 }
 
+void CCovenSDSManager::addNodeSubscriber(ISubscription *sub, SubscriptionId id)
+{
+    MemoryBuffer mb;
+    mb.setBuffer(sub->queryData().length(), (void *)sub->queryData().get());
+    StringAttr xpath;
+    bool sendValue;
+    mb.read(xpath);
+    mb.read(sendValue);
+
+    CHECKEDDALIREADLOCKBLOCK(dataRWLock, readWriteTimeout);
+    CHECKEDCRITICALBLOCK(treeRegCrit, fakeCritTimeout);
+    Owned<IPropertyTreeIterator> iter = root->getElements(xpath+1);
+    if (!iter->first())
+        throw MakeSDSException(SDSExcpt_SubscriptionNoMatch, "Failed to match any nodes: %s", xpath.get());
+    else
+    {
+        Owned<CNodeSubscriberContainer> subscriber = new CNodeSubscriberContainer(sub, id, sendValue, xpath);
+        do
+        {
+            CServerRemoteTree &node = (CServerRemoteTree &)iter->query();
+            node.setSubscribed(true);
+            subscriber->add(node);
+        }
+        while (iter->next());
+        nodeSubscriptionManager->associateSubscriber(*subscriber);
+    }
+}
+
+void CCovenSDSManager::removeNodeSubscriber(SubscriptionId id)
+{
+    CHECKEDDALIREADLOCKBLOCK(dataRWLock, readWriteTimeout);
+    CHECKEDCRITICALBLOCK(treeRegCrit, fakeCritTimeout);
+    nodeSubscriptionManager->removeSubscriberAssociation(id);
+}
+
+void CCovenSDSManager::notifyNodeDelete(CServerRemoteTree &node)
+{
+    nodeSubscriptionManager->notifyDelete(node);
+}
+
+void CCovenSDSManager::notifyNode(CServerRemoteTree &node, PDState state)
+{
+    nodeSubscriptionManager->notify(node, state);
+}
+
 ///////////////////////
 
 class CDaliSDSServer: public CInterface, public IDaliServer

dali/base/dasds.hpp

@@ -40,12 +40,16 @@
 #define RTM_MODE(X, M) ((X & M) == M)
 
 enum SDSNotifyFlags { SDSNotify_None=0x00, SDSNotify_Data=0x01, SDSNotify_Structure=0x02, SDSNotify_Added=(SDSNotify_Structure+0x04), SDSNotify_Deleted=(SDSNotify_Structure+0x08), SDSNotify_Renamed=(SDSNotify_Structure+0x10) };
-typedef __int64 SDSNotificationFlags_t;
 interface ISDSSubscription : extends IInterface
 {
     virtual void notify(SubscriptionId id, const char *xpath, SDSNotifyFlags flags, unsigned valueLen=0, const void *valueData=NULL) = 0;
 };
 
+interface ISDSNodeSubscription : extends IInterface
+{
+    virtual void notify(SubscriptionId id, SDSNotifyFlags flags, unsigned valueLen=0, const void *valueData=NULL) = 0;
+};
+
 interface ISDSConnectionSubscription : extends IInterface
 {
     virtual void notify() = 0;
@@ -93,7 +97,9 @@ interface ISDSManager
     virtual IRemoteConnection *connect(const char *xpath, SessionId id, unsigned mode, unsigned timeout) = 0;
     virtual IRemoteConnections *connect(IMultipleConnector *mConnect, SessionId id, unsigned timeout) = 0; // timeout applies to each connection
     virtual SubscriptionId subscribe(const char *xpath, ISDSSubscription &notify, bool sub=true, bool sendValue=false) = 0;
+    virtual SubscriptionId subscribeExact(const char *xpath, ISDSNodeSubscription &notify, bool sendValue=false) = 0;
     virtual void unsubscribe(SubscriptionId id) = 0;
+    virtual void unsubscribeExact(SubscriptionId id) = 0;
     virtual StringBuffer &getLocks(StringBuffer &out) = 0;
     virtual StringBuffer &getUsageStats(StringBuffer &out) = 0;
     virtual StringBuffer &getConnections(StringBuffer &out) = 0;
@@ -143,7 +149,6 @@ interface ISDSManagerServer : extends ISDSManager
     virtual bool setSDSDebug(StringArray &params, StringBuffer &reply)=0;
     virtual unsigned countConnections() = 0;
     virtual unsigned countActiveLocks() = 0;
-    virtual unsigned countSubscribers() const = 0;
     virtual unsigned queryExternalSizeThreshold() const = 0;
     virtual void setExternalSizeThreshold(unsigned _size) = 0;
     virtual bool queryRestartOnError() const = 0;
@@ -196,7 +201,8 @@ enum SDSExceptionCodes
     SDSExcpt_ClientCacheDirty,
     SDSExcpt_InvalidSessionId,
     SDSExcpt_LockHeld,
-    SDSExcpt_SubscriptionParseError
+    SDSExcpt_SubscriptionParseError,
+    SDSExcpt_SubscriptionNoMatch
 };
 
 interface ISDSException : extends IException { };

dali/base/dasubs.cpp

@@ -466,13 +466,24 @@ public:
         size32_t dlen = data.length();
         mb.append(dlen);
         mb.append(dlen,data.get());
-        try {
+        try
+        {
             queryCoven().sendRecv(mb,RANK_RANDOM,MPTAG_DALI_SUBSCRIPTION_REQUEST);
             if (mb.length())
                 throw deserializeException(mb);
         }
-        catch (IException *e) {
+        catch (IException *e)
+        {
             PrintExceptionLog(e,"Dali CDaliSubscriptionManagerStub::add");
+            {
+                CriticalBlock block(subscriptionsect);
+                unsigned idx = ids.find(id);
+                if (NotFound != idx)
+                {
+                    ids.remove(idx);
+                    subscriptions.remove(idx);
+                }
+            }
             throw;
         }
     }

dali/base/dasubs.ipp

@@ -55,6 +55,8 @@ extern da_decl void closeSubscriptionManager();
 #define SDS_PUBLISHER       2
 #define NQS_PUBLISHER       3
 #define SDSCONN_PUBLISHER   4
+#define SDSEXACT_PUBLISHER  5
+#define SDSNODE_PUBLISHER   6
 
 
 #if 0

ecl/eclcmd/eclcmd_common.hpp

@@ -75,6 +75,7 @@ typedef IEclCommand *(*EclCommandFactory)(const char *cmdname);
 #define ECLOPT_ACTIVATE_S "-A"
 #define ECLOPT_ACTIVATE_INI "activateDefault"
 #define ECLOPT_ACTIVATE_ENV NULL
+#define ECLOPT_CLONE_ACTIVE_STATE "--clone-active-state"
 #define ECLOPT_SUSPEND_PREVIOUS "--suspend-prev"
 #define ECLOPT_SUSPEND_PREVIOUS_S "-sp"
 #define ECLOPT_SUSPEND_PREVIOUS_INI "suspendPrevDefault"

ecl/eclcmd/queries/ecl-queries.cpp

@@ -488,6 +488,142 @@ private:
     bool optAllowForeign;
 };
 
+class EclCmdQueriesCopyQueryset : public EclCmdCommon
+{
+public:
+    EclCmdQueriesCopyQueryset() : optCloneActiveState(false), optAllQueries(false), optDontCopyFiles(false), optOverwrite(false), optAllowForeign(false)
+    {
+    }
+    virtual bool parseCommandLineOptions(ArgvIterator &iter)
+    {
+        if (iter.done())
+        {
+            usage();
+            return false;
+        }
+
+        for (; !iter.done(); iter.next())
+        {
+            const char *arg = iter.query();
+            if (*arg!='-')
+            {
+                if (optSourceQuerySet.isEmpty())
+                    optSourceQuerySet.set(arg);
+                else if (optDestQuerySet.isEmpty())
+                    optDestQuerySet.set(arg);
+                else
+                {
+                    fprintf(stderr, "\nunrecognized argument %s\n", arg);
+                    return false;
+                }
+                continue;
+            }
+            if (iter.matchOption(optDaliIP, ECLOPT_DALIIP))
+                continue;
+            if (iter.matchOption(optSourceProcess, ECLOPT_SOURCE_PROCESS))
+                continue;
+            if (iter.matchFlag(optCloneActiveState, ECLOPT_CLONE_ACTIVE_STATE))
+                continue;
+            if (iter.matchFlag(optDontCopyFiles, ECLOPT_DONT_COPY_FILES))
+                continue;
+            if (iter.matchFlag(optAllQueries, ECLOPT_ALL))
+                continue;
+            if (iter.matchFlag(optAllowForeign, ECLOPT_ALLOW_FOREIGN))
+                continue;
+            if (iter.matchFlag(optOverwrite, ECLOPT_OVERWRITE)||iter.matchFlag(optOverwrite, ECLOPT_OVERWRITE_S))
+                continue;
+            if (EclCmdCommon::matchCommandLineOption(iter, true)!=EclCmdOptionMatch)
+                return false;
+        }
+        return true;
+    }
+    virtual bool finalizeOptions(IProperties *globals)
+    {
+        if (!EclCmdCommon::finalizeOptions(globals))
+            return false;
+        if (optSourceQuerySet.isEmpty() || optDestQuerySet.isEmpty())
+        {
+            fputs("source and destination querysets must both be specified.\n\n", stderr);
+            return false;
+        }
+        return true;
+    }
+
+    virtual int processCMD()
+    {
+        Owned<IClientWsWorkunits> client = createCmdClient(WsWorkunits, *this);
+        Owned<IClientWUCopyQuerySetRequest> req = client->createWUCopyQuerySetRequest();
+        req->setActiveOnly(!optAllQueries);
+        req->setSource(optSourceQuerySet.get());
+        req->setTarget(optDestQuerySet.get());
+        req->setDfsServer(optDaliIP.get());
+        req->setSourceProcess(optSourceProcess);
+        req->setCloneActiveState(optCloneActiveState);
+        req->setOverwriteDfs(optOverwrite);
+        req->setCopyFiles(!optDontCopyFiles);
+        req->setAllowForeignFiles(optAllowForeign);
+
+        Owned<IClientWUCopyQuerySetResponse> resp = client->WUCopyQuerySet(req);
+        if (resp->getExceptions().ordinality())
+            outputMultiExceptions(resp->getExceptions());
+        StringArray &copied = resp->getCopiedQueries();
+        fputs("Queries copied:\n", stdout);
+        if (!copied.length())
+            fputs("  none\n\n", stdout);
+        else
+        {
+            ForEachItemIn(i, copied)
+                fprintf(stdout, "  %s\n", copied.item(i));
+            fputs("\n", stdout);
+        }
+        StringArray &existing = resp->getExistingQueries();
+        fputs("Queries already on destination target:\n", stdout);
+        if (!existing.length())
+            fputs("  none\n\n", stdout);
+        else
+        {
+            ForEachItemIn(i, existing)
+                fprintf(stdout, "  %s\n", existing.item(i));
+            fputs("\n", stdout);
+        }
+        return 0;
+    }
+    virtual void usage()
+    {
+        fputs("\nUsage:\n"
+            "\n"
+            "The 'queries copy-set' command copies a set of queries from one target to another.\n"
+            "\n"
+            "By default only active queries will be copied.  Use --all to copy all queries.\n"
+            "\n"
+            "ecl queries copy-set <source_target> <destination_target> [--clone-active-state]\n"
+            "\n"
+            " Options:\n"
+            "   <source_target>        Target cluster to copy queries from\n"
+            "   <destination_target>   Target cluster to copy queries to\n"
+            "   --all                  Copy both active and inactive queries\n"
+            "   --no-files             Do not copy files referenced by query\n"
+            "   --daliip=<ip>          Remote Dali DFS to use for copying files\n"
+            "   --source-process       Process cluster to copy files from\n"
+            "   --clone-active-state   Make copied queries active if active on source\n"
+            "   -O, --overwrite        Overwrite existing DFS file information\n"
+            "   --allow-foreign        Do not fail if foreign files are used in query (roxie)\n"
+            " Common Options:\n",
+            stdout);
+        EclCmdCommon::usage();
+    }
+private:
+    StringAttr optSourceQuerySet;
+    StringAttr optDestQuerySet;
+    StringAttr optDaliIP;
+    StringAttr optSourceProcess;
+    bool optCloneActiveState;
+    bool optOverwrite;
+    bool optDontCopyFiles;
+    bool optAllowForeign;
+    bool optAllQueries;
+};
+
 class EclCmdQueriesConfig : public EclCmdCommon
 {
 public:
@@ -639,6 +775,8 @@ IEclCommand *createEclQueriesCommand(const char *cmdname)
         return new EclCmdQueriesConfig();
     if (strieq(cmdname, "copy"))
         return new EclCmdQueriesCopy();
+    if (strieq(cmdname, "copy-set"))
+        return new EclCmdQueriesCopyQueryset();
     return NULL;
 }
 
@@ -657,9 +795,10 @@ public:
         fprintf(stdout,"\nUsage:\n\n"
             "ecl queries <command> [command options]\n\n"
             "   Queries Commands:\n"
-            "      list         list queries in queryset(s)\n"
+            "      list         list queries on target cluster(s)\n"
             "      config       update query settings\n"
-            "      copy         copy a query from one queryset to another\n"
+            "      copy         copy a query from one target cluster to another\n"
+            "      copy-set     copy queries from one target cluster to another\n"
         );
     }
 };

ecl/hthor/hthor.cpp

@@ -1009,13 +1009,18 @@ void CHThorIndexWriteActivity::execute()
 {
     size32_t maxDiskRecordSize;
     if (helper.queryDiskRecordSize()->isVariableSize())
-        maxDiskRecordSize = 0x8000;
-    else
     {
-        maxDiskRecordSize = helper.queryDiskRecordSize()->getFixedSize();
-        if (maxDiskRecordSize > 0x8000)
-            throw MakeStringException(99, "Index minimum record length (%d) exceeds 32K internal limit", maxDiskRecordSize);
+        if (helper.getFlags() & TIWmaxlength)
+            maxDiskRecordSize = helper.getMaxKeySize();
+        else
+            maxDiskRecordSize = KEYBUILD_MAXLENGTH; // Current default behaviour, could be improved in the future
     }
+    else
+        maxDiskRecordSize = helper.queryDiskRecordSize()->getFixedSize();
+
+    if (maxDiskRecordSize > KEYBUILD_MAXLENGTH)
+        throw MakeStringException(99, "Index maximum record length (%d) exceeds 32K internal limit", maxDiskRecordSize);
+
     OwnedMalloc<char> rowBuffer(maxDiskRecordSize, true);
 
     // Loop thru the results

esp/eclwatch/ws_XSLT/nls/es/hpcc.xml

@@ -1,162 +1,162 @@
 <hpcc>
 <strings>
-<st id='Action'>Action</st>
-<st id='Available'>Available</st>
-<st id='AdditionalProcessesToFilter'>Additional processes to filter</st>
-<st id='AutoRefreshEvery'>Auto Refresh every</st>
-<st id='AutoUpdateMetricsWhenViewColumnsChanging'>Auto update metrics when View Columns changing.</st>
-<st id='AvailableNodes'>Available Nodes</st>
-<st id='BrowserTimedOut'>Browser timed out due to a long time delay.</st>
-<st id='Busy'>Busy</st>
+<st id='Action'>Acción</st>
+<st id='Available'>Disponible</st>
+<st id='AdditionalProcessesToFilter'>Mas procesos para filtrar</st>
+<st id='AutoRefreshEvery'>Actualizar automaticamente cada</st>
+<st id='AutoUpdateMetricsWhenViewColumnsChanging'>Actualizar las medidas automaticamente cuando cambien las columnas.</st>
+<st id='AvailableNodes'>Nodos Disponibles</st>
+<st id='BrowserTimedOut'>Navegador expiro por respuesta lenta.</st>
+<st id='Busy'>Ocupado</st>
 <st id='Bytes'>bytes</st>
-<st id='Cancel'>Cancel</st>
-<st id='Cluster'>Cluster</st>
-<st id='ClusterProcess'>Cluster Process</st>
-<st id='Clusters'>Clusters</st>
-<st id='ClusterTopology'>Cluster Topology</st>
-<st id='Component'>Component</st>
-<st id='Computer'>Computer</st>
-<st id='ComputerUpTime'>Computer Up Time</st>
-<st id='Condition'>Condition</st>
-<st id='Configuration'>Configuration</st>
-<st id='ConfigurationFile'>Configuration file</st>
-<st id='ConfirmSwap'>Are you sure you want to swap two machines under</st>
-<st id='CPULoad'>CPU Load</st>
-<st id='Critical'>Critical</st>
-<st id='DaliServer'>Dali Server</st>
-<st id='DaliServers'>Dali Servers</st>
-<st id='Date'>Date</st>
-<st id='Description'>Description</st>
-<st id='DFUServers'>DFU Servers</st>
-<st id='Directory'>Directory</st>
-<st id='Domain'>Domain</st>
-<st id='Download'>Download</st>
-<st id='DropZone'>Drop Zone</st>
-<st id='DropZones'>Drop Zones</st>
-<st id='EarliestFirst'>EarliestFirst</st>
-<st id='ECLAgents'>ECL Agents</st>
-<st id='ECLAgentProcess'>ECL Agent Process</st>
-<st id='ECLCCServers'>ECL CC Servers</st>
-<st id='ECLSchedulers'>ECL Schedulers</st>
-<st id='ECLServers'>ECL Servers</st>
-<st id='ESPServers'>ESP Servers</st>
-<st id='FailedToRetrieveInformation'>Failed to retrieve information.  Please check configuration.</st>
+<st id='Cancel'>Cancelar</st>
+<st id='Cluster'>Clúster</st>
+<st id='ClusterProcess'>Proceso de Clúster</st>
+<st id='Clusters'>Clústeres</st>
+<st id='ClusterTopology'>Topología del Clúster</st>
+<st id='Component'>Componente</st>
+<st id='Computer'>Computador</st>
+<st id='ComputerUpTime'>Periodo de uso del Computador</st>
+<st id='Condition'>Condición</st>
+<st id='Configuration'>Configuración</st>
+<st id='ConfigurationFile'>Archivo de Configuración</st>
+<st id='ConfirmSwap'>Seguro que desea intercambiar dos máquinas debajo</st>
+<st id='CPULoad'>Carga de CPU</st>
+<st id='Critical'>Crítico</st>
+<st id='DaliServer'>Servidor de Dali</st>
+<st id='DaliServers'>Servidores de Dali</st>
+<st id='Date'>Fecha</st>
+<st id='Description'>Descripción</st>
+<st id='DFUServers'>Servidor de DFU</st>
+<st id='Directory'>Directorio</st>
+<st id='Domain'>Dominio</st>
+<st id='Download'>Descargar</st>
+<st id='DropZone'>Zona de Descarga</st>
+<st id='DropZones'>Zonas de descarga</st>
+<st id='EarliestFirst'>MásTempranoPrimero</st>
+<st id='ECLAgents'>Agentes de ECL</st>
+<st id='ECLAgentProcess'>Proceso de Agente de ECL</st>
+<st id='ECLCCServers'>ECL CC Servidores</st>
+<st id='ECLSchedulers'>Planificador de ECL</st>
+<st id='ECLServers'>Servidores de ECL</st>
+<st id='ESPServers'>Servidores de ESP</st>
+<st id='FailedToRetrieveInformation'>No se pudo conseguir la informacion. Por Favor Comprueba la Configuración.</st>
 <st id='Fatal'>Fatal</st>
-<st id='Fetched'>Fetched</st>
-<st id='Files'>Files</st>
-<st id='FirstPage'>First page</st>
-<st id='FirstRowsNotDefined'>First Rows field not defined</st>
-<st id='Folder'>Folder</st>
-<st id='FTSlaves'>FT Slaves</st>
-<st id='GenesisServers'>Genesis Servers</st>
-<st id='GetProcessorInformation'>Get processor information</st>
-<st id='GetRoxieMetrics'>Get Roxie Metrics</st>
-<st id='GetSoftwareInformation'>Get software information</st>
-<st id='GetStorageInformation'>Get storage information</st>
-<st id='Home'>Home</st>
-<st id='Hours'>hours</st>
-<st id='Instance'>Instance</st>
-<st id='InvalidFirstRows'>Invalid data in First Rows field</st>
-<st id='InvalidLastHours'>Invalid data in Last Hours field</st>
-<st id='InvalidLastRows'>Invalid data in a Last Rows field</st>
-<st id='InvalidPageNumber'>Invalid data in page number field</st>
-<st id='InvalidTime'>Invalid data in a Time field</st>
-<st id='IPAddress'>IP Address</st>
-<st id='LastHoursNotDefined'>Last Hours field not defined</st>
-<st id='LastRowsNotDefined'>Last Rows field not defined</st>
-<st id='LatestFirst'>LatestFirst</st>
-<st id='LDAPServers'>LDAP Servers</st>
-<st id='LoadingPleaseWait'>Loading, please wait...</st>
-<st id='LocalFileSystemsOnly'>Local File Systems Only</st>
-<st id='Location'>Location</st>
-<st id='LogFile'>Log file</st>
-<st id='LogDirectory'>Log Directory</st>
-<st id='LostReferenceToParentWindow'>Lost reference to parent window.  Please traverse the path again!</st>
-<st id='MachineInfo'>Machine Information</st>
-<st id='Major'>Major</st>
-<st id='Mean'>Mean</st>
-<st id='Metrics'>Metrics</st>
-<st id='Minor'>Minor</st>
-<st id='Mins'>mins</st>
-<st id='MySQLServers'>MySQL Servers</st>
+<st id='Fetched'>Obtenido</st>
+<st id='Files'>Archivos</st>
+<st id='FirstPage'>Primera Pagina</st>
+<st id='FirstRowsNotDefined'>Primera Fila no esta definida</st>
+<st id='Folder'>Carpeta</st>
+<st id='FTSlaves'>FT Esclavo</st>
+<st id='GenesisServers'>Servidores Genesis</st>
+<st id='GetProcessorInformation'>Conseguir informacion de procesesor</st>
+<st id='GetRoxieMetrics'>Conseguir Medidas de Roxie</st>
+<st id='GetSoftwareInformation'>Conseguir informacion de software</st>
+<st id='GetStorageInformation'>Conseguir informacion de almacenamiento</st>
+<st id='Home'>Inicio</st>
+<st id='Hours'>horas</st>
+<st id='Instance'>Instancia</st>
+<st id='InvalidFirstRows'>Datos invalidos en el campo de Primera Fila</st>
+<st id='InvalidLastHours'>Datos invalidos en el campo de Ultimas Horas</st>
+<st id='InvalidLastRows'>Datos invalidos en el campo de Ultima Fila</st>
+<st id='InvalidPageNumber'>Datos invalidos en el campo de Numero de Pagina</st>
+<st id='InvalidTime'>Datos invalidos en campo de Tiempo</st>
+<st id='IPAddress'>Dirección de IP</st>
+<st id='LastHoursNotDefined'>Campo de Ultima Hora no esta definido</st>
+<st id='LastRowsNotDefined'>Campo de Ultimas Fila no esta definido</st>
+<st id='LatestFirst'>MásRecientePrimero</st>
+<st id='LDAPServers'>Servidor de LDAP</st>
+<st id='LoadingPleaseWait'>Cargando, por favor espere...</st>
+<st id='LocalFileSystemsOnly'>Solamente Sistemas de Archivos Locales</st>
+<st id='Location'>Lugar</st>
+<st id='LogFile'>Archivo del Registro</st>
+<st id='LogDirectory'>Carpeta del Archivo del Registro</st>
+<st id='LostReferenceToParentWindow'>Se perdio la referencia a la ventana principal. Por favor atraviese el camino de nuevo</st>
+<st id='MachineInfo'>Informacion de Maquina</st>
+<st id='Major'>Mayor</st>
+<st id='Mean'>Media</st>
+<st id='Metrics'>Medidas</st>
+<st id='Minor'>Menor</st>
+<st id='Mins'>Minutos</st>
+<st id='MySQLServers'>Servidores de MySQL</st>
 <st id='NA'>N/A</st>
-<st id='Name'>Name</st>
-<st id='NetworkAddress'>Network Address</st>
-<st id='NextPage'>NextPage</st>
-<st id='NoClustersDefined'>No clusters defined.</st>
-<st id='Nodes'>Nodes</st>
-<st id='NoItems'>No items</st>
-<st id='NoMachinesSelected'>No machines selected!</st>
+<st id='Name'>Nombre</st>
+<st id='NetworkAddress'>Direccion de Red</st>
+<st id='NextPage'>Pagina Siguiente</st>
+<st id='NoClustersDefined'>Ningun clúster definido.</st>
+<st id='Nodes'>Nodos</st>
+<st id='NoItems'>Ninguna Sección</st>
+<st id='NoMachinesSelected'>Ninguna maquina eligida!</st>
 <st id='Normal'>Normal</st>
-<st id='NoSystemServicesDefined'>No system services defined!</st>
-<st id='NoTargetClustersSelected'>No target clusters selected!</st>
-<st id='OrFirst'>or first</st>
-<st id='OrGoToPage'>or go to page</st>
-<st id='OrLast'>or last</st>
-<st id='OrLastPage'>or last page</st>
-<st id='OrTimeFrom'>or time from</st>
-<st id='Page'>page</st>
-<st id='PageNumberNotDefined'>page number field not defined</st>
-<st id='PhysicalMemory'>Physical Memory</st>
-<st id='Platform'>Platform</st>
-<st id='Port'>Port</st>
-<st id='PrevPage'>PrevPage</st>
-<st id='Processes'>Processes</st>
-<st id='ProcessesDown'>Processes Down</st>
-<st id='Protocol'>Protocol</st>
+<st id='NoSystemServicesDefined'>Ningun servicio de sistema esta definido!</st>
+<st id='NoTargetClustersSelected'>Ningun cluster elegido!</st>
+<st id='OrFirst'>o primero</st>
+<st id='OrGoToPage'>o ir a pagina</st>
+<st id='OrLast'>o ultima</st>
+<st id='OrLastPage'>o ultima pagina</st>
+<st id='OrTimeFrom'>o tiempo desde</st>
+<st id='Page'>pagina</st>
+<st id='PageNumberNotDefined'>Campo de Numero de Pagina no esta definido</st>
+<st id='PhysicalMemory'>Memoria Fisica</st>
+<st id='Platform'>Plataforma</st>
+<st id='Port'>Puerto</st>
+<st id='PrevPage'>PaginaAnterior</st>
+<st id='Processes'>Procesos</st>
+<st id='ProcessesDown'>Procesos Terminados</st>
+<st id='Protocol'>Protocolo</st>
 <st id='Queue'>Queue</st>
-<st id='Ready'>Ready</st>
-<st id='Recycling'>Recycling</st>
-<st id='Refresh'>Refresh</st>
-<st id='Rows'>rows</st>
-<st id='RoxieServer'>Roxie Server</st>
-<st id='SashaServers'>Sasha Servers</st>
-<st id='SchedulerProcess'>Scheduler Process</st>
-<st id='SecurityString'>Security String</st>
-<st id='Select'>Select</st>
-<st id='SelectASpareNodeToSwap'>Select a spare node to swap</st>
-<st id='SelectAllOrNone'>Select All / None</st>
-<st id='SelectAllTargetClusters'>Select all target clusters</st>
-<st id='SelectDeselectAll'>Select or deselect all</st>
-<st id='SelectDeselectAllMachines'>Select or deselect all machines</st>
-<st id='SelectThisTargetCluster'>Select this target cluster</st>
-<st id='ServerProcess'>Server Process</st>
-<st id='ServiceName'>Service Name</st>
-<st id='ServiceType'>Service Type</st>
-<st id='ShowProcessesUsingFilter'>Show processes using filter</st>
-<st id='Size'>Size</st>
-<st id='SlaveNumber'>Slave Number</st>
-<st id='Starting'>Starting</st>
-<st id='State'>State</st>
-<st id='Stopping'>Stopping</st>
-<st id='Submit'>Submit</st>
-<st id='Suspended'>Suspended</st>
-<st id='Swap'>Swap</st>
-<st id='SwapNode'>Swap Node</st>
-<st id='SystemServers'>System Servers</st>
-<st id='SystemServiceNodes'>System Service Nodes</st>
-<st id='TargetClusters'>Target Clusters</st>
-<st id='TimeNotDefined'>Either Time From or To field not defined</st>
-<st id='ThisPageFromByte'>this page from byte</st>
-<st id='ThorMaster'>Thor Master</st>
-<st id='ThorSlave'>Thor Slave</st>
-<st id='ThorSlaves'>Thor Slaves</st>
+<st id='Ready'>Listo</st>
+<st id='Recycling'>Reciclaje</st>
+<st id='Refresh'>Actualizar</st>
+<st id='Rows'>Filas</st>
+<st id='RoxieServer'>Servidor de Roxie</st>
+<st id='SashaServers'>Servidores de Sasha</st>
+<st id='SchedulerProcess'>Proceso de Planeador</st>
+<st id='SecurityString'>Texto de Seguridad</st>
+<st id='Select'>Escoja</st>
+<st id='SelectASpareNodeToSwap'>Escoja un nodo disponible para intercambiar</st>
+<st id='SelectAllOrNone'>Eligir Todo / Nada</st>
+<st id='SelectAllTargetClusters'>Eligir todos los clústeres</st>
+<st id='SelectDeselectAll'>Eligir o des-elegir todo</st>
+<st id='SelectDeselectAllMachines'>Eligir o des-elegir todas las maquinas</st>
+<st id='SelectThisTargetCluster'>Eligir este clúster</st>
+<st id='ServerProcess'>Proceso de servidor</st>
+<st id='ServiceName'>Nombre del servicio</st>
+<st id='ServiceType'>Tipo del servicio</st>
+<st id='ShowProcessesUsingFilter'>Mostrar los procesos usando filtro</st>
+<st id='Size'>Tamaño</st>
+<st id='SlaveNumber'>Numero de Esclavo</st>
+<st id='Starting'>Comenzando</st>
+<st id='State'>Estado</st>
+<st id='Stopping'>Parando</st>
+<st id='Submit'>Enviar</st>
+<st id='Suspended'>Suspendido</st>
+<st id='Swap'>Intercambiar</st>
+<st id='SwapNode'>Intercambiar Nodo</st>
+<st id='SystemServers'>Servidores de Sistema</st>
+<st id='SystemServiceNodes'>Nodos de servicio de Sistema</st>
+<st id='TargetClusters'>Clusters</st>
+<st id='TimeNotDefined'>El campo de "Desde" o campo de "Asta" no estan definidos</st>
+<st id='ThisPageFromByte'>Esta pagina desde byte</st>
+<st id='ThorMaster'>Maestro Thor</st>
+<st id='ThorSlave'>Esclavo Thor</st>
+<st id='ThorSlaves'>Esclavos Thor</st>
 <st id='ThorSpare'>Thor Spare</st>
-<st id='To'>to</st>
-<st id='ToByte'>to byte</st>
+<st id='To'>hasta</st>
+<st id='ToByte'>hasta byte</st>
 <st id='Total'>Total</st>
-<st id='TotalFileSize'>Total file size</st>
-<st id='UpdateMetricsNow'>Update Metrics Now</st>
-<st id='UpTime'>Up Time</st>
-<st id='Unknown'>Unknown</st>
-<st id='ViewConfigurationFile'>View Configuration File</st>
-<st id='ViewColumns'>View Columns</st>
-<st id='ViewDetails'>View Details</st>
-<st id='ViewingPage'>Viewing page</st>
-<st id='ViewLogFile'>View Log File</st>
-<st id='WarnIfAvailableDiskSpaceIsUnder'>Warn if available disk space is under</st>
-<st id='WarnIfAvailableMemoryIsUnder'>Warn if available memory is under</st>
-<st id='WarnIfCPUUsageIsOver'>Warn if CPU usage is over</st>
-<st id='Warning'>Warning</st>
+<st id='TotalFileSize'>Tamaño de archivo</st>
+<st id='UpdateMetricsNow'>Actualizar Medidas Ahora</st>
+<st id='UpTime'>Tiempo Funcionando</st>
+<st id='Unknown'>Desconocido</st>
+<st id='ViewConfigurationFile'>Ver Archivo de Configuracion</st>
+<st id='ViewColumns'>Ver Columnas</st>
+<st id='ViewDetails'>Ver Detalles</st>
+<st id='ViewingPage'>Viendo pagina</st>
+<st id='ViewLogFile'>Ver Archivo del Registro</st>
+<st id='WarnIfAvailableDiskSpaceIsUnder'>Avisar si el espacio disponible del disco baja a menos de</st>
+<st id='WarnIfAvailableMemoryIsUnder'>Avisar si la memoria disponible baja a menos de</st>
+<st id='WarnIfCPUUsageIsOver'>Avisa si utilizacion CPU sube a mas de</st>
+<st id='Warning'>Advertencia</st>
 </strings>
 </hpcc>

esp/scm/ws_workunits.ecm

@@ -1460,6 +1460,26 @@ ESPresponse [exceptions_inline] WUQuerySetCopyQueryResponse
     string QueryId;
 };
 
+ESPrequest [nil_remove] WUCopyQuerySetRequest
+{
+    string Source;
+    string Target;
+    bool ActiveOnly(true);
+    bool CloneActiveState(true);
+    bool AllowForeignFiles(true);
+
+    string DfsServer;
+    bool CopyFiles(true);
+    bool OverwriteDfs(false);
+    string SourceProcess;
+};
+
+ESPresponse [exceptions_inline] WUCopyQuerySetResponse
+{
+    ESParray<string, QueryId> CopiedQueries;
+    ESParray<string, QueryId> ExistingQueries;
+};
+
 ESPrequest [nil_remove] WUGetZAPInfoRequest
 {
     string WUID;
@@ -1575,6 +1595,7 @@ ESPservice [
     ESPmethod WUQuerysetQueryAction(WUQuerySetQueryActionRequest, WUQuerySetQueryActionResponse);
     ESPmethod WUQuerysetAliasAction(WUQuerySetAliasActionRequest, WUQuerySetAliasActionResponse);
     ESPmethod WUQuerysetCopyQuery(WUQuerySetCopyQueryRequest, WUQuerySetCopyQueryResponse);
+    ESPmethod WUCopyQuerySet(WUCopyQuerySetRequest, WUCopyQuerySetResponse);
     ESPmethod [resp_xsl_default("/esp/xslt/WUCopyLogicalFiles.xslt")] WUCopyLogicalFiles(WUCopyLogicalFilesRequest, WUCopyLogicalFilesResponse);
     ESPmethod WUQueryConfig(WUQueryConfigRequest, WUQueryConfigResponse);
     ESPmethod WUListQueries(WUListQueriesRequest, WUListQueriesResponse);

esp/services/ws_workunits/ws_workunitsQuerySets.cpp

@@ -766,7 +766,7 @@ bool CWsWorkunitsEx::onWUPublishWorkunit(IEspContext &context, IEspWUPublishWork
 
     StringBuffer queryId;
     WUQueryActivationOptions activate = (WUQueryActivationOptions)req.getActivate();
-    addQueryToQuerySet(wu, target.str(), queryName.str(), NULL, activate, queryId, context.queryUserId());
+    addQueryToQuerySet(wu, target.str(), queryName.str(), activate, queryId, context.queryUserId());
     if (req.getMemoryLimit() || !req.getTimeLimit_isNull() || !req.getWarnTimeLimit_isNull() || req.getPriority() || req.getComment())
     {
         Owned<IPropertyTree> queryTree = getQueryById(target.str(), queryId, false);
@@ -1834,6 +1834,170 @@ bool splitQueryPath(const char *path, StringBuffer &netAddress, StringBuffer &qu
     return true;
 }
 
+class QueryCloner
+{
+public:
+    QueryCloner(IEspContext *_context, const char *source, const char *_target) :
+        context(_context), cloneFilesEnabled(false), target(_target), overwriteDfs(false)
+    {
+        srcQuerySet.setown(getQueryRegistry(source, true));
+        if (!srcQuerySet)
+            throw MakeStringException(ECLWATCH_QUERYSET_NOT_FOUND, "Source Queryset %s not found", source);
+
+        destQuerySet.setown(getQueryRegistry(target, false));
+        if (!destQuerySet) // getQueryRegistry should have created if not found
+            throw MakeStringException(ECLWATCH_QUERYSET_NOT_FOUND, "Destination Queryset %s could not be created, or found", target.sget());
+
+        factory.setown(getWorkUnitFactory(context->querySecManager(), context->queryUser()));
+    }
+    void clone(const char *name, const char *id, bool makeActive)
+    {
+        Owned<IPropertyTree> query = getQueryById(srcQuerySet, id);
+        if (!query)
+            return;
+        const char *wuid = query->queryProp("@wuid");
+        if (!wuid || !*wuid)
+            return;
+        SCMStringBuffer existingQueryId;
+        queryIdFromQuerySetWuid(destQuerySet, wuid, existingQueryId);
+        if (existingQueryId.length())
+        {
+            existingQueryIds.append(existingQueryId.str());
+            if (makeActive)
+                activateQuery(destQuerySet, ACTIVATE_SUSPEND_PREVIOUS, name, existingQueryId.str(), context->queryUserId());
+            return;
+        }
+        StringBuffer newQueryId;
+        Owned<IWorkUnit> workunit = factory->updateWorkUnit(wuid);
+        addQueryToQuerySet(workunit, destQuerySet, name, makeActive ? ACTIVATE_SUSPEND_PREVIOUS : DO_NOT_ACTIVATE, newQueryId, context->queryUserId());
+        copiedQueryIds.append(newQueryId);
+        Owned<IPropertyTree> destQuery = getQueryById(destQuerySet, newQueryId);
+        if (destQuery)
+        {
+            Owned<IAttributeIterator> aiter = query->getAttributes();
+            ForEach(*aiter)
+            {
+                const char *atname = aiter->queryName();
+                if (!destQuery->hasProp(atname))
+                    destQuery->setProp(atname, aiter->queryValue());
+            }
+            if (cloneFilesEnabled && wufiles)
+                wufiles->addFilesFromQuery(workunit, pm, newQueryId);
+        }
+    }
+
+    void cloneActive(bool makeActive)
+    {
+        Owned<IPropertyTreeIterator> activeQueries = srcQuerySet->getElements("Alias");
+        ForEach(*activeQueries)
+        {
+            IPropertyTree &alias = activeQueries->query();
+            const char *name = alias.queryProp("@name");
+            const char *id = alias.queryProp("@id");
+            clone(name, id, makeActive);
+        }
+    }
+
+    void cloneAll(bool cloneActiveState)
+    {
+        Owned<IPropertyTreeIterator> allQueries = srcQuerySet->getElements("Query");
+        ForEach(*allQueries)
+        {
+            IPropertyTree &query = allQueries->query();
+            const char *name = query.queryProp("@name");
+            const char *id = query.queryProp("@id");
+            bool makeActive = false;
+            if (cloneActiveState)
+            {
+                VStringBuffer xpath("Alias[@id='%s']", id);
+                makeActive = srcQuerySet->hasProp(xpath);
+            }
+            clone(name, id, makeActive);
+        }
+    }
+    void enableFileCloning(const char *dfsServer, const char *destProcess, const char *sourceProcess, bool _overwriteDfs, bool allowForeign)
+    {
+        cloneFilesEnabled = true;
+        overwriteDfs = _overwriteDfs;
+        splitDerivedDfsLocation(dfsServer, srcCluster, dfsIP, srcPrefix, sourceProcess, sourceProcess, NULL, NULL);
+        wufiles.setown(createReferencedFileList(context->queryUserId(), context->queryPassword(), allowForeign));
+        Owned<IHpccPackageSet> ps = createPackageSet(destProcess);
+        pm.set(ps->queryActiveMap(target));
+        process.set(destProcess);
+    }
+
+    void cloneFiles()
+    {
+        if (cloneFilesEnabled)
+        {
+            wufiles->resolveFiles(process, dfsIP, srcPrefix, srcCluster, !overwriteDfs, true);
+            Owned<IDFUhelper> helper = createIDFUhelper();
+            wufiles->cloneAllInfo(helper, overwriteDfs, true);
+        }
+    }
+private:
+    Linked<IEspContext> context;
+    Linked<IWorkUnitFactory> factory;
+    Owned<IPropertyTree> destQuerySet;
+    Owned<IPropertyTree> srcQuerySet;
+    Owned<IReferencedFileList> wufiles;
+    Owned<const IHpccPackageMap> pm;
+    StringBuffer dfsIP;
+    StringBuffer srcCluster;
+    StringBuffer srcPrefix;
+    StringAttr target;
+    StringAttr process;
+    bool cloneFilesEnabled;
+    bool overwriteDfs;
+
+public:
+    StringArray existingQueryIds;
+    StringArray copiedQueryIds;
+};
+
+
+bool CWsWorkunitsEx::onWUCopyQuerySet(IEspContext &context, IEspWUCopyQuerySetRequest &req, IEspWUCopyQuerySetResponse &resp)
+{
+    const char *source = req.getSource();
+    if (!source || !*source)
+        throw MakeStringException(ECLWATCH_MISSING_PARAMS, "No source target specified");
+    if (!isValidCluster(source))
+        throw MakeStringException(ECLWATCH_INVALID_CLUSTER_NAME, "Invalid source target name: %s", source);
+
+    const char *target = req.getTarget();
+    if (!target || !*target)
+        throw MakeStringException(ECLWATCH_MISSING_PARAMS, "No destination target specified");
+    if (!isValidCluster(target))
+        throw MakeStringException(ECLWATCH_INVALID_CLUSTER_NAME, "Invalid destination target name: %s", target);
+
+    QueryCloner cloner(&context, source, target);
+
+    SCMStringBuffer process;
+    if (req.getCopyFiles())
+    {
+        Owned <IConstWUClusterInfo> clusterInfo = getTargetClusterInfo(target);
+        if (clusterInfo && clusterInfo->getPlatform()==RoxieCluster)
+        {
+            clusterInfo->getRoxieProcess(process);
+            if (!process.length())
+                throw MakeStringException(ECLWATCH_INVALID_CLUSTER_INFO, "DFS process cluster not found for destination target %s", target);
+            cloner.enableFileCloning(req.getDfsServer(), process.str(), req.getSourceProcess(), req.getOverwriteDfs(), req.getAllowForeignFiles());
+        }
+    }
+
+    if (req.getActiveOnly())
+        cloner.cloneActive(req.getCloneActiveState());
+    else
+        cloner.cloneAll(req.getCloneActiveState());
+
+    cloner.cloneFiles();
+
+    resp.setCopiedQueries(cloner.copiedQueryIds);
+    resp.setExistingQueries(cloner.existingQueryIds);
+
+    return true;
+}
+
 bool CWsWorkunitsEx::onWUQuerysetCopyQuery(IEspContext &context, IEspWUQuerySetCopyQueryRequest &req, IEspWUQuerySetCopyQueryResponse &resp)
 {
     unsigned start = msTick();
@@ -1906,7 +2070,8 @@ bool CWsWorkunitsEx::onWUQuerysetCopyQuery(IEspContext &context, IEspWUQuerySetC
 
     StringBuffer targetQueryId;
     WUQueryActivationOptions activate = (WUQueryActivationOptions)req.getActivate();
-    addQueryToQuerySet(wu, target, targetQueryName.get(), NULL, activate, targetQueryId, context.queryUserId());
+    addQueryToQuerySet(wu, target, targetQueryName.get(), activate, targetQueryId, context.queryUserId());
+
     Owned<IPropertyTree> queryTree = getQueryById(target, targetQueryId, false);
     if (queryTree)
     {
@@ -1921,6 +2086,8 @@ bool CWsWorkunitsEx::onWUQuerysetCopyQuery(IEspContext &context, IEspWUQuerySetC
             queryTree->setProp("@comment", req.getComment());
         else if (srcInfo && srcInfo->getComment())
             queryTree->setProp("@comment", srcInfo->getComment());
+        if (srcInfo && srcInfo->getSnapshot())
+            queryTree->setProp("@snapshot", srcInfo->getSnapshot());
     }
     wu.clear();
 

esp/services/ws_workunits/ws_workunitsService.hpp

@@ -188,6 +188,7 @@ public:
     bool onWUQuerysetAliasAction(IEspContext &context, IEspWUQuerySetAliasActionRequest &req, IEspWUQuerySetAliasActionResponse &resp);
     bool onWUQueryConfig(IEspContext &context, IEspWUQueryConfigRequest &req, IEspWUQueryConfigResponse &resp);
     bool onWUQuerysetCopyQuery(IEspContext &context, IEspWUQuerySetCopyQueryRequest &req, IEspWUQuerySetCopyQueryResponse &resp);
+    bool onWUCopyQuerySet(IEspContext &context, IEspWUCopyQuerySetRequest &req, IEspWUCopyQuerySetResponse &resp);
     bool onWUCopyLogicalFiles(IEspContext &context, IEspWUCopyLogicalFilesRequest &req, IEspWUCopyLogicalFilesResponse &resp);
     bool onWUQueryDetails(IEspContext &context, IEspWUQueryDetailsRequest & req, IEspWUQueryDetailsResponse & resp);
     bool onWUListQueries(IEspContext &context, IEspWUListQueriesRequest &req, IEspWUListQueriesResponse &resp);

esp/src/eclwatch/ESPWorkunit.js

@@ -301,7 +301,7 @@ define([
         },
         resubmit: function () {
             var context = this;
-            this._resubmit(false, false).then(function (response) {
+            this._resubmit(false, true).then(function (response) {
                 if (!lang.exists("Exceptions.Source", response)) {
                     dojo.publish("hpcc/brToaster", {
                         Severity: "Message",
@@ -312,13 +312,13 @@ define([
                 return response;
             });
         },
-        restart: function () {
+        recover: function () {
             var context = this;
-            this._resubmit(false, true).then(function (response) {
+            this._resubmit(false, false).then(function (response) {
                 if (!lang.exists("Exceptions.Source", response)) {
                     dojo.publish("hpcc/brToaster", {
                         Severity: "Message",
-                        Source: "ESPWorkunit.resubmit",
+                        Source: "ESPWorkunit.recover",
                         Exceptions: [{ Source: context.Wuid, Message: context.i18n.Restarted }]
                     });
                 }

esp/src/eclwatch/InfoGridWidget.js

@@ -93,6 +93,7 @@ define([
                 if (this.showToolbar) {
                     if (has("ie") <= 9 || has("ff")) {
                         this.widget.Download.set("disabled", true);
+                        this.widget.Download.set("title", this.i18n.UnsupportedIE9FF);
                     }
                 }
 

esp/src/eclwatch/SearchResultsWidget.js

@@ -63,7 +63,7 @@ define([
             if (params.searchText) {
                 this.doSearch(params.searchText);
             }
-            this._refreshActionState();
+            this.refreshGrid();
         },
 
         getTitle: function () {
@@ -338,6 +338,7 @@ define([
             if (this.searchText) {
                 this.searchAll();
             }
+            this._refreshActionState();
         },
 
         refreshActionState: function (selection) {

esp/src/eclwatch/WUDetailsWidget.js

@@ -170,8 +170,8 @@ define([
         _onAbort: function (event) {
             this.wu.abort();
         },
-        _onRestart: function (event) {
-            this.wu.restart();
+        _onRecover: function (event) {
+            this.wu.recover();
         },
         _onPublish: function (event) {
             var allowForeign = registry.byId(this.id + "AllowForeignFiles");
@@ -451,7 +451,7 @@ define([
             registry.byId(this.id + "Abort").set("disabled", isArchived || this.wu.isComplete());
             registry.byId(this.id + "Clone").set("disabled", isArchived || !this.wu.isComplete());
             registry.byId(this.id + "Resubmit").set("disabled", isArchived || !this.wu.isComplete());
-            registry.byId(this.id + "Restart").set("disabled", isArchived || !this.wu.isComplete());
+            registry.byId(this.id + "Recover").set("disabled", isArchived || !this.wu.isComplete());
             registry.byId(this.id + "Publish").set("disabled", isArchived || !this.wu.isComplete());
 
             registry.byId(this.id + "Jobname").set("readOnly", !this.wu.isComplete());

esp/src/eclwatch/nls/hpcc.js

@@ -333,6 +333,7 @@ define({root:
     Records: "Records",
     RecordSize: "Record Size",
     RecordStructurePresent: "Record Structure Present",
+    Recover: "Recover",
     Refresh: "Refresh",
     ReleaseNotes: "Release Notes",
     Reload: "Reload",
@@ -482,6 +483,7 @@ define({root:
     undefined: "undefined",
     Unknown: "Unknown",
     Unprotect: "Unprotect",
+    UnsupportedIE9FF: "Unsupported (IE <= 9, FireFox)",
     Unsuspend: "Unsuspend",
     Unsuspended: "Unsuspended",
     Up: "Up",

esp/src/eclwatch/templates/WUDetailsWidget.html

@@ -13,9 +13,9 @@
                     <div id="${id}SetToFailed" data-dojo-attach-event="onClick:_onSetToFailed" data-dojo-type="dijit.form.Button">${i18n.SetToFailed}</div>
                     <div id="${id}Abort" data-dojo-attach-event="onClick:_onAbort" data-dojo-type="dijit.form.Button">${i18n.Abort}</div>
                     <span data-dojo-type="dijit.ToolbarSeparator"></span>
-                    <div id="${id}Clone" data-dojo-attach-event="onClick:_onClone" data-dojo-type="dijit.form.Button">${i18n.Clone}</div>
+                    <div id="${id}Recover" data-dojo-attach-event="onClick:_onRecover" data-dojo-type="dijit.form.Button">${i18n.Recover}</div>
                     <div id="${id}Resubmit" data-dojo-attach-event="onClick:_onResubmit" data-dojo-type="dijit.form.Button">${i18n.Resubmit}</div>
-                    <div id="${id}Restart" data-dojo-attach-event="onClick:_onRestart" data-dojo-type="dijit.form.Button">${i18n.Restart}</div>
+                    <div id="${id}Clone" data-dojo-attach-event="onClick:_onClone" data-dojo-type="dijit.form.Button">${i18n.Clone}</div>
                     <span data-dojo-type="dijit.ToolbarSeparator"></span>
                     <div id="${id}Publish" data-dojo-type="dijit.form.DropDownButton">
                         <span>${i18n.Publish}</span>

initfiles/sbin/install-cluster.sh.in

@@ -150,6 +150,10 @@ runPayload(){
                     exit;
             } eof {
                     exit;
+            } "${USER}@" { 
+                  # Allow non-password connection go through without changing following code.
+                  # This is will be replaced when HPCC-11156 is fixed.
+                  send \"hostname\r\" 
             }
         }
         expect "${USER}@" {
@@ -204,24 +208,13 @@ createPayload;
 for IP in $IPS; do
     if ping -c 1 -w 5 -n $IP > /dev/null 2>&1; then
         echo "$IP: Host is alive."
-        if [ ${NEW_KEY} -ne 1 ]; then
-            CAN_SSH="`ssh -i /home/hpcc/.ssh/id_rsa -o BatchMode=yes -o StrictHostKeyChecking=no hpcc@$IP exit > /dev/null 2>&1; echo $?`"
-        else
-            CAN_SSH=255
-        fi
-        if [ "$CAN_SSH" -eq 255 ]; then
-            echo "$IP: Cannot SSH to host with key..";
-            echo "$IP: Connecting with password.";
-            copyPayload $IP;
-            expandPayload $IP;
-            runPayload $IP;
-            echo "$IP: Done.";
-        else
-            echo "$IP: Has SSH Key, No install actions done.";
-        fi
+        copyPayload $IP;
+        expandPayload $IP;
+        runPayload $IP;
+        echo "$IP: Done.";
     else
         echo "$IP: Cannot Ping host? (Host Alive?)"
     fi
 done
 
-removePayload;
+removePayload;

initfiles/sbin/remote-install-engine.sh.in

@@ -75,6 +75,7 @@ pkgCmd(){
             echo "Bad option type."
             exit 1
         fi
+        [ $WITH_PLUGINS -eq 1 ] && PKGCMD="$PKGCMD --nodeps"
     else
         echo "BAD Package type."
         exit 1
@@ -194,8 +195,13 @@ if [ -z $pkgtype ]; then
     OSPKG="deb"
 else
     OSPKG="rpm"
+    WITH_PLUGINS=0
+    basename $PKG | grep -i with-plugins
+    [ $? -eq 0 ] && WITH_PLUGINS=1
 fi
 
+
+
 installPkg
 checkUser "hpcc"
 _USER=$?

roxie/ccd/ccdfile.cpp

@@ -568,8 +568,6 @@ class CRoxieFileCache : public CInterface, implements ICopyFileProgress, impleme
     bool aborting;
     bool closing;
     bool closePending[2];
-    bool needToDeleteFile;
-    StringBuffer currentTodoFile;
     StringAttrMapping fileErrorList;
     Semaphore bctStarted;
     Semaphore hctStarted;
@@ -824,31 +822,23 @@ class CRoxieFileCache : public CInterface, implements ICopyFileProgress, impleme
                 deleteTempFiles(targetFilename);
                 throw;
             }
-            if (needToDeleteFile)
+            if (!hardLinkCreated && !useTreeCopy)  // for hardlinks / treeCopy - no rename needed
             {
-                DBGLOG("%s of data file %s stopped since query has been deleted", msg, targetFilename);
-                destFile->remove();
-            }
-            else 
-            {
-                if (!hardLinkCreated && !useTreeCopy)  // for hardlinks / treeCopy - no rename needed
+                try
                 {
-                    try
-                    {
-                        destFile->rename(targetFilename);
-                    }
-                    catch(IException *)
-                    {
-                        f->setCopying(false);
-                        deleteTempFiles(targetFilename);
-                        throw;
-                    }
-
-                    DBGLOG("%s to %s complete", msg, targetFilename);
+                    destFile->rename(targetFilename);
                 }
-                
-                f->copyComplete();
+                catch(IException *)
+                {
+                    f->setCopying(false);
+                    deleteTempFiles(targetFilename);
+                    throw;
+                }
+
+                DBGLOG("%s to %s complete", msg, targetFilename);
             }
+
+            f->copyComplete();
         }
         deleteTempFiles(targetFilename);
         return fileCopied;
@@ -892,7 +882,6 @@ public:
         closing = false;
         closePending[false] = false;
         closePending[true] = false;
-        needToDeleteFile = false;
         started = false;
     }
 
@@ -955,11 +944,6 @@ public:
 
             loop
             {
-                {
-                    CriticalBlock b(crit);
-                    needToDeleteFile = false;
-                    currentTodoFile.clear();
-                }
                 fileCopied = false;
                 Linked<ILazyFileIO> next;
                 toCopy.wait();
@@ -973,8 +957,6 @@ public:
                         if (popped->isAlive())
                         {
                             next.set(popped);
-                            if (next)
-                                currentTodoFile.append(next->queryFilename());
                         }
                         atomic_dec(&numFilesToProcess);    // must decrement counter for SNMP accuracy
                     }
@@ -1179,7 +1161,6 @@ public:
                 {
                     if (copyResources) // MORE - should always copy peer files
                     {
-                        needToDeleteFile = false;
                         if (numParts==1 || (partNo==numParts && fileType==ROXIE_KEY))
                         {
                             ret->checkOpen();
@@ -1187,7 +1168,13 @@ public:
                             return ret.getLink();
                         }
 
-                        todo.append(*ret);
+                        // Copies are popped from end of the todo list
+                        // By putting the replicates on the front we ensure they are done after the primaries
+                        // and are therefore likely to result in local rather than remote copies.
+                        if (replicationLevel)
+                            todo.add(*ret, 0);
+                        else
+                            todo.append(*ret);
                         atomic_inc(&numFilesToProcess);  // must increment counter for SNMP accuracy
                         toCopy.signal();
 

roxie/ccd/ccdserver.cpp

@@ -11371,14 +11371,18 @@ public:
         bool isVariable = helper.queryDiskRecordSize()->isVariableSize();
         size32_t maxDiskRecordSize;
         if (isVariable)
-            maxDiskRecordSize = 0x8000;
-        else
         {
+            if (helper.getFlags() & TIWmaxlength)
+                maxDiskRecordSize = helper.getMaxKeySize();
+            else
+                maxDiskRecordSize = KEYBUILD_MAXLENGTH; // Current default behaviour, could be improved in the future
+        }
+        else
             maxDiskRecordSize = helper.queryDiskRecordSize()->getFixedSize();
-            if (maxDiskRecordSize > 0x8000)
-                throw MakeStringException(99, "Index minimum record length (%d) exceeds 32k internal limit", maxDiskRecordSize);
 
-        }
+        if (maxDiskRecordSize > KEYBUILD_MAXLENGTH)
+            throw MakeStringException(99, "Index maximum record length (%d) exceeds 32k internal limit", maxDiskRecordSize);
+
         OwnedMalloc<char> rowBuffer(maxDiskRecordSize, true);
 
         unsigned __int64 fileSize = 0;

roxie/roxiemem/roxiemem.cpp

@@ -385,8 +385,16 @@ static StringBuffer &memmap(StringBuffer &stats)
 
 static void throwHeapExhausted(unsigned pages)
 {
-    DBGLOG("RoxieMemMgr: Memory pool (%u pages) exhausted requested %u", heapTotalPages, pages);
-    throw MakeStringException(ROXIEMM_MEMORY_POOL_EXHAUSTED, "Memory pool exhausted");
+    VStringBuffer msg("Memory pool exhausted: pool (%u pages) exhausted, requested %u", heapTotalPages, pages);
+    DBGLOG("%s", msg.str());
+    throw MakeStringExceptionDirect(ROXIEMM_MEMORY_POOL_EXHAUSTED, msg.str());
+}
+
+static void throwHeapExhausted(unsigned newPages, unsigned oldPages)
+{
+    VStringBuffer msg("Memory pool exhausted: pool (%u pages) exhausted, requested %u, had %u", heapTotalPages, newPages, oldPages);
+    DBGLOG("%s", msg.str());
+    throw MakeStringExceptionDirect(ROXIEMM_MEMORY_POOL_EXHAUSTED, msg.str());
 }
 
 static void *suballoc_aligned(size32_t pages, bool returnNullWhenExhausted)
@@ -429,8 +437,9 @@ static void *suballoc_aligned(size32_t pages, bool returnNullWhenExhausted)
                 {
                     if (returnNullWhenExhausted)
                         return NULL;
-                    DBGLOG("RoxieMemMgr: Request for large memory denied (%u..%u)", heapLargeBlocks, largeBlocksRequired);
-                    throw MakeStringException(ROXIEMM_LARGE_MEMORY_EXHAUSTED, "Memory pool exhausted");
+                    VStringBuffer msg("Memory pool exhausted: Request for large memory denied (%u..%u)", heapLargeBlocks, largeBlocksRequired);
+                    DBGLOG("%s", msg.str());
+                    throw MakeStringException(ROXIEMM_LARGE_MEMORY_EXHAUSTED, "%s", msg.str());
                 }
 
                 heapLargeBlocks = largeBlocksRequired;
@@ -3222,9 +3231,11 @@ public:
                         if (maxSpillCost != SpillAllCost)
                             return false;
 
-                        logctx.CTXLOG("RoxieMemMgr: Memory limit exceeded - current %u, requested %u, limit %u", pageCount, numRequested, pageLimit);
-                        doOomReport();
-                        throw MakeStringException(ROXIEMM_MEMORY_LIMIT_EXCEEDED, "memory limit exceeded");
+                        VStringBuffer msg("Memory limit exceeded: current %u, requested %u, limit %u", pageCount, numRequested, pageLimit);
+                        logctx.CTXLOG("%s", msg.str());
+                        reportMemoryUsage(false);
+                        PrintStackReport();
+                        throw MakeStringException(ROXIEMM_MEMORY_LIMIT_EXCEEDED, "%s", msg.str());
                     }
                 }
             }
@@ -3680,7 +3691,7 @@ bool CHugeHeap::expandHeap(void * original, memsize_t copysize, memsize_t oldcap
             if (maxSpillCost == SpillAllCost)
             {
                 rowManager->reportMemoryUsage(false);
-                throwHeapExhausted(numPages);
+                throwHeapExhausted(newPages, oldPages);
             }
             return false;
         }

system/jlib/jptree.hpp

@@ -168,6 +168,7 @@ interface IPTreeNodeCreator : extends IInterface
     virtual IPropertyTree *create(const char *tag) = 0;
 };
 
+// NB ipt_ext4 - used by SDS
 // NB ipt_ext5 - used by SDS
 enum ipt_flags { ipt_none=0x00, ipt_caseInsensitive=0x01, ipt_binary=0x02, ipt_ordered=0x04, ipt_ext1=0x08, ipt_ext2=16, ipt_ext3=32, ipt_ext4=64, ipt_ext5=128 };
 jlib_decl IPTreeMaker *createPTreeMaker(byte flags=ipt_none, IPropertyTree *root=NULL, IPTreeNodeCreator *nodeCreator=NULL);

testing/ecl/issue10022.ecl

@@ -1,16 +0,0 @@
-//noroxie
-//this really should be supported by roxie, but it doesn't support it at the moment
-
-#option ('multiplePersistInstances', true);
-
-ds := dataset(['','!Hello',' me ', '', 'xx', '!end'], { string line });
-
-p1 := ds(line <> '') : persist('~p1', many, single);
-
-p2 := ds(line = '') : persist('~p2', multiple);
-
-p3 := ds(line[1]='!') : persist('~p3',multiple(10));
-
-p4 := ds(line[1] = ' ') : persist('~p4');
-
-output(p1 & p2 & p3 & p4);

testing/ecl/joinattr2.hql

@@ -1,175 +0,0 @@
-/*##############################################################################
-
-    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
-
-    Licensed under the Apache License, Version 2.0 (the "License");
-    you may not use this file except in compliance with the License.
-    You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-    Unless required by applicable law or agreed to in writing, software
-    distributed under the License is distributed on an "AS IS" BASIS,
-    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-    See the License for the specific language governing permissions and
-    limitations under the License.
-############################################################################## */
-
-rec :=
-RECORD
-            integer i;
-        string1 id;
-END;
-
- recout :=
-RECORD
-        string1 idL;
-        string1 idR;
-END;
-
- 
-
-ds1 := DATASET([{1,'A'}, {1,'B'}, {1,'C'}], rec);
-ds2 := DATASET([{1,'D'}], rec);
-ds3 := DATASET([{1,'E'}, {1,'F'}, {1,'G'}], rec);
-ds4 := DATASET([{1,'A'}], rec);
-ds5 := DATASET([{1,'B'}], rec);
-ds6 := DATASET([{1,'A'}, {1,'B'}], rec);
-ds7 := DATASET([{1,'C'}, {1,'D'}, {1,'E'}], rec);
-ds8 := DATASET([{1,'A'}, {1,'B'}, {1,'C'}], rec);
-ds9 := DATASET([{2,'A'}], rec);
-ds10 := DATASET([{0,'A'}], rec);
-
-recout trans(rec L, rec R) :=
-TRANSFORM
-
-            SELF.idl := L.id;
-            SELF.idr := R.id;
-END;
-
-recout transkip(rec L, rec R) :=
-TRANSFORM
-
-            SELF.idl := if(L.id='A',skip,L.id);
-            SELF.idr := if(R.id='A',skip,R.id);
-END;
-
-j1 := JOIN(ds1, ds2, LEFT.i = RIGHT.i, trans(LEFT, RIGHT), FIRSTLEFT);
-j2 := JOIN(ds1, ds2, LEFT.i = RIGHT.i, trans(LEFT, RIGHT), KEEP(1), LEFT OUTER);
-j3 := JOIN(ds1, ds2, LEFT.i = RIGHT.i, trans(LEFT, RIGHT), ATMOST(1), LEFT OUTER);
-j4 := JOIN(ds2, ds1, LEFT.i = RIGHT.i, trans(LEFT, RIGHT), FIRSTRIGHT);
-j5 := JOIN(ds2, ds1, LEFT.i = RIGHT.i, trans(LEFT, RIGHT), KEEP(1), RIGHT OUTER);
-j6 := JOIN(ds1, ds7, (LEFT.i = RIGHT.i), transkip(LEFT, RIGHT));
-j7 := JOIN(ds1, ds2, LEFT.i = RIGHT.i, trans(LEFT, RIGHT), LEFT OUTER);
-j8 := JOIN(ds2, ds1, LEFT.i = RIGHT.i, trans(LEFT, RIGHT), RIGHT OUTER);
-j9 := JOIN(ds4, ds3, (LEFT.i = RIGHT.i)and(RIGHT.id>'E'), trans(LEFT, RIGHT), KEEP(2));
-j10 := JOIN(ds1, ds5, (LEFT.i = RIGHT.i), trans(LEFT, RIGHT), FIRSTLEFT);
-j11 := JOIN(ds1, ds5, (LEFT.i = RIGHT.i)and(LEFT.id>=RIGHT.id), trans(LEFT, RIGHT), FIRSTLEFT);
-j12 := JOIN(ds1, ds3, (LEFT.i = RIGHT.i), trans(LEFT, RIGHT), KEEP(2),LEFT OUTER);
-j13 := JOIN(ds6, ds7, (LEFT.i = RIGHT.i), trans(LEFT, RIGHT), KEEP(5));
-j14 := JOIN(ds6, ds7, (LEFT.i = RIGHT.i), trans(LEFT, RIGHT), KEEP(1),RIGHT OUTER);
-j15 := JOIN(ds1, ds7, (LEFT.i = RIGHT.i), transkip(LEFT, RIGHT));
-j16 := JOIN(ds1, ds5, (LEFT.i = RIGHT.i)and(LEFT.id>=RIGHT.id), trans(LEFT, RIGHT), FIRST);
-j17 := JOIN(ds1, ds3, LEFT.i = RIGHT.i, trans(LEFT, RIGHT), KEEP(1),FULL OUTER);
-j18 := JOIN(ds1, ds3, LEFT.i = RIGHT.i, trans(LEFT, RIGHT), KEEP(1),FULL ONLY);
-j19 := JOIN(ds1, ds3, LEFT.i = RIGHT.i, trans(LEFT, RIGHT), KEEP(1),RIGHT OUTER);
-j20 := JOIN(ds1, ds3, LEFT.i = RIGHT.i, trans(LEFT, RIGHT), KEEP(1),RIGHT ONLY);
-j21 := JOIN(ds7, ds1, (LEFT.i = RIGHT.i)and(LEFT.id<=RIGHT.id), trans(LEFT, RIGHT), FIRST);
-j22 := JOIN(ds1, ds1, (LEFT.i = RIGHT.i), trans(LEFT, RIGHT), KEEP(1));
-j23 := JOIN(ds1, ds8, (LEFT.i = RIGHT.i), trans(LEFT, RIGHT), KEEP(1));
-j24 := JOIN(ds1, ds8, (LEFT.i = RIGHT.i), trans(LEFT, RIGHT), LOOKUP);
-j25 := JOIN(ds1, ds8, (LEFT.i = RIGHT.i), trans(LEFT, RIGHT), ATMOST(1));
-j26 := JOIN(ds1, ds1, (LEFT.i = RIGHT.i), trans(LEFT, RIGHT), FIRST);
-j27 := JOIN(ds1, ds8, (LEFT.i = RIGHT.i), trans(LEFT, RIGHT), FIRST);
-j28 := JOIN(ds7, ds1, (LEFT.i = RIGHT.i)and(LEFT.id<=RIGHT.id), trans(LEFT, RIGHT), LEFT OUTER, LOOKUP);
-j29 := JOIN(ds1, ds9, LEFT.i = RIGHT.i, trans(LEFT, RIGHT), FIRSTLEFT);
-j30 := JOIN(ds1, ds9, LEFT.i = RIGHT.i, trans(LEFT, RIGHT), KEEP(1), LEFT OUTER);
-j31 := JOIN(ds1, ds9, LEFT.i = RIGHT.i, trans(LEFT, RIGHT), LOOKUP, LEFT OUTER);
-j32 := JOIN(ds1, ds9, LEFT.i = RIGHT.i, trans(LEFT, RIGHT), ATMOST(1), LEFT OUTER);
-j33 := JOIN(ds1, ds9, LEFT.i = RIGHT.i, trans(LEFT, RIGHT), FIRSTRIGHT);
-j34 := JOIN(ds1, ds9, LEFT.i = RIGHT.i, trans(LEFT, RIGHT), KEEP(1), RIGHT OUTER);
-j35 := JOIN(ds1, ds9, (LEFT.i = RIGHT.i)and(RIGHT.id>'E'), trans(LEFT, RIGHT), KEEP(2));
-j36 := JOIN(ds9, ds1, LEFT.i = RIGHT.i, trans(LEFT, RIGHT), FIRSTLEFT);
-j37 := JOIN(ds9, ds1, LEFT.i = RIGHT.i, trans(LEFT, RIGHT), KEEP(1), LEFT OUTER);
-j38 := JOIN(ds9, ds1, LEFT.i = RIGHT.i, trans(LEFT, RIGHT), LOOKUP, LEFT OUTER);
-j39 := JOIN(ds9, ds1, LEFT.i = RIGHT.i, trans(LEFT, RIGHT), ATMOST(1), LEFT OUTER);
-j40 := JOIN(ds9, ds1, LEFT.i = RIGHT.i, trans(LEFT, RIGHT), FIRSTRIGHT);
-j41 := JOIN(ds9, ds1, LEFT.i = RIGHT.i, trans(LEFT, RIGHT), KEEP(1), RIGHT OUTER);
-j42 := JOIN(ds9, ds1, (LEFT.i = RIGHT.i)and(RIGHT.id>'E'), trans(LEFT, RIGHT), KEEP(2));
-j43 := JOIN(ds1, ds8, LEFT.i = RIGHT.i, transkip(LEFT, RIGHT), FIRSTLEFT);
-j44 := JOIN(ds1, ds8, LEFT.i = RIGHT.i, transkip(LEFT, RIGHT), KEEP(1), LEFT OUTER);
-j45 := JOIN(ds1, ds8, LEFT.i = RIGHT.i, trans(LEFT, RIGHT), LOOKUP);
-j46 := JOIN(ds1, ds1, LEFT.i = RIGHT.i, transkip(LEFT, RIGHT), LOOKUP);
-j47 := JOIN(ds1, ds8, LEFT.i = RIGHT.i, trans(LEFT, RIGHT), LOOKUP, LEFT OUTER);
-j48 := JOIN(ds1, ds8, LEFT.i = RIGHT.i, transkip(LEFT, RIGHT), LOOKUP, LEFT OUTER);
-j49 := JOIN(ds1, ds8, (LEFT.i = RIGHT.i)and(RIGHT.id>'E'), transkip(LEFT, RIGHT), LOOKUP);
-j50 := JOIN(ds1, ds8, (LEFT.i = RIGHT.i)and(RIGHT.id>'E'), transkip(LEFT, RIGHT), LOOKUP, LEFT OUTER);
-j51 := JOIN(ds1, ds8, LEFT.i = RIGHT.i, transkip(LEFT, RIGHT), LOOKUP, LEFT OUTER);
-j52 := JOIN(ds1, ds8, LEFT.i = RIGHT.i, transkip(LEFT, RIGHT), FIRSTLEFT);
-j53 := JOIN(ds1, ds8, LEFT.i = RIGHT.i, transkip(LEFT, RIGHT), KEEP(1), LEFT OUTER);
-j54 := JOIN(ds1, ds8, LEFT.i = RIGHT.i, transkip(LEFT, RIGHT), LOOKUP, LEFT OUTER);
-j55 := JOIN(ds1, ds8, LEFT.i = RIGHT.i, transkip(LEFT, RIGHT), ATMOST(1), LEFT OUTER);
-j56 := JOIN(ds1, ds8, LEFT.i = RIGHT.i, transkip(LEFT, RIGHT), FIRSTRIGHT);
-j57 := JOIN(ds1, ds8, LEFT.i = RIGHT.i, transkip(LEFT, RIGHT), KEEP(1), RIGHT OUTER);
-j58 := JOIN(ds1, ds8, (LEFT.i = RIGHT.i)and(RIGHT.id>'E'), transkip(LEFT, RIGHT), KEEP(2));
-
-sequential(
-    output(j1),
-    output(j2),
-    output(j3),
-    output(j4),
-    output(j5),
-    output(j6),
-    output(j7),
-    output(j8),
-    output(j9),
-    output(j10),
-    output(j11),
-    output(j12),
-    output(j13),
-    output(j14),
-    output(j15),
-    output(j16),
-    output(j17),
-    output(j18),
-    output(j19),
-    output(j20),
-    output(j21),
-    output(j22),
-    output(j23),
-    output(j24),
-    output(j25),
-    output(j26),
-    output(j27),
-    output(j29),
-    output(j30),
-    output(j31),
-    output(j32),
-    output(j33),
-    output(j34),
-    output(j35),
-    output(j36),
-    output(j37),
-    output(j38),
-    output(j39),
-    output(j40),
-    output(j41),
-    output(j42),
-    output(j43),
-    output(j44),
-    output(j45),
-    output(j46),
-    output(j47),
-    output(j48),
-    output(j49),
-    output(j50),
-    output(j51),
-    output(j52),
-    output(j53),
-    output(j54),
-    output(j55),
-    output(j56),
-    output(j57),
-    output(j58)
-);
-

testing/ecl/key/apply.xml

@@ -1,20 +0,0 @@
-<Dataset name='Result 2'>
- <Row><message>Begin Apply....</message></Row>
- <Row><message>0,</message></Row>
- <Row><message>0,</message></Row>
- <Row><message>1</message></Row>
- <Row><message>xx                  </message></Row>
- <Row><message>0,</message></Row>
- <Row><message>0,</message></Row>
- <Row><message>2</message></Row>
- <Row><message>xx                  </message></Row>
- <Row><message>0,</message></Row>
- <Row><message>0,</message></Row>
- <Row><message>3</message></Row>
- <Row><message>xx                  </message></Row>
- <Row><message>0,</message></Row>
- <Row><message>0,</message></Row>
- <Row><message>4</message></Row>
- <Row><message>xx                  </message></Row>
- <Row><message>...End Apply</message></Row>
-</Dataset>

testing/ecl/key/assert.xml

@@ -1,28 +0,0 @@
-<Error><source>user</source><line>26</line><code>100000</code><message>Assert (1 = 2) failed [val1 = 2]</message></Error>
-<Error><source>user</source><line>27</line><code>100000</code><message>Abc2</message></Error>
-<Error><source>user</source><line>28</line><code>100000</code><message>Assert (1 = 2) failed [1 = val4]</message></Error>
-<Error><source>user</source><line>29</line><code>100000</code><message>Abc3</message></Error>
-<Error><source>user</source><line>36</line><code>100000</code><message>Assert (1 = 2) failed [val1 = 2]</message></Error>
-<Error><source>user</source><line>37</line><code>100000</code><message>Abc5</message></Error>
-<Error><source>user</source><line>38</line><code>100000</code><message>Assert (1 = 2) failed [val1 = val4]</message></Error>
-<Error><source>user</source><line>39</line><code>100000</code><message>Abc6</message></Error>
-<Error><source>user</source><line>34</line><code>100000</code><message>Assert (2 = 1) failed [val1 = 1]</message></Error>
-<Error><source>user</source><line>35</line><code>100000</code><message>Abc4</message></Error>
-<Error><source>user</source><line>46</line><code>100000</code><message>Assert (1 = 2) failed [val1 = 2]</message></Error>
-<Error><source>user</source><line>47</line><code>100000</code><message>Abc8</message></Error>
-<Error><source>user</source><line>48</line><code>100000</code><message>Assert (1 = 2) failed [val1 = val4]</message></Error>
-<Error><source>user</source><line>49</line><code>100000</code><message>Abc9</message></Error>
-<Error><source>user</source><line>44</line><code>100000</code><message>Assert (2 = 1) failed [val1 = 1]</message></Error>
-<Error><source>user</source><line>45</line><code>100000</code><message>Abc7</message></Error>
-<Dataset name='Result 1'>
- <Row><val1>1</val1></Row>
- <Row><val1>2</val1></Row>
-</Dataset>
-<Dataset name='Result 2'>
- <Row><val1>1</val1></Row>
- <Row><val1>2</val1></Row>
-</Dataset>
-<Dataset name='Result 3'>
- <Row><val1>1</val1><text>One  </text></Row>
- <Row><val1>2</val1><text>Two  </text></Row>
-</Dataset>

testing/ecl/key/dbz2a.xml

@@ -1 +0,0 @@
-<Error><source>eclagent</source><code>-1</code><message>System error: -1: Division by zero</message></Error>

testing/ecl/key/dbz2b.xml

@@ -1 +0,0 @@
-<Error><source>eclagent</source><code>-1</code><message>System error: -1: Division by zero</message></Error>

testing/ecl/key/dbz2c.xml

@@ -1 +0,0 @@
-<Error><source>eclagent</source><code>-1</code><message>System error: -1: Division by zero</message></Error>

testing/ecl/key/failrestart.xml

@@ -1 +0,0 @@
-<Error><source>eclagent</source><code>0</code><message>Error: 0: Oh no!</message></Error>

testing/ecl/key/fileservice.xml

@@ -1,9 +0,0 @@
-<Dataset name='Result 1'>
-</Dataset>
-<Dataset name='Result 2'>
- <Row><name>fruit </name><blah>123</blah><value>apple    </value></Row>
- <Row><name>fruit </name><blah>246</blah><value>ford     </value></Row>
- <Row><name>os    </name><blah>680</blah><value>bsd      </value></Row>
- <Row><name>music </name><blah>369</blah><value>rhead    </value></Row>
- <Row><name>os    </name><blah>987</blah><value>os       </value></Row>
-</Dataset>

testing/ecl/key/issue10022.xml

@@ -1,13 +0,0 @@
-<Dataset name='Result 1'>
- <Row><line>!Hello</line></Row>
- <Row><line> me </line></Row>
- <Row><line>xx</line></Row>
- <Row><line>!end</line></Row>
- <Row><line></line></Row>
- <Row><line></line></Row>
- <Row><line>!Hello</line></Row>
- <Row><line>!end</line></Row>
- <Row><line></line></Row>
- <Row><line> me </line></Row>
- <Row><line></line></Row>
-</Dataset>

testing/ecl/key/optflag.xml

@@ -1,49 +0,0 @@
-<Dataset name='Result 1'>
-</Dataset>
-<Dataset name='Result 2'>
- <Row><Result_2>0</Result_2></Row>
-</Dataset>
-<Dataset name='Result 3'>
-</Dataset>
-<Dataset name='Result 4'>
- <Row><Result_4>0</Result_4></Row>
-</Dataset>
-<Dataset name='Result 5'>
-</Dataset>
-<Dataset name='Result 6'>
- <Row><Result_6>0</Result_6></Row>
-</Dataset>
-<Dataset name='Result 7'>
-</Dataset>
-<Dataset name='Result 8'>
- <Row><Result_8>0</Result_8></Row>
-</Dataset>
-<Dataset name='Result 9'>
-</Dataset>
-<Dataset name='Result 10'>
- <Row><Result_10>0</Result_10></Row>
-</Dataset>
-<Dataset name='Result 11'>
-</Dataset>
-<Dataset name='Result 12'>
- <Row><Result_12>0</Result_12></Row>
-</Dataset>
-<Dataset name='Result 13'>
-</Dataset>
-<Dataset name='Result 14'>
- <Row><Result_14>0</Result_14></Row>
-</Dataset>
-<Dataset name='Result 15'>
-</Dataset>
-<Dataset name='Result 16'>
- <Row><Result_16>0</Result_16></Row>
-</Dataset>
-<Dataset name='Result 17'>
- <Row><Result_17>          </Result_17></Row>
-</Dataset>
-<Dataset name='Result 18'>
-</Dataset>
-<Dataset name='Result 19'>
-</Dataset>
-<Dataset name='Result 20'>
-</Dataset>

testing/ecl/key/superfile1.xml

@@ -1,168 +0,0 @@
-<Dataset name='Result 1'>
-</Dataset>
-<Dataset name='Result 2'>
-</Dataset>
-<Dataset name='Result 3'>
-</Dataset>
-<Dataset name='Result 4'>
-</Dataset>
-<Dataset name='Result 5'>
-</Dataset>
-<Dataset name='Result 6'>
- <Row><i>1</i><id>A</id></Row>
- <Row><i>1</i><id>B</id></Row>
- <Row><i>1</i><id>C</id></Row>
- <Row><i>2</i><id>D</id></Row>
- <Row><i>2</i><id>E</id></Row>
- <Row><i>3</i><id>F</id></Row>
- <Row><i>3</i><id>G</id></Row>
- <Row><i>3</i><id>H</id></Row>
-</Dataset>
-<Dataset name='Result 7'>
- <Row><i>3</i><id>F</id></Row>
- <Row><i>3</i><id>G</id></Row>
- <Row><i>3</i><id>H</id></Row>
- <Row><i>2</i><id>D</id></Row>
- <Row><i>2</i><id>E</id></Row>
- <Row><i>1</i><id>A</id></Row>
- <Row><i>1</i><id>B</id></Row>
- <Row><i>1</i><id>C</id></Row>
-</Dataset>
-<Dataset name='Result 8'>
- <Row><Result_8>4</Result_8></Row>
-</Dataset>
-<Dataset name='Result 9'>
- <Row><Result_9>4</Result_9></Row>
-</Dataset>
-<Dataset name='Result 10'>
- <Row><Result_10>t1_subfile3</Result_10></Row>
-</Dataset>
-<Dataset name='Result 11'>
- <Row><Result_11>2</Result_11></Row>
-</Dataset>
-<Dataset name='Result 12'>
- <Row><Result_12>4</Result_12></Row>
-</Dataset>
-<Dataset name='Result 13'>
- <Row><Result_13>4</Result_13></Row>
-</Dataset>
-<Dataset name='Result 14'>
- <Row><i>1</i><id>A</id></Row>
- <Row><i>1</i><id>B</id></Row>
- <Row><i>1</i><id>C</id></Row>
- <Row><i>2</i><id>D</id></Row>
- <Row><i>2</i><id>E</id></Row>
- <Row><i>3</i><id>F</id></Row>
- <Row><i>3</i><id>G</id></Row>
- <Row><i>3</i><id>H</id></Row>
-</Dataset>
-<Dataset name='Result 15'>
- <Row><i>3</i><id>F</id></Row>
- <Row><i>3</i><id>G</id></Row>
- <Row><i>3</i><id>H</id></Row>
- <Row><i>2</i><id>D</id></Row>
- <Row><i>2</i><id>E</id></Row>
- <Row><i>1</i><id>A</id></Row>
- <Row><i>1</i><id>B</id></Row>
- <Row><i>1</i><id>C</id></Row>
-</Dataset>
-<Dataset name='Result 16'>
- <Row><Result_16>4</Result_16></Row>
-</Dataset>
-<Dataset name='Result 17'>
- <Row><Result_17>3</Result_17></Row>
-</Dataset>
-<Dataset name='Result 18'>
- <Row><i>3</i><id>F</id></Row>
- <Row><i>3</i><id>G</id></Row>
- <Row><i>3</i><id>H</id></Row>
- <Row><i>2</i><id>D</id></Row>
- <Row><i>2</i><id>E</id></Row>
- <Row><i>5</i><id>I</id></Row>
- <Row><i>5</i><id>J</id></Row>
- <Row><i>5</i><id>K</id></Row>
-</Dataset>
-<Dataset name='Result 19'>
- <Row><i>1</i><id>A</id></Row>
- <Row><i>1</i><id>B</id></Row>
- <Row><i>1</i><id>C</id></Row>
- <Row><i>3</i><id>F</id></Row>
- <Row><i>3</i><id>G</id></Row>
- <Row><i>3</i><id>H</id></Row>
-</Dataset>
-<Dataset name='Result 20'>
- <Row><i>1</i><id>A</id></Row>
- <Row><i>1</i><id>B</id></Row>
- <Row><i>1</i><id>C</id></Row>
- <Row><i>3</i><id>F</id></Row>
- <Row><i>3</i><id>G</id></Row>
- <Row><i>3</i><id>H</id></Row>
- <Row><i>3</i><id>F</id></Row>
- <Row><i>3</i><id>G</id></Row>
- <Row><i>3</i><id>H</id></Row>
- <Row><i>2</i><id>D</id></Row>
- <Row><i>2</i><id>E</id></Row>
- <Row><i>5</i><id>I</id></Row>
- <Row><i>5</i><id>J</id></Row>
- <Row><i>5</i><id>K</id></Row>
-</Dataset>
-<Dataset name='Result 21'>
- <Row><i>1</i><id>A</id></Row>
- <Row><i>1</i><id>B</id></Row>
- <Row><i>1</i><id>C</id></Row>
- <Row><i>3</i><id>F</id></Row>
- <Row><i>3</i><id>G</id></Row>
- <Row><i>3</i><id>H</id></Row>
-</Dataset>
-<Dataset name='Result 22'>
- <Row><i>1</i><id>A</id></Row>
- <Row><i>1</i><id>B</id></Row>
- <Row><i>1</i><id>C</id></Row>
- <Row><i>2</i><id>D</id></Row>
- <Row><i>2</i><id>E</id></Row>
- <Row><i>3</i><id>F</id></Row>
- <Row><i>3</i><id>G</id></Row>
- <Row><i>3</i><id>H</id></Row>
- <Row><i>1</i><id>A</id></Row>
- <Row><i>1</i><id>B</id></Row>
- <Row><i>1</i><id>C</id></Row>
- <Row><i>2</i><id>D</id></Row>
- <Row><i>2</i><id>E</id></Row>
- <Row><i>3</i><id>F</id></Row>
- <Row><i>3</i><id>G</id></Row>
- <Row><i>3</i><id>H</id></Row>
-</Dataset>
-<Dataset name='Result 23'>
- <Row><i>1</i><id>A</id></Row>
- <Row><i>1</i><id>B</id></Row>
- <Row><i>1</i><id>C</id></Row>
- <Row><i>2</i><id>D</id></Row>
- <Row><i>2</i><id>E</id></Row>
- <Row><i>3</i><id>F</id></Row>
- <Row><i>3</i><id>G</id></Row>
- <Row><i>3</i><id>H</id></Row>
-</Dataset>
-<Dataset name='Result 24'>
- <Row><Result_24>true</Result_24></Row>
-</Dataset>
-<Dataset name='Result 25'>
- <Row><Result_25>false</Result_25></Row>
-</Dataset>
-<Dataset name='Result 26'>
- <Row><Result_26>1</Result_26></Row>
-</Dataset>
-<Dataset name='Result 27'>
- <Row><Result_27>true</Result_27></Row>
-</Dataset>
-<Dataset name='Result 28'>
- <Row><Result_28>0</Result_28></Row>
-</Dataset>
-<Dataset name='Result 29'>
- <Row><Result_29>false</Result_29></Row>
-</Dataset>
-<Dataset name='Result 30'>
- <Row><Result_30>0</Result_30></Row>
-</Dataset>
-<Dataset name='Result 31'>
- <Row><Result_31>false</Result_31></Row>
-</Dataset>

testing/ecl/key/superfile10.xml

@@ -1,28 +0,0 @@
-<Dataset name='Result 1'>
-</Dataset>
-<Dataset name='Result 2'>
-</Dataset>
-<Dataset name='Result 3'>
- <Row><Result_3>true</Result_3></Row>
-</Dataset>
-<Dataset name='Result 4'>
- <Row><Result_4>true</Result_4></Row>
-</Dataset>
-<Dataset name='Result 5'>
- <Row><Result_5>false</Result_5></Row>
-</Dataset>
-<Dataset name='Result 6'>
- <Row><Result_6>true</Result_6></Row>
-</Dataset>
-<Dataset name='Result 7'>
- <Row><Result_7>false</Result_7></Row>
-</Dataset>
-<Dataset name='Result 8'>
- <Row><Result_8>true</Result_8></Row>
-</Dataset>
-<Dataset name='Result 9'>
- <Row><Result_9>false</Result_9></Row>
-</Dataset>
-<Dataset name='Result 10'>
- <Row><Result_10>false</Result_10></Row>
-</Dataset>

testing/ecl/key/superfile2.xml

@@ -1,25 +0,0 @@
-<Dataset name='Result 1'>
-</Dataset>
-<Dataset name='Result 2'>
- <Row><Result_2>false</Result_2></Row>
-</Dataset>
-<Dataset name='Result 3'>
- <Row><Result_3>false</Result_3></Row>
-</Dataset>
-<Dataset name='Result 4'>
-</Dataset>
-<Dataset name='Result 5'>
- <Row><Result_5>true</Result_5></Row>
-</Dataset>
-<Dataset name='Result 6'>
- <Row><Result_6>true</Result_6></Row>
-</Dataset>
-<Dataset name='Result 7'>
- <Row><Result_7>false</Result_7></Row>
-</Dataset>
-<Dataset name='Result 8'>
- <Row><Result_8>false</Result_8></Row>
-</Dataset>
-<Dataset name='Result 9'>
- <Row><Result_9>done</Result_9></Row>
-</Dataset>

testing/ecl/key/superfile6.xml

@@ -1 +0,0 @@
-<Error><source>eclagent</source><code>0</code><message>System error: 0: AddSuperFile: Adding super file t6::superfile to itself!</message></Error>

testing/ecl/key/superfile8.xml

@@ -1,53 +0,0 @@
-<Dataset name='Result 1'>
-</Dataset>
-<Dataset name='Result 2'>
-</Dataset>
-<Dataset name='Result 3'>
-</Dataset>
-<Dataset name='Result 4'>
-</Dataset>
-<Dataset name='Result 5'>
- <Row><Result_5>false</Result_5></Row>
-</Dataset>
-<Dataset name='Result 6'>
- <Row><Result_6>true</Result_6></Row>
-</Dataset>
-<Dataset name='Result 7'>
- <Row><Result_7>true</Result_7></Row>
-</Dataset>
-<Dataset name='Result 8'>
- <Row><Result_8>true</Result_8></Row>
-</Dataset>
-<Dataset name='Result 9'>
- <Row><Result_9>true</Result_9></Row>
-</Dataset>
-<Dataset name='Result 10'>
- <Row><Result_10>false</Result_10></Row>
-</Dataset>
-<Dataset name='Result 11'>
- <Row><Result_11>true</Result_11></Row>
-</Dataset>
-<Dataset name='Result 12'>
- <Row><Result_12>true</Result_12></Row>
-</Dataset>
-<Dataset name='Result 13'>
- <Row><Result_13>true</Result_13></Row>
-</Dataset>
-<Dataset name='Result 14'>
- <Row><Result_14>true</Result_14></Row>
-</Dataset>
-<Dataset name='Result 15'>
- <Row><Result_15>false</Result_15></Row>
-</Dataset>
-<Dataset name='Result 16'>
- <Row><Result_16>false</Result_16></Row>
-</Dataset>
-<Dataset name='Result 17'>
- <Row><Result_17>false</Result_17></Row>
-</Dataset>
-<Dataset name='Result 18'>
- <Row><Result_18>false</Result_18></Row>
-</Dataset>
-<Dataset name='Result 19'>
- <Row><Result_19>false</Result_19></Row>
-</Dataset>

testing/ecl/key/superfile9.xml

@@ -1,34 +0,0 @@
-<Dataset name='Result 1'>
-</Dataset>
-<Dataset name='Result 2'>
-</Dataset>
-<Dataset name='Result 3'>
- <Row><Result_3>true</Result_3></Row>
-</Dataset>
-<Dataset name='Result 4'>
- <Row><Result_4>true</Result_4></Row>
-</Dataset>
-<Dataset name='Result 5'>
- <Row><Result_5>false</Result_5></Row>
-</Dataset>
-<Dataset name='Result 6'>
- <Row><Result_6>false</Result_6></Row>
-</Dataset>
-<Dataset name='Result 7'>
- <Row><Result_7>false</Result_7></Row>
-</Dataset>
-<Dataset name='Result 8'>
- <Row><Result_8>true</Result_8></Row>
-</Dataset>
-<Dataset name='Result 9'>
- <Row><Result_9>true</Result_9></Row>
-</Dataset>
-<Dataset name='Result 10'>
- <Row><Result_10>false</Result_10></Row>
-</Dataset>
-<Dataset name='Result 11'>
- <Row><Result_11>false</Result_11></Row>
-</Dataset>
-<Dataset name='Result 12'>
- <Row><Result_12>true</Result_12></Row>
-</Dataset>

testing/ecl/key/update.xml

@@ -1,19 +0,0 @@
-<Dataset name='Result 1'>
-</Dataset>
-<Dataset name='Result 2'>
-</Dataset>
-<Dataset name='Result 3'>
-</Dataset>
-<Dataset name='Result 4'>
-</Dataset>
-<Dataset name='Result 5'>
-</Dataset>
-<Dataset name='Result 6'>
-</Dataset>
-<Dataset name='Result 7'>
-</Dataset>
-<Dataset name='Result 8'>
-</Dataset>
-<Dataset name='Result 16'>
- <Row><Result_16>non-update branch</Result_16></Row>
-</Dataset>

testing/ecl/setup/setup.ecl

@@ -186,294 +186,6 @@ createBookIndex := INDEX(allBooks, { string20 title := title }, { dataset(Serial
 
 BUILD(createBookIndex, overwrite);
 
-//******************************** Child query setup code ***********************
-
-udecimal8 baseDate := 20050101;
-
-rawHouse := dataset([
-    { '99 Maltings Road', 'SW1A0AA', 1720, 
-        [{ 'Gavin', 'Halliday', 19700101, 1000, 0,
-            [
-            { 'To kill a mocking bird', 'Harper Lee', 95},
-            { 'Clarion Language Manual', 'Richard Taylor', 1, 399.99 },
-            { 'The bible', 'Various', 98 },
-            { 'Compilers', 'Aho, Sethi, Ullman', 80 },
-            { 'Lord of the Rings', 'JRR Tolkien', 95 }
-            ]
-        },
-        { 'Abigail', 'Halliday', 20000101, 40, 0,
-            [
-            { 'The thinks you can think', 'Dr. Seuss', 90, 5.99 },
-            { 'Where is flop?', '', 85, 4.99 },
-            { 'David and Goliath', '', 20, 2.99 },
-            { 'The story of Jonah', '', 80, 6.99 }
-            ]
-        },
-        { 'Liz', 'Halliday', 19700909, 0, 0,
-            [
-            { 'The Life of Pi', 'Yan Martel', 90 },
-            { 'BNF', 'Various', 60 },
-            { 'The Third Policeman', 'Flann O\'Brien', 85 }
-            ]
-        }]
-    },
-    { 'Buckingham Palace', 'WC1', 1702,
-        [{'Elizabeth', 'Windsor', 19260421, 0, 0,
-            [
-            { 'The bible', 'Various', 93 },
-            { 'The Prince', 'Machiavelli', 40 },
-            { 'Atlas of the world', 'Times', 70 },
-            { 'Girl guide annual', 'Various', 50 },
-            { 'Rwandan war dances', 'Unknown', 30 }
-            ]
-        },
-        {'Philip', 'Windsor', 19210610, 0, 0,
-            [
-            { 'Greek tragedies', 'Various', 30 },
-            { 'Social etiquette', 'RU Correct', 10},
-            { 'The Rituals of Dinner: The Origins, Evolution, Eccentricities and the Meaning of Table Manners', 'Margaret Visser', 60 },
-            { 'The cat in the hat', 'Dr. Seuss', 85 }
-            ]
-        }]
-    },
-    { 'Bedrock', '', 0,
-        [{'Fred', 'Flintstone', 00000101, 0, 0, [] },
-        {'Wilma', 'Flintstone', 00020202, 0, 0, 
-            [
-            { 'Dinosaur stews', 'Trog', 55 }
-            ]
-        }]
-    },
-    { 'Wimpole Hall', 'AG1 6QT', 1203,
-        [{'John', 'Grabdale', 19361008, 0, 0,
-            [
-            { 'Pride and prejudice, 1st edition', 'Jane Austen', 95, 12000 },
-            { 'Mein Kampf, 1st edition', 'Adolph Hitler', 80, 10000 }
-            ]
-        }]
-    },
-    { '56 New Road', 'SG8 1S2', 2003,
-        [{'Brian', 'Jones', 19540206, 0, 0,
-            [
-            { 'All quiet on the western front', 'Erich Maria Remarque', 85, 4.99 },
-            { 'Fox in Socks', 'Dr. Seuss', 99, 4.99 }
-            ]
-        },
-        {'Julia', 'Jones', 19550312, 0, 0,
-            [
-            { 'Zen and the art of motorcyle maintenance', 'Robert Pirsig', 90, 7.99 },
-            { 'Where the wild things are', 'Maurice Sendak', 70, 4.79 },
-            { 'The bible', 'Various', 10 , 5.99 },
-            { 'The cat in the hat', 'Dr. Seuss', 80 }
-            ]
-        }]
-    }
-    ], sqHousePersonBookRec);
-
-
-//First reproject the datasets to 
-
-sqBookIdRec addIdToBook(sqBookRec l) := 
-            transform
-                self.id := 0;
-                self := l;
-            end;
-
-sqPersonBookIdRec addIdToPerson(sqPersonBookRec l) := 
-            transform
-                unsigned2 aage := if (l.dob < baseDate, (unsigned2)((baseDate - l.dob) / 10000), 0);
-                self.id := 0;
-                self.books := project(l.books, addIdToBook(LEFT));
-                self.aage := if (aage > 200, 99, aage);
-                self := l;
-            end;
-
-sqHousePersonBookIdRec addIdToHouse(sqHousePersonBookRec l) := 
-            transform
-                self.id := 0;
-                self.persons := project(l.persons, addIdToPerson(LEFT));
-                self := l;
-            end;
-
-
-projected := project(rawHouse, addIdToHouse(LEFT));
-
-//version 1 assign unique ids a really inefficient way...
-//doesn't actually work....
-
-sqBookIdRec setBookId(sqHousePersonBookIdRec lh, sqBookIdRec l, unsigned4 basebookid) := 
-            transform
-                unsigned maxbookid := max(lh.persons, max(lh.persons.books, id));
-                self.id := if(maxbookid=0, basebookid, maxbookid)+1;
-                self := l;
-            end;
-
-sqPersonBookIdRec setPersonId(sqHousePersonBookIdRec lh, sqPersonBookIdRec l, unsigned4 basepersonid, unsigned4 basebookid) := 
-            transform
-                unsigned4 maxpersonid := max(lh.persons, id);
-                self.id := if(maxpersonid=0, basepersonid, maxpersonid)+1;
-                self.books := project(l.books, setBookId(lh, LEFT, basebookid));
-                self := l;
-            end;
-
-sqHousePersonBookIdRec setHouseId(sqHousePersonBookIdRec l, sqHousePersonBookIdRec r, unsigned4 id) := 
-            transform
-                unsigned prevmaxpersonid := max(l.persons, id);
-                unsigned prevmaxbookid := max(l.persons, max(l.persons.books, id));
-                self.id := id;
-                self.persons := project(r.persons, setPersonId(r, LEFT, prevmaxpersonid, prevmaxbookid));
-                self := r;
-            end;
-
-
-final1 := iterate(projected, setHouseId(LEFT, RIGHT, counter));
-
-
-//------------------ Common extraction functions... ---------------
-
-sqHouseIdRec extractHouse(sqHousePersonBookIdRec l) :=
-            TRANSFORM
-                SELF := l;
-            END;
-
-sqPersonBookRelatedIdRec extractPersonBook(sqHousePersonBookIdRec l, sqPersonBookIdRec r) :=
-            TRANSFORM
-                SELF.houseid := l.id;
-                SELF := r;
-            END;
-
-sqPersonRelatedIdRec extractPerson(sqPersonBookRelatedIdRec l) :=
-            TRANSFORM
-                SELF := l;
-            END;
-
-sqBookRelatedIdRec extractBook(sqBookIdRec l, unsigned4 personid) :=
-            TRANSFORM
-                SELF.personid := personid;
-                SELF := l;
-            END;
-
-
-
-//------------------- Add Sequence numbers by normalized/project/denormalize
-
-//normalize, adding parent ids as we do it.  Once all normalized and sequenced then combine them back together
-
-DoAssignSeq(ds, o) := macro
-#uniquename (trans)
-typeof(ds) %trans%(ds l, unsigned c) := 
-        transform
-            self.id := c;
-            self := l;
-        end;
-o := sorted(project(ds, %trans%(LEFT, COUNTER)), id);
-endmacro;
-
-DoAssignSeq(projected, projectedSeq);
-normSeqHouse := project(projectedSeq, extractHouse(LEFT));
-normPersonBooks := normalize(projectedSeq, left.persons, extractPersonBook(LEFT, RIGHT));
-DoAssignSeq(normPersonBooks, normSeqPersonBooks);
-normSeqPerson := project(normSeqPersonBooks, extractPerson(LEFT));
-normBook := normalize(normSeqPersonBooks, count(left.books), extractBook(LEFT.books[COUNTER], LEFT.id));
-DoAssignSeq(normBook, normSeqBook);
-
-// finally denormalize by joining back together.
-
-sqPersonBookRelatedIdRec expandPerson(sqPersonRelatedIdRec l) :=
-        TRANSFORM
-            SELF := l;
-            SELF.books := [];
-        END;
-
-sqHousePersonBookIdRec expandHouse(sqHouseIdRec l) :=
-        TRANSFORM
-            SELF := l;
-            SELF.persons := [];
-        END;
-
-sqPersonBookRelatedIdRec combinePersonBook(sqPersonBookRelatedIdRec l, sqBookRelatedIdRec r) :=
-        TRANSFORM
-            SELF.books := l.books + row({r.id, r.name, r.author, r.rating100, r.price}, sqBookIdRec);
-            SELF := l;
-        END;
-
-sqHousePersonBookIdRec combineHousePerson(sqHousePersonBookIdRec l, sqPersonBookRelatedIdRec r) :=
-        TRANSFORM
-            SELF.persons := l.persons + row(r, sqPersonBookIdRec);
-            SELF := l;
-        END;
-
-normSeqHouseEx := project(normSeqHouse, expandHouse(LEFT));
-normSeqPersonEx := project(normSeqPerson, expandPerson(LEFT));
-normSeqPersonBook := denormalize(normSeqPersonEx, sorted(normSeqBook, personid), left.id = right.personid, combinePersonBook(left, right), local);
-final3 := denormalize(normSeqHouseEx, sorted(normSeqPersonBook, houseid), left.id = right.houseid, combineHousePerson(left, right), local);
-
-
-
-//------------ Now generate the different output files.... -----------------
-// Try and do everything as many different ways as possible...!
-
-final := final3;
-
-houseOut := project(final, extractHouse(LEFT));
-personBooks := normalize(final, left.persons, extractPersonBook(LEFT, RIGHT));
-personOut := project(personBooks, extractPerson(LEFT));
-bookOut := normalize(personBooks, count(left.books), extractBook(LEFT.books[COUNTER], LEFT.id));
-
-simplePersonBooks := project(personBooks, transform(sqSimplePersonBookRec, SELF := LEFT, SELF.limit.booklimit := LEFT.booklimit));
-
-output(final,, sqHousePersonBookName,overwrite);
-output(personBooks,, sqPersonBookName,overwrite);
-output(houseOut,,sqHouseName,overwrite);
-output(personOut,,sqPersonName,overwrite);
-output(bookOut,,sqBookName,overwrite);
-
-output(simplePersonBooks,, sqSimplePersonBookName,overwrite);
-buildindex(
-#if (useLocal=true)
-  DISTRIBUTE(sqSimplePersonBookDs, IF(surname > 'G', 0, 1)),
-#else
-  sqSimplePersonBookDs, 
-#end
-  { surname, forename, aage  }, { sqSimplePersonBookDs }, sqSimplePersonBookIndexName, overwrite
-#if (useLocal=true)
- , NOROOT
-#end
-);
-fileServices.AddFileRelationship( __nameof__(sqSimplePersonBookDs), sqSimplePersonBookIndexName, '', '', 'view', '1:1', false);
-fileServices.AddFileRelationship( __nameof__(sqSimplePersonBookDs), sqSimplePersonBookIndexName, '__fileposition__', 'filepos', 'link', '1:1', true);
-
-fileServices.AddFileRelationship( sqHouseName, sqPersonName, 'id', 'houseid', 'link', '1:M', false);
-fileServices.AddFileRelationship( sqPersonName, sqBookName, 'id', 'personid', 'link', '1:M', false);
-
-fileServices.AddFileRelationship( sqHouseName, sqHousePersonBookName, 'id', 'id', 'link', '1:1', false);
-fileServices.AddFileRelationship( sqHouseName, sqPersonBookName, 'id', 'houseid', 'link', '1:M', false);
-
-//Now build some indexes - with numeric fields in the key
-buildindex(sqHouseExDs, { id }, { addr, filepos }, sqHouseIndexName+'ID', overwrite);
-buildindex(sqPersonExDs, { id }, { filepos }, sqPersonIndexName+'ID', overwrite);
-buildindex(sqBookExDs, { id }, { filepos }, sqBookIndexName+'ID', overwrite);
-
-fileServices.AddFileRelationship( __nameof__(sqHouseExDs), sqHouseIndexName+'ID', '', '', 'view', '1:1', false);
-fileServices.AddFileRelationship( __nameof__(sqHouseExDs), sqHouseIndexName+'ID', '__fileposition__', 'filepos', 'link', '1:1', true);
-fileServices.AddFileRelationship( __nameof__(sqPersonExDs), sqPersonIndexName+'ID', '', '', 'view', '1:1', false);
-fileServices.AddFileRelationship( __nameof__(sqPersonExDs), sqPersonIndexName+'ID', '__fileposition__', 'filepos', 'link', '1:1', true);
-fileServices.AddFileRelationship( __nameof__(sqBookExDs), sqBookIndexName+'ID', '', '', 'view', '1:1', false);
-fileServices.AddFileRelationship( __nameof__(sqBookExDs), sqBookIndexName+'ID', '__fileposition__', 'filepos', 'link', '1:1', true);
-
-//Some more conventional indexes - some requiring a double lookup to resolve the payload
-buildindex(sqHouseExDs, { string40 addr := sqHouseExDs.addr, postcode }, { filepos }, sqHouseIndexName, overwrite);
-buildindex(sqPersonExDs, { string40 forename := sqPersonExDs.forename, string40 surname := sqPersonExDs.surname }, { id }, sqPersonIndexName, overwrite);
-buildindex(sqBookExDs, { string40 name := sqBookExDs.name, string40 author := sqBookExDs.author }, { id }, sqBookIndexName, overwrite);
-
-fileServices.AddFileRelationship( __nameof__(sqHouseExDs), sqHouseIndexName, '', '', 'view', '1:1', false);
-fileServices.AddFileRelationship( __nameof__(sqHouseExDs), sqHouseIndexName, '__fileposition__', 'filepos', 'link', '1:1', true);
-fileServices.AddFileRelationship( __nameof__(sqPersonExDs), sqPersonIndexName, '', '', 'view', '1:1', false);
-fileServices.AddFileRelationship( sqPersonIndexName+'ID', sqPersonIndexName, 'id', 'id', 'link', '1:1', true);
-fileServices.AddFileRelationship( __nameof__(sqBookExDs), sqBookIndexName, '', '', 'view', '1:1', false);
-fileServices.AddFileRelationship( sqBookIndexName+'ID', sqBookIndexName, 'id', 'id', 'link', '1:1', true);
-
-//Should try creating a dataset with a set of ids which are used as a link...  (e.g., bookids->bookfile)
 
 DG_MemFileRec t_u2(DG_MemFileRec l, integer c) := transform self.u2 := c-2; self := l; END;
 DG_MemFileRec t_u3(DG_MemFileRec l, integer c) := transform self.u3 := c-2; self := l; END;
@@ -508,7 +220,3 @@ DG_IntegerRecord createIntegerRecord(unsigned8 c) := transform
     SELF.i5 := c;
     SELF.i3 := c;
 END;
-
-singleNullRowDs := dataset([transform({unsigned1 i}, self.i := 0;)]);
-output(normalize(singleNullRowDs, 100, createIntegerRecord(counter)),,DG_IntegerDatasetName,overwrite);
-build(DG_IntegerIndex,overwrite);

testing/ecl/setup/setup.txt

@@ -226,33 +226,6 @@ SET OF STRING3 DG_MONTHS := ['JAN','FEB','MAR','APR','MAY','JUN','JUL','AUG',
                              'SEP','OCT','NOV','DEC','ABC','DEF','GHI','JKL'];
 
 
-t_personfile := DATASET('t_personfile', RECORD
-  unsigned integer4 hhid;
-  unsigned integer4 personid;
-  string20 firstname;
-  string20 lastname;
-  string20 middlename;
-  unsigned integer1 age;
-  unsigned integer8 ssn;
-END, THOR);
-
-t_tradesfile := DATASET('t_tradesfile', RECORD
-  unsigned integer4 personid;
-  string20 tradeid;
-  real4 amount;
-  string8 date;
-END, THOR);
-
-t_hhfile := DATASET('t_hhfile', RECORD
-  unsigned integer4 hhid;
-  string2 State;
-  string5 zip;
-  string20 City;
-  string40 street;
-  unsigned integer4 houseNumber;
-END, THOR);
-
-
 //----------------------------- Definitions for testing dictionary serialization -----------------
 
 serialTest := MODULE
@@ -298,214 +271,6 @@ serialTest := MODULE
     
 END; /* serial */
 
-//----------------------------- Child query related definitions ----------------------------------
-
-// Raw record definitions:
-
-sqHouseRec := 
-            record
-string          addr;
-string10        postcode;
-unsigned2       yearBuilt := 0;
-            end;
-
-
-sqPersonRec := 
-            record
-string          forename;
-string          surname;
-udecimal8       dob;
-udecimal8       booklimit := 0;
-unsigned2       aage := 0;
-            end;
-
-sqBookRec := 
-            record
-string          name;
-string          author;
-unsigned1       rating100;
-udecimal8_2     price := 0;
-            end;
-
-
-// Nested record definitions
-sqPersonBookRec :=
-            record
-sqPersonRec;
-dataset(sqBookRec)      books;
-            end;
-
-sqHousePersonBookRec :=
-            record
-sqHouseRec;
-dataset(sqPersonBookRec) persons;
-            end;
-
-
-// Record definitions with additional ids
-
-sqHouseIdRec :=
-            record
-unsigned4       id;
-sqHouseRec;
-            end;
-
-
-sqPersonIdRec :=
-            record
-unsigned4       id;
-sqPersonRec;
-            end;
-
-
-sqBookIdRec :=
-            record
-unsigned4       id;
-sqBookRec;
-            end;
-
-
-// Same with parent linking field.
-
-sqPersonRelatedIdRec :=
-            record
-sqPersonIdRec;
-unsigned4       houseid;
-            end;
-
-
-sqBookRelatedIdRec :=
-            record
-sqBookIdRec;
-unsigned4       personid;
-            end;
-
-
-// Nested definitions with additional ids...
-
-sqPersonBookIdRec :=
-            record
-sqPersonIdRec;
-dataset(sqBookIdRec)        books;
-            end;
-
-sqHousePersonBookIdRec :=
-            record
-sqHouseIdRec;
-dataset(sqPersonBookIdRec) persons;
-            end;
-
-
-sqPersonBookRelatedIdRec := 
-            RECORD
-                sqPersonBookIdRec;
-unsigned4       houseid;
-            END;
-
-sqNestedBlob := 
-            RECORD
-udecimal8       booklimit := 0;
-            END;
-
-sqSimplePersonBookRec := 
-            RECORD
-string20        surname;
-string10        forename;
-udecimal8       dob;
-//udecimal8     booklimit := 0;
-sqNestedBlob    limit{blob};
-unsigned1       aage := 0;
-dataset(sqBookIdRec)        books{blob};
-            END;
-sqNamePrefix := '~REGRESS::' + filePrefix + '::';
-sqHousePersonBookName := sqNamePrefix + 'HousePersonBook';
-sqPersonBookName := sqNamePrefix + 'PersonBook';
-sqHouseName := sqNamePrefix + 'House';
-sqPersonName := sqNamePrefix + 'Person';
-sqBookName := sqNamePrefix + 'Book';
-sqSimplePersonBookName := sqNamePrefix + 'SimplePersonBook';
-
-sqHousePersonBookIndexName := sqNamePrefix + 'HousePersonBookIndex';
-sqPersonBookIndexName := sqNamePrefix + 'PersonBookIndex';
-sqHouseIndexName := sqNamePrefix + 'HouseIndex';
-sqPersonIndexName := sqNamePrefix + 'PersonIndex';
-sqBookIndexName := sqNamePrefix + 'BookIndex';
-sqSimplePersonBookIndexName := sqNamePrefix + 'SimplePersonBookIndex';
-sqHousePersonBookIdExRec := record
-sqHousePersonBookIdRec;
-unsigned8           filepos{virtual(fileposition)};
-                end;
-
-sqPersonBookRelatedIdExRec := record
-sqPersonBookRelatedIdRec;
-unsigned8           filepos{virtual(fileposition)};
-                end;
-
-sqHouseIdExRec := record
-sqHouseIdRec;
-unsigned8           filepos{virtual(fileposition)};
-                end;
-
-sqPersonRelatedIdExRec := record
-sqPersonRelatedIdRec;
-unsigned8           filepos{virtual(fileposition)};
-                end;
-
-sqBookRelatedIdExRec := record
-sqBookRelatedIdRec;
-unsigned8           filepos{virtual(fileposition)};
-                end;
-
-sqSimplePersonBookExRec := record
-sqSimplePersonBookRec;
-unsigned8           filepos{virtual(fileposition)};
-                end;
-
-// Dataset definitions:
-
-
-sqHousePersonBookDs := dataset(sqHousePersonBookName, sqHousePersonBookIdExRec, thor);
-sqPersonBookDs := dataset(sqPersonBookName, sqPersonBookRelatedIdRec, thor);
-sqHouseDs := dataset(sqHouseName, sqHouseIdExRec, thor);
-sqPersonDs := dataset(sqPersonName, sqPersonRelatedIdRec, thor);
-sqBookDs := dataset(sqBookName, sqBookRelatedIdRec, thor);
-
-sqHousePersonBookExDs := dataset(sqHousePersonBookName, sqHousePersonBookIdExRec, thor);
-sqPersonBookExDs := dataset(sqPersonBookName, sqPersonBookRelatedIdExRec, thor);
-sqHouseExDs := dataset(sqHouseName, sqHouseIdExRec, thor);
-sqPersonExDs := dataset(sqPersonName, sqPersonRelatedIdExRec, thor);
-sqBookExDs := dataset(sqBookName, sqBookRelatedIdExRec, thor);
-
-sqSimplePersonBookDs := dataset(sqSimplePersonBookName, sqSimplePersonBookExRec, thor);
-sqSimplePersonBookIndex := index(sqSimplePersonBookDs, { surname, forename, aage  }, { sqSimplePersonBookDs }, sqSimplePersonBookIndexName);
-
-//related datasets:
-//Don't really work because inheritance structure isn't preserved.
-
-relatedBooks(sqPersonIdRec parentPerson) := sqBookDs(personid = parentPerson.id);
-relatedPersons(sqHouseIdRec parentHouse) := sqPersonDs(houseid = parentHouse.id);
-
-sqNamesTable1 := dataset(sqSimplePersonBookDs, sqSimplePersonBookName, FLAT);
-sqNamesTable2 := dataset(sqSimplePersonBookDs, sqSimplePersonBookName, FLAT);
-sqNamesTable3 := dataset(sqSimplePersonBookDs, sqSimplePersonBookName, FLAT);
-sqNamesTable4 := dataset(sqSimplePersonBookDs, sqSimplePersonBookName, FLAT);
-sqNamesTable5 := dataset(sqSimplePersonBookDs, sqSimplePersonBookName, FLAT);
-sqNamesTable6 := dataset(sqSimplePersonBookDs, sqSimplePersonBookName, FLAT);
-sqNamesTable7 := dataset(sqSimplePersonBookDs, sqSimplePersonBookName, FLAT);
-sqNamesTable8 := dataset(sqSimplePersonBookDs, sqSimplePersonBookName, FLAT);
-sqNamesTable9 := dataset(sqSimplePersonBookDs, sqSimplePersonBookName, FLAT);
-
-sqNamesIndex1 := index(sqSimplePersonBookIndex,sqSimplePersonBookIndexName);
-sqNamesIndex2 := index(sqSimplePersonBookIndex,sqSimplePersonBookIndexName);
-sqNamesIndex3 := index(sqSimplePersonBookIndex,sqSimplePersonBookIndexName);
-sqNamesIndex4 := index(sqSimplePersonBookIndex,sqSimplePersonBookIndexName);
-sqNamesIndex5 := index(sqSimplePersonBookIndex,sqSimplePersonBookIndexName);
-sqNamesIndex6 := index(sqSimplePersonBookIndex,sqSimplePersonBookIndexName);
-sqNamesIndex7 := index(sqSimplePersonBookIndex,sqSimplePersonBookIndexName);
-sqNamesIndex8 := index(sqSimplePersonBookIndex,sqSimplePersonBookIndexName);
-sqNamesIndex9 := index(sqSimplePersonBookIndex,sqSimplePersonBookIndexName);
-
-
 //----------------------------- Text search definitions ----------------------------------
 TS_MaxTerms             := 50;
 TS_MaxStages            := 50;
@@ -566,22 +331,4 @@ END;
 
 DG_MemFile := DATASET(DG_MemFileName,DG_MemFileRec,FLAT);
 
-
-//record structures
-DG_NestedIntegerRecord := RECORD
-  big_endian UNSIGNED4 i4;
-  big_endian UNSIGNED3 u3;
-END;
-
-DG_IntegerRecord := RECORD
-    INTEGER6    i6;
-    DG_NestedIntegerRecord nested;
-    integer5    i5;
-    integer3    i3;
-END;
-
-DG_IntegerDataset := DATASET(DG_IntegerDatasetName, DG_IntegerRecord, thor);
-DG_IntegerIndex := INDEX(DG_IntegerDataset, { i6, nested }, { DG_IntegerDataset }, DG_IntegerIndexName);
-
-
 #line(0)

testing/ecl/stepping7h.ecl

@@ -1,33 +0,0 @@
-/*##############################################################################
-
-    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
-
-    Licensed under the Apache License, Version 2.0 (the "License");
-    you may not use this file except in compliance with the License.
-    You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-    Unless required by applicable law or agreed to in writing, software
-    distributed under the License is distributed on an "AS IS" BASIS,
-    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-    See the License for the specific language governing permissions and
-    limitations under the License.
-############################################################################## */
-
-//UseStandardFiles
-//UseIndexes
-//varskip payload
-//varskip varload
-//varskip trans
-//varskip dynamic
-
-//Multi level smart stepping, with priorities in the correct order
-
-i1 := STEPPED(TS_searchIndex(kind=1 AND word='the'), doc, PRIORITY(3),HINT(maxseeklookahead(50)));
-i2 := STEPPED(TS_searchIndex(kind=1 AND word='walls'), doc, PRIORITY(2),HINT(maxseeklookahead(50)));
-i3 := STEPPED(TS_searchIndex(kind=1 AND word='jericho'), doc, PRIORITY(2),HINT(maxseeklookahead(50)));
-
-output(COUNT(TABLE(i1, { src := TS_docid2source(doc); UNSIGNED doc := TS_docid2doc(doc), cnt := COUNT(GROUP)},doc)));
-output(COUNT(TABLE(i2, { src := TS_docid2source(doc); UNSIGNED doc := TS_docid2doc(doc), cnt := COUNT(GROUP)},doc)));
-output(COUNT(TABLE(i3, { src := TS_docid2source(doc); UNSIGNED doc := TS_docid2doc(doc), cnt := COUNT(GROUP)},doc)));

testing/regress/README.rst

@@ -1,4 +1,4 @@
-Overview of Regression Suite usage (v:0.0.17)
+Overview of Regression Suite usage (v:0.0.21)
 ==============================================
 
 To use Regression Suite change directory to HPCC-Platform/testing/regress subdirectory.
@@ -20,6 +20,7 @@ Result:
 |                       [--timeout [TIMEOUT]] [--keyDir [KEYDIR]]
 |                       [--ignoreResult]
 |                       [-X name1=value1[,name2=value2...]]
+|                       [-f optionA=valueA[,optionB=valueB...]]
 |                       {list,setup,run,query} ...
 | 
 |       HPCC Platform Regression suite
@@ -45,6 +46,21 @@ Result:
 |            --ignoreResult, -i    completely ignore the result.
 |            -X name1=value1[,name2=value2...]
 |                                  sets the stored input value (stored('name')).
+|            -f optionA=valueA[,optionB=valueB...]
+|                                  set an ECL option (equivalent to #option).
+
+Important!
+    There is a bug in Python argparse library whichis impacts the quoted parameters. So either in -X or -f or both contains a value with space(s) inside then the whole argument should be put in double quote!
+
+    Example: We should pass these names values pairs to set stored input values:
+                param1 = 1
+                param2 = A string
+                param2 = Other string
+
+    The proper ecl-test command is:
+            ./ecl-test -X"param1=1,param2=A string,param3=Other String" ...
+
+    Same format should use for -f option(s) and values. This problem doesn't impact parameters are stored in ecl-test.json config file. (See 9.)
 
 
 Parameters of Regression Suite list sub-command:

testing/regress/ecl-test

@@ -31,7 +31,7 @@ from hpcc.util.ecl.file import ECLFile
 from hpcc.util.util import checkPqParam,  getVersionNumbers,  checkXParam
 from hpcc.common.error import Error
 
-prog_version = "0.0.20"
+prog_version = "0.0.21"
 
 # For coverage
 if ('coverage' in os.environ) and (os.environ['coverage'] == '1'):
@@ -172,6 +172,8 @@ class RegressMain:
                             action='store_true')
         parser.add_argument('-X', help="sets the stored input value (stored('name')).",
                             nargs=1, type=checkXParam,  default='None',  metavar="name1=value1[,name2=value2...]")
+        parser.add_argument('-f', help="set an ECL option (equivalent to #option).",
+                            nargs=1, type=checkXParam,  default='None',  metavar="optionA=valueA[,optionB=valueB...]")
 
         subparsers = parser.add_subparsers(help='sub-command help')
 

testing/regress/ecl/common/TextSearch.ecl

@@ -1839,13 +1839,20 @@ MaxResults := 10000;
 
 publicExports := MODULE
 
+    EXPORT getWordIndex(string source, boolean useLocal) := FUNCTION
+        Files := Setup.Files(source, useLocal);
+        RETURN Files.getWordIndex();
+    END;
+    
     EXPORT queryInputRecord := { string query{maxlength(2048)}; };
 
     EXPORT processedRecord := record(queryInputRecord)
         dataset(searchRecord) request{maxcount(MaxActions)};
         dataset(simpleUserOutputRecord) result{maxcount(MaxResults)};
     END;
-
+    
+    EXPORT GetSearchExecutor(dataset(TS.wordIndexRecord) wordIndex, unsigned4 internalFlags = 0) := SearchExecutor(wordIndex, internalFlags);
+    
     EXPORT processedRecord doBatchExecute(dataset(TS.wordIndexRecord) wordIndex, queryInputRecord l, boolean useLocal, unsigned4 internalFlags=0) := transform
         processed := queryProcessor(wordIndex, l.query, useLocal, internalFlags);
         self.request := processed.processed;
@@ -1861,8 +1868,7 @@ publicExports := MODULE
     end;
 
     EXPORT executeBatchAgainstWordIndex(DATASET(queryInputRecord) queries, boolean useLocal, string source, unsigned4 internalFlags=0) := FUNCTION
-        Files := Setup.Files(source);
-        wordIndex := index(TS.textSearchIndex, Files.NameWordIndex(useLocal));
+        wordIndex := getWordIndex(source, useLocal);
         p := project(nofold(queries), doBatchExecute(wordIndex, LEFT, useLocal, internalFlags));
         RETURN p;
     END;

testing/regress/ecl/common/dict15.ecl

@@ -0,0 +1,44 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//Find all anagrams of a word, that match the list of known words
+import $.^.setup.TS;
+
+export dict15(string searchWord, DICTIONARY({TS.wordType word}) knownWords) := FUNCTION
+
+    R := RECORD
+      STRING SoFar;
+      STRING Rest;
+    END;
+    
+    Initial := DATASET([{'',searchWord}],R);
+    
+    R Pluck1(DATASET(R) infile) := FUNCTION
+      R TakeOne(R le, UNSIGNED1 c) := TRANSFORM
+        SELF.SoFar := le.SoFar + le.Rest[c];
+        SELF.Rest := le.Rest[..c-1]+le.Rest[c+1..]; // Boundary Conditions handled automatically
+      END;
+      RETURN NORMALIZE(infile,LENGTH(LEFT.Rest),TakeOne(LEFT,COUNTER));
+    END;
+    
+    Processed := LOOP(Initial,LENGTH(searchWord),Pluck1(ROWS(LEFT)));
+    
+    Anagrams := TABLE(Processed, { string word := SoFar; });
+    
+    RETURN Anagrams(Word in knownWords);
+END;
+

testing/ecl/dict15a.ecl

@@ -15,20 +15,16 @@
     limitations under the License.
 ############################################################################## */
 
-//UseStandardFiles
 //Find all anagrams of a word, that match the list of known words
 
-allWordsDs := DEDUP(SORTED(TS_wordIndex), word);
+import $.^.setup.TS;
 
-knownWords := DICTIONARY(allWordsDs, { word });
+export dict15a(string searchWord, DICTIONARY({TS.wordType word}) knownWords) := FUNCTION
 
-string searchWord := 'gabs' : stored('word');
-
-R := RECORD
-  STRING Word;
-END;
-
-findAnagrams(string searchWord) := FUNCTION
+  R := RECORD
+    STRING Word;
+  END;
+    
   Initial := DATASET([{searchWord}],R);
     
   R Pluck1(DATASET(R) infile, unsigned4 numDone) := FUNCTION
@@ -43,5 +39,3 @@ findAnagrams(string searchWord) := FUNCTION
   RETURN Anagrams(Word in knownWords);
 END;
 
-OUTPUT(findAnagrams(searchWord));
-

testing/ecl/dict15b.ecl

@@ -15,18 +15,16 @@
     limitations under the License.
 ############################################################################## */
 
-//UseStandardFiles
 //Find all anagrams of a word, that match the list of known words
 
-allWordsDs := DEDUP(SORTED(TS_wordIndex(word[1]='t'), word), word);  // Restrict to words starting T for speed
+import $.^.setup.TS;
 
-knownWords := DICTIONARY(allWordsDs, { word });
+export dict15b(string searchWord, DICTIONARY({TS.wordType word}) knownWords) := FUNCTION
 
-R := RECORD
-  STRING Word;
-END;
+  R := RECORD
+    STRING Word;
+  END;
 
-findAnagrams(string searchWord) := FUNCTION
   trimmedWord := TRIM(searchWord);
   Initial := DATASET([{trimmedWord}],R);
     
@@ -42,10 +40,3 @@ findAnagrams(string searchWord) := FUNCTION
   uniqueAnagrams := DEDUP(anagrams, Word, HASH);
   RETURN uniqueAnagrams(Word in knownWords);
 END;
-
-shortWords := TABLE(allWordsDs, { Word })(LENGTH(TRIM(Word)) <= 5);
-
-//BUG: Without the NOFOLD the code generator merges the projects, and introduces an ambiguous dataset
-moreThanOne := NOFOLD(shortWords)(count(findAnagrams(Word))>1);
-
-OUTPUT(moreThanOne);

testing/ecl/dict15c.ecl

@@ -15,20 +15,16 @@
     limitations under the License.
 ############################################################################## */
 
-//UseStandardFiles
 //Find all anagrams of a word, that match the list of known words
 
-#option ('showMetaIngraph', true);
+import $.^.setup.TS;
 
-allWordsDs := DEDUP(TS_wordIndex, word, HASH);
+export dict15c(string searchWord, DICTIONARY({TS.wordType word}) knownWords) := FUNCTION
 
-knownWords := DICTIONARY(allWordsDs, { word });
-
-R := RECORD
-  STRING Word;
-END;
+  R := RECORD
+    STRING Word;
+  END;
 
-findAnagrams(string searchWord) := FUNCTION
   trimmedWord := TRIM(searchWord);
   Initial := DATASET([{trimmedWord}],R);
     
@@ -42,16 +38,5 @@ findAnagrams(string searchWord) := FUNCTION
   anagrams := LOOP(Initial,LENGTH(trimmedWord),Pluck1(ROWS(LEFT),COUNTER-1));
    
   uniqueAnagrams := DEDUP(anagrams, Word, HASH);
-   
   RETURN uniqueAnagrams(Word in knownWords);
 END;
-
-shortWords := TABLE(allWordsDs, { Word })(LENGTH(TRIM(Word)) <= 6); 
-
-//Find all words that have anagrams 
-//BUG: Without the NOFOLD the code generator merges the projects, and introduces an ambiguous dataset
-withAnagrams := TABLE(NOFOLD(shortWords), { word, anagrams := findAnagrams(Word) });
-
-moreThanOne := withAnagrams(count(anagrams)>1);
-
-OUTPUT(sort(moreThanOne, RECORD));

testing/regress/ecl/dict15_hthor.ecl

@@ -0,0 +1,28 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//Find all anagrams of a word, that match the list of known words
+import $.Common;
+import $.Common.TextSearch;
+
+wordIndex := TextSearch.getWordIndex('hthor', false);
+allWordsDs := DEDUP(SORTED(wordIndex), word);
+knownWords := DICTIONARY(allWordsDs, { word });
+
+string searchWord := 'gabs' : stored('word');
+
+OUTPUT(Common.Dict15(searchWord, knownWords));

testing/regress/ecl/dict15_thor.ecl

@@ -0,0 +1,28 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//Find all anagrams of a word, that match the list of known words
+import $.Common;
+import $.Common.TextSearch;
+
+wordIndex := TextSearch.getWordIndex('thorlcr', false);
+allWordsDs := DEDUP(SORTED(wordIndex), word);
+knownWords := DICTIONARY(allWordsDs, { word });
+
+string searchWord := 'gabs' : stored('word');
+
+OUTPUT(Common.Dict15(searchWord, knownWords));

testing/regress/ecl/dict15a_hthor.ecl

@@ -0,0 +1,28 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//Find all anagrams of a word, that match the list of known words
+import $.Common;
+import $.Common.TextSearch;
+
+wordIndex := TextSearch.getWordIndex('hthor', false);
+allWordsDs := DEDUP(SORTED(wordIndex), word);
+knownWords := DICTIONARY(allWordsDs, { word });
+
+string searchWord := 'gabs' : stored('word');
+
+OUTPUT(Common.Dict15a(searchWord, knownWords));

testing/regress/ecl/dict15a_thor.ecl

@@ -0,0 +1,28 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//Find all anagrams of a word, that match the list of known words
+import $.Common;
+import $.Common.TextSearch;
+
+wordIndex := TextSearch.getWordIndex('thorlcr', false);
+allWordsDs := DEDUP(SORTED(wordIndex), word);
+knownWords := DICTIONARY(allWordsDs, { word });
+
+string searchWord := 'gabs' : stored('word');
+
+OUTPUT(Common.Dict15a(searchWord, knownWords));

testing/regress/ecl/dict15b_hthor.ecl

@@ -0,0 +1,31 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2014 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//Find all anagrams of a word, that match the list of known words
+import $.Common;
+import $.Common.TextSearch;
+
+wordIndex := TextSearch.getWordIndex('hthor', false);
+allWordsDs := DEDUP(SORTED(wordIndex(word[1]='t')), word);
+knownWords := DICTIONARY(allWordsDs, { word });
+
+shortWords := TABLE(allWordsDs, { Word })(LENGTH(TRIM(Word)) <= 5);
+
+//BUG: Without the NOFOLD the code generator merges the projects, and introduces an ambiguous dataset
+moreThanOne := NOFOLD(shortWords)(count(Common.Dict15b(Word, knownWords))>1);
+
+OUTPUT(moreThanOne);

testing/regress/ecl/dict15b_thor.ecl

@@ -0,0 +1,31 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2014 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//Find all anagrams of a word, that match the list of known words
+import $.Common;
+import $.Common.TextSearch;
+
+wordIndex := TextSearch.getWordIndex('thorlcr', false);
+allWordsDs := DEDUP(SORTED(wordIndex(word[1]='t')), word);
+knownWords := DICTIONARY(allWordsDs, { word });
+
+shortWords := TABLE(allWordsDs, { Word })(LENGTH(TRIM(Word)) <= 5);
+
+//BUG: Without the NOFOLD the code generator merges the projects, and introduces an ambiguous dataset
+moreThanOne := NOFOLD(shortWords)(count(Common.Dict15b(Word, knownWords))>1);
+
+OUTPUT(moreThanOne);

testing/ecl/dict15.ecl

@@ -1,6 +1,6 @@
 /*##############################################################################
 
-    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+    HPCC SYSTEMS software Copyright (C) 2014 HPCC Systems.
 
     Licensed under the Apache License, Version 2.0 (the "License");
     you may not use this file except in compliance with the License.
@@ -15,32 +15,20 @@
     limitations under the License.
 ############################################################################## */
 
-//UseStandardFiles
 //Find all anagrams of a word, that match the list of known words
+import $.Common;
+import $.Common.TextSearch;
 
-allWordsDs := DEDUP(SORTED(TS_wordIndex), word);
-
+wordIndex := TextSearch.getWordIndex('hthor', false);
+allWordsDs := DEDUP(wordIndex, word, HASH);
 knownWords := DICTIONARY(allWordsDs, { word });
 
-string searchWord := 'gabs' : stored('word');
-
-R := RECORD
-  STRING SoFar;
-  STRING Rest;
-END;
-
-Initial := DATASET([{'',searchWord}],R);
-
-R Pluck1(DATASET(R) infile) := FUNCTION
-  R TakeOne(R le, UNSIGNED1 c) := TRANSFORM
-    SELF.SoFar := le.SoFar + le.Rest[c];
-    SELF.Rest := le.Rest[..c-1]+le.Rest[c+1..]; // Boundary Conditions handled automatically
-  END;
-  RETURN NORMALIZE(infile,LENGTH(LEFT.Rest),TakeOne(LEFT,COUNTER));
-END;
+shortWords := TABLE(allWordsDs, { Word })(LENGTH(TRIM(Word)) <= 6); 
 
-Processed := LOOP(Initial,LENGTH(searchWord),Pluck1(ROWS(LEFT)));
+//Find all words that have anagrams 
+//BUG: Without the NOFOLD the code generator merges the projects, and introduces an ambiguous dataset
+withAnagrams := TABLE(NOFOLD(shortWords), { word, anagrams := Common.Dict15c(Word, knownWords) });
 
-Anagrams := TABLE(Processed, { string word := SoFar; });
+moreThanOne := withAnagrams(count(anagrams)>1);
 
-OUTPUT(Anagrams(Word in knownWords));
+OUTPUT(sort(moreThanOne, RECORD));

testing/regress/ecl/dict15c_thor.ecl

@@ -0,0 +1,34 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2014 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//Find all anagrams of a word, that match the list of known words
+import $.Common;
+import $.Common.TextSearch;
+
+wordIndex := TextSearch.getWordIndex('thorlcr', false);
+allWordsDs := DEDUP(wordIndex, word, HASH);
+knownWords := DICTIONARY(allWordsDs, { word });
+
+shortWords := TABLE(allWordsDs, { Word })(LENGTH(TRIM(Word)) <= 6); 
+
+//Find all words that have anagrams 
+//BUG: Without the NOFOLD the code generator merges the projects, and introduces an ambiguous dataset
+withAnagrams := TABLE(NOFOLD(shortWords), { word, anagrams := Common.Dict15c(Word, knownWords) });
+
+moreThanOne := withAnagrams(count(anagrams)>1);
+
+OUTPUT(sort(moreThanOne, RECORD));

testing/ecl/fileservice.ecl

@@ -15,8 +15,6 @@
     limitations under the License.
 ############################################################################## */
 
-//noRoxie
-
 import Std.File;
 
 rec := RECORD

testing/regress/ecl/hthor/platform.xml

@@ -0,0 +1,10 @@
+<Dataset name='Result 1'>
+ <Row><Result_1>hthor</Result_1></Row>
+</Dataset>
+<Dataset name='Result 2'>
+ <Row><x>hthor1</x></Row>
+ <Row><x>hthor2</x></Row>
+ <Row><x>hthor3</x></Row>
+ <Row><x>hthor4</x></Row>
+ <Row><x>hthor5</x></Row>
+</Dataset>

testing/ecl/imgkey.ecl

@@ -61,9 +61,9 @@ d1 := dataset([
     {'id2', '20030910', 3, x'313233'}, 
     {'id2', '20030905', 3, x'333231'}], 
     rawLayout);
-output(d1,,'imgfile', overwrite);
+output(d1,,'~REGRESS::imgfile', overwrite);
 
-d := dataset('imgfile', rawLayout1, FLAT);
+d := dataset('~REGRESS::imgfile', rawLayout1, FLAT);
 i := index(d, keylayout, 'imgindex');
 
 rawtrim := table(d, { dl, date, unsigned2 seq:=0, unsigned2 num := 0, imgLength, _fpos});