Merge remote-tracking branch 'origin/candidate-3.8.x'

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

cmake_modules/dependencies/el6.cmake

@@ -1 +1 @@
-set ( CPACK_RPM_PACKAGE_REQUIRES "boost141-regex, openldap, libicu, m4, libtool, libxslt, libxml2, gcc-c++, openssh-server, openssh-clients, expect, libarchive")
+set ( CPACK_RPM_PACKAGE_REQUIRES "boost-regex, openldap, libicu, m4, libtool, libxslt, libxml2, gcc-c++, openssh-server, openssh-clients, expect, libarchive")

cmake_modules/dependencies/squeeze.cmake

@@ -1 +1 @@
-set ( CPACK_DEBIAN_PACKAGE_DEPENDS "libboost-regex1.42.0, libicu44, libxalan110, libxerces-c28, binutils, libldap-2.4-2, openssl, zlib1g, g++, openssh-client, openssh-server, expect, libarchive")
+set ( CPACK_DEBIAN_PACKAGE_DEPENDS "libboost-regex1.42.0, libicu44, libxalan110, libxerces-c28, binutils, libldap-2.4-2, openssl, zlib1g, g++, openssh-client, openssh-server, expect, libarchive1")

common/deftype/deftype.cpp

@@ -1673,6 +1673,9 @@ extern DEFTYPE_API ITypeInfo *makeUnicodeType(unsigned len, _ATOM locale)
 
 extern DEFTYPE_API ITypeInfo *makeVarUnicodeType(unsigned len, _ATOM locale)
 {
+    if(!locale)
+        locale = emptyAtom;
+
     CUnicodeTypeKey key;
     key.length = len;
     key.locale.set(locale);

common/workunit/workunit.cpp

@@ -4119,16 +4119,29 @@ IConstWUClusterInfo* getTargetClusterInfo(IPropertyTree *environment, IPropertyT
     return new CEnvironmentClusterInfo(clustname, prefix, querySetName, agent, thors, roxie);
 }
 
-IConstWUClusterInfo* getTargetClusterInfo(const char *clustname)
+IPropertyTree* getTopologyCluster(Owned<IRemoteConnection> &conn, const char *clustname)
 {
-    if (!clustname)
+    if (!clustname || !*clustname)
         return NULL;
-    Owned<IRemoteConnection> conn = querySDS().connect("Environment", myProcessSession(), RTM_LOCK_READ, SDS_LOCK_TIMEOUT);
+    conn.setown(querySDS().connect("Environment", myProcessSession(), RTM_LOCK_READ, SDS_LOCK_TIMEOUT));
     if (!conn)
         return NULL;
     StringBuffer xpath;
     xpath.appendf("Software/Topology/Cluster[@name=\"%s\"]", clustname);
-    Owned<IPropertyTree> cluster = conn->queryRoot()->getPropTree(xpath.str());
+    return conn->queryRoot()->getPropTree(xpath.str());
+}
+
+bool validateTargetClusterName(const char *clustname)
+{
+    Owned<IRemoteConnection> conn;
+    Owned<IPropertyTree> cluster = getTopologyCluster(conn, clustname);
+    return (cluster.get()!=NULL);
+}
+
+IConstWUClusterInfo* getTargetClusterInfo(const char *clustname)
+{
+    Owned<IRemoteConnection> conn;
+    Owned<IPropertyTree> cluster = getTopologyCluster(conn, clustname);
     if (!cluster)
         return NULL;
     return getTargetClusterInfo(conn->queryRoot(), cluster);

common/workunit/workunit.hpp

@@ -1142,6 +1142,7 @@ extern WORKUNIT_API StringBuffer &getClusterEclCCServerQueueName(StringBuffer &r
 extern WORKUNIT_API StringBuffer &getClusterEclServerQueueName(StringBuffer &ret, const char *cluster);
 extern WORKUNIT_API StringBuffer &getClusterEclAgentQueueName(StringBuffer &ret, const char *cluster);
 extern WORKUNIT_API IStringIterator *getTargetClusters(const char *processType, const char *processName);
+extern WORKUNIT_API bool validateTargetClusterName(const char *clustname);
 extern WORKUNIT_API IConstWUClusterInfo* getTargetClusterInfo(const char *clustname);
 typedef IArrayOf<IConstWUClusterInfo> CConstWUClusterInfoArray;
 extern WORKUNIT_API unsigned getEnvironmentClusterInfo(CConstWUClusterInfoArray &clusters);

docs/ECLPlayground/ECLPlay-Mods/ECL_Playground.xml

@@ -78,7 +78,7 @@
 
                   <mediaobject>
                     <imageobject>
-                      <imagedata fileref="../../images/ECLP_001.JPG" />
+                      <imagedata fileref="../../images/ECLP_001.jpg" />
                     </imageobject>
                   </mediaobject>
                 </figure></para>
@@ -103,7 +103,7 @@
 
             <mediaobject>
               <imageobject>
-                <imagedata fileref="../../images/ECLP_002d.JPG" />
+                <imagedata fileref="../../images/ECLP_002d.jpg" />
               </imageobject>
             </mediaobject>
           </figure></para>
@@ -122,7 +122,7 @@
 
             <mediaobject>
               <imageobject>
-                <imagedata fileref="../../images/ECLP_003b.JPG" />
+                <imagedata fileref="../../images/ECLP_003b.jpg" />
               </imageobject>
             </mediaobject>
           </figure>The selected code displays in the <emphasis>Editor
@@ -147,7 +147,7 @@
 
             <mediaobject>
               <imageobject>
-                <imagedata fileref="../../images/ECLP_005.JPG" />
+                <imagedata fileref="../../images/ECLP_005.jpg" />
               </imageobject>
             </mediaobject>
           </figure></para>
@@ -168,7 +168,7 @@
 
             <mediaobject>
               <imageobject>
-                <imagedata fileref="../../images/ECLP_004a.JPG" />
+                <imagedata fileref="../../images/ECLP_004a.jpg" />
               </imageobject>
             </mediaobject>
           </figure>The status area displays the job status. If a job fails,
@@ -189,7 +189,7 @@
 
               <mediaobject>
                 <imageobject>
-                  <imagedata fileref="../../images/ECLP_006a.JPG" />
+                  <imagedata fileref="../../images/ECLP_006a.jpg" />
                 </imageobject>
               </mediaobject>
             </figure></para>
@@ -212,7 +212,7 @@
 
                   <mediaobject>
                     <imageobject>
-                      <imagedata fileref="../../images/ECLP_007.JPG" />
+                      <imagedata fileref="../../images/ECLP_007.jpg" />
                     </imageobject>
                   </mediaobject>
                 </figure></para>
@@ -232,7 +232,7 @@
 
                   <mediaobject>
                     <imageobject>
-                      <imagedata fileref="../../images/ECLP_008.JPG" />
+                      <imagedata fileref="../../images/ECLP_008.jpg" />
                     </imageobject>
                   </mediaobject>
                 </figure></para>
@@ -247,7 +247,7 @@
 
                   <mediaobject>
                     <imageobject>
-                      <imagedata fileref="../../images/ECLP_009.JPG" />
+                      <imagedata fileref="../../images/ECLP_009.jpg" />
                     </imageobject>
                   </mediaobject>
                 </figure></para>

docs/HPCCClientTools/CT_Mods/CT_Comm_Line_DFU.xml

@@ -182,6 +182,41 @@
                   to split file parts to multiple target parts. If omitted,
                   the default is 0.</entry>
                 </row>
+
+                <row>
+                  <entry>compress</entry>
+
+                  <entry>Optional. A boolean flag (0 | 1) indicating whether
+                  to compress the target file.</entry>
+                </row>
+
+                <row>
+                  <entry>push</entry>
+
+                  <entry>Optional. A boolean flag (0 | 1) indicating whether
+                  to override push/pull default.</entry>
+                </row>
+
+                <row>
+                  <entry>encrypt=&lt;password&gt;</entry>
+
+                  <entry>Optional. Specifies to encrypt the target filename
+                  using the supplied password.</entry>
+                </row>
+
+                <row>
+                  <entry>decrypt=&lt;password&gt;</entry>
+
+                  <entry>Optional. Specifies to decrypt the source filename
+                  using the supplied password.</entry>
+                </row>
+
+                <row>
+                  <entry>jobname=&lt;jobname&gt;</entry>
+
+                  <entry>Specify a jobname for the DFU operation's
+                  workunit.</entry>
+                </row>
               </tbody>
             </tgroup>
           </informaltable>
@@ -213,8 +248,8 @@ replicate=1</programlisting>
 
                 <tbody>
                   <row>
-                    <entry><graphic xml:base="../../" fileref="images/caution.png"
-                    scale="noin" /></entry>
+                    <entry><graphic fileref="images/caution.png" scale="noin"
+                    xml:base="../../" /></entry>
 
                     <entry>We do not recommend storing your password in the
                     INI file (which is clear text). The password is included

ecl/hqlcpp/hqliproj.cpp

@@ -1448,66 +1448,64 @@ void ImplicitProjectTransformer::analyseExpr(IHqlExpression * expr)
             return;
         default:
             {
+                unsigned numArgs = expr->numChildren();
+                unsigned first = 0;
+                unsigned last = numArgs;
+                unsigned start = 0;
                 if (!expr->isAction() && !expr->isDataset() && !expr->isDatarow())
                 {
                     switch (op)
                     {
                     case NO_AGGREGATE:
-                    case no_call:
-                    case no_externalcall:
                     case no_createset:
+                        last = 1;
                         break;
                     case no_sizeof:
-                        //MORE: Is this the best way to handle this?
-                        allowActivity = false;
-                        Parent::analyseExpr(expr);
-                        allowActivity = true;
-                        return;
+                        last = 0;
+                        break;
                     default:
                         extra->kind = NonActivity;
-                        Parent::analyseExpr(expr);
-                        gatherFieldsUsed(expr, extra);
-                        return;
+                        break;
                     }
                 }
-
-                IHqlExpression * record = expr->queryRecord();
-                if (!record && expr->queryChild(0))
-                    record = expr->queryChild(0)->queryRecord();
-                if (!record || !isSensibleRecord(record))
-                    extra->preventOptimization();
-
-                unsigned first = getFirstActivityArgument(expr);
-                unsigned last = first + getNumActivityArguments(expr);
-                unsigned numArgs = expr->numChildren();
-                unsigned start = 0;
-                switch (expr->getOperator())
+                else
                 {
-                case no_dedup:
-                    if (dedupMatchesWholeRecord(expr))
+                    IHqlExpression * record = expr->queryRecord();
+                    if (!record && expr->queryChild(0))
+                        record = expr->queryChild(0)->queryRecord();
+                    if (!record || !isSensibleRecord(record))
                         extra->preventOptimization();
-                    break;
-                case no_process:
-                    extra->preventOptimization();
-                    break;
-                case no_executewhen:
-                    last = 1;
-                    break;
-                case no_newkeyindex:
-//              case no_dataset:
-                    //No point walking the transform for an index
-                    start = 3;
-                    numArgs = 4;
-                    break;
-                case no_compound_diskaggregate:
-                case no_compound_diskcount:
-                case no_compound_diskgroupaggregate:
-                case no_compound_indexaggregate:
-                case no_compound_indexcount:
-                case no_compound_indexgroupaggregate:
-                    //walk inside these... they're not compoundable, but they may be able to lose some fields from the transform.
-                    last = 1;
-                    break;
+
+                    first = getFirstActivityArgument(expr);
+                    last = first + getNumActivityArguments(expr);
+                    switch (expr->getOperator())
+                    {
+                    case no_dedup:
+                        if (dedupMatchesWholeRecord(expr))
+                            extra->preventOptimization();
+                        break;
+                    case no_process:
+                        extra->preventOptimization();
+                        break;
+                    case no_executewhen:
+                        last = 1;
+                        break;
+                    case no_newkeyindex:
+    //              case no_dataset:
+                        //No point walking the transform for an index
+                        start = 3;
+                        numArgs = 4;
+                        break;
+                    case no_compound_diskaggregate:
+                    case no_compound_diskcount:
+                    case no_compound_diskgroupaggregate:
+                    case no_compound_indexaggregate:
+                    case no_compound_indexcount:
+                    case no_compound_indexgroupaggregate:
+                        //walk inside these... they're not compoundable, but they may be able to lose some fields from the transform.
+                        last = 1;
+                        break;
+                    }
                 }
 
                 for (unsigned i =start; i < numArgs; i++)
@@ -1515,12 +1513,23 @@ void ImplicitProjectTransformer::analyseExpr(IHqlExpression * expr)
                     IHqlExpression * cur = expr->queryChild(i);
                     allowActivity = (i >= first) && (i < last);
                     analyseExpr(cur);
-                    if (allowActivity && !cur->isAction() && !cur->isAttribute())
+                    if (allowActivity)
                     {
-                        assertex(queryBodyExtra(cur)->activityKind() != NonActivity);
-                        connect(cur, expr);
+                        if (extra->kind == NonActivity)
+                        {
+                            ImplicitProjectInfo * childExtra = queryBodyExtra(cur);
+                            childExtra->preventOptimization();
+                        }
+                        else if (!cur->isAction() && !cur->isAttribute())
+                        {
+                            connect(cur, expr);
+                        }
                     }
                 }
+
+                if (extra->kind == NonActivity)
+                    gatherFieldsUsed(expr, extra);
+
                 allowActivity = true;
             }
         }

ecl/hqlcpp/hqlttcpp.cpp

@@ -949,6 +949,11 @@ YesNoOption HqlThorBoundaryTransformer::calcNormalizeThor(IHqlExpression * expr)
             {
             case no_fromxml:
                 return option;
+            case no_createrow:
+                //MORE: There are more cases that could be evaluated outside of thor, but playing safe.
+                if (expr->queryChild(0)->isConstant())
+                    return option;
+                break;
             }
             return OptionYes;
         case type_groupedtable:

ecl/regress/bug104762.ecl

@@ -0,0 +1,40 @@
+/*##############################################################################
+
+    Copyright (C) 2011 HPCC Systems.
+
+    All rights reserved. This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as
+    published by the Free Software Foundation, either version 3 of the
+    License, or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+############################################################################## */
+
+// ---- Start ECL code ----
+
+tempRec := record
+        unicode overflow   {maxlength(5000000)};
+end;
+tempVRec := record
+        varunicode overflow   {maxlength(5000000)};
+end;
+
+ds1 := dataset([{'This is some string'},
+                {''}], tempRec);
+ds2 := dataset([{'This is some string'},
+                {''}], tempVRec);
+
+
+output(ds1(overflow!=u''));     // fine
+output(ds1(overflow!=''));     // fine
+output(ds2(overflow!=u''));    // fine
+
+output(ds2(overflow!=''));  // EclServer Terminated Unexpectedly
+
+// ---- End ECL code ----

ecl/regress/rowcall.ecl

@@ -0,0 +1,34 @@
+/*##############################################################################
+
+    Copyright (C) 2011 HPCC Systems.
+
+    All rights reserved. This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as
+    published by the Free Software Foundation, either version 3 of the
+    License, or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+############################################################################## */
+
+
+r := { unsigned value1, unsigned value2 };
+makeRow(unsigned v1, unsigned v2) := ROW(TRANSFORM(r, SELF.value1 := v1; SELF.value2 := v2));
+
+x := SERVICE
+       dataset(r) myFunction(dataset(r) x) : entrypoint('doesnotexist');
+       integer myFunction2(dataset(r) x) : entrypoint('doesnotexist');
+   END;
+
+row1 := makeRow(1,2);
+
+output(x.myFunction(dataset(row1)));
+
+row2 := makeRow(5,6);
+
+output(x.myFunction2(dataset(row2)));

ecl/regress/rowcompare.ecl

@@ -0,0 +1,26 @@
+/*##############################################################################
+
+    Copyright (C) 2011 HPCC Systems.
+
+    All rights reserved. This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as
+    published by the Free Software Foundation, either version 3 of the
+    License, or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+############################################################################## */
+
+
+r := { unsigned value1, unsigned value2 };
+makeRow(unsigned v1, unsigned v2) := ROW(TRANSFORM(r, SELF.value1 := v1; SELF.value2 := v2));
+
+row1 := makeRow(1,2);
+row2 := makeRow(2,3);
+
+output(row1 = row2);

esp/bindings/http/platform/httpbinding.cpp

@@ -654,8 +654,6 @@ int EspHttpBinding::onGet(CHttpRequest* request, CHttpResponse* response)
             return onGetRoot(context, request, response);
         case sub_serv_config:
             return onGetConfig(context, request, response);
-        case sub_serv_relogin:
-            return onRelogin(context, request, response);
         case sub_serv_getversion:
             return onGetVersion(context, request, response, serviceName.str());
         case sub_serv_index:
@@ -1126,82 +1124,6 @@ int EspHttpBinding::onGetVersion(IEspContext &context, CHttpRequest* request, CH
     return 0;
 }
 
-int EspHttpBinding::onRelogin(IEspContext &context, CHttpRequest* request, CHttpResponse* response)
-{
-    StringBuffer action;
-    request->getParameter("action", action);
-
-    if(action.length() == 0 || stricmp(action.str(), "ok") == 0)
-    {
-        CEspCookie* logincookie = request->queryCookie("RELOGIN");
-        if(logincookie == NULL || stricmp(logincookie->getValue(), "1") == 0)
-        {
-            response->addCookie(new CEspCookie("RELOGIN", "0"));
-
-            StringBuffer content(
-            "<html xmlns=\"http://www.w3.org/1999/xhtml\">"
-                "<head>"
-                    "<meta http-equiv=\"Content-Type\" content=\"text/html; charset=utf-8\"/>"
-                    "<title>Enterprise Services Platform</title>"
-                "</head>"
-                "<body onLoad=\"location.href='relogin_?action=cancel'\">"
-                "</body>"
-            "</html>");
-
-            response->sendBasicChallenge(getChallengeRealm(), content.str());
-        }
-        else
-        {
-            response->addCookie(new CEspCookie("RELOGIN", "1"));
-            response->setContentType("text/html; charset=UTF-8");
-            StringBuffer content(
-            "<html xmlns=\"http://www.w3.org/1999/xhtml\">"
-                "<head>"
-                    "<meta http-equiv=\"Content-Type\" content=\"text/html; charset=utf-8\"/>"
-                    "<title>Enterprise Services Platform</title>"
-                "</head>"
-                "<body>"
-                "<br/><b>Relogin successful, you're now logged in as ");
-            content.append(context.queryUserId()).append(
-                "</b>"
-                "</body>"
-                "</html>");
-
-            response->setContent(content.str());
-            response->send();
-        }
-            
-        return 0;
-    }
-    else if(stricmp(action.str(), "cancel") == 0)
-    {
-        CEspCookie* logincookie = request->queryCookie("RELOGIN");
-        response->addCookie(new CEspCookie("RELOGIN", "1"));
-        response->setContentType("text/html; charset=UTF-8");
-        StringBuffer content(
-        "<html xmlns=\"http://www.w3.org/1999/xhtml\">"
-            "<head>"
-            "<script type='text/javascript'>"
-            "function closeWin() { top.opener=top; top.close(); }"
-            "</script>"
-                "<meta http-equiv=\"Content-Type\" content=\"text/html; charset=utf-8\"/>"
-                "<title>Enterprise Services Platform</title>"
-            "</head>"
-            "<body onload=\"javascript:closeWin();\">"
-                "<br/><b>Relogin canceled, you're now still logged in as ");
-        content.append(context.queryUserId()).append(
-            "</b>"
-            "</body>"
-        "</html>");
-
-        response->setContent(content.str());
-        response->send();
-        return 0;
-    }
-
-    return 0;
-}
-
 int EspHttpBinding::onGetXsd(IEspContext &context, CHttpRequest* request, CHttpResponse* response, const char *service, const char *method)
 {
     return getWsdlOrXsd(context,request,response,service,method,false);

esp/bindings/http/platform/httpbinding.hpp

@@ -91,7 +91,6 @@ interface IEspHttpBinding
     virtual int onGetContent(IEspContext &context, CHttpRequest* request, CHttpResponse* response, const char *serviceName, const char *methodName)=0;
     virtual int onGetWsdl(IEspContext &context, CHttpRequest* request, CHttpResponse* response, const char *serviceName, const char *methodName)=0;
     virtual int onGetXsd(IEspContext &context, CHttpRequest* request, CHttpResponse* response, const char *serviceName, const char *methodName)=0;
-    virtual int onRelogin(IEspContext &context, CHttpRequest* request, CHttpResponse* response)=0;
     virtual int onGetSoapBuilder(IEspContext &context, CHttpRequest* request, CHttpResponse* response,  const char *serv, const char *method)=0;
     virtual int onGetReqSampleXml(IEspContext &context, CHttpRequest* request, CHttpResponse* response, const char *serv, const char *method)=0;
     virtual int onGetRespSampleXml(IEspContext &context, CHttpRequest* request, CHttpResponse* response,    const char *serv, const char *method)=0;
@@ -222,7 +221,6 @@ public:
     virtual int onGetItext(IEspContext &context, CHttpRequest* request, CHttpResponse* response, const char *path);
     virtual int onGetIframe(IEspContext &context, CHttpRequest* request, CHttpResponse* response, const char *path);
     virtual int onGetContent(IEspContext &context, CHttpRequest* request, CHttpResponse* response, const char *serv, const char *method);
-    virtual int onRelogin(IEspContext &context, CHttpRequest* request, CHttpResponse* response);
     virtual int onGetSoapBuilder(IEspContext &context, CHttpRequest* request, CHttpResponse* response,  const char *serv, const char *method);
     
     virtual int onSoapRequest(CHttpRequest* request, CHttpResponse* response){return 0;}

esp/bindings/http/platform/httpservice.cpp

@@ -218,7 +218,6 @@ const char* getSubServiceDesc(sub_service stype)
     DEF_CASE(sub_serv_xsd)
     DEF_CASE(sub_serv_config)
     DEF_CASE(sub_serv_php)
-    DEF_CASE(sub_serv_relogin)
     DEF_CASE(sub_serv_getversion)
     DEF_CASE(sub_serv_reqsamplexml)
     DEF_CASE(sub_serv_respsamplexml)

esp/bindings/http/platform/httptransport.cpp

@@ -1507,8 +1507,6 @@ void CHttpRequest::parseEspPathInfo()
                     m_sstype=sub_serv_iframe;
                 else if (!stricmp(m_espMethodName.str(), "itext"))
                     m_sstype=sub_serv_itext;
-                else if (!stricmp(m_espMethodName.str(), "relogin_"))
-                    m_sstype=sub_serv_relogin;
                 else if (!stricmp(m_espMethodName.str(), "version_"))
                     m_sstype=sub_serv_getversion;
             }
@@ -2411,13 +2409,22 @@ bool CHttpResponse::handleExceptions(IXslProcessor *xslp, IMultiException *me, c
         text.append('\n');
         WARNLOG("Exception(s) in %s::%s - %s", serv, meth, text.str());
 
-        if (errorXslt)
+        bool returnXml = context->queryRequestParameters()->hasProp("rawxml_");
+        if (errorXslt || returnXml)
         {
             me->serialize(text.clear());
-            StringBuffer theOutput;
-            xslTransformHelper(xslp, text.str(), errorXslt, theOutput, context->queryXslParameters());
-            setContent(theOutput.str());
-            setContentType("text/html");
+            if (returnXml)
+            {
+                setContent(text.str());
+                setContentType("text/xml");
+            }
+            else
+            {
+                StringBuffer theOutput;
+                xslTransformHelper(xslp, text.str(), errorXslt, theOutput, context->queryXslParameters());
+                setContent(theOutput.str());
+                setContentType("text/html");
+            }
             send();
             return true;
         }

esp/bindings/http/platform/httptransport.ipp

@@ -296,7 +296,6 @@ typedef enum sub_service_
     sub_serv_xsd,
     sub_serv_config,
     sub_serv_php,
-    sub_serv_relogin,
     sub_serv_getversion,
     sub_serv_reqsamplexml,
     sub_serv_respsamplexml,

esp/eclwatch/ws_XSLT/dropzonefile.xslt

@@ -251,7 +251,7 @@
                                 <xsl:attribute name="title">
                                   <xsl:value-of select="Path"/>;<xsl:value-of select="Linux"/>
                                 </xsl:attribute>
-                                <xsl:if test="$netAddress0=NetAddress and $path0=Path">
+                                <xsl:if test="$netAddress0=NetAddress and ($path0=Path or $path0=concat(Path,'/') or $path0=concat(Path,'\\'))">
                                   <xsl:attribute name="selected">selected</xsl:attribute>
                                 </xsl:if>
                                 <xsl:value-of select="Computer"/>/<xsl:value-of select="Name"/>

esp/files/esp_app.html

@@ -425,17 +425,6 @@ function writeESPappname(appname)
                                     { text: "Refresh", onclick: { fn: refresh_main }}
                                 ]
                         ] }
-                    },
-                    {
-                        text: "<b>User</b>",
-                        submenu: {
-                            id: "usermenu",
-                            itemdata: [
-                                [
-                                    { text: "re-login", url: 'esp/relogin_?action=ok', target: "main"}
-                                ]
-                        ] }
-
                     }
                 ];
 

esp/files/esp_app_tree.html

@@ -444,17 +444,6 @@ function writeESPappname(appname)
                                     { text: "Refresh", onclick: { fn: refresh_main }}
                                 ]
                         ] }
-                    },
-                    {
-                        text: "<b>User</b>",
-                        submenu: {
-                            id: "usermenu",
-                            itemdata: [
-                                [
-                                    { text: "re-login", url: 'esp/relogin_?action=ok', target: "main"}
-                                ]
-                        ] }
-
                     }
                 ];
 

esp/files/scripts/configmgr/configmgr.js

@@ -1609,6 +1609,8 @@ function createMultiColTreeCtrlForComp(rows, compName, subRecordIndex) {
         lazyload: true
       });
 
+      top.document.ContextMenu = oContextMenu;
+
       oContextMenu.dt = dt;
       oContextMenu.subscribe("beforeShow", onContextMenuBeforeShow);
       dt.expandRecord = function(id) {

esp/files/scripts/configmgr/navtree.js

@@ -51,6 +51,9 @@
         invokeWiz = false;
       }
       document.getElementById('top1').style.display = 'none';
+      document.getElementById('top1').addEventListener("click", function() {
+            top.document.ContextMenu.clearContent()  } );
+
       getWaitDlg().show();
       var params = "queryType=customType::params=environment,laststarted,defenvfile,username,wizops";
 

esp/services/WsDeploy/WsDeployService.hpp

@@ -206,7 +206,7 @@ private:
             {
               bool bDoNotify = true;
 
-              if ( (diffIter->getFlags() == IDDIunchanged) || (pConfigFileObserver != NULL && (strcmp( pConfigFileObserver->getConfigFilePath(), diffIter->query().queryFilename() ) != 0)) )
+              if ( diffIter->getFlags() == IDDIunchanged || diffIter->query().queryFilename() == NULL || pConfigFileObserver == NULL || pConfigFileObserver->getConfigFilePath() == NULL || (strcmp( pConfigFileObserver->getConfigFilePath(), diffIter->query().queryFilename() ) != 0) )
               {
                 bDoNotify = false;
               }
@@ -448,7 +448,12 @@ public:
     };
     virtual const char* getConfigFilePath()
     {
-       return m_pFile->queryFilename();
+      if (m_pFile == NULL)
+      {
+        return NULL;
+      } 
+       
+      return m_pFile->queryFilename();
     };
     virtual void updateConfigFromFile();
     virtual bool deploy(IEspContext &context, IEspDeployRequest &req, IEspDeployResponse &resp);

esp/services/ecldirect/EclDirectService.cpp

@@ -68,6 +68,32 @@ EclDirectWUExceptions::EclDirectWUExceptions(IConstWorkUnit& cw)
     }
 }
 
+void CEclDirectEx::refreshValidClusters()
+{
+    validClusters.kill();
+    Owned<IStringIterator> it = getTargetClusters(NULL, NULL);
+    ForEach(*it)
+    {
+        SCMStringBuffer s;
+        IStringVal &val = it->str(s);
+        if (!validClusters.getValue(val.str()))
+            validClusters.setValue(val.str(), true);
+    }
+}
+
+bool CEclDirectEx::isValidCluster(const char *cluster)
+{
+    CriticalBlock block(crit);
+    if (validClusters.getValue(cluster))
+        return true;
+    if (validateTargetClusterName(cluster))
+    {
+        refreshValidClusters();
+        return true;
+    }
+    return false;
+}
+
 void CEclDirectEx::init(IPropertyTree *cfg, const char *process, const char *service)
 {
     StringBuffer xpath;
@@ -84,6 +110,8 @@ void CEclDirectEx::init(IPropertyTree *cfg, const char *process, const char *ser
 
     defaultWait = srvcfg->getPropInt("WuTimeout", 60000);
     deleteWorkunits = cfg->getPropBool("DeleteWorkUnits", false);
+
+    refreshValidClusters();
 }
 
 CEclDirectSoapBindingEx::CEclDirectSoapBindingEx(IPropertyTree* cfg, const char *binding, const char *process):CEclDirectSoapBinding(cfg, binding, process)
@@ -142,9 +170,15 @@ bool CEclDirectEx::onRunEcl(IEspContext &context, IEspRunEclRequest & req, IEspR
     workunit->setUser((user.length()) ? user.str() : "user");
 
     const char* clustername = req.getCluster();
-    if (!clustername || *clustername==0 || strieq(clustername, "default")) 
+    if (!clustername || !*clustername || strieq(clustername, "default"))
         clustername = defaultCluster.str();
 
+    if (!clustername || !*clustername)
+        throw MakeStringException(-1, "No Cluster Specified");
+
+    if (!isValidCluster(clustername))
+        throw MakeStringException(-1, "Invalid TargetCluster %s Specified", clustername);
+
     workunit->setClusterName(clustername);
     if (req.getLimitResults())
         workunit->setResultLimit(100);
@@ -215,6 +249,13 @@ bool CEclDirectEx::onRunEclEx(IEspContext &context, IEspRunEclExRequest & req, I
     const char* cluster = req.getCluster();
     if (!cluster || !*cluster || !stricmp(cluster, "default"))
         cluster = defaultCluster.str();
+
+    if (!cluster || !*cluster)
+        throw MakeStringException(-1, "No Cluster Specified");
+
+    if (!isValidCluster(cluster))
+        throw MakeStringException(-1, "Invalid TargetCluster %s Specified", cluster);
+
     workunit->setClusterName(cluster);
 
     const char* snapshot = req.getSnapshot();

esp/services/ecldirect/EclDirectService.hpp

@@ -27,6 +27,8 @@ private:
     StringBuffer defaultCluster;
     int defaultWait;
     bool deleteWorkunits;
+    BoolHash validClusters;
+    CriticalSection crit;
 
 public:
    IMPLEMENT_IINTERFACE;
@@ -34,6 +36,8 @@ public:
     CEclDirectEx() : defaultWait(0){}
 
     virtual void init(IPropertyTree *cfg, const char *process, const char *service);
+    void refreshValidClusters();
+    bool isValidCluster(const char *cluster);
 
     bool onRunEcl(IEspContext &context, IEspRunEclRequest &req, IEspRunEclResponse &resp);
     bool onRunEclEx(IEspContext &context, IEspRunEclExRequest &req, IEspRunEclExResponse &resp);

esp/services/ws_account/ws_accountService.hpp

@@ -51,97 +51,14 @@ public:
         {
             ensureNavLink(*folder, "My Account", "/Ws_Access/SecurityNotEnabled?form_", "My Account", NULL, NULL, 0, true);//Force the menu to use this setting
             ensureNavLink(*folder, "Change Password", "/Ws_Access/SecurityNotEnabled?form_", "Change Password", NULL, NULL, 0, true);//Force the menu to use this setting
-            if (!isFF)
-                ensureNavLink(*folder, "Relogin", "/Ws_Access/SecurityNotEnabled?form_", "Relogin", NULL, NULL, 0, true);//Force the menu to use this setting
-            else
-                ensureNavLink(*folder, "Relogin", "/Ws_Access/FirefoxNotSupport?form_", "Relogin", NULL, NULL, 0, true);//Force the menu to use this setting
         }
         else
         {
             ensureNavLink(*folder, "My Account", "/Ws_Account/MyAccount", "MyAccount", NULL, NULL, 0, true);//Force the menu to use this setting
             ensureNavLink(*folder, "Change Password", "/Ws_Account/UpdateUserInput", "Change Password", NULL, NULL, 0, true);//Force the menu to use this setting
-            if (!isFF)
-                ensureNavLink(*folder, "Relogin", "/Ws_Account/LogoutUser", "Relogin", NULL, NULL, 0, true);//Force the menu to use this setting
-            else
-                ensureNavLink(*folder, "Relogin", "/Ws_Access/FirefoxNotSupport?form_", "Relogin", NULL, NULL, 0, true);//Force the menu to use this setting
         }
 #endif
     }
-
-#ifdef _USE_OPENLDAP
-    int onGetInstantQuery(IEspContext &context, CHttpRequest* request, CHttpResponse* response, const char *service, const char *method)
-    {
-        if(!stricmp(method, "LogoutUser")||!stricmp(method, "LogoutUserRequest"))
-        {
-            CEspCookie* logincookie = request->queryCookie("RELOGIN");
-            if(logincookie == NULL || stricmp(logincookie->getValue(), "1") == 0)
-            {
-                response->addCookie(new CEspCookie("RELOGIN", "0"));
-
-                StringBuffer content(
-                "<html xmlns=\"http://www.w3.org/1999/xhtml\">"
-                    "<head>"
-                        "<meta http-equiv=\"Content-Type\" content=\"text/html; charset=utf-8\"/>"
-                        "<title>Enterprise Services Platform</title>"
-                    "</head>"
-                    "<body onLoad=\"location.href='/ws_account/LogoutUserCancel'\">"
-                    "</body>"
-                "</html>");
-
-                response->sendBasicChallenge("ESP", content.str());
-            }
-            else
-            {
-                response->addCookie(new CEspCookie("RELOGIN", "1"));
-                response->setContentType("text/html; charset=UTF-8");
-                StringBuffer content(
-                "<html xmlns=\"http://www.w3.org/1999/xhtml\">"
-                    "<head>"
-                        "<meta http-equiv=\"Content-Type\" content=\"text/html; charset=utf-8\"/>"
-                        "<title>Enterprise Services Platform</title>"
-                    "</head>"
-                    "<body>"
-                    "<br/><b>Relogin successful, you're now logged in as ");
-                content.append(context.queryUserId()).append(
-                    "</b>"
-                    "</body>"
-                    "</html>");
-
-                response->setContent(content.str());
-                response->send();
-            }
-
-            return 0;
-        }
-        else if(!stricmp(method, "LogoutUserCancel")||!stricmp(method, "LogoutUserRequest"))
-        {
-            CEspCookie* logincookie = request->queryCookie("RELOGIN");
-            response->addCookie(new CEspCookie("RELOGIN", "1"));
-            response->setContentType("text/html; charset=UTF-8");
-            StringBuffer content(
-            "<html xmlns=\"http://www.w3.org/1999/xhtml\">"
-                "<head>"
-                    "<meta http-equiv=\"Content-Type\" content=\"text/html; charset=utf-8\"/>"
-                    "<title>Enterprise Services Platform</title>"
-                    "<script type='text/javascript'>"
-                        "function closeWin() { top.opener=top; top.close(); }"
-                    "</script>"
-                "</head>"
-                "<body onload=\"javascript:closeWin();\">"
-                    "<br/><b>Relogin canceled, you're now still logged in as ");
-            content.append(context.queryUserId()).append(
-                "</b>"
-                "</body>"
-            "</html>");
-
-            response->setContent(content.str());
-            response->send();
-            return 0;
-        }
-        else
-            return Cws_accountSoapBinding::onGetInstantQuery(context, request, response, service, method);
-    }
-#endif
 };
 
 class Cws_accountEx : public Cws_account

esp/services/ws_ecl/ws_ecl_service.cpp

@@ -2381,34 +2381,6 @@ int CWsEclBinding::getWsEclExample(CHttpRequest* request, CHttpResponse* respons
     return 0;
 }
 
-int CWsEclBinding::onRelogin(IEspContext &context, CHttpRequest* request, CHttpResponse* response)
-{
-    const char* build_level = getBuildLevel();
-    if (!build_level || !*build_level || streq(build_level, "COMMUNITY") || 
-        strieq(wsecl->auth_method.sget(), "none") || strieq(wsecl->auth_method.sget(), "local"))
-    {
-        StringBuffer html;
-
-        html.append(
-          "<html>"
-            "<head><meta http-equiv=\"Content-Type\" content=\"text/html; charset=utf-8\" /><title>Advanced feature in Enterprise Edition</title></head>"
-            "<body>"
-              "<h3 style=\"text-align:centre;\">Advanced feature in the Enterprise Edition</h4>"
-              "<p style=\"text-align:centre;\">Support for this feature is coming soon. Further information can be found at ");
-        html.appendf("<a href=\"%s\" target=\"_blank\">%s</a>.", wsecl->portal_URL.sget(), wsecl->portal_URL.sget());
-        html.append(
-              "</p>"
-            "</body>"
-          "</html>");
-        response->setContent(html.str());
-        response->setContentType(HTTP_TYPE_TEXT_HTML_UTF8);
-        response->send();
-        return 0;
-    }
-    
-    return EspHttpBinding::onRelogin(context, request, response);
-}
-
 int CWsEclBinding::onGet(CHttpRequest* request, CHttpResponse* response)
 {
     Owned<IMultiException> me = MakeMultiException("WsEcl");

esp/services/ws_ecl/ws_ecl_service.hpp

@@ -192,7 +192,6 @@ public:
     void getWsEclJsonRequest(StringBuffer& soapmsg, IEspContext &context, CHttpRequest* request, WsEclWuInfo &wsinfo, const char *xmltype, const char *ns, unsigned flags);
     void getWsEclJsonResponse(StringBuffer& jsonmsg, IEspContext &context, CHttpRequest *request, const char *xml, WsEclWuInfo &wsinfo);
     
-    int onRelogin(IEspContext &context, CHttpRequest* request, CHttpResponse* response);
     void sendRoxieRequest(const char *process, StringBuffer &req, StringBuffer &resp, StringBuffer &status);
 };
 

esp/services/ws_workunits/ws_workunitsHelpers.hpp

@@ -134,7 +134,7 @@ public:
         Owned<IWorkUnitFactory> factory = getWorkUnitFactory(ctx.querySecManager(), ctx.queryUser());
         cw.setown(factory->openWorkUnit(wuid_, false));
         if(!cw)
-            throw MakeStringException(ECLWATCH_CANNOT_UPDATE_WORKUNIT,"Cannot open workunit %s.", wuid_);
+            throw MakeStringException(ECLWATCH_CANNOT_OPEN_WORKUNIT,"Cannot open workunit %s.", wuid_);
     }
 
     bool getResultViews(StringArray &resultViews, unsigned flags);

esp/services/ws_workunits/ws_workunitsService.cpp

@@ -533,6 +533,8 @@ void CWsWorkunitsEx::init(IPropertyTree *cfg, const char *process, const char *s
 
     DBGLOG("Initializing %s service [process = %s]", service, process);
 
+    refreshValidClusters();
+
     wuActionTable.setValue("delete", ActionDelete);
     wuActionTable.setValue("abort", ActionAbort);
     wuActionTable.setValue("pausenow", ActionPauseNow);
@@ -566,6 +568,32 @@ void CWsWorkunitsEx::init(IPropertyTree *cfg, const char *process, const char *s
     m_sched.start();
 }
 
+void CWsWorkunitsEx::refreshValidClusters()
+{
+    validClusters.kill();
+    Owned<IStringIterator> it = getTargetClusters(NULL, NULL);
+    ForEach(*it)
+    {
+        SCMStringBuffer s;
+        IStringVal &val = it->str(s);
+        if (!validClusters.getValue(val.str()))
+            validClusters.setValue(val.str(), true);
+    }
+}
+
+bool CWsWorkunitsEx::isValidCluster(const char *cluster)
+{
+    CriticalBlock block(crit);
+    if (validClusters.getValue(cluster))
+        return true;
+    if (validateTargetClusterName(cluster))
+    {
+        refreshValidClusters();
+        return true;
+    }
+    return false;
+}
+
 bool CWsWorkunitsEx::onWUCreate(IEspContext &context, IEspWUCreateRequest &req, IEspWUCreateResponse &resp)
 {
     try
@@ -981,6 +1009,8 @@ bool CWsWorkunitsEx::onWUSchedule(IEspContext &context, IEspWUScheduleRequest &r
         const char* cluster = req.getCluster();
         if (isEmpty(cluster))
              throw MakeStringException(ECLWATCH_INVALID_INPUT,"No Cluster defined.");
+        if (!isValidCluster(cluster))
+            throw MakeStringException(ECLWATCH_INVALID_CLUSTER_NAME, "Invalid cluster name: %s", cluster);
 
         Owned<IWorkUnitFactory> factory = getWorkUnitFactory(context.querySecManager(), context.queryUser());
         WorkunitUpdate wu(factory->updateWorkUnit(req.getWuid()));
@@ -1037,9 +1067,11 @@ bool CWsWorkunitsEx::onWUSubmit(IEspContext &context, IEspWUSubmitRequest &req,
             throw MakeStringException(ECLWATCH_INVALID_INPUT, "No workunit ID provided.");
 
         DBGLOG("Submit workunit: %s", req.getWuid());
-
-        if (isEmpty(req.getCluster()))
+        const char *cluster = req.getCluster();
+        if (isEmpty(cluster))
             throw MakeStringException(ECLWATCH_INVALID_INPUT,"No Cluster defined.");
+        if (!isValidCluster(cluster))
+            throw MakeStringException(ECLWATCH_INVALID_CLUSTER_NAME, "Invalid cluster name: %s", cluster);
 
         Owned<IWorkUnitFactory> factory = getWorkUnitFactory(context.querySecManager(), context.queryUser());
         Owned<IConstWorkUnit> cw = factory->openWorkUnit(req.getWuid(), false);
@@ -1052,10 +1084,10 @@ bool CWsWorkunitsEx::onWUSubmit(IEspContext &context, IEspWUSubmitRequest &req,
                 throw noteException(wu, MakeStringException(ECLWATCH_INVALID_INPUT,"Queryset and/or query not specified"));
             }
 
-            runWsWuQuery(context, cw, info.queryset.sget(), info.query.sget(), req.getCluster(), NULL);
+            runWsWuQuery(context, cw, info.queryset.sget(), info.query.sget(), cluster, NULL);
         }
         else
-            submitWsWorkunit(context, cw, req.getCluster(), req.getSnapshot(), req.getMaxRunTime(), true, false);
+            submitWsWorkunit(context, cw, cluster, req.getSnapshot(), req.getMaxRunTime(), true, false);
 
         if (req.getBlockTillFinishTimer() != 0)
             waitForWorkUnitToComplete(req.getWuid(), req.getBlockTillFinishTimer());
@@ -1073,21 +1105,25 @@ bool CWsWorkunitsEx::onWURun(IEspContext &context, IEspWURunRequest &req, IEspWU
 {
     try
     {
+        const char *cluster = req.getCluster();
+        if (notEmpty(cluster) && !isValidCluster(cluster))
+            throw MakeStringException(ECLWATCH_INVALID_CLUSTER_NAME, "Invalid cluster name: %s", cluster);
+
         const char *runWuid = req.getWuid();
         StringBuffer wuid;
 
         if (runWuid && *runWuid)
         {
             if (req.getCloneWorkunit())
-                runWsWorkunit(context, wuid, runWuid, req.getCluster(), req.getInput());
+                runWsWorkunit(context, wuid, runWuid, cluster, req.getInput());
             else
             {
-                submitWsWorkunit(context, runWuid, req.getCluster(), NULL, 0, false, true, req.getInput());
+                submitWsWorkunit(context, runWuid, cluster, NULL, 0, false, true, req.getInput());
                 wuid.set(runWuid);
             }
         }
         else if (notEmpty(req.getQuerySet()) && notEmpty(req.getQuery()))
-            runWsWuQuery(context, wuid, req.getQuerySet(), req.getQuery(), req.getCluster(), req.getInput());
+            runWsWuQuery(context, wuid, req.getQuerySet(), req.getQuery(), cluster, req.getInput());
         else
             throw MakeStringException(ECLWATCH_MISSING_PARAMS,"Workunit or Query required");
 
@@ -1487,32 +1523,32 @@ bool CWsWorkunitsEx::onWUInfo(IEspContext &context, IEspWUInfoRequest &req, IEsp
             getArchivedWUInfo(context, req, resp);
         else
         {
+            unsigned flags=0;
+            if (req.getTruncateEclTo64k())
+                flags|=WUINFO_TruncateEclTo64k;
+            if (req.getIncludeExceptions())
+                flags|=WUINFO_IncludeExceptions;
+            if (req.getIncludeGraphs())
+                flags|=WUINFO_IncludeGraphs;
+            if (req.getIncludeSourceFiles())
+                flags|=WUINFO_IncludeSourceFiles;
+            if (req.getIncludeResults())
+                flags|=WUINFO_IncludeResults;
+            if (req.getIncludeVariables())
+                flags|=WUINFO_IncludeVariables;
+            if (req.getIncludeTimers())
+                flags|=WUINFO_IncludeTimers;
+            if (req.getIncludeDebugValues())
+                flags|=WUINFO_IncludeDebugValues;
+            if (req.getIncludeApplicationValues())
+                flags|=WUINFO_IncludeApplicationValues;
+            if (req.getIncludeWorkflows())
+                flags|=WUINFO_IncludeWorkflows;
+            if (!req.getSuppressResultSchemas())
+                flags|=WUINFO_IncludeEclSchemas;
+
             try
             {
-                unsigned flags=0;
-                if (req.getTruncateEclTo64k())
-                    flags|=WUINFO_TruncateEclTo64k;
-                if (req.getIncludeExceptions())
-                    flags|=WUINFO_IncludeExceptions;
-                if (req.getIncludeGraphs())
-                    flags|=WUINFO_IncludeGraphs;
-                if (req.getIncludeSourceFiles())
-                    flags|=WUINFO_IncludeSourceFiles;
-                if (req.getIncludeResults())
-                    flags|=WUINFO_IncludeResults;
-                if (req.getIncludeVariables())
-                    flags|=WUINFO_IncludeVariables;
-                if (req.getIncludeTimers())
-                    flags|=WUINFO_IncludeTimers;
-                if (req.getIncludeDebugValues())
-                    flags|=WUINFO_IncludeDebugValues;
-                if (req.getIncludeApplicationValues())
-                    flags|=WUINFO_IncludeApplicationValues;
-                if (req.getIncludeWorkflows())
-                    flags|=WUINFO_IncludeWorkflows;
-                if (!req.getSuppressResultSchemas())
-                    flags|=WUINFO_IncludeEclSchemas;
-
                 WsWuInfo winfo(context, req.getWuid());
                 winfo.getInfo(resp.updateWorkunit(), flags);
 
@@ -1522,37 +1558,36 @@ bool CWsWorkunitsEx::onWUInfo(IEspContext &context, IEspWUInfoRequest &req, IEsp
                     winfo.getResultViews(views, WUINFO_IncludeResultsViewNames);
                     resp.setResultViews(views);
                 }
-
-                switch (resp.getWorkunit().getStateID())
-                {
-                    case WUStateCompiling:
-                    case WUStateCompiled:
-                    case WUStateScheduled:
-                    case WUStateSubmitted:
-                    case WUStateRunning:
-                    case WUStateAborting:
-                    case WUStateWait:
-                    case WUStateUploadingFiles:
-                    case WUStateDebugPaused:
-                    case WUStateDebugRunning:
-                        resp.setAutoRefresh(WUDETAILS_REFRESH_MINS);
-                        break;
-                    case WUStateBlocked:
-                        resp.setAutoRefresh(WUDETAILS_REFRESH_MINS*5);
-                        break;
-                }
-
-                resp.setCanCompile(notEmpty(context.queryUserId()));
-                if (context.getClientVersion() > 1.24 && notEmpty(req.getThorSlaveIP()))
-                    resp.setThorSlaveIP(req.getThorSlaveIP());
             }
             catch (IException *e)
             {
-                StringBuffer errMsg;
-                if (strnicmp(e->errorMessage(errMsg), "Could not open workunit", 23))
+                if (e->errorCode() != ECLWATCH_CANNOT_OPEN_WORKUNIT)
                     throw e;
                 getArchivedWUInfo(context, req, resp);
             }
+
+            switch (resp.getWorkunit().getStateID())
+            {
+                case WUStateCompiling:
+                case WUStateCompiled:
+                case WUStateScheduled:
+                case WUStateSubmitted:
+                case WUStateRunning:
+                case WUStateAborting:
+                case WUStateWait:
+                case WUStateUploadingFiles:
+                case WUStateDebugPaused:
+                case WUStateDebugRunning:
+                    resp.setAutoRefresh(WUDETAILS_REFRESH_MINS);
+                    break;
+                case WUStateBlocked:
+                    resp.setAutoRefresh(WUDETAILS_REFRESH_MINS*5);
+                    break;
+            }
+
+            resp.setCanCompile(notEmpty(context.queryUserId()));
+            if (context.getClientVersion() > 1.24 && notEmpty(req.getThorSlaveIP()))
+                resp.setThorSlaveIP(req.getThorSlaveIP());
         }
     }
     catch(IException* e)
@@ -3460,8 +3495,11 @@ void writeSharedObject(const char *srcpath, const MemoryBuffer &obj, const char
     io->write(0, obj.length(), obj.toByteArray());
 }
 
-void deploySharedObject(IEspContext &context, StringBuffer &wuid, const char *filename, const char *cluster, const char *name, const MemoryBuffer &obj, const char *dir, const char *xml)
+void CWsWorkunitsEx::deploySharedObject(IEspContext &context, StringBuffer &wuid, const char *filename, const char *cluster, const char *name, const MemoryBuffer &obj, const char *dir, const char *xml)
 {
+    if (notEmpty(cluster) && !isValidCluster(cluster))
+        throw MakeStringException(ECLWATCH_INVALID_CLUSTER_NAME, "Invalid cluster name: %s", cluster);
+
     StringBuffer dllpath, dllname;
     StringBuffer srcname(filename);
     if (!srcname.length())
@@ -3507,16 +3545,17 @@ void deploySharedObject(IEspContext &context, StringBuffer &wuid, const char *fi
     AuditSystemAccess(context.queryUserId(), true, "Updated %s", wuid.str());
 }
 
-void deploySharedObject(IEspContext &context, IEspWUDeployWorkunitRequest & req, IEspWUDeployWorkunitResponse & resp, const char *dir, const char *xml=NULL)
+void CWsWorkunitsEx::deploySharedObject(IEspContext &context, IEspWUDeployWorkunitRequest & req, IEspWUDeployWorkunitResponse & resp, const char *dir, const char *xml)
 {
     if (isEmpty(req.getFileName()))
        throw MakeStringException(ECLWATCH_INVALID_INPUT, "File name required when deploying a shared object.");
 
-    if (isEmpty(req.getCluster()))
+    const char *cluster = req.getCluster();
+    if (isEmpty(cluster))
        throw MakeStringException(ECLWATCH_INVALID_INPUT, "Cluster name required when deploying a shared object.");
 
     StringBuffer wuid;
-    deploySharedObject(context, wuid, req.getFileName(), req.getCluster(), req.getName(), req.getObject(), dir, xml);
+    deploySharedObject(context, wuid, req.getFileName(), cluster, req.getName(), req.getObject(), dir, xml);
 
     WsWuInfo winfo(context, wuid.str());
     winfo.getCommon(resp.updateWorkunit(), WUINFO_All);

esp/services/ws_workunits/ws_workunitsService.hpp

@@ -37,6 +37,10 @@ public:
         CWsWorkunits::setContainer(container);
         m_sched.setContainer(container);
     }
+    void refreshValidClusters();
+    bool isValidCluster(const char *cluster);
+    void deploySharedObject(IEspContext &context, IEspWUDeployWorkunitRequest & req, IEspWUDeployWorkunitResponse & resp, const char *dir, const char *xml=NULL);
+    void deploySharedObject(IEspContext &context, StringBuffer &wuid, const char *filename, const char *cluster, const char *name, const MemoryBuffer &obj, const char *dir, const char *xml=NULL);
 
     bool onWUQuery(IEspContext &context, IEspWUQueryRequest &req, IEspWUQueryResponse &resp);
     bool onWUPublishWorkunit(IEspContext &context, IEspWUPublishWorkunitRequest & req, IEspWUPublishWorkunitResponse & resp);
@@ -105,6 +109,8 @@ private:
     StringBuffer queryDirectory;
     Owned<DataCache> dataCache;
     Owned<ArchivedWuCache> archivedWuCache;
+    BoolHash validClusters;
+    CriticalSection crit;
     WUSchedule m_sched;
     unsigned short port;
 };
@@ -147,6 +153,4 @@ private:
     bool batchWatchFeaturesOnly;
 };
 
-void deploySharedObject(IEspContext &context, StringBuffer &newWuid, const char *filename, const char *cluster, const char *name, const MemoryBuffer &obj, const char *dir, const char *xml=NULL);
-
 #endif

esp/xslt/espheader.xsl

@@ -130,18 +130,6 @@
                     <xsl:text disable-output-escaping="yes">&amp;nbsp;</xsl:text>
                     <img border="0" src="files_/img/topurl.png" title="No Frames" width="13" height="15" style="cursor:pointer"
                     onclick="top.location.href=top.frames['main'].location.href"/>
-                    <xsl:if test="NoUser != '1'">
-                    <xsl:text disable-output-escaping="yes">&amp;nbsp;</xsl:text>
-                    <a href="relogin_?action=ok" onclick="return go('relogin_?action=ok'); return false;">
-                      <img border="0" src="files_/img/relogin.png" width="13" height="15">
-                        <xsl:attribute name="title">
-                          <xsl:text disable-output-escaping="yes">You are logged in as '</xsl:text>
-                          <xsl:value-of select="LoginId"/>
-                          <xsl:text disable-output-escaping="yes">'. Click here to log in as a different user.</xsl:text>
-                        </xsl:attribute>
-                      </img>
-                    </a>
-                    </xsl:if>
                     <noscript>
                       <span style="color:red;">
                         <small>JavaScript needs to be enabled for the Enterprise Services Platform to work correctly.</small>

plugins/dbconnectors/hpccjdbc/src/com/hpccsystems/jdbcdriver/ECLEngine.java

@@ -30,7 +30,7 @@ public class ECLEngine
 
 	private  		String		urlString;
 	private final 	String 		basicAuth;
-	private 	 	String 		eclqueryname;
+	private 	 	HPCCQuery 	hpccquery;
 	private final 	String 		datasetname;
 	private 		NodeList 	resultschema;
 	private final 	Properties 	props;
@@ -40,13 +40,15 @@ public class ECLEngine
 	private 		String 		indexToUseName;
 	private 		HashMap<String, String> 	eclEnteties;
 
-	public ECLEngine(String eclqueryname, String datasetname, Properties props)
+	public ECLEngine(SQLParser parser, HPCCDatabaseMetaData dbmetadata, Properties props, HPCCQuery query)
 	{
 		this.props = props;
-		this.datasetname = datasetname;
-		this.eclqueryname = eclqueryname;
+		this.dbMetadata = dbmetadata;
+		this.parser = parser;
 		basicAuth = props.getProperty("BasicAuth");
+		datasetname = null;
 		resultschema = null;
+		hpccquery = query;
 		eclEnteties = new HashMap<String, String>();
 	}
 
@@ -59,7 +61,7 @@ public class ECLEngine
 		this.indexToUseName = indextouse;
 		datasetname = null;
 		resultschema = null;
-		eclqueryname = null;
+		hpccquery = null;
 		eclEnteties = new HashMap<String, String>();
 	}
 
@@ -336,8 +338,7 @@ public class ECLEngine
 			}
 			case SQLParser.SQL_TYPE_CALL:
 			{
-				eclqueryname = HPCCJDBCUtils.handleQuotedString(parser.getStoredProcName());
-				if(!dbMetadata.eclQueryExists("", eclqueryname))
+				if(hpccquery == null)
 					throw new Exception("Invalid store procedure found");
 
 				return executeCall(null);
@@ -701,7 +702,11 @@ public class ECLEngine
 	{
 		try
 		{
-			urlString = "http://" + props.getProperty("WsECLAddress") + ":" + props.getProperty("WsECLPort") + "/WsEcl/submit/query/" + props.getProperty("Cluster") + "/" + eclqueryname + "/expanded";
+			urlString = "http://" + props.getProperty("WsECLAddress") + ":" +
+						props.getProperty("WsECLPort") +
+						"/WsEcl/submit/query/" +
+						hpccquery.getQuerySet() + "/" +
+						hpccquery.getName() + "/expanded";			
 			System.out.println("WSECL:executeCall: " + urlString);
 
 			// Construct data
@@ -709,7 +714,7 @@ public class ECLEngine
 			sb.append(URLEncoder.encode("submit_type_=xml", "UTF-8"));
 			sb.append("&").append(URLEncoder.encode("S1=Submit", "UTF-8"));
 
-			ArrayList<HPCCColumnMetaData> storeProcInParams = dbMetadata.getStoredProcInColumns("",eclqueryname);
+			ArrayList<HPCCColumnMetaData> storeProcInParams = hpccquery.getAllInFields();
 			String[] procInParamValues = parser.getStoredProcInParamVals();
 
 			for (int i = 0; i < procInParamValues.length; i++)

plugins/dbconnectors/hpccjdbc/src/com/hpccsystems/jdbcdriver/HPCCConnection.java

@@ -30,7 +30,6 @@ public class HPCCConnection implements Connection
     private HPCCDatabaseMetaData metadata;
     private Properties props;
     private String serverAddress;
-    //private String cluster;
     private Properties clientInfo;
 
     public HPCCConnection(Properties props)
@@ -49,6 +48,9 @@ public class HPCCConnection implements Connection
 		if (!this.props.containsKey("Cluster"))
 			this.props.setProperty("Cluster", "hthor");
 
+		if (!this.props.containsKey("QuerySet"))
+			this.props.setProperty("QuerySet", "hthor");
+
 		if (!this.props.containsKey("WsECLWatchAddress"))
 			this.props.setProperty("WsECLWatchAddress", serverAddress);
 
@@ -109,8 +111,11 @@ public class HPCCConnection implements Connection
 				+ props.getProperty("password");
 
        String basicAuth = "Basic " + HPCCJDBCUtils.Base64Encode(userPassword.getBytes(), false);
-
        this.props.put("BasicAuth", basicAuth);
+
+       if (!this.props.containsKey("LazyLoad"))
+			this.props.setProperty("LazyLoad", "True");
+
        metadata = new HPCCDatabaseMetaData(props);
 
        //TODO not doing anything w/ this yet, just exposing it to comply w/ API definition...

plugins/dbconnectors/hpccjdbc/src/com/hpccsystems/jdbcdriver/HPCCDriver.java

@@ -5,6 +5,7 @@ import java.sql.Driver;
 import java.sql.DriverManager;
 import java.sql.DriverPropertyInfo;
 import java.sql.PreparedStatement;
+import java.sql.ResultSet;
 import java.sql.ResultSetMetaData;
 import java.sql.SQLException;
 import java.util.Properties;
@@ -102,7 +103,9 @@ public class HPCCDriver implements Driver
 			Properties info = new Properties();
 			info.put("ServerAddress", "192.168.124.128");
 
-			//info.put("Cluster", "thor");
+			info.put("LazyLoad", "false");
+			info.put("Cluster", "myroxie");
+			info.put("QuerySet", "thor");
 			info.put("WsECLWatchPort", "8010");
 			info.put("EclResultLimit", "ALL");
 			info.put("WsECLPort", "8002");
@@ -135,8 +138,10 @@ public class HPCCDriver implements Driver
 			//"select MAX(firstname), lastname from tutorial::rp::tutorialperson  limit 1000"
 			//"select 1"
 			//"select count(persons.zip) as zipcount, persons.city as mycity from tutorial::rp::tutorialperson as persons where persons.city = 'ABBEVILLE' "
-				"select min(zip) as maxzip from tutorial::rp::tutorialperson as persons where persons.city = 'ABBEVILLE' "
+			//"select min(zip) as maxzip from tutorial::rp::tutorialperson as persons where persons.city = 'ABBEVILLE' "
 			//"select 1 as ONE"
+			"call myroxie::fetchpeoplebyzipservice(33445)"
+			//"call fetchpeoplebyzipservice(33445)"
 
 			//"select MIN(zip), city from tutorial::rp::tutorialperson where zip  > '33445'"
 
@@ -284,15 +289,19 @@ public class HPCCDriver implements Driver
 			//	System.out.println("   " + tables.getString("TABLE_NAME") + " Remarks: \'" + tables.getString("REMARKS")+"\'");
 		//	}
 
-
-			/*
 			ResultSet procs = conn.getMetaData().getProcedures(null, null, null);
 
 			System.out.println("procs found: ");
 			while (procs.next()) {
 				System.out.println("   " + procs.getString("PROCEDURE_NAME"));
 			}
-			*/
+
+			ResultSet procs2 = conn.getMetaData().getProcedures(null, null, null);
+
+			System.out.println("procs found: ");
+			while (procs2.next()) {
+				System.out.println("   " + procs2.getString("PROCEDURE_NAME"));
+			}
 			/*
 			ResultSet proccols = conn.getMetaData().getProcedureColumns(null, null, null, null);
 

plugins/dbconnectors/hpccjdbc/src/com/hpccsystems/jdbcdriver/HPCCQueries.java

@@ -6,30 +6,53 @@ import java.util.Properties;
 public class HPCCQueries {
 
 	private Properties queries;
-	private String clusterName;
 
-	public HPCCQueries(String cluster)
+	public HPCCQueries()
 	{
-		clusterName = cluster;
 		queries = new Properties();
 	}
 
-	public void put(String name, HPCCQuery query)
+	public void put(HPCCQuery query)
 	{
-		queries.put(name, query);
+		queries.put(query.getQuerySet()+"::"+query.getName(), query);
 	}
 
-	public String getClusterName()
-	{
-		return clusterName;
-	}
 	public Enumeration<Object> getQueries()
 	{
 		return queries.elements();
 	}
 
+	public HPCCQuery getQuerysetQuery(String eclqueryname)
+	{
+		String querysplit [] = eclqueryname.split("::");
+		if(querysplit.length > 2)
+		{
+			String name ="";
+			for (int i = 1; i < querysplit.length; i++)
+				name += "::" + querysplit[i];
+
+			return getQuery(querysplit[0], name);
+		}
+		else if (querysplit.length == 2)
+			return getQuery(querysplit[0], querysplit[1]);
+		else if (querysplit.length == 1)
+			return getQuery(querysplit[0]);
+		else
+			return null;
+	}
+
+	public HPCCQuery getQuery(String queryset, String eclqueryname)
+	{
+		return (HPCCQuery)queries.get(queryset+"::"+eclqueryname);
+	}
+
 	public HPCCQuery getQuery(String eclqueryname)
 	{
 		return (HPCCQuery)queries.get(eclqueryname);
 	}
+
+	public int getLength()
+	{
+		return queries.size();
+	}
 }

plugins/dbconnectors/hpccjdbc/src/com/hpccsystems/jdbcdriver/HPCCQuery.java

@@ -12,6 +12,7 @@ public class HPCCQuery
 	private String WUID;
 	private String DLL;
 	private String Alias;
+	private String QuerySet;
 
 	private boolean Suspended;
 	private List<String> ResultDatasets;
@@ -28,10 +29,11 @@ public class HPCCQuery
 		WUID = "";
 		Suspended = false;
 		Alias = "";
+		QuerySet = "";
 		ResultDatasets = new ArrayList<String>();
 		schema = new ArrayList<HPCCColumnMetaData>();
 		defaulttable = null;
-		defaultfields = new ArrayList<HPCCColumnMetaData>();;
+		defaultfields = new ArrayList<HPCCColumnMetaData>();
 	}
 
 	public String getAlias() {
@@ -174,7 +176,6 @@ public class HPCCQuery
 
 	public boolean containsField(String tablename, String fieldname)
 	{
-		//Iterator<EclColumnMetaData> it = schema.iterator();
 		Iterator<HPCCColumnMetaData> it = defaultfields.iterator();
 		while (it.hasNext())
 		{
@@ -250,4 +251,14 @@ public class HPCCQuery
 
 		return inparams;
 	}
+
+	public String getQuerySet()
+	{
+		return QuerySet;
+	}
+
+	public void setQueryset(String qsname)
+	{
+		QuerySet = qsname;
+	}
 }

plugins/dbconnectors/hpccjdbc/src/com/hpccsystems/jdbcdriver/HPCCResultSet.java

@@ -37,7 +37,7 @@ public class HPCCResultSet implements ResultSet
 	private List<List>				rows;
 	private int						index		= -1;
 	private HPCCResultSetMetadata	resultMetadata;
-	private HPCCDatabaseMetaData		dbMetadata;
+	private HPCCDatabaseMetaData	dbMetadata;
 	private Statement				statement;
 	private String					test_query	= "SELECT 1";
 	private String 					defaultEclQueryReturnDatasetName;
@@ -47,6 +47,7 @@ public class HPCCResultSet implements ResultSet
 	private static final int LeftMostKeyIndexPosition = 1;
 	private static final int NumberofColsKeyedInThisIndex = 2;
 	private static final int INDEXSCORECRITERIA = 3;
+
 	public HPCCResultSet(List recrows, ArrayList<HPCCColumnMetaData> metadatacols, String tablename) throws SQLException
 	{
 		resultMetadata = new HPCCResultSetMetadata(metadatacols, tablename);
@@ -54,6 +55,7 @@ public class HPCCResultSet implements ResultSet
 		lastResult = new Object();
 		warnings = new ArrayList<SQLWarning>();
 	}
+
 	public HPCCResultSet(Statement statement, String query, Map inParameters) throws SQLException
 	{
 		warnings = new ArrayList<SQLWarning>();
@@ -72,6 +74,7 @@ public class HPCCResultSet implements ResultSet
 			int sqlreqtype = parser.getSqlType();
 			//not sure this is actually needed...
 			parser.populateParametrizedExpressions(inParameters);
+			ECLEngine eclengine;
 			if(sqlreqtype == SQLParser.SQL_TYPE_SELECT)
 			{
 				String hpccfilename = HPCCJDBCUtils.handleQuotedString(parser.getTableName());
@@ -110,30 +113,34 @@ public class HPCCResultSet implements ResultSet
 				}
 				//columns are base 1 indexed
 				resultMetadata = new HPCCResultSetMetadata(expectedretcolumns, hpccfilename);
+				eclengine = new ECLEngine(parser, dbMetadata, connection.getProperties(), indextousename);
 			}
 			else if(sqlreqtype == SQLParser.SQL_TYPE_SELECTCONST)
 			{
 				expectedretcolumns = parser.getSelectColumns();
 				resultMetadata = new HPCCResultSetMetadata(expectedretcolumns, "Constants");
+				eclengine = new ECLEngine(parser, dbMetadata, connection.getProperties(), indextousename);
 			}
 			else if(sqlreqtype == SQLParser.SQL_TYPE_CALL)
 			{
 				ArrayList<HPCCColumnMetaData> storeProcInParams = new ArrayList();
-				String hpccQuery = HPCCJDBCUtils.handleQuotedString(parser.getStoredProcName());
-				if(!dbMetadata.eclQueryExists("", hpccQuery))
+				HPCCQuery hpccQuery = dbMetadata.getHpccQuery(HPCCJDBCUtils.handleQuotedString(parser.getStoredProcName()));
+				if (hpccQuery == null)
 					throw new Exception("Invalid store procedure found");
-				defaultEclQueryReturnDatasetName = dbMetadata.getdefaultECLQueryResultDatasetName("", hpccQuery);
-				expectedretcolumns = dbMetadata.getStoredProcOutColumns("", hpccQuery);
-				storeProcInParams = dbMetadata.getStoredProcInColumns("",hpccQuery);
+				defaultEclQueryReturnDatasetName = hpccQuery.getDefaultTableName();
+				expectedretcolumns = hpccQuery.getAllNonInFields();
+				storeProcInParams = hpccQuery.getAllInFields();
+
 				//columns are base 1 indexed
-				resultMetadata = new HPCCResultSetMetadata(expectedretcolumns, hpccQuery);
+				resultMetadata = new HPCCResultSetMetadata(expectedretcolumns, hpccQuery.getName());
+				eclengine = new ECLEngine(parser, dbMetadata, connection.getProperties(), hpccQuery);
 			}
 			else
 			{
 				throw new SQLException("SQL request type not determined");
 			}
-			ECLEngine eclengine = new ECLEngine(parser, dbMetadata, connection.getProperties(), indextousename);
 			dsList = eclengine.execute();
+
 			// Get the data
 			fetchData(dsList,expectedretcolumns);
 			return;
@@ -155,7 +162,6 @@ public class HPCCResultSet implements ResultSet
 			ArrayList inRowList = (ArrayList) dsList.get(i);
 			for (int j = 0; j < inRowList.size(); j++)
 			{
-				//ArrayList rowValues = new ArrayList();
 				//create row with default values b/c HPCC will not return a column for empty fields
 				ArrayList rowValues = resultMetadata.createDefaultResultRow();
 				rows.add(rowValues);
@@ -228,7 +234,6 @@ public class HPCCResultSet implements ResultSet
 		}
 		for (int i = 0; i<relindexes.size(); i++)
 		{
-			//if (indexscore[i][NumberOfCommonParamInThisIndex] == 0 || indexscore[i][NumberofColsKeyedInThisIndex] == 0) //does one imply the other?
 			if (indexscore[i][NumberofColsKeyedInThisIndex] == 0) //does one imply the other?
 				continue; //not good enough
 			if (payloadIdxWithAtLeast1KeyedFieldFound && indexscore[i][NumberOfCommonParamInThisIndex]<expectedretcolumns.size())

plugins/dbconnectors/hpccjdbc/src/com/hpccsystems/jdbcdriver/HPCCResultSetMetadata.java

@@ -98,7 +98,6 @@ public class HPCCResultSetMetadata implements ResultSetMetaData{
         return tableName;
     }
 
-
     public String getCatalogName(int column) throws SQLException {
         return catalogName;
     }

roxie/roxiemem/roxiemem.cpp

@@ -1633,6 +1633,29 @@ public:
         callbackRanges.append(rowBufferCallbacks.ordinality());
     }
 
+    bool callbackReleasesRows(IBufferedRowCallback * callback, bool critical)
+    {
+        //If already processing this callback then don't call it again
+        if (activeCallbacks.contains(callback))
+        {
+            DBGLOG("RoxieMemMgr: allocation in IBufferedRowCallback[%p]::freeBufferedRows() requested new page", callback);
+            return false;
+        }
+
+        activeCallbacks.append(callback);
+        try
+        {
+            bool ok = callback->freeBufferedRows(critical);
+            activeCallbacks.pop();
+            return ok;
+        }
+        catch (...)
+        {
+            activeCallbacks.pop();
+            throw;
+        }
+    }
+
     bool doReleaseBuffers(const bool critical, const unsigned minSuccess)
     {
         const unsigned numCallbacks = rowBufferCallbacks.ordinality();
@@ -1656,7 +1679,8 @@ public:
                 if (next == last)
                     next = first;
 
-                if (curCallback->freeBufferedRows(critical))
+
+                if (callbackReleasesRows(curCallback, critical))
                 {
                     if (++numSuccess >= minSuccess)
                     {
@@ -1744,6 +1768,7 @@ protected:
     CriticalSection callbackCrit;
     Semaphore releaseBuffersSem;
     PointerArrayOf<IBufferedRowCallback> rowBufferCallbacks;
+    PointerArrayOf<IBufferedRowCallback> activeCallbacks;
     Owned<ReleaseBufferThread> releaseBuffersThread;
     UnsignedArray callbackRanges;  // the maximum index of the callbacks for the nth priority
     UnsignedArray nextCallbacks;  // the next call back to try and free for the nth priority
@@ -3002,6 +3027,74 @@ protected:
     unsigned priority;
 };
 
+//A buffered row class - used for testing
+class CallbackBlockAllocator : implements IBufferedRowCallback
+{
+public:
+    CallbackBlockAllocator(IRowManager * _rowManager, unsigned _size, unsigned _priority) : priority(_priority), rowManager(_rowManager), size(_size)
+    {
+        rowManager->addRowBuffer(this);
+    }
+    ~CallbackBlockAllocator()
+    {
+        rowManager->removeRowBuffer(this);
+    }
+
+//interface IBufferedRowCallback
+    virtual unsigned getPriority() const { return priority; }
+
+    void allocate()
+    {
+        row.setown(rowManager->allocate(size, 0));
+    }
+
+protected:
+    OwnedRoxieRow row;
+    IRowManager * rowManager;
+    unsigned size;
+    unsigned priority;
+};
+
+
+//Free the block as soon as requested
+class SimpleCallbackBlockAllocator : public CallbackBlockAllocator
+{
+public:
+    SimpleCallbackBlockAllocator(IRowManager * _rowManager, unsigned _size, unsigned _priority)
+        : CallbackBlockAllocator(_rowManager, _size, _priority)
+    {
+    }
+
+    virtual bool freeBufferedRows(bool critical)
+    {
+        if (!row)
+            return false;
+        row.clear();
+        return true;
+    }
+};
+
+//Allocate another row before disposing of the first
+class NastyCallbackBlockAllocator : public CallbackBlockAllocator
+{
+public:
+    NastyCallbackBlockAllocator(IRowManager * _rowManager, unsigned _size, unsigned _priority)
+        : CallbackBlockAllocator(_rowManager, _size, _priority)
+    {
+    }
+
+    virtual bool freeBufferedRows(bool critical)
+    {
+        if (!row)
+            return false;
+
+        OwnedRoxieRow tempRow = rowManager->allocate(size, 0);
+        row.clear();
+        return true;
+    }
+};
+
+
 class IStdException : extends std::exception
 {
     Owned<IException> jException;
@@ -3022,6 +3115,7 @@ class RoxieMemTests : public CppUnit::TestFixture
         CPPUNIT_TEST(testBitmap);
         CPPUNIT_TEST(testDatamanagerThreading);
         CPPUNIT_TEST(testCallbacks);
+        CPPUNIT_TEST(testRecursiveCallbacks);
     CPPUNIT_TEST_SUITE_END();
     const IContextLogger &logctx;
 
@@ -3939,6 +4033,49 @@ protected:
         testCallback(128, 10, 5, 0.25, RHFunique);  // 4 at each priority level
         testCallback(1024, 10, 5, 0.25, RHFunique);  // 4 at each priority level
     }
+    void testRecursiveCallbacks1()
+    {
+        const size32_t bigRowSize = HEAP_ALIGNMENT_SIZE * 2 / 3;
+        Owned<IRowManager> rowManager = createRowManager(2 * HEAP_ALIGNMENT_SIZE, NULL, logctx, NULL);
+
+        //The lower priority allocator allocates an extra row when it is called to free all its rows.
+        //this will only succeed if the higher priority allocator is then called to free its data.
+        NastyCallbackBlockAllocator alloc1(rowManager, bigRowSize, 10);
+        SimpleCallbackBlockAllocator alloc2(rowManager, bigRowSize, 20);
+
+        alloc1.allocate();
+        alloc2.allocate();
+        OwnedRoxieRow tempRow = rowManager->allocate(bigRowSize, 0);
+    }
+    void testRecursiveCallbacks2()
+    {
+        const size32_t bigRowSize = HEAP_ALIGNMENT_SIZE * 2 / 3;
+        Owned<IRowManager> rowManager = createRowManager(2 * HEAP_ALIGNMENT_SIZE, NULL, logctx, NULL);
+
+        //Both allocators allocate extra memory when they are requested to free.  Ensure that an exception
+        //is thrown instead of the previous stack fault.
+        NastyCallbackBlockAllocator alloc1(rowManager, bigRowSize, 10);
+        NastyCallbackBlockAllocator alloc2(rowManager, bigRowSize, 20);
+
+        alloc1.allocate();
+        alloc2.allocate();
+        bool ok = false;
+        try
+        {
+            OwnedRoxieRow tempRow = rowManager->allocate(bigRowSize, 0);
+        }
+        catch (IException * e)
+        {
+            e->Release();
+            ok = true;
+        }
+        ASSERT(ok);
+    }
+    void testRecursiveCallbacks()
+    {
+        testRecursiveCallbacks1();
+        testRecursiveCallbacks2();
+    }
 };
 
 

rtl/eclrtl/eclrtl.cpp

@@ -225,6 +225,7 @@ bool rtlGetNormalizedUnicodeLocaleName(unsigned len, char const * in, char * out
 
 RTLLocale * queryRTLLocale(char const * locale)
 {
+    if (!locale) locale = "";
     CriticalBlock b(localeCrit);
     RTLLocale * loc = localeMap->getValue(locale);
     if(!loc)

thorlcr/activities/aggregate/thaggregateslave.cpp

@@ -59,7 +59,7 @@ protected:
         if (1 == numPartialResults)
             return firstRow;
 
-        CThorExpandingRowArray partialResults(*this, this, true, false, true, numPartialResults);
+        CThorExpandingRowArray partialResults(*this, this, true, stableSort_none, true, numPartialResults);
         if (hadElement)
             partialResults.setRow(0, firstRow);
         --numPartialResults;

thorlcr/activities/thdiskbase.cpp

@@ -360,7 +360,7 @@ rowcount_t getCount(CActivityBase &activity, unsigned partialResults, rowcount_t
 const void *getAggregate(CActivityBase &activity, unsigned partialResults, IRowInterfaces &rowIf, IHThorCompoundAggregateExtra &aggHelper, mptag_t mpTag)
 {
     // JCSMORE - pity this isn't common routine with similar one in aggregate, but helper is not common
-    CThorExpandingRowArray slaveResults(activity, &activity, true, false, true, partialResults);
+    CThorExpandingRowArray slaveResults(activity, &activity, true, stableSort_none, true, partialResults);
     unsigned _partialResults = partialResults;
     while (_partialResults--)
     {

thorlcr/thorutil/thmem.cpp

@@ -367,7 +367,7 @@ public:
 
 //====
 
-void CThorExpandingRowArray::init(rowidx_t initialSize, bool _stableSort)
+void CThorExpandingRowArray::init(rowidx_t initialSize, StableSortFlag _stableSort)
 {
     rowManager = activity.queryJob().queryRowManager();
     stableSort = _stableSort;
@@ -376,10 +376,12 @@ void CThorExpandingRowArray::init(rowidx_t initialSize, bool _stableSort)
     if (initialSize)
     {
         rows = static_cast<const void * *>(rowManager->allocate(initialSize * sizeof(void*), activity.queryContainer().queryId()));
-        maxRows = RoxieRowCapacity(rows) / sizeof(void *);
+        maxRows = getRowsCapacity();
         memset(rows, 0, maxRows * sizeof(void *));
-        if (stableSort)
-            stableSortTmp = static_cast<void **>(rowManager->allocate(initialSize * sizeof(void*), activity.queryContainer().queryId()));
+        if (stableSort_earlyAlloc == stableSort)
+            stableSortTmp = static_cast<void **>(rowManager->allocate(maxRows * sizeof(void*), activity.queryContainer().queryId()));
+        else
+            stableSortTmp = NULL;
     }
     else
     {
@@ -389,9 +391,28 @@ void CThorExpandingRowArray::init(rowidx_t initialSize, bool _stableSort)
     numRows = 0;
 }
 
-const void *CThorExpandingRowArray::allocateNewRows(rowidx_t requiredRows, OwnedConstThorRow &newStableSortTmp)
+const void *CThorExpandingRowArray::allocateRowTable(rowidx_t num)
 {
-    unsigned newSize = maxRows;
+    try
+    {
+        return rowManager->allocate(num * sizeof(void*), activity.queryContainer().queryId());
+    }
+    catch (IException * e)
+    {
+        //Pahological cases - not enough memory to reallocate the target row buffer, or no contiguous pages available.
+        unsigned code = e->errorCode();
+        if ((code == ROXIEMM_MEMORY_LIMIT_EXCEEDED) || (code == ROXIEMM_MEMORY_POOL_EXHAUSTED))
+        {
+            e->Release();
+            return NULL;
+        }
+        throw;
+    }
+}
+
+const void *CThorExpandingRowArray::allocateNewRows(rowidx_t requiredRows)
+{
+    rowidx_t newSize = maxRows;
     //This condition must be <= at least 1/scaling factor below otherwise you'll get an infinite loop.
     if (newSize <= 4)
         newSize = requiredRows;
@@ -405,40 +426,44 @@ const void *CThorExpandingRowArray::allocateNewRows(rowidx_t requiredRows, Owned
         while (newSize < requiredRows)
             newSize += newSize/4;
     }
-    OwnedConstThorRow newRows;
-    try
-    {
-        newRows.setown(rowManager->allocate(newSize * sizeof(void*), activity.queryContainer().queryId()));
-        if (!newRows)
-            return NULL;
-        if (stableSort)
-        {
-            newStableSortTmp.setown(rowManager->allocate(newSize * sizeof(void*), activity.queryContainer().queryId()));
-            if (!newStableSortTmp)
-                return NULL;
-        }
-    }
-    catch (IException * e)
+    return allocateRowTable(newSize);
+}
+
+void **CThorExpandingRowArray::allocateStableTable(bool error)
+{
+    dbgassertex(NULL != rows);
+    rowidx_t rowsCapacity = getRowsCapacity();
+    OwnedConstThorRow newStableSortTmp = allocateRowTable(rowsCapacity);
+    if (!newStableSortTmp)
     {
-        //Pahological cases - not enough memory to reallocate the target row buffer, or no contiguous pages available.
-        unsigned code = e->errorCode();
-        if ((code == ROXIEMM_MEMORY_LIMIT_EXCEEDED) || (code == ROXIEMM_MEMORY_POOL_EXHAUSTED))
-        {
-            e->Release();
-            return NULL;
-        }
-        throw;
+        if (error)
+            throw MakeActivityException(&activity, 0, "Out of memory, allocating stable row array, trying to allocate %"RIPF"d elements", rowsCapacity);
+        return NULL;
     }
-    return newRows.getClear();
+    return (void **)newStableSortTmp.getClear();
 }
 
-void CThorExpandingRowArray::doSort(unsigned n, void **const rows, ICompare &compare, unsigned maxCores)
+void CThorExpandingRowArray::doSort(rowidx_t n, void **const rows, ICompare &compare, unsigned maxCores)
 {
-    if (stableSort)
+    // NB: will only be called if numRows>1
+    if (stableSort_none != stableSort)
     {
+        OwnedConstThorRow newStableSortTmp;
+        void **stableTable;
+        if (stableSort_lateAlloc == stableSort)
+        {
+            dbgassertex(NULL == stableSortTmp);
+            newStableSortTmp.setown(allocateStableTable(true));
+            stableTable = (void **)newStableSortTmp.get();
+        }
+        else
+        {
+            dbgassertex(NULL != stableSortTmp);
+            stableTable = stableSortTmp;
+        }
         void **_rows = rows;
-        memcpy(stableSortTmp, _rows, n*sizeof(void **));
-        parqsortvecstable(stableSortTmp, n, compare, (void ***)_rows, maxCores);
+        memcpy(stableTable, _rows, n*sizeof(void **));
+        parqsortvecstable(stableTable, n, compare, (void ***)_rows, maxCores);
         while (n--)
         {
             *_rows = **((void ***)_rows);
@@ -449,7 +474,7 @@ void CThorExpandingRowArray::doSort(unsigned n, void **const rows, ICompare &com
         parqsortvec((void **const)rows, n, compare, maxCores);
 }
 
-CThorExpandingRowArray::CThorExpandingRowArray(CActivityBase &_activity, IRowInterfaces *_rowIf, bool _allowNulls, bool _stableSort, bool _throwOnOom, rowidx_t initialSize) : activity(_activity)
+CThorExpandingRowArray::CThorExpandingRowArray(CActivityBase &_activity, IRowInterfaces *_rowIf, bool _allowNulls, StableSortFlag _stableSort, bool _throwOnOom, rowidx_t initialSize) : activity(_activity)
 {
     init(initialSize, _stableSort);
     setup(_rowIf, _allowNulls, _stableSort, _throwOnOom);
@@ -462,7 +487,7 @@ CThorExpandingRowArray::~CThorExpandingRowArray()
     ReleaseThorRow(stableSortTmp);
 }
 
-void CThorExpandingRowArray::setup(IRowInterfaces *_rowIf, bool _allowNulls, bool _stableSort, bool _throwOnOom)
+void CThorExpandingRowArray::setup(IRowInterfaces *_rowIf, bool _allowNulls, StableSortFlag _stableSort, bool _throwOnOom)
 {
     rowIf = _rowIf;
     stableSort = _stableSort;
@@ -506,7 +531,7 @@ void CThorExpandingRowArray::swap(CThorExpandingRowArray &other)
     const void **otherRows = other.rows;
     void **otherStableSortTmp = other.stableSortTmp;
     bool otherAllowNulls = other.allowNulls;
-    bool otherStableSort = other.stableSort;
+    StableSortFlag otherStableSort = other.stableSort;
     bool otherThrowOnOom = other.throwOnOom;
     rowidx_t otherMaxRows = other.maxRows;
     rowidx_t otherNumRows = other.numRows;
@@ -542,7 +567,7 @@ void CThorExpandingRowArray::transferFrom(CThorExpandingRowArray &donor)
     kill();
     donor.transferRows(numRows, rows);
     maxRows = numRows;
-    if (stableSort && maxRows)
+    if (maxRows && (stableSort_earlyAlloc == stableSort))
         ensure(maxRows);
 }
 
@@ -574,21 +599,31 @@ void CThorExpandingRowArray::clearUnused()
 
 bool CThorExpandingRowArray::ensure(rowidx_t requiredRows)
 {
-    OwnedConstThorRow newStableSortTmp;
-    OwnedConstThorRow newRows = allocateNewRows(requiredRows, newStableSortTmp);
-    if (!newRows)
-        throw MakeActivityException(&activity, 0, "Out of memory, allocating row array, had %"RIPF"d, trying to allocate %"RIPF"d elements", ordinality(), requiredRows);
-
-    const void **oldRows = rows;
-    void **oldStableSortTmp = stableSortTmp;
-
-    memcpy((void *)newRows.get(), rows, numRows * sizeof(void*));
+    if (getRowsCapacity() < requiredRows) // check, because may have expanded previously, but failed to allocate stableSortTmp and set new maxRows
+    {
+        OwnedConstThorRow newRows = allocateNewRows(requiredRows);
+        if (!newRows)
+        {
+            if (throwOnOom)
+                throw MakeActivityException(&activity, 0, "Out of memory, allocating row array, had %"RIPF"d, trying to allocate %"RIPF"d elements", ordinality(), requiredRows);
+            return false;
+        }
 
-    rows = (const void **)newRows.getClear();
-    maxRows = RoxieRowCapacity(rows) / sizeof(void *);
-    stableSortTmp = (void **)newStableSortTmp.getClear();
-    ReleaseThorRow(oldRows);
-    ReleaseThorRow(oldStableSortTmp);
+        const void **oldRows = rows;
+        memcpy((void *)newRows.get(), rows, numRows * sizeof(void*));
+        rows = (const void **)newRows.getClear();
+        ReleaseThorRow(oldRows);
+    }
+    if (stableSort_earlyAlloc == stableSort)
+    {
+        OwnedConstThorRow newStableSortTmp = allocateStableTable(throwOnOom);
+        if (!newStableSortTmp)
+            return false;
+        void **oldStableSortTmp = stableSortTmp;
+        stableSortTmp = (void **)newStableSortTmp.getClear();
+        ReleaseThorRow(oldStableSortTmp);
+    }
+    maxRows = getRowsCapacity();
 
     return true;
 }
@@ -599,7 +634,7 @@ void CThorExpandingRowArray::sort(ICompare &compare, unsigned maxCores)
         doSort(numRows, (void **const)rows, compare, maxCores);
 }
 
-void CThorExpandingRowArray::reorder(rowidx_t start, rowidx_t num, unsigned *neworder)
+void CThorExpandingRowArray::reorder(rowidx_t start, rowidx_t num, rowidx_t *neworder)
 {
     if (start>=numRows)
         return;
@@ -611,17 +646,17 @@ void CThorExpandingRowArray::reorder(rowidx_t start, rowidx_t num, unsigned *new
     void **tmp = (void **)ma.allocate(num*sizeof(void *));
     const void **p = rows + start;
     memcpy(tmp, p, num*sizeof(void *));
-    for (unsigned i=0; i<num; i++)
+    for (rowidx_t i=0; i<num; i++)
         p[i] = tmp[neworder[i]];
 }
 
 bool CThorExpandingRowArray::equal(ICompare *icmp, CThorExpandingRowArray &other)
 {
     // slow but better than prev!
-    unsigned n = other.ordinality();
+    rowidx_t n = other.ordinality();
     if (n!=ordinality())
         return false;
-    for (unsigned i=0;i<n;i++)
+    for (rowidx_t i=0;i<n;i++)
     {
         const void *p1 = rows[i];
         const void *p2 = other.query(i);
@@ -633,9 +668,8 @@ bool CThorExpandingRowArray::equal(ICompare *icmp, CThorExpandingRowArray &other
 
 bool CThorExpandingRowArray::checkSorted(ICompare *icmp)
 {
-    unsigned i;
-    unsigned n=ordinality();
-    for (i=1; i<n; i++)
+    rowidx_t n=ordinality();
+    for (rowidx_t i=1; i<n; i++)
     {
         if (icmp->docompare(rows[i-1], rows[i])>0)
             return false;
@@ -685,15 +719,16 @@ IRowStream *CThorExpandingRowArray::createRowStream(rowidx_t start, rowidx_t num
 
 void CThorExpandingRowArray::partition(ICompare &compare, unsigned num, UnsignedArray &out)
 {
-    unsigned p=0;
-    unsigned n = ordinality();
+    rowidx_t p=0;
+    rowidx_t n = ordinality();
     while (num)
     {
         out.append(p);
         if (p<n)
         {
-            unsigned q = p+(n-p)/num;
-            if (p==q){ // skip to next group
+            rowidx_t q = p+(n-p)/num;
+            if (p==q) // skip to next group
+            {
                 while (q<n)
                 {
                     q++;
@@ -718,7 +753,7 @@ offset_t CThorExpandingRowArray::serializedSize()
     rowidx_t c = ordinality();
     assertex(serializer);
     offset_t total = 0;
-    for (unsigned i=0; i<c; i++)
+    for (rowidx_t i=0; i<c; i++)
     {
         CSizingSerializer ssz;
         serializer->serialize(ssz, (const byte *)rows[i]);
@@ -780,18 +815,18 @@ void CThorExpandingRowArray::serializeCompress(MemoryBuffer &mb)
     fastLZCompressToBuffer(mb,exp.length(), exp.toByteArray());
 }
 
-unsigned CThorExpandingRowArray::serializeBlock(MemoryBuffer &mb, size32_t dstmax, unsigned idx, unsigned count)
+rowidx_t CThorExpandingRowArray::serializeBlock(MemoryBuffer &mb, size32_t dstmax, rowidx_t idx, rowidx_t count)
 {
     assertex(serializer);
     CMemoryRowSerializer out(mb);
     bool warnnull = true;
-    unsigned num=ordinality();
+    rowidx_t num=ordinality();
     if (idx>=num)
         return 0;
     if (num-idx<count)
         count = num-idx;
-    unsigned ret = 0;
-    for (unsigned i=0;i<count;i++)
+    rowidx_t ret = 0;
+    for (rowidx_t i=0;i<count;i++)
     {
         size32_t ln = mb.length();
         const void *row = query(i+idx);
@@ -866,8 +901,8 @@ void CThorSpillableRowArray::unregisterWriteCallback(IWritePosCallback &cb)
     writeCallbacks.zap(cb);
 }
 
-CThorSpillableRowArray::CThorSpillableRowArray(CActivityBase &activity, IRowInterfaces *rowIf, bool allowNulls, bool stable, rowidx_t initialSize, size32_t _commitDelta)
-    : CThorExpandingRowArray(activity, rowIf, false, stable, false, initialSize), commitDelta(_commitDelta)
+CThorSpillableRowArray::CThorSpillableRowArray(CActivityBase &activity, IRowInterfaces *rowIf, bool allowNulls, StableSortFlag stableSort, rowidx_t initialSize, size32_t _commitDelta)
+    : CThorExpandingRowArray(activity, rowIf, false, stableSort, false, initialSize), commitDelta(_commitDelta)
 {
     commitRows = 0;
     firstRow = 0;
@@ -897,35 +932,47 @@ bool CThorSpillableRowArray::ensure(rowidx_t requiredRows)
 {
     //Only the writer is allowed to reallocate rows (otherwise append can't be optimized), so rows is valid outside the lock
 
-    OwnedConstThorRow newStableSortTmp;
-    OwnedConstThorRow newRows = allocateNewRows(requiredRows, newStableSortTmp);
-    if (!newRows)
-        return false;
+    OwnedConstThorRow newRows;
+    if (getRowsCapacity() < requiredRows) // check, because may have expanded previously, but failed to allocate stableSortTmp and set new maxRows
+    {
+        newRows.setown(allocateNewRows(requiredRows));
+        if (!newRows)
+            return false;
+    }
 
-    const void **oldRows;
-    void **oldStableSortTmp;
     {
         CThorSpillableRowArrayLock block(*this);
+        if (newRows)
+        {
+            const void **oldRows = rows;
+            memcpy((void *)newRows.get(), rows+firstRow, (numRows - firstRow) * sizeof(void*));
+            numRows -= firstRow;
+            commitRows -= firstRow;
+            firstRow = 0;
 
-        oldRows = rows;
-        oldStableSortTmp = stableSortTmp;
-        memcpy((void *)newRows.get(), rows+firstRow, (numRows - firstRow) * sizeof(void*));
-        numRows -= firstRow;
-        commitRows -= firstRow;
-        firstRow = 0;
+            rows = (const void **)newRows.getClear();
+            ReleaseThorRow(oldRows);
+        }
 
-        rows = (const void **)newRows.getClear();
-        maxRows = RoxieRowCapacity(rows) / sizeof(void *);
-        stableSortTmp = (void **)newStableSortTmp.getClear();
+        // NB: can't release lock, or change maxRows, until know this succeeds
+        if (stableSort_earlyAlloc == stableSort)
+        {
+            OwnedConstThorRow newStableSortTmp = allocateStableTable(false);
+            if (!newStableSortTmp)
+                return false;
+            void **oldStableSortTmp = stableSortTmp;
+            stableSortTmp = (void **)newStableSortTmp.getClear();
+            ReleaseThorRow(oldStableSortTmp);
+        }
+        maxRows = getRowsCapacity();
     }
-    ReleaseThorRow(oldRows);
-    ReleaseThorRow(oldStableSortTmp);
     return true;
 }
 
 void CThorSpillableRowArray::sort(ICompare &compare, unsigned maxCores)
 {
-    unsigned n = numCommitted();
+    // NB: only to be called inside lock
+    rowidx_t n = numCommitted();
     if (n>1)
     {
         void **const rows = (void **const)getBlock(n);
@@ -933,7 +980,7 @@ void CThorSpillableRowArray::sort(ICompare &compare, unsigned maxCores)
     }
 }
 
-unsigned CThorSpillableRowArray::save(IFile &iFile, rowidx_t watchRecNum, offset_t *watchFilePosResult)
+rowidx_t CThorSpillableRowArray::save(IFile &iFile, rowidx_t watchRecNum, offset_t *watchFilePosResult)
 {
     rowidx_t n = numCommitted();
     if (0 == n)
@@ -1233,7 +1280,7 @@ public:
         }
         maxCores = activity.queryMaxCores();
 
-        spillableRows.setup(rowIf, false, isStable);
+        spillableRows.setup(rowIf, false, isStable?stableSort_earlyAlloc:stableSort_none);
     }
     ~CThorRowCollectorBase()
     {
@@ -1284,7 +1331,7 @@ public:
             mmRegistered = false;
             activity.queryJob().queryRowManager()->removeRowBuffer(this);
         }
-        spillableRows.setup(rowIf, false, isStable);
+        spillableRows.setup(rowIf, false, isStable?stableSort_earlyAlloc:stableSort_none);
     }
 // IBufferedRowCallback
     virtual unsigned getPriority() const

thorlcr/thorutil/thmem.hpp

@@ -210,6 +210,7 @@ graph_decl StringBuffer &getRecordString(const void *key, IOutputRowSerializer *
 #define SPILL_PRIORITY_SPILLABLE_STREAM SPILL_PRIORITY_DEFAULT
 #define SPILL_PRIORITY_RESULT SPILL_PRIORITY_DEFAULT
 
+enum StableSortFlag { stableSort_none, stableSort_earlyAlloc, stableSort_lateAlloc };
 class CThorSpillableRowArray;
 class graph_decl CThorExpandingRowArray : public CSimpleInterface
 {
@@ -223,21 +224,25 @@ protected:
     roxiemem::IRowManager *rowManager;
     const void **rows;
     void **stableSortTmp;
-    bool stableSort, throwOnOom, allowNulls;
+    bool throwOnOom; // tested during array expansion (ensure())
+    bool allowNulls;
+    StableSortFlag stableSort;
     rowidx_t maxRows;  // Number of rows that can fit in the allocated memory.
     rowidx_t numRows;  // rows that have been added can only be updated by writing thread.
 
-    void init(rowidx_t initialSize, bool stable);
-    const void *allocateNewRows(rowidx_t requiredRows, OwnedConstThorRow &newStableSortTmp);
+    void init(rowidx_t initialSize, StableSortFlag stableSort);
+    void **allocateStableTable(bool error); // allocates stable table based on std. ptr table
+    const void *allocateRowTable(rowidx_t num);
+    const void *allocateNewRows(rowidx_t requiredRows);
     void serialize(IRowSerializerTarget &out);
-    void doSort(unsigned n, void **const rows, ICompare &compare, unsigned maxCores);
-
+    void doSort(rowidx_t n, void **const rows, ICompare &compare, unsigned maxCores);
+    inline rowidx_t getRowsCapacity() const { return rows ? RoxieRowCapacity(rows) / sizeof(void *) : 0; }
 public:
-    CThorExpandingRowArray(CActivityBase &activity, IRowInterfaces *rowIf, bool allowNulls=false, bool stableSort=false, bool throwOnOom=true, rowidx_t initialSize=InitialSortElements);
+    CThorExpandingRowArray(CActivityBase &activity, IRowInterfaces *rowIf, bool allowNulls=false, StableSortFlag stableSort=stableSort_none, bool throwOnOom=true, rowidx_t initialSize=InitialSortElements);
     ~CThorExpandingRowArray();
     CActivityBase &queryActivity() { return activity; }
     // NB: throws error on OOM by default
-    void setup(IRowInterfaces *rowIf, bool allowNulls=false, bool stableSort=false, bool throwOnOom=true);
+    void setup(IRowInterfaces *rowIf, bool allowNulls=false, StableSortFlag stableSort=stableSort_none, bool throwOnOom=true);
     inline void setAllowNulls(bool b) { allowNulls = b; }
 
     void clearRows();
@@ -303,7 +308,7 @@ public:
     void removeRows(rowidx_t start, rowidx_t n);
     void clearUnused();
     void sort(ICompare &compare, unsigned maxCores);
-    void reorder(rowidx_t start, rowidx_t num, unsigned *neworder);
+    void reorder(rowidx_t start, rowidx_t num, rowidx_t *neworder);
 
     bool equal(ICompare *icmp, CThorExpandingRowArray &other);
     bool checkSorted(ICompare *icmp);
@@ -315,7 +320,7 @@ public:
     offset_t serializedSize();
     void serialize(MemoryBuffer &mb);
     void serializeCompress(MemoryBuffer &mb);
-    unsigned serializeBlock(MemoryBuffer &mb, size32_t dstmax, unsigned idx, unsigned count);
+    rowidx_t serializeBlock(MemoryBuffer &mb, size32_t dstmax, rowidx_t idx, rowidx_t count);
     void deserializeRow(IRowDeserializerSource &in); // NB single row not NULL
     void deserialize(size32_t sz, const void *buf);
     void deserializeExpand(size32_t sz, const void *data);
@@ -351,12 +356,12 @@ public:
         inline ~CThorSpillableRowArrayLock() { rows.unlock(); }
     };
 
-    CThorSpillableRowArray(CActivityBase &activity, IRowInterfaces *rowIf, bool allowNulls=false, bool stableSort=false, rowidx_t initialSize=InitialSortElements, size32_t commitDelta=CommitStep);
+    CThorSpillableRowArray(CActivityBase &activity, IRowInterfaces *rowIf, bool allowNulls=false, StableSortFlag stableSort=stableSort_none, rowidx_t initialSize=InitialSortElements, size32_t commitDelta=CommitStep);
     ~CThorSpillableRowArray();
     // NB: throwOnOom false
-    void setup(IRowInterfaces *rowIf, bool allowNulls=false, bool stableSort=false, bool throwOnOom=false)
+    void setup(IRowInterfaces *rowIf, bool allowNulls=false, StableSortFlag stableSort=stableSort_none)
     {
-        CThorExpandingRowArray::setup(rowIf, allowNulls, stableSort, throwOnOom);
+        CThorExpandingRowArray::setup(rowIf, allowNulls, stableSort, false);
     }
     void registerWriteCallback(IWritePosCallback &cb);
     void unregisterWriteCallback(IWritePosCallback &cb);
@@ -401,7 +406,7 @@ public:
 
     //A thread calling the following functions must own the lock, or guarantee no other thread will access
     void sort(ICompare & compare, unsigned maxcores);
-    unsigned save(IFile &file, rowidx_t watchRecNum=(rowidx_t)-1, offset_t *watchFilePosResult=NULL);
+    rowidx_t save(IFile &file, rowidx_t watchRecNum=(rowidx_t)-1, offset_t *watchFilePosResult=NULL);
     const void **getBlock(rowidx_t readRows);
     inline void noteSpilled(rowidx_t spilledRows)
     {