Merge branch 'candidate-5.2.0'

Conflicts:
	common/thorhelper/roxiehelper.hpp

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

cmake_modules/commonSetup.cmake

@@ -315,7 +315,6 @@ IF ("${COMMONSETUP_DONE}" STREQUAL "")
   endmacro(HPCC_ADD_SUBDIRECTORY)
 
   set ( SCM_GENERATED_DIR ${CMAKE_BINARY_DIR}/generated )
-  include_directories (${SCM_GENERATED_DIR})
 
   ###############################################################
   # Macro for Logging Plugin build in CMake

common/thorhelper/roxiehelper.cpp

@@ -563,6 +563,8 @@ bool CSafeSocket::readBlock(StringBuffer &ret, unsigned timeout, HttpHelper *pHt
                 payload += 4;
                 char *str;
 
+                pHttpHelper->setUrlPath(header);
+
                 // capture authentication token
                 if ((str = strstr(header, "Authorization: Basic ")) != NULL)
                     pHttpHelper->setAuthToken(str+21);
@@ -1460,8 +1462,35 @@ StringBuffer & mangleLocalTempFilename(StringBuffer & out, char const * in)
     return out;
 }
 
+static const char *skipLfnForeign(const char *lfn)
+{
+    const char *finger = lfn;
+    while (*finger=='~')
+        finger++;
+    const char *scope = strstr(finger, "::");
+    if (scope)
+    {
+        StringBuffer cmp;
+        if (strieq("foreign", cmp.append(scope-finger, finger).trim()))
+        {
+            // foreign scope - need to strip off the ip and port
+            scope += 2;  // skip ::
+            finger = strstr(scope,"::");
+            if (finger)
+            {
+                finger += 2;
+                while (*finger == ' ')
+                    finger++;
+                return finger;
+            }
+        }
+    }
+    return lfn;
+}
+
 StringBuffer & expandLogicalFilename(StringBuffer & logicalName, const char * fname, IConstWorkUnit * wu, bool resolveLocally)
 {
+    fname = skipLfnForeign(fname); //foreign location should already be reflected in local dali dfs meta data
     if (fname[0]=='~')
         logicalName.append(fname+1);
     else if (resolveLocally)
@@ -1493,3 +1522,32 @@ void IRoxieContextLogger::CTXLOGae(IException *E, const char *file, unsigned lin
     CTXLOGaeva(E, file, line, prefix, format, args);
     va_end(args);
 }
+
+void HttpHelper::gatherUrlParameters()
+{
+    const char *finger = strchr(urlPath, '?');
+    if (!finger)
+        return;
+    finger++;
+    while (*finger)
+    {
+        StringBuffer s, prop, val;
+        while (*finger && *finger != '&' && *finger != '=')
+            s.append(*finger++);
+        appendDecodedURL(prop, s.trim());
+        if (!*finger || *finger == '&')
+            val.set("1");
+        else
+        {
+            s.clear();
+            finger++;
+            while (*finger && *finger != '&')
+                s.append(*finger++);
+            appendDecodedURL(val, s.trim());
+        }
+        if (prop.length())
+            parameters->setProp(prop, val);
+        if (*finger)
+            finger++;
+    }
+}

common/thorhelper/roxiehelper.hpp

@@ -31,8 +31,10 @@ class THORHELPER_API HttpHelper : public CInterface
 {
 private:
     bool _isHttp;
+    StringAttr urlPath;
     StringAttr authToken;
     StringAttr contentType;
+    Owned<IProperties> parameters;
 private:
     inline void setHttpHeaderValue(StringAttr &s, const char *v, bool ignoreExt)
     {
@@ -44,12 +46,15 @@ private:
         if (len)
             s.set(v, len);
     }
+    void gatherUrlParameters();
+
 public:
     IMPLEMENT_IINTERFACE;
-    HttpHelper() { _isHttp = false; };
-    bool isHttp() { return _isHttp; };
-    void setIsHttp(bool __isHttp) { _isHttp = __isHttp; };
-    const char *queryAuthToken() { return authToken.str(); };
+    HttpHelper() { _isHttp = false; parameters.setown(createProperties(true));}
+    bool isHttp() { return _isHttp; }
+    bool getTrim() {return parameters->getPropBool(".trim", true); /*http currently defaults to true, maintain compatibility */}
+    void setIsHttp(bool __isHttp) { _isHttp = __isHttp; }
+    const char *queryAuthToken() { return authToken.str(); }
     inline void setAuthToken(const char *v)
     {
         setHttpHeaderValue(authToken, v, false);
@@ -59,7 +64,17 @@ public:
     {
         setHttpHeaderValue(contentType, v, true);
     };
+    inline void setUrlPath(const char *v)
+    {
+        const char *end = strstr(v, " HTTP");
+        if (end)
+        {
+            urlPath.set(v, end - v);
+            gatherUrlParameters();
+        }
+    }
     TextMarkupFormat queryContentFormat(){return (strieq(queryContentType(), "application/json")) ? MarkupFmt_JSON : MarkupFmt_XML;}
+    IProperties *queryUrlParameters(){return parameters;}
 };
 
 //========================================================================================= 

common/thorhelper/roxierow.cpp

@@ -697,7 +697,7 @@ protected:
 
     void testSetup()
     {
-        setTotalMemoryLimit(false, true, 40*HEAP_ALIGNMENT_SIZE, 0, NULL, NULL);
+        setTotalMemoryLimit(false, true, false, 40*HEAP_ALIGNMENT_SIZE, 0, NULL, NULL);
     }
 
     void testCleanup()

common/thorhelper/thorcommon.hpp

@@ -360,6 +360,10 @@ public:
     {
         return ctx->getResultHash(name, sequence);
     }
+    virtual unsigned getExternalResultHash(const char * wuid, const char * name, unsigned sequence)
+    {
+        return ctx->getExternalResultHash(wuid, name, sequence);
+    }
     virtual char *getWuid()
     {
         return ctx->getWuid();

docs/ECLLanguageReference/ECLR_mods/BltInFunc-BUILD.xml

@@ -196,7 +196,10 @@
     role="bold">] ) [, THRESHOLD<indexterm>
         <primary>THRESHOLD</primary>
       </indexterm>(</emphasis><emphasis>size</emphasis><emphasis role="bold">)
-    ] ]</emphasis></emphasis></para>
+    ] ] <emphasis role="bold">[, MAXLENGTH<indexterm>
+        <primary>MAXLENGTH</primary>
+      </indexterm>[(</emphasis><emphasis>value</emphasis><emphasis
+    role="bold">)] ] ]</emphasis></emphasis></emphasis></para>
 
     <para><informaltable colsep="1" frame="all" rowsep="1">
         <tgroup cols="2">
@@ -303,12 +306,19 @@
             </row>
 
             <row>
-              <entry><emphasis
-              role="bold">FILEPOSITION(false)</emphasis></entry>
+              <entry><emphasis role="bold">FILEPOSITION</emphasis></entry>
 
-              <entry>Optional. Prevents the implicit fileposition field from
-              being created and will not treat a trailing integer field any
-              differently from the rest of the payload.</entry>
+              <entry>Optional. If <emphasis>flag</emphasis> is FALSE, prevents
+              the implicit fileposition field from being created and will not
+              treat a trailing integer field any differently from the rest of
+              the payload.</entry>
+            </row>
+
+            <row>
+              <entry><emphasis>flag</emphasis></entry>
+
+              <entry>Optional. TRUE or FALSE, indicating whether or not to
+              create the implicit fileposition field.</entry>
             </row>
 
             <row>
@@ -445,9 +455,31 @@
               <entry>An integer value indicating the minimum number of bytes
               for a single part. Default is 1GB.</entry>
             </row>
+
+            <row>
+              <entry><emphasis role="bold">MAXLENGTH</emphasis></entry>
+
+              <entry>Optional. This option is used to create indexes that are
+              backward compatible for platform versions prior to 3.0.
+              Specifies the maximum length of a variable-length index record.
+              Fixed length records always use the minimum size required. If
+              the default maximum length causes inefficiency problems, it can
+              be explicitly overridden.</entry>
+            </row>
+
+            <row>
+              <entry><emphasis>value</emphasis></entry>
+
+              <entry>Optional. An integer value indicating the maximum length.
+              If omitted, the maximum size is calculated from the record
+              structure. Variable-length records that do not specify MAXLENGTH
+              may be slightly inefficient</entry>
+            </row>
           </tbody>
         </tgroup>
       </informaltable></para>
+
+    <para></para>
   </sect2>
 
   <sect2 id="BUILD_an_Access_Index">

docs/ECLLanguageReference/ECLR_mods/BltInFunc-LoadXML.xml

@@ -84,7 +84,7 @@ LOADXML(myxmlText);
      'OUTPUT(FETCH(Files.Vehicle,Files.VehicleIDX(personid= '
          + %'id'% + '),RIGHT.RecPos));\n' )
 #END
-%OutputStr%
+%OutStr%
 ENDMACRO;
     
 //this is an example of code for a drilldown (1 per row)

docs/ECLLanguageReference/ECLR_mods/RecordStructure.xml

@@ -66,20 +66,23 @@
         <row>
           <entry><emphasis role="bold">MAXLENGTH</emphasis></entry>
 
-          <entry>Optional. Specifies the maximum number of characters allowed
-          in the RECORD structure or field. MAXLENGTH on the RECORD structure
-          overrides any MAXLENGTH on a field definition, which overrides any
-          MAXLENGTH specified in the TYPE structure<indexterm>
+          <entry>Optional. This option is used to create indexes that are
+          backward compatible for platform versions prior to 3.0. Specifies
+          the maximum number of characters allowed in the RECORD structure or
+          field. MAXLENGTH on the RECORD structure overrides any MAXLENGTH on
+          a field definition, which overrides any MAXLENGTH specified in the
+          TYPE structure<indexterm>
               <primary>TYPE structure</primary>
             </indexterm> if the <emphasis>datatype</emphasis> names an alien
           data type. This option defines the maximum size of variable-length
-          records. If omitted, a warning is generated. The default maximum
-          size of a record containing variable-length fields is 4096 bytes
-          (this may be overridden by using
+          records. If omitted, fixed size records use the minimum size
+          required and variable length records produce a warning. The default
+          maximum size of a record containing variable-length fields is 4096
+          bytes (this may be overridden by using
           <emphasis>#OPTION(maxLength,####)</emphasis> to change the default).
           The maximum record size should be set as conservatively as possible,
           and is better set on a per-field basis (see the <emphasis
-          role="bold">Field Modifiers</emphasis>section below).</entry>
+          role="bold">Field Modifiers</emphasis>section below). </entry>
         </row>
 
         <row>
@@ -779,7 +782,7 @@ END;</programlisting>
     case field name, and the default value for Repeated is "Row." For example,
     this demonstrates "Container/Repeated":</para>
 
-    <programlisting>DATASET(PeopleNames) People{xpath('people/name])};
+    <programlisting>DATASET(PeopleNames) People{xpath('people/name'])};
           /*matches: &lt;people&gt;
                         &lt;name&gt;Gavin&lt;/name&gt;
                         &lt;name&gt;Ricardo&lt;/name&gt;
@@ -787,14 +790,14 @@ END;</programlisting>
 
     <para>This demonstrates "/Repeated":</para>
 
-    <programlisting>DATASET(Names) Names{xpath('/name])};
+    <programlisting>DATASET(Names) Names{xpath('/name'])};
           /*matches: &lt;name&gt;Gavin&lt;/name&gt;
                      &lt;name&gt;Ricardo&lt;/name&gt; */</programlisting>
 
     <para>"Container" and "Repeated" may also contain xpath filters, like
     this:</para>
 
-    <programlisting>DATASET(doctorRec) doctors{xpath('person[@job=\'doctor\'])};
+    <programlisting>DATASET(doctorRec) doctors{xpath('person[@job=\'doctor\']')};
           /*matches: &lt;person job='doctor'&gt;
                        &lt;FName&gt;Kevin&lt;/FName&gt;
                        &lt;LName&gt;Richards&lt;/LName&gt;

docs/ECLLanguageReference/ECLR_mods/Recrd-Index.xml

@@ -24,20 +24,30 @@
       <primary>FIRST</primary>
     </indexterm>) ]</emphasis><emphasis role="bold"> [,DISTRIBUTED<indexterm>
       <primary>DISTRIBUTED</primary>
-    </indexterm>]) <emphasis
-  role="bold">[,FILEPOSITION(<emphasis>false</emphasis>);</emphasis></emphasis><emphasis></emphasis></para>
+    </indexterm>] [,FILEPOSITION( [
+  </emphasis><emphasis>flag</emphasis><emphasis role="bold">] ) ] [,
+  MAXLENGTH<indexterm>
+      <primary>MAXLENGTH</primary>
+    </indexterm>[</emphasis>(<emphasis>**value</emphasis>**)<emphasis
+  role="bold">] ] );</emphasis></para>
 
   <para><emphasis> attr</emphasis><emphasis role="bold"> :=
   INDEX([</emphasis><emphasis> baserecset, </emphasis><emphasis
   role="bold">]</emphasis><emphasis> keys, payload, indexfile
-  </emphasis><emphasis role="bold">[,SORTED] [,PRELOAD]</emphasis><emphasis
-  role="bold"> [,OPT] [,COMPRESSED( LZW | ROW | FIRST) ]</emphasis><emphasis
-  role="bold"> [,DISTRIBUTED]) </emphasis><emphasis
-  role="bold">[,FILEPOSITION(<emphasis>false</emphasis>);</emphasis><emphasis></emphasis></para>
+  </emphasis><emphasis role="bold">[,SORTED] [,PRELOAD][,OPT] [,COMPRESSED(
+  LZW | ROW | FIRST) ] [,DISTRIBUTED] [,FILEPOSITION( [
+  </emphasis><emphasis>flag</emphasis><emphasis role="bold">] ) ] [,
+  MAXLENGTH<indexterm>
+      <primary>MAXLENGTH</primary>
+    </indexterm>[</emphasis>(<emphasis>value</emphasis>)<emphasis
+  role="bold">] ]</emphasis>);</para>
 
   <para><emphasis> attr</emphasis><emphasis role="bold"> :=
   INDEX(</emphasis><emphasis>index,newindexfile</emphasis><emphasis
-  role="bold">);</emphasis></para>
+  role="bold"> [, MAXLENGTH<indexterm>
+      <primary>MAXLENGTH</primary>
+    </indexterm>[</emphasis>(<emphasis>value</emphasis>)<emphasis
+  role="bold">] ]); </emphasis></para>
 
   <informaltable colsep="1" frame="all" rowsep="1">
     <tgroup cols="2">
@@ -160,11 +170,19 @@
         </row>
 
         <row>
-          <entry><emphasis role="bold">FILEPOSITION(false)</emphasis></entry>
+          <entry><emphasis role="bold">FILEPOSITION</emphasis></entry>
 
-          <entry>Optional. Prevents the implicit fileposition field from being
-          created and will not treat a trailing integer field any differently
-          from the rest of the payload.</entry>
+          <entry>Optional. If <emphasis>flag</emphasis> is FALSE, prevents the
+          normal behavior of implicit fileposition field being created and
+          will not treat a trailing integer field any differently from the
+          rest of the payload.</entry>
+        </row>
+
+        <row>
+          <entry><emphasis>flag</emphasis></entry>
+
+          <entry>Optional. TRUE or FALSE, indicating whether or not to create
+          the implicit fileposition field.</entry>
         </row>
 
         <row>
@@ -182,6 +200,26 @@
           Filenames</emphasis> section for more on logical
           filenames.</para></entry>
         </row>
+
+        <row>
+          <entry><emphasis role="bold">MAXLENGTH</emphasis></entry>
+
+          <entry>Optional. This option is used to create indexes that are
+          backward compatible for platform versions prior to 3.0. Specifies
+          the maximum length of a variable-length index record. Fixed length
+          records always use the minimum size required. If the default maximum
+          length causes inefficiency problems, it can be explicitly
+          overridden.</entry>
+        </row>
+
+        <row>
+          <entry><emphasis>value</emphasis></entry>
+
+          <entry>Optional. An integer value indicating the maximum length. If
+          omitted, the maximum size is calculated from the record structure.
+          Variable-length records that do not specify MAXLENGTH may be
+          slightly inefficient</entry>
+        </row>
       </tbody>
     </tgroup>
   </informaltable>

docs/ECLWatch/ECLWa_mods/ECLWatchSrc.xml

@@ -90,7 +90,7 @@
 
           <mediaobject>
             <imageobject>
-              <imagedata fileref="../../images/ECLWA00A.jpg"
+              <imagedata fileref="../../images/ECLWa00a.jpg"
                          vendor="eclwatchSS" />
             </imageobject>
           </mediaobject>
@@ -1422,6 +1422,57 @@
             the drop menu.</para>
           </listitem>
         </itemizedlist>
+
+        <sect3>
+          <title>Superfiles</title>
+
+          <para>A superfile is a managed list of sub-files (Logical Files)
+          treated as a single logical entity. When a file is a superfile, the
+          Summary tab displays the superfile details, such as each subfile.
+          Select a superfile from the Logical Files list, then press the Open
+          action button. This displays the superfile details page.<figure>
+              <title>Superfile Details page</title>
+
+              <mediaobject>
+                <imageobject>
+                  <imagedata fileref="../../images/ECLWAS00a.jpg"
+                             vendor="eclwatchSS" />
+                </imageobject>
+              </mediaobject>
+            </figure></para>
+
+          <para>On the superfile details page you can:</para>
+
+          <para><itemizedlist>
+              <listitem>
+                <para>See a summary of the sub-files the superfile
+                contains.</para>
+              </listitem>
+
+              <listitem>
+                <para>See the details of the sub-files.</para>
+              </listitem>
+
+              <listitem>
+                <para>Select one or more sub-file(s).</para>
+              </listitem>
+
+              <listitem>
+                <para>Press the <emphasis role="bold">Save</emphasis> button
+                to save any changes the superfile.</para>
+              </listitem>
+
+              <listitem>
+                <para>Press the <emphasis role="bold">Delete</emphasis> button
+                delete the superfile.</para>
+              </listitem>
+
+              <listitem>
+                <para>Press the <emphasis role="bold">Remove</emphasis> button
+                to remove a sub-file from the superfile.</para>
+              </listitem>
+            </itemizedlist></para>
+        </sect3>
       </sect2>
     </sect1>
 

ecl/eclagent/eclagent.cpp

@@ -857,6 +857,16 @@ unsigned EclAgent::getResultHash(const char * name, unsigned sequence)
     return r->getResultHash();
 }
 
+unsigned EclAgent::getExternalResultHash(const char * wuid, const char * name, unsigned sequence)
+{
+    logGetResult("ExternalHash", name, sequence);
+    Owned<IConstWUResult> r = getExternalResult(wuid, name, sequence);
+    if (!r)
+        failv(0, "Failed to retrieve hash value %s from workunit %s", name, wuid);
+    return r->getResultHash();
+}
+
+
 
 //---------------------------------------------------------------------------
 
@@ -2019,6 +2029,9 @@ void EclAgent::runProcess(IEclProcess *process)
     bool allowTransparentHugePages = agentTopology->getPropBool("@heapUseTransparentHugePages", true);
     allowTransparentHugePages = globals->getPropBool("heapUseTransparentHugePages", allowTransparentHugePages);
 
+    bool retainMemory = agentTopology->getPropBool("@heapRetainMemory", false);
+    retainMemory = globals->getPropBool("heapRetainMemory", retainMemory);
+
 #ifndef __64BIT__
     if (memLimitMB > 4096)
     {
@@ -2028,7 +2041,7 @@ void EclAgent::runProcess(IEclProcess *process)
     }
 #endif
     memsize_t memLimitBytes = (memsize_t)memLimitMB * 1024 * 1024;
-    roxiemem::setTotalMemoryLimit(allowHugePages, allowTransparentHugePages, memLimitBytes, 0, NULL, NULL);
+    roxiemem::setTotalMemoryLimit(allowHugePages, allowTransparentHugePages, retainMemory, memLimitBytes, 0, NULL, NULL);
 
     rowManager.setown(roxiemem::createRowManager(0, NULL, queryDummyContextLogger(), allocatorMetaCache, false));
     setHThorRowManager(rowManager.get());

ecl/eclagent/eclagent.ipp

@@ -470,6 +470,7 @@ public:
     virtual double getResultReal(const char * name, unsigned sequence);
     virtual unsigned getResultHash(const char * name, unsigned sequence);
     virtual void getExternalResultRaw(unsigned & tlen, void * & tgt, const char * wuid, const char * stepname, unsigned sequence, IXmlToRowTransformer * xmlTransformer, ICsvToRowTransformer * csvTransformer);
+    virtual unsigned getExternalResultHash(const char * wuid, const char * name, unsigned sequence);
     virtual void getResultRowset(size32_t & tcount, byte * * & tgt, const char * name, unsigned sequence, IEngineRowAllocator * _rowAllocator, bool isGrouped, IXmlToRowTransformer * xmlTransformer, ICsvToRowTransformer * csvTransformer);
     virtual void getResultDictionary(size32_t & tcount, byte * * & tgt, IEngineRowAllocator * _rowAllocator, const char * name, unsigned sequence, IXmlToRowTransformer * xmlTransformer, ICsvToRowTransformer * csvTransformer, IHThorHashLookupInfo * hasher);
     virtual char *getJobName();

ecl/hql/hqlopt.cpp

@@ -2410,7 +2410,7 @@ IHqlExpression * CTreeOptimizer::doCreateTransformed(IHqlExpression * transforme
                             {
                                 //This test should not be required, but it avoids problems with elements from rows
                                 //being used conditionally within transforms.  See HPCC-11018 for details.
-                                if (isIndependentOfScope(match))
+                                if ((options & HOOexpandselectcreaterow) || isIndependentOfScope(match))
                                 {
                                     DBGLOG("Optimizer: Extract value %s from %s", queryNode0Text(cur), queryNode1Text(transformed));
                                     noteUnused(child);

ecl/hql/hqlopt.hpp

@@ -32,6 +32,7 @@ enum
     HOOfoldconstantdatasets     = 0x0080,
     HOOalwayslocal              = 0x0100,
     HOOexpensive                = 0x0200,   // include potentially expensive optimizations
+    HOOexpandselectcreaterow    = 0x0400,
 };
 
 extern HQL_API IHqlExpression * optimizeHqlExpression(IErrorReceiver & errorProcessor, IHqlExpression * expr, unsigned options);

ecl/hql/hqlutil.cpp

@@ -2387,12 +2387,12 @@ void DependenciesUsed::addFilenameWrite(IHqlExpression * expr)
         allWritten = true;
 }
 
-void DependenciesUsed::addResultRead(IHqlExpression * seq, IHqlExpression * name, bool isGraphResult)
+void DependenciesUsed::addResultRead(IHqlExpression * wuid, IHqlExpression * seq, IHqlExpression * name, bool isGraphResult)
 {
     if (!isGraphResult)
         if (!seq || !seq->queryValue())
             return;         //Can be called in parser when no sequence has been allocated
-    OwnedHqlExpr result = createAttribute(resultAtom, LINK(seq), LINK(name));
+    OwnedHqlExpr result = createAttribute(resultAtom, LINK(seq), LINK(name), LINK(wuid));
     if (resultsWritten.find(*result) == NotFound)
         appendUniqueExpr(resultsRead, LINK(result));
 }
@@ -2516,12 +2516,13 @@ void DependenciesUsed::extractDependencies(IHqlExpression * expr, unsigned flags
         {
             IHqlExpression * sequence = queryAttributeChild(expr, sequenceAtom, 0);
             IHqlExpression * name = queryAttributeChild(expr, nameAtom, 0);
-            addResultRead(sequence, name, false);
+            IHqlExpression * wuid = expr->queryAttribute(wuidAtom);
+            addResultRead(wuid, sequence, name, false);
         }
         break;
     case no_getgraphresult:
         if (flags & GatherGraphResultRead)
-            addResultRead(expr->queryChild(1), expr->queryChild(2), true);
+            addResultRead(NULL, expr->queryChild(1), expr->queryChild(2), true);
         break;
     case no_setgraphresult:
         if (flags & GatherGraphResultWrite)
@@ -2532,7 +2533,8 @@ void DependenciesUsed::extractDependencies(IHqlExpression * expr, unsigned flags
         {
             IHqlExpression * sequence = queryAttributeChild(expr, sequenceAtom, 0);
             IHqlExpression * name = queryAttributeChild(expr, namedAtom, 0);
-            addResultRead(sequence, name, false);
+            IHqlExpression * wuid = expr->queryAttribute(wuidAtom);
+            addResultRead(wuid, sequence, name, false);
         }
         break;
     case no_ensureresult:
@@ -2554,7 +2556,7 @@ void DependenciesUsed::extractDependencies(IHqlExpression * expr, unsigned flags
     case no_callsideeffect:
         if (flags & GatherResultRead)
         {
-            addResultRead(expr->queryAttribute(_uid_Atom), NULL, false);
+            addResultRead(NULL, expr->queryAttribute(_uid_Atom), NULL, false);
         }
         break;
     }

ecl/hql/hqlutil.hpp

@@ -299,7 +299,7 @@ protected:
     void addFilenameRead(IHqlExpression * expr);
     void addFilenameWrite(IHqlExpression * expr);
     void addRefDependency(IHqlExpression * expr);
-    void addResultRead(IHqlExpression * seq, IHqlExpression * name, bool isGraphResult);
+    void addResultRead(IHqlExpression * wuid, IHqlExpression * seq, IHqlExpression * name, bool isGraphResult);
     void addResultWrite(IHqlExpression * seq, IHqlExpression * name, bool isGraphResult);
     IHqlExpression * getNormalizedFilename(IHqlExpression * filename);
 

ecl/hqlcpp/hqlcatom.cpp

@@ -340,6 +340,7 @@ IIdAtom * getEnvId;
 IIdAtom * getEventExtraId;
 IIdAtom * getEventNameId;
 IIdAtom * getExpandLogicalNameId;
+IIdAtom * getExternalResultHashId;
 IIdAtom * getFailMessageId;
 IIdAtom * getFilePositionId;
 IIdAtom * getGraphLoopCounterId;
@@ -989,6 +990,7 @@ MODULE_INIT(INIT_PRIORITY_HQLATOM-1)
     MAKEID(getEventExtra);
     MAKEID(getEventName);
     MAKEID(getExpandLogicalName);
+    MAKEID(getExternalResultHash);
     MAKEID(getFailMessage);
     MAKEID(getFilePosition);
     MAKEID(getGraphLoopCounter);

ecl/hqlcpp/hqlcatom.hpp

@@ -340,6 +340,7 @@ extern IIdAtom * getEnvId;
 extern IIdAtom * getEventExtraId;
 extern IIdAtom * getEventNameId;
 extern IIdAtom * getExpandLogicalNameId;
+extern IIdAtom * getExternalResultHashId;
 extern IIdAtom * getFailMessageId;
 extern IIdAtom * getFilePositionId;
 extern IIdAtom * getGraphLoopCounterId;

ecl/hqlcpp/hqlcpp.cpp

@@ -1732,6 +1732,7 @@ void HqlCppTranslator::cacheOptions()
         DebugOption(options.reportAssertFilenameTail,"reportAssertFilenameTail",false),        
         DebugOption(options.newBalancedSpotter,"newBalancedSpotter",true),
         DebugOption(options.keyedJoinPreservesOrder,"keyedJoinPreservesOrder",true),
+        DebugOption(options.expandSelectCreateRow,"expandSelectCreateRow",false),
     };
 
     //get options values from workunit
@@ -1864,6 +1865,8 @@ unsigned HqlCppTranslator::getOptimizeFlags() const
         optFlags |= HOOfoldconstantdatasets;
     if (options.optimizeMax)
         optFlags |= HOOexpensive;
+    if (options.expandSelectCreateRow)
+        optFlags  |= HOOexpandselectcreaterow;
     return optFlags;
 }
 

ecl/hqlcpp/hqlcpp.ipp

@@ -737,6 +737,7 @@ struct HqlCppOptions
     bool                reportAssertFilenameTail;
     bool                newBalancedSpotter;
     bool                keyedJoinPreservesOrder;
+    bool                expandSelectCreateRow;
 };
 
 //Any information gathered while processing the query should be moved into here, rather than cluttering up the translator class

ecl/hqlcpp/hqlcppsys.ecl

@@ -659,6 +659,7 @@ const char * cppSystemText[]  = {
     "   varstring   getResultVarString(const varstring stepname, unsigned4 sequence) : ctxmethod,pure,entrypoint='getResultVarString';",
     "   varunicode  getResultVarUnicode(const varstring stepname, unsigned4 sequence) : ctxmethod,pure,entrypoint='getResultVarUnicode';",
     "   set of any getResultSet(const varstring stepname, unsigned4 sequence, boolean xmltransformer, boolean csvtransformer) : ctxmethod,pure,entrypoint='getResultSet',newset;",
+    "   unsigned4 getExternalResultHash(const varstring wuid, const varstring stepname, unsigned4 sequence) : ctxmethod,pure;",
 
     "   _linkcounted_ dataset getResultRowset(const varstring stepname, unsigned4 sequence, boolean _allocator, boolean isGrouped, boolean xmltransformer, boolean csvtransformer) : ctxmethod,allocator(false),pure,entrypoint='getResultRowset';",
     "   linkcounted dictionary getResultDictionary(const varstring stepname, unsigned4 sequence, boolean xmltransformer, boolean csvtransformer, boolean hasher) : ctxmethod,pure,entrypoint='getResultDictionary';",

ecl/hqlcpp/hqlhtcpp.cpp

@@ -7724,14 +7724,23 @@ IHqlExpression * HqlCppTranslator::calculatePersistInputCrc(BuildCtx & ctx, Depe
         IHqlExpression & cur = dependencies.resultsRead.item(idx2);
         IHqlExpression * seq = cur.queryChild(0);
         IHqlExpression * name = cur.queryChild(1);
+        IHqlExpression * wuid = cur.queryChild(2);
+        if (name->isAttribute())
+        {
+            assertex(name->queryName() == wuidAtom);
+            wuid = name;
+            name = NULL;
+        }
 
         //Not sure if we need to do this if the result is internal.  Leave on for the moment.
         //if (seq->queryValue()->getIntValue() != ResultSequenceInternal)
         bool expandLogical = matchesConstantValue(seq, ResultSequencePersist) && !cur.hasAttribute(_internal_Atom);
         HqlExprArray args;
+        if (wuid)
+            args.append(*LINK(wuid->queryChild(0)));
         args.append(*createResultName(name, expandLogical));
         args.append(*LINK(seq));
-        OwnedHqlExpr call = bindFunctionCall(getResultHashId, args);
+        OwnedHqlExpr call = bindFunctionCall(wuid ? getExternalResultHashId : getResultHashId, args);
         OwnedHqlExpr value = createValue(no_bxor, crcExpr->getType(), LINK(crcExpr), ensureExprType(call, crcExpr->queryType()));
         buildAssignToTemp(ctx, crcExpr, value);
     }

esp/CMakeLists.txt

@@ -13,6 +13,7 @@
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
 ################################################################################
+include_directories (${SCM_GENERATED_DIR})
 HPCC_ADD_SUBDIRECTORY (bindings)
 HPCC_ADD_SUBDIRECTORY (clients "PLATFORM")
 HPCC_ADD_SUBDIRECTORY (eclwatch "PLATFORM")

esp/bindings/http/platform/httptransport.cpp

@@ -450,14 +450,8 @@ void CHttpMessage::addParameter(const char* paramname, const char *value)
     if (strcmp(paramname,"form")==0)
         m_isForm = true;
 
-    if (!m_isForm)
-    {
-        // remove the leading '.'
-        if (*paramname=='.') 
-            paramname++;
-    }
-        m_queryparams->setProp(paramname, value);
-        m_paramCount++;
+    m_queryparams->setProp(paramname, value);
+    m_paramCount++;
 }
 
 StringBuffer& CHttpMessage::getParameter(const char* paramname, StringBuffer& paramval)

esp/scm/ws_workunits.ecm

@@ -1571,6 +1571,43 @@ ESPresponse [exceptions_inline] WUCheckFeaturesResponse
     ESPStruct DeploymentFeatures Deployment;
 };
 
+ESPStruct [nil_remove] WUStatisticItem
+{
+    string Creator;
+    string CreatorType;
+    string Scope;
+    string ScopeType;
+    string Description;
+    string TimeStamp;
+    string Measure;
+    string Kind;
+    string Value;
+    int64 RawValue;
+    int64 Count;
+    int64 Max;
+};
+
+ESPrequest [nil_remove] WUGetStatsRequest
+{
+    string WUID;
+    string CreatorType;
+    string Creator;
+    string ScopeType;
+    string Scope;
+    string Kind;
+    string Measure;
+    unsigned MinScopeDepth;
+    unsigned MaxScopeDepth;
+    boolean IncludeGraphs;
+    boolean CreateDescriptions;
+};
+
+ESPresponse [exceptions_inline] WUGetStatsResponse
+{
+    string WUID;
+    ESParray<ESPstruct WUStatisticItem> Statistics;
+};
+
 ESPservice [
     version("1.54"), default_client_version("1.54"),
     noforms,exceptions_inline("./smc_xslt/exceptions.xslt"),use_method_name] WsWorkunits
@@ -1647,6 +1684,7 @@ ESPservice [
     ESPmethod WUCreateZAPInfo(WUCreateZAPInfoRequest, WUCreateZAPInfoResponse);
     ESPmethod [resp_xsl_default("/esp/xslt/WUZAPInfoForm.xslt")] WUGetZAPInfo(WUGetZAPInfoRequest, WUGetZAPInfoResponse);
     ESPmethod WUCheckFeatures(WUCheckFeaturesRequest, WUCheckFeaturesResponse);
+    ESPmethod WUGetStats(WUGetStatsRequest, WUGetStatsResponse);
 };
 
 

esp/services/ws_ecl/ws_ecl_service.cpp

@@ -1832,7 +1832,7 @@ int CWsEclBinding::submitWsEclWorkunit(IEspContext & context, WsEclWuInfo &wsinf
     return submitWsEclWorkunit(context, wsinfo, reqTree, out, flags, fmt, viewname, xsltname);
 }
 
-void CWsEclBinding::sendRoxieRequest(const char *target, StringBuffer &req, StringBuffer &resp, StringBuffer &status, const char *query, const char *contentType)
+void CWsEclBinding::sendRoxieRequest(const char *target, StringBuffer &req, StringBuffer &resp, StringBuffer &status, const char *query, bool trim, const char *contentType)
 {
     ISmartSocketFactory *conn = NULL;
     SocketEndpoint ep;
@@ -1845,7 +1845,9 @@ void CWsEclBinding::sendRoxieRequest(const char *target, StringBuffer &req, Stri
 
         Owned<IHttpClientContext> httpctx = getHttpClientContext();
         StringBuffer url("http://");
-        ep.getIpText(url).append(':').append(ep.port);
+        ep.getIpText(url).append(':').append(ep.port).append('/').append(target);
+        if (!trim)
+            url.append("?.trim=0");
 
         Owned<IHttpClient> httpclient = httpctx->createHttpClient(NULL, url);
         httpclient->setTimeOut(wsecl->roxieTimeout);
@@ -1890,13 +1892,15 @@ int CWsEclBinding::onSubmitQueryOutput(IEspContext &context, CHttpRequest* reque
     bool isRoxieReq = wsecl->connMap.getValue(wsinfo.qsetname.get())!=NULL;
     bool outputJSON = (format && strieq(format, "json"));
     const char *jsonp = context.queryRequestParameters()->queryProp("jsonp");
+    bool trim = context.queryRequestParameters()->getPropBool(".trim", true);
+    bool trim2 = context.queryRequestParameters()->getPropBool("trim", true);
     if (isRoxieReq && outputJSON)
     {
         StringBuffer jsonmsg;
         getWsEclJsonRequest(jsonmsg, context, request, wsinfo, "json", NULL, 0, false);
         if (jsonp && *jsonp)
             output.append(jsonp).append('(');
-        sendRoxieRequest(wsinfo.qsetname.get(), jsonmsg, output, status, wsinfo.queryname, "application/json");
+        sendRoxieRequest(wsinfo.qsetname.get(), jsonmsg, output, status, wsinfo.queryname, trim, "application/json");
         if (jsonp && *jsonp)
             output.append(");");
     }
@@ -1917,7 +1921,7 @@ int CWsEclBinding::onSubmitQueryOutput(IEspContext &context, CHttpRequest* reque
         else
         {
             StringBuffer roxieresp;
-            sendRoxieRequest(wsinfo.qsetname, soapmsg, roxieresp, status, wsinfo.queryname);
+            sendRoxieRequest(wsinfo.qsetname, soapmsg, roxieresp, status, wsinfo.queryname, trim, "text/xml");
             if (xmlflags & WWV_OMIT_SCHEMAS)
                 expandWuXmlResults(output, wsinfo.queryname, roxieresp.str(), xmlflags);
             else
@@ -1965,7 +1969,7 @@ int CWsEclBinding::onSubmitQueryOutputView(IEspContext &context, CHttpRequest* r
     const char *view = context.queryRequestParameters()->queryProp("view");
     if (strieq(clustertype.str(), "roxie"))
     {
-        sendRoxieRequest(wsinfo.qsetname.get(), soapmsg, output, status, wsinfo.queryname);
+        sendRoxieRequest(wsinfo.qsetname.get(), soapmsg, output, status, wsinfo.queryname, false, "text/xml");
         Owned<IWuWebView> web = createWuWebView(*wu, wsinfo.qsetname.get(), wsinfo.queryname.get(), getCFD(), true);
         if (!view)
             web->applyResultsXSLT(xsltfile.str(), output.str(), html);
@@ -2360,7 +2364,7 @@ int CWsEclBinding::onGet(CHttpRequest* request, CHttpResponse* response)
             StringBuffer status;
             if (getEspLogLevel()>LogNormal)
                 DBGLOG("roxie req: %s", soapreq.str());
-            sendRoxieRequest(target, soapreq, output, status, qid);
+            sendRoxieRequest(target, soapreq, output, status, qid, parms->getPropBool(".trim", true), "text/xml");
             if (getEspLogLevel()>LogNormal)
                 DBGLOG("roxie resp: %s", output.str());
 
@@ -2531,6 +2535,7 @@ void CWsEclBinding::handleJSONPost(CHttpRequest *request, CHttpResponse *respons
             nextPathNode(thepath, queryname);
         }
 
+        bool trim = ctx->queryRequestParameters()->getPropBool(".trim", true);
         const char *jsonp = ctx->queryRequestParameters()->queryProp("jsonp");
         if (jsonp && *jsonp)
             jsonresp.append(jsonp).append('(');
@@ -2541,7 +2546,7 @@ void CWsEclBinding::handleJSONPost(CHttpRequest *request, CHttpResponse *respons
 
         StringBuffer status;
         if (wsecl->connMap.getValue(queryset.str()))
-            sendRoxieRequest(queryset.str(), content, jsonresp, status, queryname.str(), "application/json");
+            sendRoxieRequest(queryset.str(), content, jsonresp, status, queryname.str(), trim, "application/json");
         else
         {
             WsEclWuInfo wsinfo(wuid.str(), queryset.str(), queryname.str(), ctx->queryUserId(), ctx->queryPassword());
@@ -2655,9 +2660,10 @@ int CWsEclBinding::HandleSoapRequest(CHttpRequest* request, CHttpResponse* respo
 
     if (wsecl->connMap.getValue(target))
     {
+        bool trim = ctx->queryRequestParameters()->getPropBool(".trim", true);
         StringBuffer content(request->queryContent());
         StringBuffer output;
-        sendRoxieRequest(target, content, output, status, queryname);
+        sendRoxieRequest(target, content, output, status, queryname, trim, "text/xml");
         if (!(xmlflags  & WWV_CDATA_SCHEMAS))
             soapresp.swapWith(output);
         else

esp/services/ws_ecl/ws_ecl_service.hpp

@@ -196,7 +196,7 @@ public:
     int getJsonTestForm(IEspContext &context, CHttpRequest* request, CHttpResponse* response, WsEclWuInfo &wsinfo, const char *formtype);
     void getWsEclJsonRequest(StringBuffer& soapmsg, IEspContext &context, CHttpRequest* request, WsEclWuInfo &wsinfo, const char *xmltype, const char *ns, unsigned flags, bool validate);
     
-    void sendRoxieRequest(const char *process, StringBuffer &req, StringBuffer &resp, StringBuffer &status, const char *query, const char *contentType="text/xml");
+    void sendRoxieRequest(const char *process, StringBuffer &req, StringBuffer &resp, StringBuffer &status, const char *query, bool trim, const char *contentType);
 };
 
 #endif //_WS_ECL_SERVICE_HPP__

esp/services/ws_workunits/ws_workunitsHelpers.cpp

@@ -1620,6 +1620,60 @@ void WsWuInfo::getResults(IEspECLWorkunit &info, unsigned flags)
     }
 }
 
+void WsWuInfo::getStats(StatisticsFilter& filter, bool createDescriptions, IArrayOf<IEspWUStatisticItem>& statistics)
+{
+    Owned<IConstWUStatisticIterator> stats = &cw->getStatistics(&filter);
+    ForEach(*stats)
+    {
+        IConstWUStatistic & cur = stats->query();
+        StringBuffer xmlBuf, tsValue;
+        SCMStringBuffer curCreator, curScope, curDescription, curFormattedValue;
+
+        StatisticCreatorType curCreatorType = cur.getCreatorType();
+        StatisticScopeType curScopeType = cur.getScopeType();
+        StatisticMeasure curMeasure = cur.getMeasure();
+        StatisticKind curKind = cur.getKind();
+        unsigned __int64 value = cur.getValue();
+        unsigned __int64 count = cur.getCount();
+        unsigned __int64 max = cur.getMax();
+        unsigned __int64 ts = cur.getTimestamp();
+        cur.getCreator(curCreator);
+        cur.getScope(curScope);
+        cur.getDescription(curDescription, createDescriptions);
+        cur.getFormattedValue(curFormattedValue);
+
+        Owned<IEspWUStatisticItem> wuStatistic = createWUStatisticItem();
+
+        if (curCreatorType != SCTnone)
+            wuStatistic->setCreatorType(queryCreatorTypeName(curCreatorType));
+        if (curCreator.length())
+            wuStatistic->setCreator(curCreator.str());
+        if (curScopeType != SSTnone)
+            wuStatistic->setScopeType(queryScopeTypeName(curScopeType));
+        if (curScope.length())
+            wuStatistic->setScope(curScope.str());
+        if (curMeasure != SMeasureNone)
+            wuStatistic->setMeasure(queryMeasureName(curMeasure));
+        if (curKind != StKindNone)
+            wuStatistic->setKind(queryStatisticName(curKind));
+        wuStatistic->setRawValue(value);
+        wuStatistic->setValue(curFormattedValue.str());
+        if (count != 1)
+            wuStatistic->setCount(count);
+        if (max)
+            wuStatistic->setMax(max);
+        if (ts)
+        {
+            formatStatistic(tsValue, ts, SMeasureTimestampUs);
+            wuStatistic->setTimeStamp(tsValue.str());
+        }
+        if (curDescription.length())
+            wuStatistic->setDescription(curDescription.str());
+
+        statistics.append(*wuStatistic.getClear());
+    }
+}
+
 bool WsWuInfo::getFileSize(const char* fileName, const char* IPAddress, offset_t& fileSize)
 {
     if (!fileName || !*fileName)

esp/services/ws_workunits/ws_workunitsHelpers.hpp

@@ -175,6 +175,7 @@ public:
     void getEclSchemaFields(IArrayOf<IEspECLSchemaItem>& schemas, IHqlExpression * expr, bool isConditional);
     bool getResultEclSchemas(IConstWUResult &r, IArrayOf<IEspECLSchemaItem>& schemas);
     void getResult(IConstWUResult &r, IArrayOf<IEspECLResult>& results, unsigned flags);
+    void getStats(StatisticsFilter& filter, bool createDescriptions, IArrayOf<IEspWUStatisticItem>& statistics);
 
     void getWorkunitEclAgentLog(const char* eclAgentInstance, const char* agentPid, MemoryBuffer& buf);
     void getWorkunitThorLog(const char *processName, MemoryBuffer& buf);

esp/services/ws_workunits/ws_workunitsQuerySets.cpp

@@ -689,9 +689,9 @@ void copyQueryFilesToCluster(IEspContext &context, IConstWorkUnit *cw, const cha
         if (queryname && *queryname)
             queryname = queryid.append(queryname).append(".0").str(); //prepublish dummy version number to support fuzzy match like queries="myquery.*" in package
         wufiles->addFilesFromQuery(cw, (ps) ? ps->queryActiveMap(target) : NULL, queryname);
-        wufiles->resolveFiles(process.str(), remoteIP, remotePrefix, srcCluster, !overwrite, true);
+        wufiles->resolveFiles(process.str(), remoteIP, remotePrefix, srcCluster, !overwrite, true, true);
         Owned<IDFUhelper> helper = createIDFUhelper();
-        wufiles->cloneAllInfo(helper, overwrite, true);
+        wufiles->cloneAllInfo(helper, overwrite, true, true);
     }
 }
 
@@ -2242,9 +2242,9 @@ public:
     {
         if (cloneFilesEnabled)
         {
-            wufiles->resolveFiles(process, dfsIP, srcPrefix, srcCluster, !overwriteDfs, true);
+            wufiles->resolveFiles(process, dfsIP, srcPrefix, srcCluster, !overwriteDfs, true, true);
             Owned<IDFUhelper> helper = createIDFUhelper();
-            wufiles->cloneAllInfo(helper, overwriteDfs, true);
+            wufiles->cloneAllInfo(helper, overwriteDfs, true, true);
         }
     }
 private:

esp/services/ws_workunits/ws_workunitsService.cpp

@@ -4234,3 +4234,51 @@ bool CWsWorkunitsEx::onWUCheckFeatures(IEspContext &context, IEspWUCheckFeatures
     resp.updateDeployment().setUseCompression(true);
     return true;
 }
+
+static const char * checkGetStatsInput(const char * s)
+{
+    if (!s || !*s)
+        return "*";
+    return s;
+}
+
+bool CWsWorkunitsEx::onWUGetStats(IEspContext &context, IEspWUGetStatsRequest &req, IEspWUGetStatsResponse &resp)
+{
+    try
+    {
+        StringBuffer wuid = req.getWUID();
+        WsWuHelpers::checkAndTrimWorkunit("WUInfo", wuid);
+
+        ensureWsWorkunitAccess(context, wuid.str(), SecAccess_Read);
+
+        const char* creatorType = checkGetStatsInput(req.getCreatorType());
+        const char* creator = checkGetStatsInput(req.getCreator());
+        const char* scopeType = checkGetStatsInput(req.getScopeType());
+        const char* scope = checkGetStatsInput(req.getScope());
+        const char* kind = checkGetStatsInput(req.getKind());
+        const char* measure = req.getMeasure();
+
+        StatisticsFilter filter(creatorType, creator, scopeType, scope, measure, kind);
+        if (!req.getMinScopeDepth_isNull() && !req.getMaxScopeDepth_isNull())
+            filter.setScopeDepth(req.getMinScopeDepth(), req.getMaxScopeDepth());
+        else if (!req.getMinScopeDepth_isNull())
+            filter.setScopeDepth(req.getMinScopeDepth());
+        if (!req.getIncludeGraphs_isNull())
+            filter.setMergeSources(req.getIncludeGraphs());
+
+        bool createDescriptions = false;
+        if (!req.getCreateDescriptions_isNull())
+            createDescriptions = req.getCreateDescriptions();
+
+        WsWuInfo winfo(context, wuid.str());
+        IArrayOf<IEspWUStatisticItem> statistics;
+        winfo.getStats(filter, createDescriptions, statistics);
+        resp.setStatistics(statistics);
+        resp.setWUID(wuid.str());
+    }
+    catch(IException* e)
+    {
+        FORWARDEXCEPTION(context, e,  ECLWATCH_INTERNAL_ERROR);
+    }
+    return true;
+}

esp/services/ws_workunits/ws_workunitsService.hpp

@@ -254,6 +254,7 @@ public:
     bool onWUCreateZAPInfo(IEspContext &context, IEspWUCreateZAPInfoRequest &req, IEspWUCreateZAPInfoResponse &resp);
     bool onWUGetZAPInfo(IEspContext &context, IEspWUGetZAPInfoRequest &req, IEspWUGetZAPInfoResponse &resp);
     bool onWUCheckFeatures(IEspContext &context, IEspWUCheckFeaturesRequest &req, IEspWUCheckFeaturesResponse &resp);
+    bool onWUGetStats(IEspContext &context, IEspWUGetStatsRequest &req, IEspWUGetStatsResponse &resp);
 
 private:
     void addProcessLogfile(Owned<IConstWorkUnit> &cwu, WsWuInfo &winfo, const char * process, const char* path);

initfiles/bash/etc/init.d/hpcc_common.in

@@ -420,6 +420,9 @@ startCmd() {
         echo "compName=$compName compPath=$compPath compProcessName=$compType"
     fi
 
+    # use less heap when threaded
+    export MALLOC_ARENA_MAX=8
+
     # Creating logfiles for component
     logDir=$log/${compName}
     logFile=$log/${compName}/${compName}_init.log

initfiles/componentfiles/configxml/eclagent_config.xsd.in

@@ -198,6 +198,14 @@
       </xs:annotation>
     </xs:attribute>
 
+    <xs:attribute name="heapRetainMemory" type="xs:boolean" default="false">
+      <xs:annotation>
+        <xs:appinfo>
+          <tooltip>Retain and do not return unused memory to the operating system.</tooltip>
+        </xs:appinfo>
+      </xs:annotation>
+    </xs:attribute>
+
     <xs:attribute name="pluginDirectory" type="absolutePath" use="optional" default="${PLUGINS_PATH}/">
       <xs:annotation>
         <xs:appinfo>

initfiles/componentfiles/configxml/roxie.xsd.in

@@ -765,6 +765,13 @@
         </xs:appinfo>
       </xs:annotation>
     </xs:attribute>
+    <xs:attribute name="heapRetainMemory" type="xs:boolean" default="false">
+      <xs:annotation>
+        <xs:appinfo>
+          <tooltip>Retain and do not return unused memory to the operating system.</tooltip>
+        </xs:appinfo>
+      </xs:annotation>
+    </xs:attribute>
     <xs:attribute name="trapTooManyActiveQueries" type="xs:boolean" use="optional" default="true">
       <xs:annotation>
         <xs:appinfo>

initfiles/componentfiles/configxml/thor.xsd.in

@@ -381,6 +381,13 @@
           </xs:appinfo>
         </xs:annotation>
       </xs:attribute>
+      <xs:attribute name="heapMasterUseHugePages" type="xs:boolean" use="optional">
+        <xs:annotation>
+          <xs:appinfo>
+            <tooltip>Allow heapUseHugePages to be overridden for the master.  Useful to disable if running on a single machine.</tooltip>
+          </xs:appinfo>
+        </xs:annotation>
+      </xs:attribute>
       <xs:attribute name="heapUseTransparentHugePages" type="xs:boolean" default="true">
         <xs:annotation>
           <xs:appinfo>
@@ -388,6 +395,13 @@
           </xs:appinfo>
         </xs:annotation>
       </xs:attribute>
+      <xs:attribute name="heapRetainMemory" type="xs:boolean" default="false">
+        <xs:annotation>
+          <xs:appinfo>
+            <tooltip>Retain and do not return unused memory to the operating system.</tooltip>
+          </xs:appinfo>
+        </xs:annotation>
+      </xs:attribute>
       <xs:attribute name="pluginsPath" type="relativePath" default="${PLUGINS_PATH}/"/>
       <xs:attribute name="nodeGroup" type="xs:string" use="optional">
         <xs:annotation>

initfiles/sbin/install-hpcc.exp

@@ -53,7 +53,8 @@ proc checkSSHConnection {} {
    global ip user password prompt
 
    set timeout 60
-   spawn ssh ${user}@${ip}
+   # JIRA 12585 in chance root user cannot ssh with key pairs
+   spawn su ${user} -c "ssh ${user}@${ip}"
    expect {
       *?assword:* {
          send "${password}\r";
@@ -82,7 +83,7 @@ proc copyPayload {} {
    global ip user password prompt
 
    set timeout 300
-   spawn scp /tmp/remote_install.tgz ${user}@${ip}:~;
+   spawn su ${user} -c "scp /tmp/remote_install.tgz ${user}@${ip}:~;"
    expect {
       *?assword:* {
          send "${password}\r";
@@ -109,7 +110,7 @@ proc expandPayload {} {
    global ip user password prompt
 
    set timeout 180
-   spawn ssh ${user}@${ip} "cd /; tar -zxf ~/remote_install.tgz"
+   spawn su ${user} -c "ssh ${user}@${ip} \"cd /; tar -zxf ~/remote_install.tgz\""
    expect {
       *?assword:* {
          send "${password}\r"
@@ -139,7 +140,7 @@ proc runPayload {} {
    set timeout 60
 
    expect -re $
-   spawn ssh ${user}@${ip}
+   spawn su ${user} -c "ssh ${user}@${ip}"
    expect {
       *?assword:* {
          send "${password}\r"
@@ -160,7 +161,7 @@ proc runPayload {} {
    expect -re $
    send "${cmd_prefix} ${remote_install}/remote-install-engine.sh ${remote_install}/${basepkg}\r"
    expect {
-      *?assword:* {
+      "*password for*" {
          send "${password}\r"
          exp_continue
       } "*?ermission denied*" {

plugins/cassandra/CMakeLists.txt

@@ -56,9 +56,14 @@ if (USE_CASSANDRA)
     option(CASS_INSTALL_HEADER "Install header file" OFF)
     option(CASS_BUILD_STATIC "Build static library" OFF)
     option(CASS_BUILD_EXAMPLES "Build examples" OFF)
+
+    SET (_SAVE_Boost_FOUND "${Boost_FOUND}")  # Working around a bug in the cassandra cmake file
+    UNSET(Boost_FOUND)
     add_subdirectory (cpp-driver ${PROJECT_BINARY_DIR}/cassandra)
     add_dependencies( cassandra libuv )
 
+    SET (Boost_FOUND "${_SAVE_Boost_FOUND}")
+
     set(SRCS cassandraembed.cpp)
 
     include_directories (

plugins/cassandra/cassandraembed.cpp

@@ -165,6 +165,7 @@ public:
         {
             CassandraFuture close_future(cass_session_close(session));
             cass_future_wait(close_future);
+            cass_session_free(session);
         }
         session = _session;
     }
@@ -1130,12 +1131,6 @@ protected:
 
 // Each call to a Cassandra function will use a new CassandraEmbedFunctionContext object
 
-static void cassandraLogCallback(cass_uint64_t time, CassLogLevel severity, CassString message, void* data)
-{
-    const IContextLogger *logctx = (const IContextLogger *) data;
-    logctx->CTXLOG("cassandra: %s: %.*s", cass_log_level_string(severity), (int) message.length, message.data);
-}
-
 class CassandraEmbedFunctionContext : public CInterfaceOf<IEmbedFunctionContext>
 {
 public:
@@ -1143,7 +1138,6 @@ public:
       : logctx(_logctx), flags(_flags), nextParam(0), numParams(0), batchMode((CassBatchType) -1)
     {
         cluster.setown(new CassandraCluster(cass_cluster_new()));
-        cass_cluster_set_log_callback(*cluster, cassandraLogCallback, (void *) &logctx);
         const char *contact_points = "localhost";
         const char *user = "";
         const char *password = "";
@@ -1188,7 +1182,7 @@ public:
                 else if (stricmp(optName, "num_threads_io")==0)
                 {
                     unsigned num_threads_io = getUnsignedOption(val, "num_threads_io");
-                    checkSetOption(cass_cluster_set_num_threads_io(*cluster, num_threads_io), "num_threads_io");
+                    cass_cluster_set_num_threads_io(*cluster, num_threads_io);  // No status return
                 }
                 else if (stricmp(optName, "queue_size_io")==0)
                 {
@@ -1205,35 +1199,35 @@ public:
                     unsigned max_connections_per_host = getUnsignedOption(val, "max_connections_per_host");
                     checkSetOption(cass_cluster_set_max_connections_per_host(*cluster, max_connections_per_host), "max_connections_per_host");
                 }
-                else if (stricmp(optName, "max_simultaneous_creation")==0)
+                else if (stricmp(optName, "max_concurrent_creation")==0)
                 {
-                    unsigned max_simultaneous_creation = getUnsignedOption(val, "max_simultaneous_creation");
-                    checkSetOption(cass_cluster_set_max_simultaneous_creation(*cluster, max_simultaneous_creation), "max_simultaneous_creation");
+                    unsigned max_concurrent_creation = getUnsignedOption(val, "max_concurrent_creation");
+                    checkSetOption(cass_cluster_set_max_concurrent_creation(*cluster, max_concurrent_creation), "max_concurrent_creation");
                 }
-                else if (stricmp(optName, "max_pending_requests")==0)
+                else if (stricmp(optName, "pending_requests_high_water_mark")==0)
                 {
-                    unsigned max_pending_requests = getUnsignedOption(val, "max_pending_requests");
-                    checkSetOption(cass_cluster_set_max_pending_requests(*cluster, max_pending_requests), "max_pending_requests");
+                    unsigned pending_requests_high_water_mark = getUnsignedOption(val, "pending_requests_high_water_mark");
+                    checkSetOption(cass_cluster_set_pending_requests_high_water_mark(*cluster, pending_requests_high_water_mark), "pending_requests_high_water_mark");
                 }
-                else if (stricmp(optName, "max_simultaneous_requests_threshold")==0)
+                else if (stricmp(optName, "pending_requests_low_water_mark")==0)
                 {
-                    unsigned max_simultaneous_requests_threshold = getUnsignedOption(val, "max_simultaneous_requests_threshold");
-                    checkSetOption(cass_cluster_set_max_simultaneous_requests_threshold(*cluster, max_simultaneous_requests_threshold), "max_simultaneous_requests_threshold");
+                    unsigned pending_requests_low_water_mark = getUnsignedOption(val, "pending_requests_low_water_mark");
+                    checkSetOption(cass_cluster_set_pending_requests_low_water_mark(*cluster, pending_requests_low_water_mark), "pending_requests_low_water_mark");
+                }
+                else if (stricmp(optName, "max_concurrent_requests_threshold")==0)
+                {
+                    unsigned max_concurrent_requests_threshold = getUnsignedOption(val, "max_concurrent_requests_threshold");
+                    checkSetOption(cass_cluster_set_max_concurrent_requests_threshold(*cluster, max_concurrent_requests_threshold), "max_concurrent_requests_threshold");
                 }
                 else if (stricmp(optName, "connect_timeout")==0)
                 {
                     unsigned connect_timeout = getUnsignedOption(val, "connect_timeout");
-                    checkSetOption(cass_cluster_set_connect_timeout(*cluster, connect_timeout), "connect_timeout");
+                    cass_cluster_set_connect_timeout(*cluster, connect_timeout);
                 }
                 else if (stricmp(optName, "request_timeout")==0)
                 {
                     unsigned request_timeout = getUnsignedOption(val, "request_timeout");
-                    checkSetOption(cass_cluster_set_request_timeout(*cluster, request_timeout), "request_timeout");
-                }
-                else if (stricmp(optName, "log_level")==0)
-                {
-                    unsigned log_level = getUnsignedOption(val, "log_level");
-                    checkSetOption(cass_cluster_set_log_level(*cluster, (CassLogLevel) log_level), "log_level");
+                    cass_cluster_set_request_timeout(*cluster, request_timeout);
                 }
                 else
                     failx("Unrecognized option %s", optName.str());
@@ -1243,9 +1237,9 @@ public:
         if (*user || *password)
             cass_cluster_set_credentials(*cluster, user, password);
 
-        CassandraFuture future(keyspace ? cass_cluster_connect_keyspace(*cluster, keyspace) : cass_cluster_connect(*cluster));
+        session.setown(new CassandraSession(cass_session_new()));
+        CassandraFuture future(keyspace ? cass_session_connect_keyspace(*session, *cluster, keyspace) : cass_session_connect(*session, *cluster));
         future.wait("connect");
-        session.setown(new CassandraSession(cass_future_get_session(future)));
     }
     virtual bool getBooleanResult()
     {

plugins/cassandra/cpp-driver

@@ -1 +1 @@
-Subproject commit 5822a469fdca3bdc0ca034dd61c01d1966acc6ef
+Subproject commit 019d3f62b39eb6ebf0a8605faa7130bd9863a9f9

roxie/ccd/ccdactivities.cpp

@@ -538,6 +538,7 @@ public:
     }
 
     virtual unsigned getResultHash(const char * name, unsigned sequence) { throwUnexpected(); }
+    virtual unsigned getExternalResultHash(const char * wuid, const char * name, unsigned sequence) { throwUnexpected(); }
 
     // Not yet thought about these....
 

roxie/ccd/ccdcontext.cpp

@@ -324,6 +324,12 @@ private:
         return r->getResultHash();
     }
 
+    unsigned getExternalResultHash(const char * wuid, const char * name, unsigned sequence)
+    {
+        //GH->RKC Roxie should implement reading external results when connected to dali
+        UNIMPLEMENTED;
+    }
+
     unsigned __int64 getResultInt(const char * name, unsigned sequence)
     {
         Owned<IConstWUResult> r = getWorkUnitResult(workunit, name, sequence);
@@ -1516,6 +1522,7 @@ public:
     virtual void setResultVarUnicode(const char * name, unsigned sequence, UChar const * value) { throwUnexpected(); }
 
     virtual unsigned getResultHash(const char * name, unsigned sequence) { throwUnexpected(); }
+    virtual unsigned getExternalResultHash(const char * wuid, const char * name, unsigned sequence) { throwUnexpected(); }
     virtual void printResults(IXmlWriter *output, const char *name, unsigned sequence) { throwUnexpected(); }
 
     virtual char *getWuid() { throwUnexpected(); }
@@ -3713,7 +3720,7 @@ private:
 
 public:
     CSoapRoxieServerContext(IPropertyTree *_context, const IQueryFactory *_factory, SafeSocket &_client, HttpHelper &httpHelper, const ContextLogger &_logctx, PTreeReaderOptions xmlReadFlags, const char *_querySetName)
-        : CRoxieServerContext(_context, _factory, _client, MarkupFmt_XML, false, false, httpHelper, true, _logctx, xmlReadFlags, _querySetName)
+        : CRoxieServerContext(_context, _factory, _client, MarkupFmt_XML, false, false, httpHelper, httpHelper.getTrim(), _logctx, xmlReadFlags, _querySetName)
     {
         queryName.set(_context->queryName());
     }
@@ -3772,7 +3779,7 @@ private:
 
 public:
     CJsonRoxieServerContext(IPropertyTree *_context, const IQueryFactory *_factory, SafeSocket &_client, HttpHelper &httpHelper, const ContextLogger &_logctx, PTreeReaderOptions xmlReadFlags, const char *_querySetName)
-        : CRoxieServerContext(_context, _factory, _client, MarkupFmt_JSON, false, false, httpHelper, true, _logctx, xmlReadFlags, _querySetName)
+        : CRoxieServerContext(_context, _factory, _client, MarkupFmt_JSON, false, false, httpHelper, httpHelper.getTrim(), _logctx, xmlReadFlags, _querySetName)
     {
         queryName.set(_context->queryName());
     }

roxie/ccd/ccdlistener.cpp

@@ -170,7 +170,7 @@ public:
         try
         {
             IPropertyTree &request = requestArray.item(idx);
-            Owned<IRoxieServerContext> ctx = f->createContext(&request, client, httpHelper.queryContentFormat(), false, false, httpHelper, true, logctx, xmlReadFlags, querySetName);
+            Owned<IRoxieServerContext> ctx = f->createContext(&request, client, httpHelper.queryContentFormat(), false, false, httpHelper, httpHelper.getTrim(), logctx, xmlReadFlags, querySetName);
             ctx->process();
             ctx->flush(idx);
             CriticalBlock b(crit);

roxie/ccd/ccdmain.cpp

@@ -759,9 +759,10 @@ int STARTQUERY_API start_query(int argc, const char *argv[])
         memsize_t totalMemoryLimit = (memsize_t) topology->getPropInt64("@totalMemoryLimit", 0);
         bool allowHugePages = topology->getPropBool("@heapUseHugePages", false);
         bool allowTransparentHugePages = topology->getPropBool("@heapUseTransparentHugePages", true);
+        bool retainMemory = topology->getPropBool("@heapRetainMemory", false);
         if (!totalMemoryLimit)
             totalMemoryLimit = 1024 * 0x100000;  // 1 Gb;
-        roxiemem::setTotalMemoryLimit(allowHugePages, allowTransparentHugePages, totalMemoryLimit, 0, NULL, NULL);
+        roxiemem::setTotalMemoryLimit(allowHugePages, allowTransparentHugePages, retainMemory, totalMemoryLimit, 0, NULL, NULL);
 
         traceStartStop = topology->getPropBool("@traceStartStop", false);
         traceServerSideCache = topology->getPropBool("@traceServerSideCache", false);

roxie/ccd/ccdserver.cpp

@@ -26603,7 +26603,7 @@ protected:
 
     void testSetup()
     {
-        roxiemem::setTotalMemoryLimit(false, true, 100 * 1024 * 1024, 0, NULL, NULL);
+        roxiemem::setTotalMemoryLimit(false, true, false, 100 * 1024 * 1024, 0, NULL, NULL);
     }
 
     void testCleanup()

roxie/roxiemem/roxiemem.cpp

@@ -138,7 +138,7 @@ typedef MapBetween<unsigned, unsigned, memsize_t, memsize_t> MapActivityToMemsiz
 
 static CriticalSection heapBitCrit;
 
-static void initializeHeap(bool allowHugePages, bool allowTransparentHugePages, unsigned pages, unsigned largeBlockGranularity, ILargeMemCallback * largeBlockCallback)
+static void initializeHeap(bool allowHugePages, bool allowTransparentHugePages, bool retainMemory, unsigned pages, unsigned largeBlockGranularity, ILargeMemCallback * largeBlockCallback)
 {
     if (heapBase) return;
 
@@ -148,6 +148,9 @@ static void initializeHeap(bool allowHugePages, bool allowTransparentHugePages,
     heapLargeBlockGranularity = largeBlockGranularity;
     heapLargeBlockCallback = largeBlockCallback;
     memsize_t memsize = memsize_t(heapTotalPages) * HEAP_ALIGNMENT_SIZE;
+
+    heapNotifyUnusedEachFree = !retainMemory;
+    heapNotifyUnusedEachBlock = false;
 #ifdef _WIN32
     // Not the world's best code but will do 
     char *next = (char *) HEAP_ALIGNMENT_SIZE;
@@ -174,8 +177,7 @@ static void initializeHeap(bool allowHugePages, bool allowTransparentHugePages,
         {
             heapUseHugePages = true;
             heapNotifyUnusedEachFree = false;
-            //MORE: At the moment I'm not sure calling madvise() has any benefit, it may be better for the following to stay false;
-            heapNotifyUnusedEachBlock = true;
+            heapNotifyUnusedEachBlock = !retainMemory;
             DBGLOG("Using Huge Pages for roxiemem");
         }
         else
@@ -234,13 +236,12 @@ static void initializeHeap(bool allowHugePages, bool allowTransparentHugePages,
                 if ((heapBlockSize % hugePageSize) == 0)
                 {
                     //If we notify heapBlockSize items at a time it will always be a multiple of hugePageSize so shouldn't trigger defragmentation
-                    heapNotifyUnusedEachBlock = true;
-                    DBGLOG("Transparent huge pages used for roxiemem heap - memory released in blocks");
+                    heapNotifyUnusedEachBlock = !retainMemory;
+                    DBGLOG("Transparent huge pages used for roxiemem heap");
                 }
                 else
                 {
-                    DBGLOG("Transparent huge pages used for roxiemem heap- MEMORY WILL NOT BE RELEASED.");
-                    DBGLOG("Increase HEAP_ALIGNMENT_SIZE so HEAP_ALIGNMENT_SIZE*32 is a multiple of huge page size"");");
+                    DBGLOG("Transparent huge pages used for roxiemem heap");
                 }
             }
         }
@@ -260,6 +261,17 @@ static void initializeHeap(bool allowHugePages, bool allowTransparentHugePages,
     }
 #endif
 
+    if (heapNotifyUnusedEachFree)
+        DBGLOG("Memory released to OS on each %uk 'page'", (unsigned)(HEAP_ALIGNMENT_SIZE/1024));
+    else if (heapNotifyUnusedEachBlock)
+        DBGLOG("Memory released to OS in %uk blocks", (unsigned)(HEAP_ALIGNMENT_SIZE*UNSIGNED_BITS/1024));
+    else
+    {
+        DBGLOG("MEMORY WILL NOT BE RELEASED TO OS");
+        if (!retainMemory)
+            DBGLOG("Increase HEAP_ALIGNMENT_SIZE so HEAP_ALIGNMENT_SIZE*32 is a multiple of system huge page size");
+    }
+
     assertex(((memsize_t)heapBase & (HEAP_ALIGNMENT_SIZE-1)) == 0);
 
     heapEnd = heapBase + memsize;
@@ -4351,7 +4363,7 @@ extern void setMemoryStatsInterval(unsigned secs)
     lastStatsCycles = get_cycles_now();
 }
 
-extern void setTotalMemoryLimit(bool allowHugePages, bool allowTransparentHugePages, memsize_t max, memsize_t largeBlockSize, const unsigned * allocSizes, ILargeMemCallback * largeBlockCallback)
+extern void setTotalMemoryLimit(bool allowHugePages, bool allowTransparentHugePages, bool retainMemory, memsize_t max, memsize_t largeBlockSize, const unsigned * allocSizes, ILargeMemCallback * largeBlockCallback)
 {
     assertex(largeBlockSize == align_pow2(largeBlockSize, HEAP_ALIGNMENT_SIZE));
     unsigned totalMemoryLimit = (unsigned) (max / HEAP_ALIGNMENT_SIZE);
@@ -4360,7 +4372,7 @@ extern void setTotalMemoryLimit(bool allowHugePages, bool allowTransparentHugePa
         totalMemoryLimit = 1;
     if (memTraceLevel)
         DBGLOG("RoxieMemMgr: Setting memory limit to %"I64F"d bytes (%u pages)", (unsigned __int64) max, totalMemoryLimit);
-    initializeHeap(allowHugePages, allowTransparentHugePages, totalMemoryLimit, largeBlockGranularity, largeBlockCallback);
+    initializeHeap(allowHugePages, allowTransparentHugePages, retainMemory, totalMemoryLimit, largeBlockGranularity, largeBlockCallback);
     initAllocSizeMappings(allocSizes ? allocSizes : defaultAllocSizes);
 }
 
@@ -4616,7 +4628,7 @@ protected:
         ASSERT(HugeHeaplet::dataOffset() >= sizeof(HugeHeaplet));
 
         memsize_t memory = (useLargeMemory ? largeMemory : smallMemory) * (unsigned __int64)0x100000U;
-        initializeHeap(false, true, (unsigned)(memory / HEAP_ALIGNMENT_SIZE), 0, NULL);
+        initializeHeap(false, true, false, (unsigned)(memory / HEAP_ALIGNMENT_SIZE), 0, NULL);
         initAllocSizeMappings(defaultAllocSizes);
     }
 
@@ -5986,7 +5998,7 @@ public:
 protected:
     void testSetup()
     {
-        setTotalMemoryLimit(true, true, memorySize, 0, NULL, NULL);
+        setTotalMemoryLimit(true, true, false, memorySize, 0, NULL, NULL);
     }
 
     void testCleanup()
@@ -6182,7 +6194,7 @@ public:
 protected:
     void testSetup()
     {
-        setTotalMemoryLimit(true, true, hugeMemorySize, 0, NULL, NULL);
+        setTotalMemoryLimit(true, true, false, hugeMemorySize, 0, NULL, NULL);
     }
 
     void testCleanup()

roxie/roxiemem/roxiemem.hpp

@@ -511,7 +511,7 @@ interface IDataBufferManager : extends IInterface
 
 extern roxiemem_decl IDataBufferManager *createDataBufferManager(size32_t size);
 extern roxiemem_decl void setMemoryStatsInterval(unsigned secs);
-extern roxiemem_decl void setTotalMemoryLimit(bool allowHugePages, bool allowTransparentHugePages, memsize_t max, memsize_t largeBlockSize, const unsigned * allocSizes, ILargeMemCallback * largeBlockCallback);
+extern roxiemem_decl void setTotalMemoryLimit(bool allowHugePages, bool allowTransparentHugePages, bool retainMemory, memsize_t max, memsize_t largeBlockSize, const unsigned * allocSizes, ILargeMemCallback * largeBlockCallback);
 extern roxiemem_decl memsize_t getTotalMemoryLimit();
 extern roxiemem_decl void releaseRoxieHeap();
 extern roxiemem_decl bool memPoolExhausted();

roxie/udplib/uttest.cpp

@@ -754,7 +754,7 @@ int main(int argc, char * argv[] )
             printf("ERROR: my ip does not appear to be in range\n");
             usage();
         }
-        roxiemem::setTotalMemoryLimit(false, true, 1048576000, 0, NULL, NULL);
+        roxiemem::setTotalMemoryLimit(false, true, false, 1048576000, 0, NULL, NULL);
         testNxN();
         roxiemem::releaseRoxieHeap();
     }

rtl/include/eclhelper.hpp

@@ -645,6 +645,7 @@ interface ICodeContext : public IResourceContext
 
     virtual const void * fromJson(IEngineRowAllocator * _rowAllocator, size32_t len, const char * utf8, IXmlToRowTransformer * xmlTransformer, bool stripWhitespace) = 0;
     virtual void getRowJSON(size32_t & lenResult, char * & result, IOutputMetaData & info, const void * row, unsigned flags) = 0;
+    virtual unsigned getExternalResultHash(const char * wuid, const char * name, unsigned sequence) = 0;
 };
 
 

testing/regress/ecl/aaawriteresult.ecl

@@ -0,0 +1,28 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2015 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#workunit ('name','ExportResult');
+
+namesRecord := 
+            RECORD
+string10        forename;
+string20        surname;
+            END;
+
+ds := dataset([{'Jo','Smith'},{'Jim','Smithe'},{'Joe','Schmitt'}], namesRecord);
+    
+output(ds,NAMED('ExportedNames'));

testing/regress/ecl/key/aaawriteresult.xml

@@ -0,0 +1,5 @@
+<Dataset name='ExportedNames'>
+ <Row><forename>Jo        </forename><surname>Smith               </surname></Row>
+ <Row><forename>Jim       </forename><surname>Smithe              </surname></Row>
+ <Row><forename>Joe       </forename><surname>Schmitt             </surname></Row>
+</Dataset>

testing/regress/ecl/key/readresult.xml

@@ -0,0 +1,8 @@
+<Dataset name='Result 1'>
+ <Row><Result_1>W</Result_1></Row>
+</Dataset>
+<Dataset name='Result 2'>
+ <Row><forename>Jo        </forename><surname>Smith               </surname></Row>
+ <Row><forename>Jim       </forename><surname>Smithe              </surname></Row>
+ <Row><forename>Joe       </forename><surname>Schmitt             </surname></Row>
+</Dataset>

testing/regress/ecl/readresult.ecl

@@ -0,0 +1,48 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2015 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//noroxie       - roxie doesn't currently support reading from external workunits
+
+namesRecord := 
+            RECORD
+string10        forename;
+string20        surname;
+            END;
+
+
+//Horrible code - get a list of workunits that match the name of the job that creates the result
+//which needs to be inside a nothor.
+
+import Std.System.Workunit as Wu;
+myWuid := workunit;
+startOfDay := myWuid[1..9] + '-000000';
+writers := Wu.WorkunitList(lowWuid := startOfDay,jobname := 'aaawriteresult*');
+
+//Now sort and extract the most recent wuid that matches the condition
+lastWriter := sort(nothor(writers), -wuid);
+wuid := TRIM(lastWriter[1].wuid) : independent;   // trim should not be needed
+
+ds := dataset(workunit(WUID,'ExportedNames'), namesRecord);
+
+p := ds : persist('readExported', single);
+
+import Std.File;
+sequential(
+    output(NOFOLD(wuid)[1..1]); /// yuk - output the wuid so it gets evaluated before the check in the persist call
+    File.DeleteLogicalFile('~readExported'),
+    output(p);
+);

testing/regress/ecl/setup/setupsearchindex.ecl

@@ -15,6 +15,8 @@
     limitations under the License.
 ############################################################################## */
 
+//class=textsearch
+
 #option ('checkAsserts',false);
 
 import $.SetupText;

testing/regress/ecl/steplimit1_hthor.ecl

@@ -15,6 +15,8 @@
     limitations under the License.
 ############################################################################## */
 
+//class=textsearch
+
 import $.Setup;
 import $.Setup.TS;
 searchIndex := Setup.Files('hthor', false).getSearchIndex();

testing/regress/ecl/steplimit1_thor.ecl

@@ -15,6 +15,8 @@
     limitations under the License.
 ############################################################################## */
 
+//class=textsearch
+
 import $.Setup;
 import $.Setup.TS;
 searchIndex := Setup.Files('thorlcr', false).getSearchIndex();

testing/regress/ecl/steplimit2_hthor.ecl

@@ -15,6 +15,8 @@
     limitations under the License.
 ############################################################################## */
 
+//class=textsearch
+
 import $.Setup;
 import $.Setup.TS;
 searchIndex := Setup.Files('hthor', false).getSearchIndex();

testing/regress/ecl/steplimit2_thor.ecl

@@ -15,6 +15,8 @@
     limitations under the License.
 ############################################################################## */
 
+//class=textsearch
+
 import $.Setup;
 import $.Setup.TS;
 searchIndex := Setup.Files('thorlcr', false).getSearchIndex();

testing/regress/ecl/steplimit3_hthor.ecl

@@ -15,6 +15,8 @@
     limitations under the License.
 ############################################################################## */
 
+//class=textsearch
+
 //MORE: This really should be supported on thor....
 //Stepped global joins unsupported, see issue HPCC-8148
 //skip type==thorlcr TBD

testing/regress/ecl/steplimit3_thor.ecl

@@ -15,6 +15,8 @@
     limitations under the License.
 ############################################################################## */
 
+//class=textsearch
+
 //Stepped global joins unsupported, see issue HPCC-8148
 //skip type==thorlcr TBD
 

testing/regress/ecl/stepping7a.ecl

@@ -15,6 +15,8 @@
     limitations under the License.
 ############################################################################## */
 
+//class=textsearch
+
 //Stepped global joins unsupported, see issue HPCC-8148
 //skip type==thorlcr TBD
 

testing/regress/ecl/stepping7a_thor.ecl

@@ -15,6 +15,8 @@
     limitations under the License.
 ############################################################################## */
 
+//class=textsearch
+
 //Stepped global joins unsupported, see issue HPCC-8148
 //skip type==thorlcr TBD
 

testing/regress/ecl/stepping7b.ecl

@@ -15,6 +15,8 @@
     limitations under the License.
 ############################################################################## */
 
+//class=textsearch
+
 //Stepped global joins unsupported, see issue HPCC-8148
 //skip type==thorlcr TBD
 

testing/regress/ecl/stepping7b_thor.ecl

@@ -15,6 +15,8 @@
     limitations under the License.
 ############################################################################## */
 
+//class=textsearch
+
 //Stepped global joins unsupported, see issue HPCC-8148
 //skip type==thorlcr TBD
 

testing/regress/ecl/stepping7c.ecl

@@ -15,6 +15,8 @@
     limitations under the License.
 ############################################################################## */
 
+//class=textsearch
+
 //Stepped global joins unsupported, see issue HPCC-8148
 //skip type==thorlcr TBD
 

testing/regress/ecl/stepping7c_thor.ecl

@@ -15,6 +15,8 @@
     limitations under the License.
 ############################################################################## */
 
+//class=textsearch
+
 //Stepped global joins unsupported, see issue HPCC-8148
 //skip type==thorlcr TBD
 

testing/regress/ecl/stepping7d.ecl

@@ -15,6 +15,8 @@
     limitations under the License.
 ############################################################################## */
 
+//class=textsearch
+
 //Stepped global joins unsupported, see issue HPCC-8148
 //skip type==thorlcr TBD
 

testing/regress/ecl/stepping7d_thor.ecl

@@ -15,6 +15,8 @@
     limitations under the License.
 ############################################################################## */
 
+//class=textsearch
+
 //Stepped global joins unsupported, see issue HPCC-8148
 //skip type==thorlcr TBD
 

testing/regress/ecl/stepping7e.ecl

@@ -15,6 +15,8 @@
     limitations under the License.
 ############################################################################## */
 
+//class=textsearch
+
 //Stepped global joins unsupported, see issue HPCC-8148
 //skip type==thorlcr TBD
 

testing/regress/ecl/stepping7e_thor.ecl

@@ -15,6 +15,8 @@
     limitations under the License.
 ############################################################################## */
 
+//class=textsearch
+
 //Stepped global joins unsupported, see issue HPCC-8148
 //skip type==thorlcr TBD
 

testing/regress/ecl/stepping7f.ecl

@@ -15,6 +15,8 @@
     limitations under the License.
 ############################################################################## */
 
+//class=textsearch
+
 //Stepped global joins unsupported, see issue HPCC-8148
 //skip type==thorlcr TBD
 

testing/regress/ecl/stepping7f_thor.ecl

@@ -15,6 +15,8 @@
     limitations under the License.
 ############################################################################## */
 
+//class=textsearch
+
 //Stepped global joins unsupported, see issue HPCC-8148
 //skip type==thorlcr TBD
 

testing/regress/ecl/stepping7g.ecl

@@ -15,6 +15,8 @@
     limitations under the License.
 ############################################################################## */
 
+//class=textsearch
+
 import $.Setup;
 import $.Setup.TS;
 searchIndex := Setup.Files('hthor', false).getSearchIndex();

testing/regress/ecl/stepping7g_thor.ecl

@@ -15,6 +15,8 @@
     limitations under the License.
 ############################################################################## */
 
+//class=textsearch
+
 import $.Setup;
 import $.Setup.TS;
 searchIndex := Setup.Files('thorlcr', false).getSearchIndex();

testing/regress/ecl/stepping7h.ecl

@@ -15,6 +15,8 @@
     limitations under the License.
 ############################################################################## */
 
+//class=textsearch
+
 import $.Setup;
 import $.Setup.TS;
 searchIndex := Setup.Files('hthor', false).getSearchIndex();

testing/regress/ecl/stepping7h_thor.ecl

@@ -15,6 +15,8 @@
     limitations under the License.
 ############################################################################## */
 
+//class=textsearch
+
 import $.Setup;
 import $.Setup.TS;
 searchIndex := Setup.Files('thorlcr', false).getSearchIndex();

testing/regress/ecl/stepping8.ecl

@@ -15,6 +15,8 @@
     limitations under the License.
 ############################################################################## */
 
+//class=textsearch
+
 //Stepped global joins unsupported, see issue HPCC-8148
 //skip type==thorlcr TBD
 

testing/regress/ecl/stepping8_thor.ecl

@@ -15,6 +15,8 @@
     limitations under the License.
 ############################################################################## */
 
+//class=textsearch
+
 //Stepped global joins unsupported, see issue HPCC-8148
 //skip type==thorlcr TBD
 

testing/regress/ecl/stepping8a.ecl

@@ -15,6 +15,8 @@
     limitations under the License.
 ############################################################################## */
 
+//class=textsearch
+
 //Stepped global joins unsupported, see issue HPCC-8148
 //skip type==thorlcr TBD
 

testing/regress/ecl/stepping8a_thor.ecl

@@ -15,6 +15,8 @@
     limitations under the License.
 ############################################################################## */
 
+//class=textsearch
+
 //Stepped global joins unsupported, see issue HPCC-8148
 //skip type==thorlcr TBD
 

testing/regress/ecl/stepping8b.ecl

@@ -15,6 +15,8 @@
     limitations under the License.
 ############################################################################## */
 
+//class=textsearch
+
 //Stepped global joins unsupported, see issue HPCC-8148
 //skip type==thorlcr TBD
 

testing/regress/ecl/stepping8b_thor.ecl

@@ -15,6 +15,8 @@
     limitations under the License.
 ############################################################################## */
 
+//class=textsearch
+
 //Stepped global joins unsupported, see issue HPCC-8148
 //skip type==thorlcr TBD
 

testing/regress/ecl/stepping9a.ecl

@@ -15,6 +15,8 @@
     limitations under the License.
 ############################################################################## */
 
+//class=textsearch
+
 //Stepped global joins unsupported, see issue HPCC-8148
 //skip type==thorlcr TBD
 

testing/regress/ecl/stepping9a_thor.ecl

@@ -15,6 +15,8 @@
     limitations under the License.
 ############################################################################## */
 
+//class=textsearch
+
 //Stepped global joins unsupported, see issue HPCC-8148
 //skip type==thorlcr TBD
 

testing/regress/ecl/stepping9b.ecl

@@ -15,6 +15,8 @@
     limitations under the License.
 ############################################################################## */
 
+//class=textsearch
+
 //Stepped global joins unsupported, see issue HPCC-8148
 //skip type==thorlcr TBD
 

testing/regress/ecl/stepping9b_thor.ecl

@@ -15,6 +15,8 @@
     limitations under the License.
 ############################################################################## */
 
+//class=textsearch
+
 //Stepped global joins unsupported, see issue HPCC-8148
 //skip type==thorlcr TBD
 

testing/regress/ecl/textsearch4.ecl

@@ -15,6 +15,8 @@
     limitations under the License.
 ############################################################################## */
 
+//class=textsearch
+
 //nothor
 
 #option ('checkAsserts',false);

testing/regress/ecl/textsearch4_thor.ecl

@@ -15,6 +15,8 @@
     limitations under the License.
 ############################################################################## */
 
+//class=textsearch
+
 //nothor
 
 #option ('checkAsserts',false);

thorlcr/graph/thgraph.hpp

@@ -471,6 +471,7 @@ class graph_decl CGraphBase : public CInterface, implements IEclGraphResults, im
         virtual char *getResultVarString(const char * name, unsigned sequence) { return ctx->getResultVarString(name, sequence); }
         virtual UChar *getResultVarUnicode(const char * name, unsigned sequence) { return ctx->getResultVarUnicode(name, sequence); }
         virtual unsigned getResultHash(const char * name, unsigned sequence) { return ctx->getResultHash(name, sequence); }
+        virtual unsigned getExternalResultHash(const char * wuid, const char * name, unsigned sequence) { return ctx->getExternalResultHash(wuid, name, sequence); }
         virtual const char *cloneVString(const char *str) const { return ctx->cloneVString(str); }
         virtual const char *cloneVString(size32_t len, const char *str) const { return ctx->cloneVString(len, str); }
         virtual char *getWuid() { return ctx->getWuid(); }

thorlcr/graph/thgraphmaster.cpp

@@ -1062,6 +1062,20 @@ public:
             return r->getResultHash();
         );
     }
+    virtual unsigned getExternalResultHash(const char * wuid, const char * stepname, unsigned sequence)
+    {
+        try
+        {
+            LOG(MCdebugProgress, unknownJob, "getExternalResultRaw %s", stepname);
+
+            Owned<IConstWUResult> r = getExternalResult(wuid, stepname, sequence);
+            return r->getResultHash();
+        }
+        catch (CATCHALL)
+        {
+            throw MakeStringException(TE_FailedToRetrieveWorkunitValue, "Failed to retrieve external data hash %s from workunit %s", stepname, wuid);
+        }
+    }
     virtual void getResultRowset(size32_t & tcount, byte * * & tgt, const char * stepname, unsigned sequence, IEngineRowAllocator * _rowAllocator, bool isGrouped, IXmlToRowTransformer * xmlTransformer, ICsvToRowTransformer * csvTransformer)
     {
         tgt = NULL;

thorlcr/graph/thgraphslave.cpp

@@ -951,6 +951,7 @@ public:
     virtual unsigned getResultHash(const char * name, unsigned sequence) { throwUnexpected(); }
 
     virtual void getExternalResultRaw(unsigned & tlen, void * & tgt, const char * wuid, const char * stepname, unsigned sequence, IXmlToRowTransformer * xmlTransformer, ICsvToRowTransformer * csvTransformer) { throwUnexpected(); }
+    virtual unsigned getExternalResultHash(const char * wuid, const char * name, unsigned sequence) { throwUnexpected(); }
 
     virtual void addWuException(const char * text, unsigned code, unsigned severity, const char * source)
     {

thorlcr/master/thmastermain.cpp

@@ -683,13 +683,15 @@ int main( int argc, char *argv[]  )
                 mmemSize = gmemSize;
         }
         bool gmemAllowHugePages = globals->getPropBool("@heapUseHugePages", false);
+        gmemAllowHugePages = globals->getPropBool("@heapMasterUseHugePages", gmemAllowHugePages);
         bool gmemAllowTransparentHugePages = globals->getPropBool("@heapUseTransparentHugePages", true);
+        bool gmemRetainMemory = globals->getPropBool("@heapRetainMemory", false);
 
         // if @masterMemorySize and @globalMemorySize unspecified gmemSize will be default based on h/w
         globals->setPropInt("@masterMemorySize", mmemSize);
 
         PROGLOG("Global memory size = %d MB", mmemSize);
-        roxiemem::setTotalMemoryLimit(gmemAllowHugePages, gmemAllowTransparentHugePages, ((memsize_t)mmemSize) * 0x100000, 0, thorAllocSizes, NULL);
+        roxiemem::setTotalMemoryLimit(gmemAllowHugePages, gmemAllowTransparentHugePages, gmemRetainMemory, ((memsize_t)mmemSize) * 0x100000, 0, thorAllocSizes, NULL);
 
         const char * overrideBaseDirectory = globals->queryProp("@thorDataDirectory");
         const char * overrideReplicateDirectory = globals->queryProp("@thorReplicateDirectory");

thorlcr/slave/slavmain.cpp

@@ -664,11 +664,13 @@ void slaveMain()
     unsigned gmemSize = globals->getPropInt("@globalMemorySize");
     bool gmemAllowHugePages = globals->getPropBool("@heapUseHugePages", false);
     bool gmemAllowTransparentHugePages = globals->getPropBool("@heapUseTransparentHugePages", true);
+    bool gmemRetainMemory = globals->getPropBool("@heapRetainMemory", false);
+
     if (gmemSize >= hdwInfo.totalMemory)
     {
         // should prob. error here
     }
-    roxiemem::setTotalMemoryLimit(gmemAllowHugePages, gmemAllowTransparentHugePages, ((memsize_t)gmemSize) * 0x100000, 0, thorAllocSizes, NULL);
+    roxiemem::setTotalMemoryLimit(gmemAllowHugePages, gmemAllowTransparentHugePages, gmemRetainMemory, ((memsize_t)gmemSize) * 0x100000, 0, thorAllocSizes, NULL);
 
     CJobListener jobListener;
     CThorResourceSlave slaveResource;