Merge remote-tracking branch 'origin/candidate-5.2.0'

Signed-off-by: Gavin Halliday <gavin.halliday@lexisnexis.com>

Conflicts:
	roxie/roxiemem/roxiemem.cpp

common/thorhelper/roxierow.cpp

@@ -697,7 +697,7 @@ protected:
 
     void testSetup()
     {
-        setTotalMemoryLimit(false, 40*HEAP_ALIGNMENT_SIZE, 0, NULL, NULL);
+        setTotalMemoryLimit(false, true, 40*HEAP_ALIGNMENT_SIZE, 0, NULL, NULL);
     }
 
     void testCleanup()

docs/ECLLanguageReference/ECLR_mods/Recrd-DICTIONARY.xml

@@ -98,8 +98,8 @@ CodeColorDCT  := DICTIONARY(ColorCodes,{Code => Color});
 MapCode2Color(UNSIGNED1 code) := CodeColorDCT[code].color;
 MapColor2Code(STRING color)   := ColorCodeDCT[color].code;
 
-OUTPUT(MapColor2Code('Red'));     //true
-OUTPUT(MapCode2Color(4));         //true
+OUTPUT(MapColor2Code('Red'));     //2
+OUTPUT(MapCode2Color(4));         //'Yellow'
 
 //Search term examples
 OUTPUT('Green' IN ColorCodeDCT);  //true

docs/Installing_and_RunningTheHPCCPlatform/Inst-Mods/hpcc_ldap.xml

@@ -60,7 +60,7 @@
 
         <listitem>
           <para>Verify that they are stopped. You can use a single command,
-          such as : <programlisting>sudo -u hpcc /opt/HPCCSystems/sbin/hpcc-run.sh -a hpcc-init status</programlisting></para>
+          such as : <programlisting>sudo /opt/HPCCSystems/sbin/hpcc-run.sh -a hpcc-init status</programlisting></para>
         </listitem>
 
         <listitem>
@@ -223,7 +223,7 @@
 
         <listitem>
           <para>Verify that they are stopped. You can use a single command,
-          such as : <programlisting>sudo -u hpcc /opt/HPCCSystems/sbin/hpcc-run.sh -a hpcc-init status</programlisting></para>
+          such as : <programlisting>sudo /opt/HPCCSystems/sbin/hpcc-run.sh -a hpcc-init status</programlisting></para>
         </listitem>
 
         <listitem>

docs/Installing_and_RunningTheHPCCPlatform/Inst-Mods/ssl-esp.xml

@@ -343,7 +343,7 @@ sudo cp server.key /var/lib/HPCCSystems/myesp/privatekey.cer
         <para>Back up the original environment.xml file</para>
 
         <programlisting># for example
-sudo -u hpcc cp /etc/HPCCSystems/environment.xml /etc/HPCCSystems/environment.bak
+sudo cp /etc/HPCCSystems/environment.xml /etc/HPCCSystems/environment.bak
 </programlisting>
 
         <para>Note: the "live" environment.xml file is located in your
@@ -359,7 +359,7 @@ sudo -u hpcc cp /etc/HPCCSystems/environment.xml /etc/HPCCSystems/environment.ba
         the /etc/HPCCSystems and rename the file to environment.xml</para>
 
         <programlisting># for example
-sudo -u hpcc cp /etc/HPCCSystems/source/NewEnvironment.xml /etc/HPCCSystems/environment.xml
+sudo cp /etc/HPCCSystems/source/NewEnvironment.xml /etc/HPCCSystems/environment.xml
 </programlisting>
       </listitem>
 

docs/Installing_and_RunningTheHPCCPlatform/Installing_and_RunningTheHPCCPlatform.xml

@@ -27,7 +27,7 @@
       message.</para>
 
       <para>LexisNexis and the Knowledge Burst logo are registered trademarks
-      of Reed Elsevier Properties Inc., used under license. </para>
+      of Reed Elsevier Properties Inc., used under license.</para>
 
       <para>HPCC Systems is a registered trademark of LexisNexis Risk Data
       Management Inc.</para>
@@ -1562,7 +1562,7 @@
             environment.xml</para>
 
             <programlisting># for example
-sudo -u hpcc cp /etc/HPCCSystems/source/NewEnvironment.xml /etc/HPCCSystems/environment.xml</programlisting>
+sudo cp /etc/HPCCSystems/source/NewEnvironment.xml /etc/HPCCSystems/environment.xml</programlisting>
 
             <para><informaltable colsep="1" frame="all" rowsep="1">
                 <?dbfo keep-together="always"?>
@@ -1581,9 +1581,7 @@ sudo -u hpcc cp /etc/HPCCSystems/source/NewEnvironment.xml /etc/HPCCSystems/envi
                       write file(s) to the destination directory before
                       attempting to copy. If prompted to overwrite the
                       destination file, you should answer <emphasis
-                      role="bold">yes</emphasis>. The environment.xml file
-                      MUST be owned by the <emphasis
-                      role="bold">hpcc</emphasis> user.</entry>
+                      role="bold">yes</emphasis>. </entry>
                     </row>
                   </tbody>
                 </tgroup>
@@ -1617,7 +1615,7 @@ sudo -u hpcc cp /etc/HPCCSystems/source/NewEnvironment.xml /etc/HPCCSystems/envi
             all nodes. A sample script is provided with HPCC. The following
             command copies the XML files out to all nodes as required:</para>
 
-            <para><programlisting>sudo -u hpcc /opt/HPCCSystems/sbin/hpcc-push.sh &lt;sourcefile&gt; &lt;destinationfile&gt;
+            <para><programlisting>sudo /opt/HPCCSystems/sbin/hpcc-push.sh &lt;sourcefile&gt; &lt;destinationfile&gt;
 </programlisting>See the appendix for more information on using this
             script.</para>
           </listitem>
@@ -1660,7 +1658,7 @@ sudo -u hpcc cp /etc/HPCCSystems/source/NewEnvironment.xml /etc/HPCCSystems/envi
                     <entry><para>You may want to create a script to push this
                     command out to every node. A sample script is provided
                     with HPCC. Use the following command to start HPCC on all
-                    nodes:</para><para><programlisting>sudo -u hpcc /opt/HPCCSystems/sbin/hpcc-run.sh -a hpcc-init start</programlisting></para></entry>
+                    nodes:</para><para><programlisting>sudo /opt/HPCCSystems/sbin/hpcc-run.sh -a hpcc-init start</programlisting></para></entry>
                   </row>
                 </tbody>
               </tgroup>
@@ -2883,11 +2881,11 @@ OUTPUT(ValidWords)
         <para>The IP addresses were defined when editing the environment in
         ConfigMgr.</para>
 
-        <para><programlisting>sudo -u hpcc /opt/HPCCSystems/sbin/hpcc-push.sh &lt;sourcefile&gt; &lt;destinationfile&gt;
+        <para><programlisting>sudo /opt/HPCCSystems/sbin/hpcc-push.sh &lt;sourcefile&gt; &lt;destinationfile&gt;
 </programlisting>For example:</para>
 
-        <para><programlisting>sudo -u hpcc /opt/HPCCSystems/sbin/hpcc-push.sh \
-             /etc/HPCCSystems/environment.xml /etc/HPCCSystems/environment.xml
+        <para><programlisting>sudo /opt/HPCCSystems/sbin/hpcc-push.sh \
+     /etc/HPCCSystems/environment.xml /etc/HPCCSystems/environment.xml
 </programlisting></para>
       </sect2>
 
@@ -2958,24 +2956,24 @@ OUTPUT(ValidWords)
 
         <para>This example starts all components on the nodes</para>
 
-        <para><programlisting>sudo -u hpcc /opt/HPCCSystems/sbin/hpcc-run.sh -a hpcc-init start
+        <para><programlisting>sudo /opt/HPCCSystems/sbin/hpcc-run.sh -a hpcc-init start
 </programlisting></para>
 
         <para>This example starts all components on all the nodes, using 8
         concurrent executions</para>
 
-        <para><programlisting>sudo -u hpcc /opt/HPCCSystems/sbin/hpcc-run.sh -a hpcc-init start -n 8
+        <para><programlisting>sudo /opt/HPCCSystems/sbin/hpcc-run.sh -a hpcc-init start -n 8
 </programlisting>This example starts all components of the esp type on the
         nodes</para>
 
-        <para><programlisting>sudo -u hpcc /opt/HPCCSystems/sbin/hpcc-run.sh -c esp -a hpcc-init start 
+        <para><programlisting>sudo /opt/HPCCSystems/sbin/hpcc-run.sh -c esp -a hpcc-init start 
 </programlisting>This example starts all components with a component name
         myesp on the nodes</para>
 
-        <para><programlisting>sudo -u hpcc /opt/HPCCSystems/sbin/hpcc-run.sh -c myesp -a hpcc-init start 
+        <para><programlisting>sudo /opt/HPCCSystems/sbin/hpcc-run.sh -c myesp -a hpcc-init start 
 </programlisting>This example starts the dafilesrv helper application</para>
 
-        <para><programlisting>sudo -u hpcc /opt/HPCCSystems/sbin/hpcc-run.sh -a dafilesrv start
+        <para><programlisting>sudo /opt/HPCCSystems/sbin/hpcc-run.sh -a dafilesrv start
 </programlisting></para>
       </sect2>
     </sect1>

ecl/eclagent/eclagent.cpp

@@ -2016,6 +2016,9 @@ void EclAgent::runProcess(IEclProcess *process)
     bool allowHugePages = agentTopology->getPropBool("@heapUseHugePages", false);
     allowHugePages = globals->getPropBool("heapUseHugePages", allowHugePages);
 
+    bool allowTransparentHugePages = agentTopology->getPropBool("@heapUseTransparentHugePages", true);
+    allowTransparentHugePages = globals->getPropBool("heapUseTransparentHugePages", allowTransparentHugePages);
+
 #ifndef __64BIT__
     if (memLimitMB > 4096)
     {
@@ -2025,7 +2028,7 @@ void EclAgent::runProcess(IEclProcess *process)
     }
 #endif
     memsize_t memLimitBytes = (memsize_t)memLimitMB * 1024 * 1024;
-    roxiemem::setTotalMemoryLimit(allowHugePages, memLimitBytes, 0, NULL, NULL);
+    roxiemem::setTotalMemoryLimit(allowHugePages, allowTransparentHugePages, memLimitBytes, 0, NULL, NULL);
 
     rowManager.setown(roxiemem::createRowManager(0, NULL, queryDummyContextLogger(), allocatorMetaCache, false));
     setHThorRowManager(rowManager.get());

ecl/eclccserver/vchooks/git.sh

@@ -19,7 +19,7 @@
 # Hook to use a git repository for eclcc compilation
 # Files will be fetched from the remote repo before each compile
 #
-# This hook expecte environment variables to be set as follows:
+# This hook expects environment variables to be set as follows:
 # GIT_URL       - The remote repository location
 # GIT_BRANCH    - The branch/commit/tag to use  (if ommitted, master is assumed)
 # GIT_DIRECTORY - can be set to allow the dir to be maintained separately - it is
@@ -143,7 +143,14 @@ for repo in ${repositories[@]} ; do
 done
 
 cd $originalDir
-if [ -n "$GIT_VERBOSE" ]; then
+if [ -n "$IS_ECLSERVER" ]; then
+  if [ -n "$GIT_VERBOSE" ]; then
+    echo GIT: returning settings $GIT_INCLUDE_PATH 1>&2
+  fi
+  echo $GIT_INCLUDE_PATH
+else
+  if [ -n "$GIT_VERBOSE" ]; then
     echo GIT: calling eclcc $GIT_INCLUDE_PATH "$@"  1>&2
-fi
-eclcc $GIT_INCLUDE_PATH "$@"
+  fi
+  eclcc $GIT_INCLUDE_PATH "$@"
+fi

ecl/hql/hqlexpr.cpp

@@ -4421,7 +4421,9 @@ void CHqlExpression::beforeDispose()
 #endif
     if (infoFlags & HEFobserved)
     {
+#ifdef HQLEXPR_MULTI_THREADED
         CriticalBlock block(*exprCacheCS);
+#endif
         if (infoFlags & HEFobserved)
             exprCache->removeExact(this);
     }
@@ -4472,7 +4474,9 @@ IHqlExpression * CHqlExpression::commonUpExpression()
 
     IHqlExpression * match;
     {
+#ifdef HQLEXPR_MULTI_THREADED
         CriticalBlock block(*exprCacheCS);
+#endif
         match = exprCache->addOrFind(*this);
 #ifndef GATHER_COMMON_STATS
         if (match == this)

ecl/hql/hqlexpr.ipp

@@ -17,10 +17,16 @@
 #ifndef HQLEXPR_IPP_INCL
 #define HQLEXPR_IPP_INCL
 
+//The following needs to be defined to allow multi-threaded access to the expressions
+//Required inside esp and other areas for parsing record definitions etc.
+#define HQLEXPR_MULTI_THREADED
+
 #define NUM_PARALLEL_TRANSFORMS 1
 //I'm not sure if the following is needed or not - I'm slight concerned that remote scopes (e.g.,, plugins)
 //may be accessed in parallel from multiple threads, causing potential conflicts
+#ifdef HQLEXPR_MULTI_THREADED
 #define THREAD_SAFE_SYMBOLS
+#endif
 
 //The following flag is to allow tracing when expressions are created, linked, released, destroyed
 //It allocates a unique id to each expression that is created, then add the unique ids into the
@@ -145,11 +151,17 @@ protected:
     UsedExpressionHashTable newScopeTables;
 };
 
-class HQL_API CHqlExpression : public CInterfaceOf<IHqlExpression>
+#ifdef HQLEXPR_MULTI_THREADED
+typedef CInterfaceOf<IHqlExpression> LinkedBaseIHqlExpression;
+#else
+typedef CSingleThreadInterfaceOf<IHqlExpression> LinkedBaseIHqlExpression;
+#endif
+
+class HQL_API CHqlExpression : public LinkedBaseIHqlExpression
 {
 public:
     friend class CHqlExprMeta;
-    typedef CInterfaceOf<IHqlExpression> Parent;
+    typedef LinkedBaseIHqlExpression Parent;
 
 #ifdef USE_TBB
     void *operator new(size32_t size) { return scalable_malloc(size); }

ecl/hql/hqlgram.hpp

@@ -646,6 +646,7 @@ public:
     void addDatasetField(const attribute &errpos, IIdAtom * name, ITypeInfo * type, IHqlExpression *value, IHqlExpression * attrs);
     void addDictionaryField(const attribute &errpos, IIdAtom * name, ITypeInfo * type, IHqlExpression *value, IHqlExpression * attrs);
     void addField(const attribute &errpos, IIdAtom * name, ITypeInfo *type, IHqlExpression *value, IHqlExpression *attrs);
+    void addFieldFromValue(const attribute &errpos, attribute & valueAttr);
     void addFields(const attribute &errpos, IHqlExpression *record, IHqlExpression * dataset, bool clone);
     void addIfBlockToActive(const attribute &errpos, IHqlExpression * ifblock);
     void addToActiveRecord(IHqlExpression * newField);

ecl/hql/hqlgram.y

@@ -4241,15 +4241,11 @@ fieldDefs
 
 fieldDef
     : expression        {
-                            parser->normalizeExpression($1);
-                            IHqlExpression *value = $1.getExpr();
-
-                            IIdAtom * name = parser->createFieldNameFromExpr(value);
-
-                            IHqlExpression * attrs = extractAttrsFromExpr(value);
-
-                            ITypeInfo * type = value->queryType();
-                            parser->addField($1, name, LINK(type), value, attrs);
+                            parser->addFieldFromValue($1, $1);
+                            $$.clear();
+                        }
+    | DATASET '(' dataSet ')' {
+                            parser->addFieldFromValue($1, $3);
                             $$.clear();
                         }
     | VALUE_ID_REF      {
@@ -4313,7 +4309,7 @@ fieldDef
                             parser->addField($1, $1.getId(), makeRowType(LINK(value->queryRecordType())), value, attrs);
                             $$.clear();
                         }
-    | typeDef knownOrUnknownId optFieldAttrs defaultValue
+    | typeDef knownOrUnknownId optFieldAttrs optDefaultValue
                         {
                             $$.clear($1);
                             IHqlExpression *value = $4.getExpr();
@@ -4324,7 +4320,7 @@ fieldDef
                                 type.setown(makeRowType(type.getClear()));
                             parser->addField($2, $2.getId(), type.getClear(), value, $3.getExpr());
                         }
-    | ANY knownOrUnknownId optFieldAttrs defaultValue
+    | ANY knownOrUnknownId optFieldAttrs optDefaultValue
                         {
                             $$.clear($1);
                             IHqlExpression *value = $4.getExpr();
@@ -4333,7 +4329,7 @@ fieldDef
                             OwnedITypeInfo type = makeAnyType();
                             parser->addField($2, $2.getId(), type.getClear(), value, $3.getExpr());
                         }
-    | typeDef knownOrUnknownId '[' expression ']' optFieldAttrs defaultValue
+    | typeDef knownOrUnknownId '[' expression ']' optFieldAttrs optDefaultValue
                         {
                             $$.clear($1);
                             parser->normalizeExpression($4, type_int, false);
@@ -4353,7 +4349,7 @@ fieldDef
                             Owned<ITypeInfo> datasetType = makeTableType(makeRowType(createRecordType(record)));
                             parser->addDatasetField($2, $2.getId(), datasetType, value, attrs.getClear());
                         }
-    | setType knownOrUnknownId optFieldAttrs defaultValue
+    | setType knownOrUnknownId optFieldAttrs optDefaultValue
                         {
                             $$.clear($1);
                             IHqlExpression *value = $4.getExpr();
@@ -4361,7 +4357,7 @@ fieldDef
                             ITypeInfo *type = $1.getType();
                             parser->addField($2, $2.getId(), type, value, $3.getExpr());
                         }
-    | explicitDatasetType knownOrUnknownId optFieldAttrs defaultValue
+    | explicitDatasetType knownOrUnknownId optFieldAttrs optDefaultValue
                         {
                             $$.clear($1);
                             Owned<ITypeInfo> type = $1.getType();
@@ -4373,7 +4369,7 @@ fieldDef
                             parser->addDatasetField($1, $1.getId(), NULL, value, $2.getExpr());
                             $$.clear();
                         }
-    | explicitDictionaryType knownOrUnknownId optFieldAttrs defaultValue
+    | explicitDictionaryType knownOrUnknownId optFieldAttrs optDefaultValue
                         {
                             $$.clear($1);
                             Owned<ITypeInfo> type = $1.getType();
@@ -4385,7 +4381,7 @@ fieldDef
                             parser->addDictionaryField($1, $1.getId(), value->queryType(), value, $2.getExpr());
                             $$.clear();
                         }
-    | alienTypeInstance knownOrUnknownId optFieldAttrs defaultValue
+    | alienTypeInstance knownOrUnknownId optFieldAttrs optDefaultValue
                         {
                             $$.clear($1);
                             parser->addField($2, $2.getId(), $1.getType(), $4.getExpr(), $3.getExpr());
@@ -4398,15 +4394,7 @@ fieldDef
                             $$.clear();
                         }
     | dictionary        {
-                            parser->normalizeExpression($1);
-                            IHqlExpression *value = $1.getExpr();
-
-                            IIdAtom * name = parser->createFieldNameFromExpr(value);
-
-                            IHqlExpression * attrs = extractAttrsFromExpr(value);
-
-                            ITypeInfo * type = value->queryType();
-                            parser->addField($1, name, LINK(type), value, attrs);
+                            parser->addFieldFromValue($1, $1);
                             $$.clear();
                         }
     | dataSet               {
@@ -4414,12 +4402,8 @@ fieldDef
                             parser->addFields($1, e->queryRecord(), e, true);
                             $$.clear();
                         }
-    | dataRow               {
-                            IHqlExpression *e = $1.getExpr();
-                            IHqlExpression * attrs = extractAttrsFromExpr(e);
-                            IIdAtom * name = parser->createFieldNameFromExpr(e);
-                            parser->addField($1, name, makeRowType(LINK(e->queryRecordType())), e, attrs);
-                            //e->Release();
+    | dataRow           {
+                            parser->addFieldFromValue($1, $1);
                             $$.clear();
                         }
     | ifblock
@@ -4611,6 +4595,14 @@ optParams
     |                   {   $$.setNullExpr();   }
     ;
 
+optDefaultValue
+    : defaultValue
+    |                       
+                        {
+                            $$.setNullExpr();
+                        }
+    ;
+    
 defaultValue
     : ASSIGN expression 
                         {
@@ -4633,10 +4625,6 @@ defaultValue
                         {
                             $$.inherit($2);
                         }
-    |                       
-                        {
-                            $$.setNullExpr();
-                        }
     ;
 
 setType

ecl/hql/hqlgram2.cpp

@@ -2586,6 +2586,20 @@ void HqlGram::addDictionaryField(const attribute &errpos, IIdAtom * name, ITypeI
     addToActiveRecord(newField);
 }
 
+void HqlGram::addFieldFromValue(const attribute &errpos, attribute & valueAttr)
+{
+    normalizeExpression(valueAttr);
+    IHqlExpression *value = valueAttr.getExpr();
+
+    IIdAtom * name = createFieldNameFromExpr(value);
+    IHqlExpression * attrs = extractAttrsFromExpr(value);
+
+    if (value->isDataset())
+        addDatasetField(errpos, name, NULL, value, attrs);
+    else
+        addField(errpos, name, value->getType(), value, attrs);
+}
+
 void HqlGram::addIfBlockToActive(const attribute &errpos, IHqlExpression * ifblock)
 {
     activeRecords.tos().addOperand(LINK(ifblock));

ecl/hql/hqltrans.ipp

@@ -300,7 +300,11 @@ enum
     NTFselectorOriginal     = 0x0002,
 };
 
-class HQL_API ANewTransformInfo : implements CInterfaceOf<IInterface> 
+
+//Unless we start using tbb or something similar to parallelise transformations
+//[which would be tricky because queryTransformExtra() assumes single threaded access]
+//all Link() and Release() calls must be from the same thread => this can be lightweight
+class HQL_API ANewTransformInfo : implements CSingleThreadSimpleInterfaceOf<IInterface>
 {
 public:
     ANewTransformInfo(IHqlExpression * _original);

ecl/hqlcpp/hqlstmt.ipp

@@ -24,7 +24,8 @@
 class HqlStmts;
 class PeepHoleOptimizer;
 
-class HqlStmt : public CInterfaceOf<IHqlStmt>
+//All generation is single threaded.  Even if multiple wus were generated at once the statements aren't shared
+class HqlStmt : public CSingleThreadSimpleInterfaceOf<IHqlStmt>
 {
 public:
     HqlStmt(StmtKind _kind, HqlStmts * _container);

ecl/regress/indexcrc.ecl

@@ -61,9 +61,9 @@ ds1 := DATASET('crcbug1.d00', {prec, UNSIGNED8 fp{virtual(fileposition)}}, FLAT)
 ds2 := DATASET('crcbug2.d00', {prec_combined, UNSIGNED8 fp{virtual(fileposition)}}, FLAT);
 ds3 := DATASET('crcbug3.d00', {prec_mod, UNSIGNED8 fp{virtual(fileposition)}}, FLAT);
 
-idx1 := INDEX(ds1, {i}, {DATASET chld := chld, fp}, 'crcbug1.idx');
-idx2 := INDEX(ds2, {i}, {DATASET chld := chld, fp}, 'crcbug2.idx');
-idx3 := INDEX(ds3, {i}, {DATASET chld := chld, fp}, 'crcbug3.idx');
+idx1 := INDEX(ds1, {i}, { DATASET(chld), fp}, 'crcbug1.idx');
+idx2 := INDEX(ds2, {i}, { DATASET(chld), fp}, 'crcbug2.idx');
+idx3 := INDEX(ds3, {i}, { DATASET chld := chld, fp}, 'crcbug3.idx');
 
 SEQUENTIAL(
     OUTPUT(baseds, , 'crcbug1.d00', OVERWRITE),

esp/CMakeLists.txt

@@ -26,5 +26,5 @@ HPCC_ADD_SUBDIRECTORY (smc "PLATFORM")
 HPCC_ADD_SUBDIRECTORY (test "PLATFORM")
 HPCC_ADD_SUBDIRECTORY (tools "CLIENTTOOLS")
 HPCC_ADD_SUBDIRECTORY (xslt "PLATFORM")
-HPCC_ADD_SUBDIRECTORY (esdllib "PLATFORM")
+HPCC_ADD_SUBDIRECTORY (esdllib)
 

esp/src/eclwatch/FileBelongsToWidget.js

@@ -0,0 +1,59 @@
+/*##############################################################################
+#    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+############################################################################## */
+define([
+    "dojo/_base/declare",
+    "dojo/i18n",
+    "dojo/i18n!./nls/hpcc",
+
+    "hpcc/GridDetailsWidget",
+    "hpcc/ESPLogicalFile",
+    "hpcc/ESPUtil"
+
+], function (declare, i18n, nlsHPCC,
+                GridDetailsWidget, ESPLogicalFile, ESPUtil) {
+    return declare("FileBelongsToWidget", [GridDetailsWidget], {
+        i18n: nlsHPCC,
+        logicalFile: null,
+
+        gridTitle: nlsHPCC.SuperFilesBelongsTo,
+        idProperty: "Name",
+
+        init: function (params) {
+            if (this.inherited(arguments))
+                return;
+            this.logicalFile = ESPLogicalFile.Get(params.NodeGroup, params.Name);
+            this.refreshGrid();
+        },
+
+        createGrid: function (domID) {
+            var retVal = new declare([ESPUtil.Grid(false, true)])({
+                store: this.store,
+                columns: {
+                    Name: { label: this.i18n.Name }
+                }
+            }, domID);
+            return retVal;
+        },
+
+        refreshGrid: function (args) {
+            var context = this;
+            if (this.logicalFile.Superfiles.DFULogicalFile) {
+                context.store.setData(this.logicalFile.Superfiles.DFULogicalFile);
+                context.grid.refresh();
+            }
+        }
+    });
+});

esp/src/eclwatch/LFDetailsWidget.js

@@ -44,6 +44,7 @@ define([
     "hpcc/TargetSelectWidget",
     "hpcc/ESPLogicalFile",
     "hpcc/ESPDFUWorkunit",
+    "hpcc/FileBelongsToWidget",
 
     "dojo/text!../templates/LFDetailsWidget.html",
 
@@ -57,7 +58,7 @@ define([
 
 ], function (exports, declare, lang, i18n, nlsHPCC, arrayUtil, dom, domAttr, domClass, domForm, query,
                 BorderContainer, TabContainer, ContentPane, Toolbar, TooltipDialog, Form, SimpleTextarea, TextBox, Button, DropDownButton, TitlePane, registry,
-                _TabContainerWidget, DelayLoadWidget, TargetSelectWidget, ESPLogicalFile, ESPDFUWorkunit,
+                _TabContainerWidget, DelayLoadWidget, TargetSelectWidget, ESPLogicalFile, ESPDFUWorkunit, FileBelongsToWidget,
                 template) {
     exports.fixCircularDependency = declare("LFDetailsWidget", [_TabContainerWidget], {
         templateString: template,
@@ -77,6 +78,7 @@ define([
         filePartsWidget: null,
         workunitWidget: null,
         dfuWorkunitWidget: null,
+        fileBelongsTo: null,
 
         logicalFile: null,
         prevState: "",
@@ -96,6 +98,7 @@ define([
             this.dfuWorkunitWidget = registry.byId(this.id + "_DFUWorkunit");
             this.copyTargetSelect = registry.byId(this.id + "CopyTargetSelect");
             this.desprayTargetSelect = registry.byId(this.id + "DesprayTargetSelect");
+            this.fileBelongsToWidget = registry.byId(this.id + "_FileBelongs");
         },
 
         //  Hitched actions  ---
@@ -241,7 +244,12 @@ define([
                     this.dfuWorkunitWidget.init({
                         Wuid: this.logicalFile.Wuid
                     });
-                } else {
+                } else if (currSel.id == this.fileBelongsToWidget.id) {
+                    this.fileBelongsToWidget.init({
+                        NodeGroup: this.logicalFile.NodeGroup,
+                        Name: this.logicalFile.Name
+                    });
+                 } else {
                     currSel.init(currSel.params);
                 }
             }
@@ -299,6 +307,8 @@ define([
                 domClass.remove(this.id + "StateIdImage");
                 domClass.add(this.id + "StateIdImage", this.logicalFile.getStateIconClass());
                 this.refreshActionState();
+            } else if (name === "Superfiles") {
+                this.setDisabled(this.id + "_FileBelongs", false);
             }
         },
 

esp/src/eclwatch/nls/hpcc.js

@@ -441,8 +441,10 @@ define({root:
     Summary: "Summary",
     SummaryMessage: "Summary Message",
     Superfile: "Superfile",
+    Superfiles: "Superfiles",
     SuperFile: "Super File",
     SuperFiles: "Super Files",
+    SuperFilesBelongsTo: "Member of Superfile(s)",
     SuperfilesOnly: "Superfiles Only",
     Suspend: "Suspend",
     Suspended: "Suspended",

esp/src/eclwatch/templates/LFDetailsWidget.html

@@ -158,6 +158,8 @@
             </div>
             <div id="${id}_XML" title="${i18n.XML}" data-dojo-props="delayProps: {WUXml: true}, delayWidget: 'ECLSourceWidget', disabled: true" data-dojo-type="DelayLoadWidget">
             </div>
+            <div id="${id}_FileBelongs" title="${i18n.Superfiles}" data-dojo-props="delayWidget: 'FileBelongsToWidget', disabled: true" data-dojo-type="DelayLoadWidget">
+            </div>
             <div id="${id}_FileParts" title="${i18n.FileParts}" data-dojo-props="delayWidget: 'FilePartsWidget'" data-dojo-type="DelayLoadWidget">
             </div>
             <div id="${id}_Queries" title="${i18n.Queries}" data-dojo-props="delayWidget: 'QuerySetQueryWidget', iconClass:'iconFind'" data-dojo-type="DelayLoadWidget">

initfiles/componentfiles/configxml/eclagent_config.xsd.in

@@ -190,6 +190,14 @@
         </xs:annotation>
     </xs:attribute>
 
+    <xs:attribute name="heapUseTransparentHugePages" type="xs:boolean" default="true">
+      <xs:annotation>
+        <xs:appinfo>
+          <tooltip>Use memory from transparent huge pages.</tooltip>
+        </xs:appinfo>
+      </xs:annotation>
+    </xs:attribute>
+
     <xs:attribute name="pluginDirectory" type="absolutePath" use="optional" default="${PLUGINS_PATH}/">
       <xs:annotation>
         <xs:appinfo>

initfiles/componentfiles/configxml/roxie.xsd.in

@@ -758,6 +758,13 @@
         </xs:appinfo>
       </xs:annotation>
     </xs:attribute>
+    <xs:attribute name="heapUseTransparentHugePages" type="xs:boolean" default="true">
+      <xs:annotation>
+        <xs:appinfo>
+          <tooltip>Allow roxie to use memory from transparent huge pages.</tooltip>
+        </xs:appinfo>
+      </xs:annotation>
+    </xs:attribute>
     <xs:attribute name="trapTooManyActiveQueries" type="xs:boolean" use="optional" default="true">
       <xs:annotation>
         <xs:appinfo>

initfiles/componentfiles/configxml/thor.xsd.in

@@ -381,6 +381,13 @@
           </xs:appinfo>
         </xs:annotation>
       </xs:attribute>
+      <xs:attribute name="heapUseTransparentHugePages" type="xs:boolean" default="true">
+        <xs:annotation>
+          <xs:appinfo>
+            <tooltip>Allow Thor master and slave to use memory from transparent huge pages.</tooltip>
+          </xs:appinfo>
+        </xs:annotation>
+      </xs:attribute>
       <xs:attribute name="pluginsPath" type="relativePath" default="${PLUGINS_PATH}/"/>
       <xs:attribute name="nodeGroup" type="xs:string" use="optional">
         <xs:annotation>

plugins/memcached/memcachedplugin.cpp

@@ -326,7 +326,7 @@ MemCachedPlugin::MCached::MCached(ICodeContext * ctx, const char * _options)
         throw;
     }
 #else
-    pool = memcached_pool(_servers, strlen(_servers));
+    pool = memcached_pool(_options, strlen(_options));
 #endif
     assertPool();
 

plugins/pyembed/pyembed.cpp

@@ -381,6 +381,7 @@ PyObject *PythonThreadContext::compileEmbeddedScript(size32_t lenChars, const ch
     if (!prevtext || strcmp(text, prevtext) != 0)
     {
         prevtext.clear();
+        text.stripChar('\r');
         script.setown(globalState.compileScript(text));
         prevtext.set(utf, bytes);
     }

roxie/ccd/ccdmain.cpp

@@ -758,9 +758,10 @@ int STARTQUERY_API start_query(int argc, const char *argv[])
         socketCheckInterval = topology->getPropInt("@socketCheckInterval", 5000);
         memsize_t totalMemoryLimit = (memsize_t) topology->getPropInt64("@totalMemoryLimit", 0);
         bool allowHugePages = topology->getPropBool("@heapUseHugePages", false);
+        bool allowTransparentHugePages = topology->getPropBool("@heapUseTransparentHugePages", true);
         if (!totalMemoryLimit)
             totalMemoryLimit = 1024 * 0x100000;  // 1 Gb;
-        roxiemem::setTotalMemoryLimit(allowHugePages, totalMemoryLimit, 0, NULL, NULL);
+        roxiemem::setTotalMemoryLimit(allowHugePages, allowTransparentHugePages, totalMemoryLimit, 0, NULL, NULL);
 
         traceStartStop = topology->getPropBool("@traceStartStop", false);
         traceServerSideCache = topology->getPropBool("@traceServerSideCache", false);

roxie/ccd/ccdserver.cpp

@@ -26586,7 +26586,7 @@ protected:
 
     void testSetup()
     {
-        roxiemem::setTotalMemoryLimit(false, 100 * 1024 * 1024, 0, NULL, NULL);
+        roxiemem::setTotalMemoryLimit(false, true, 100 * 1024 * 1024, 0, NULL, NULL);
     }
 
     void testCleanup()

roxie/roxiemem/roxiemem.cpp

@@ -97,6 +97,8 @@ static unsigned heapLWM;
 static unsigned heapLargeBlocks;
 static unsigned heapLargeBlockGranularity;
 static ILargeMemCallback * heapLargeBlockCallback;
+static bool heapNotifyUnusedEachFree = true;
+static bool heapNotifyUnusedEachBlock = false;
 static unsigned __int64 lastStatsCycles;
 static unsigned __int64 statsCyclesInterval;
 
@@ -107,6 +109,7 @@ static atomic_t dataBuffersActive;
 const unsigned UNSIGNED_BITS = sizeof(unsigned) * 8;
 const unsigned UNSIGNED_ALLBITS = (unsigned) -1;
 const unsigned TOPBITMASK = 1<<(UNSIGNED_BITS-1);
+const memsize_t heapBlockSize = UNSIGNED_BITS*HEAP_ALIGNMENT_SIZE;
 
 template <typename VALUE_TYPE, typename ALIGN_TYPE>
 inline VALUE_TYPE align_pow2(VALUE_TYPE value, ALIGN_TYPE alignment)
@@ -116,13 +119,25 @@ inline VALUE_TYPE align_pow2(VALUE_TYPE value, ALIGN_TYPE alignment)
 
 #define PAGES(x, alignment)    (((x) + ((alignment)-1)) / (alignment))           // hope the compiler converts to a shift
 
+inline void notifyMemoryUnused(void * address, memsize_t size)
+{
+#ifdef NOTIFY_UNUSED_PAGES_ON_FREE
+#ifdef _WIN32
+        VirtualAlloc(address, size, MEM_RESET, PAGE_READWRITE);
+#else
+        // for linux mark as unwanted
+        madvise(address,size,MADV_DONTNEED);
+#endif
+#endif
+}
+
 //---------------------------------------------------------------------------------------------------------------------
 
 typedef MapBetween<unsigned, unsigned, memsize_t, memsize_t> MapActivityToMemsize;
 
 static CriticalSection heapBitCrit;
 
-static void initializeHeap(bool allowHugePages, unsigned pages, unsigned largeBlockGranularity, ILargeMemCallback * largeBlockCallback)
+static void initializeHeap(bool allowHugePages, bool allowTransparentHugePages, unsigned pages, unsigned largeBlockGranularity, ILargeMemCallback * largeBlockCallback)
 {
     if (heapBase) return;
 
@@ -157,6 +172,9 @@ static void initializeHeap(bool allowHugePages, unsigned pages, unsigned largeBl
         if (heapBase != MAP_FAILED)
         {
             heapUseHugePages = true;
+            heapNotifyUnusedEachFree = false;
+            //MORE: At the moment I'm not sure calling madvise() has any benefit, it may be better for the following to stay false;
+            heapNotifyUnusedEachBlock = true;
             DBGLOG("Using Huge Pages for roxiemem");
         }
         else
@@ -172,8 +190,14 @@ static void initializeHeap(bool allowHugePages, unsigned pages, unsigned largeBl
 
     if (!heapBase)
     {
+        const memsize_t hugePageSize = getHugePageSize();
+        bool useTransparentHugePages = allowTransparentHugePages && areTransparentHugePagesEnabled();
+        memsize_t heapAlignment = useTransparentHugePages ? hugePageSize : HEAP_ALIGNMENT_SIZE;
+        if (heapAlignment < HEAP_ALIGNMENT_SIZE)
+            heapAlignment = HEAP_ALIGNMENT_SIZE;
+
         int ret;
-        if ((ret = posix_memalign((void **) &heapBase, HEAP_ALIGNMENT_SIZE, memsize)) != 0) {
+        if ((ret = posix_memalign((void **) &heapBase, heapAlignment, memsize)) != 0) {
 
         	switch (ret)
         	{
@@ -197,6 +221,37 @@ static void initializeHeap(bool allowHugePages, unsigned pages, unsigned largeBl
         	}
             HEAPERROR("RoxieMemMgr: Unable to create heap");
         }
+
+        //If system supports transparent huge pages, use madvise to mark the memory as request huge pages
+        if (useTransparentHugePages)
+        {
+            if (madvise(heapBase,memsize,MADV_HUGEPAGE) == 0)
+            {
+                //Prevent the transparent huge page code from working hard trying to defragment memory when single heaplets are released
+                heapNotifyUnusedEachFree = false;
+                if ((heapBlockSize % hugePageSize) == 0)
+                {
+                    //If we notify heapBlockSize items at a time it will always be a multiple of hugePageSize so shouldn't trigger defragmentation
+                    heapNotifyUnusedEachBlock = true;
+                    DBGLOG("Transparent huge pages used for roxiemem heap - memory released in blocks");
+                }
+                else
+                {
+                    DBGLOG("Transparent huge pages used for roxiemem heap- MEMORY WILL NOT BE RELEASED.");
+                    DBGLOG("Increase HEAP_ALIGNMENT_SIZE so HEAP_ALIGNMENT_SIZE*32 is a multiple of huge page size"");");
+                }
+            }
+        }
+        else
+        {
+            if (!allowTransparentHugePages)
+            {
+                madvise(heapBase,memsize,MADV_NOHUGEPAGE);
+                DBGLOG("Transparent huge pages disabled in configuration by user.");
+            }
+            else
+                DBGLOG("Transparent huge pages unsupported or disabled by system.");
+        }
     }
 #endif
 
@@ -565,18 +620,15 @@ static void subfree_aligned(void *ptr, unsigned pages = 1)
         DBGLOG("RoxieMemMgr: Incorrect alignment of freed area (ptr=%p)", ptr);
         HEAPERROR("RoxieMemMgr: Incorrect alignment of freed area");
     }
-#ifdef NOTIFY_UNUSED_PAGES_ON_FREE
-#ifdef _WIN32
-    VirtualAlloc(ptr, pages*HEAP_ALIGNMENT_SIZE, MEM_RESET, PAGE_READWRITE);
-#else
-    // for linux mark as unwanted
-    madvise(ptr,pages*HEAP_ALIGNMENT_SIZE,MADV_DONTNEED);
-#endif
-#endif
+    if (heapNotifyUnusedEachFree)
+        notifyMemoryUnused(ptr, pages*HEAP_ALIGNMENT_SIZE);
+
     unsigned wordOffset = (unsigned) (pageOffset / UNSIGNED_BITS);
     unsigned bitOffset = (unsigned) (pageOffset % UNSIGNED_BITS);
     unsigned mask = 1<<bitOffset;
     unsigned nextPageOffset = (pageOffset+pages + (UNSIGNED_BITS-1)) / UNSIGNED_BITS;
+    char * firstReleaseBlock = NULL;
+    char * lastReleaseBlock = NULL;
     {
         CriticalBlock b(heapBitCrit);
         heapAllocated -= pages;
@@ -602,7 +654,17 @@ static void subfree_aligned(void *ptr, unsigned pages = 1)
         {
             unsigned prev = heapBitmap[wordOffset];
             if ((prev & mask) == 0)
-                heapBitmap[wordOffset] = (prev|mask);
+            {
+                unsigned next = prev | mask;
+                heapBitmap[wordOffset] = next;
+                if ((next == UNSIGNED_ALLBITS) && heapNotifyUnusedEachBlock)
+                {
+                    char * address = heapBase + wordOffset * heapBlockSize;
+                    if (!firstReleaseBlock)
+                        firstReleaseBlock = address;
+                    lastReleaseBlock = address;
+                }
+            }
             else
                 HEAPERROR("RoxieMemMgr: Page freed twice");
             if (!--pages)
@@ -616,6 +678,10 @@ static void subfree_aligned(void *ptr, unsigned pages = 1)
                 mask <<= 1;
         }
     }
+
+    if (firstReleaseBlock)
+        notifyMemoryUnused(firstReleaseBlock, (lastReleaseBlock - firstReleaseBlock) + heapBlockSize);
+
     if (memTraceLevel >= 2)
         DBGLOG("RoxieMemMgr: subfree_aligned() %u pages ok - addr=%p heapLWM=%u totalPages=%u", _pages, ptr, heapLWM, heapTotalPages);
 }
@@ -4260,7 +4326,7 @@ extern void setMemoryStatsInterval(unsigned secs)
     lastStatsCycles = get_cycles_now();
 }
 
-extern void setTotalMemoryLimit(bool allowHugePages, memsize_t max, memsize_t largeBlockSize, const unsigned * allocSizes, ILargeMemCallback * largeBlockCallback)
+extern void setTotalMemoryLimit(bool allowHugePages, bool allowTransparentHugePages, memsize_t max, memsize_t largeBlockSize, const unsigned * allocSizes, ILargeMemCallback * largeBlockCallback)
 {
     assertex(largeBlockSize == align_pow2(largeBlockSize, HEAP_ALIGNMENT_SIZE));
     unsigned totalMemoryLimit = (unsigned) (max / HEAP_ALIGNMENT_SIZE);
@@ -4269,7 +4335,7 @@ extern void setTotalMemoryLimit(bool allowHugePages, memsize_t max, memsize_t la
         totalMemoryLimit = 1;
     if (memTraceLevel)
         DBGLOG("RoxieMemMgr: Setting memory limit to %"I64F"d bytes (%u pages)", (unsigned __int64) max, totalMemoryLimit);
-    initializeHeap(allowHugePages, totalMemoryLimit, largeBlockGranularity, largeBlockCallback);
+    initializeHeap(allowHugePages, allowTransparentHugePages, totalMemoryLimit, largeBlockGranularity, largeBlockCallback);
     initAllocSizeMappings(allocSizes ? allocSizes : defaultAllocSizes);
 }
 
@@ -4525,7 +4591,7 @@ protected:
         ASSERT(HugeHeaplet::dataOffset() >= sizeof(HugeHeaplet));
 
         memsize_t memory = (useLargeMemory ? largeMemory : smallMemory) * (unsigned __int64)0x100000U;
-        initializeHeap(false, (unsigned)(memory / HEAP_ALIGNMENT_SIZE), 0, NULL);
+        initializeHeap(false, true, (unsigned)(memory / HEAP_ALIGNMENT_SIZE), 0, NULL);
         initAllocSizeMappings(defaultAllocSizes);
     }
 
@@ -4636,6 +4702,8 @@ protected:
             _heapLWM = heapLWM;
             _heapAllocated = heapAllocated;
             _heapUseHugePages = heapUseHugePages;
+            _heapNotifyUnusedEachFree = heapNotifyUnusedEachFree;
+            _heapNotifyUnusedEachBlock = heapNotifyUnusedEachBlock;
         }
         ~HeapPreserver()
         {
@@ -4647,6 +4715,8 @@ protected:
             heapLWM = _heapLWM;
             heapAllocated = _heapAllocated;
             heapUseHugePages = _heapUseHugePages;
+            heapNotifyUnusedEachFree = _heapNotifyUnusedEachFree;
+            heapNotifyUnusedEachBlock = _heapNotifyUnusedEachBlock;
         }
         char *_heapBase;
         char *_heapEnd;
@@ -4656,6 +4726,8 @@ protected:
         unsigned _heapLWM;
         unsigned _heapAllocated;
         bool _heapUseHugePages;
+        bool _heapNotifyUnusedEachFree;
+        bool _heapNotifyUnusedEachBlock;
     };
     void initBitmap(unsigned size)
     {
@@ -4850,6 +4922,8 @@ protected:
         HeapPreserver preserver;
 
         initBitmap(maxBitmapSize);
+        heapNotifyUnusedEachFree = false; // prevent calls to map out random chunks of memory!
+        heapNotifyUnusedEachBlock = false;
 
         Semaphore sem;
         BitmapAllocatorThread * threads[numBitmapThreads];
@@ -4880,14 +4954,9 @@ protected:
     }
     void testBitmapThreading()
     {
-#ifndef NOTIFY_UNUSED_PAGES_ON_FREE
-        //Don't run this with NOTIFY_UNUSED_PAGES_ON_FREE enabled - I'm not sure what the calls to map out random memory are likely to do!
         testBitmapThreading(1);
         testBitmapThreading(3);
         testBitmapThreading(11);
-#else
-        DBGLOG("testBitmapThreading not executed since NOTIFY_UNUSED_PAGES_ON_FREE is enabled");
-#endif
     }
 
     void testCompressSize()
@@ -5884,7 +5953,7 @@ public:
 protected:
     void testSetup()
     {
-        setTotalMemoryLimit(false, memorySize, 0, NULL, NULL);
+        setTotalMemoryLimit(true, true, memorySize, 0, NULL, NULL);
     }
 
     void testCleanup()
@@ -6080,7 +6149,7 @@ public:
 protected:
     void testSetup()
     {
-        setTotalMemoryLimit(false, hugeMemorySize, 0, NULL, NULL);
+        setTotalMemoryLimit(true, true, hugeMemorySize, 0, NULL, NULL);
     }
 
     void testCleanup()

roxie/roxiemem/roxiemem.hpp

@@ -511,7 +511,7 @@ interface IDataBufferManager : extends IInterface
 
 extern roxiemem_decl IDataBufferManager *createDataBufferManager(size32_t size);
 extern roxiemem_decl void setMemoryStatsInterval(unsigned secs);
-extern roxiemem_decl void setTotalMemoryLimit(bool allowHugePages, memsize_t max, memsize_t largeBlockSize, const unsigned * allocSizes, ILargeMemCallback * largeBlockCallback);
+extern roxiemem_decl void setTotalMemoryLimit(bool allowHugePages, bool allowTransparentHugePages, memsize_t max, memsize_t largeBlockSize, const unsigned * allocSizes, ILargeMemCallback * largeBlockCallback);
 extern roxiemem_decl memsize_t getTotalMemoryLimit();
 extern roxiemem_decl void releaseRoxieHeap();
 extern roxiemem_decl bool memPoolExhausted();

roxie/udplib/uttest.cpp

@@ -754,7 +754,7 @@ int main(int argc, char * argv[] )
             printf("ERROR: my ip does not appear to be in range\n");
             usage();
         }
-        roxiemem::setTotalMemoryLimit(false, 1048576000, 0, NULL, NULL);
+        roxiemem::setTotalMemoryLimit(false, true, 1048576000, 0, NULL, NULL);
         testNxN();
         roxiemem::releaseRoxieHeap();
     }

rtl/eclrtl/eclinclude4.hpp

@@ -41,7 +41,6 @@
 #endif
 
 #include <string.h>
-#include <malloc.h>
 #include <stdlib.h>
 #include <ctype.h>
 #include <math.h>

system/jlib/jdebug.cpp

@@ -2778,6 +2778,50 @@ void PrintMemoryReport(bool full)
 #endif
 
 
+bool areTransparentHugePagesEnabled()
+{
+#ifdef __linux__
+    StringBuffer contents;
+    try
+    {
+        contents.loadFile("/sys/kernel/mm/transparent_hugepage/enabled");
+        return !strstr(contents.str(), "[never]");
+    }
+    catch (IException * e)
+    {
+        e->Release();
+    }
+#endif
+    return false;
+}
+
+memsize_t getHugePageSize()
+{
+#ifdef __linux__
+    StringBuffer contents;
+    try
+    {
+        //Search for an entry   Hugepagesize:      xxxx kB
+        const char * const tag = "Hugepagesize:";
+        contents.loadFile("/proc/meminfo");
+        const char * hugepage = strstr(contents.str(), tag);
+        if (hugepage)
+        {
+            const char * next = hugepage + strlen(tag);
+            char * end;
+            memsize_t size = strtoul(next, &end, 10);
+            if (strncmp(end, " kB", 3) == 0)
+                return size * 0x400;
+        }
+    }
+    catch (IException * e)
+    {
+        e->Release();
+    }
+#endif
+    return 0x200000; // Default for an x86 system
+}
+
 //===========================================================================
 
 #ifdef LEAK_CHECK

system/jlib/jdebug.hpp

@@ -307,7 +307,8 @@ extern jlib_decl unsigned getAffinityCpus();
 extern jlib_decl void printProcMap(const char *fn, bool printbody, bool printsummary, StringBuffer *lnout, MemoryBuffer *mb, bool useprintf);
 extern jlib_decl void PrintMemoryReport(bool full=true);
 extern jlib_decl void printAllocationSummary();
-
+extern jlib_decl bool areTransparentHugePagesEnabled();
+extern jlib_decl memsize_t getHugePageSize();
 
 #endif
 

system/jlib/jiface.hpp

@@ -119,15 +119,43 @@ public:
     }
 };
 
+//---------------------------------------------------------------------------------------------------------------------
+
+// An implementation of IInterface for objects that are never shared.  Only likely to be used in a few situations.
+// It still allows use with IArrayOf and Owned classes etc.
+template <class INTERFACE>
+class CUnsharedInterfaceOf : public INTERFACE
+{
+public:
+    inline virtual ~CUnsharedInterfaceOf() {}
+
+    inline CUnsharedInterfaceOf()         { }
+    inline bool IsShared(void) const    { return false; }
+    inline int getLinkCount(void) const { return 1; }
+
+    inline void Link() const            { assert(!"Unshared::Link"); }
+
+    inline bool Release(void) const
+    {
+        delete this;
+        return true;
+    }
+};
+
 //- Template variants -------------------------------------------------------------------------------------------------
 
 template <class INTERFACE>
 class CInterfaceOf;
 
 template <class INTERFACE>
+class CReusableInterfaceOf;
+
+// A thread safe basic implementation of IInterface that destroys the object when last Release() occurs.
+template <class INTERFACE>
 class CSimpleInterfaceOf : public INTERFACE
 {
     friend class CInterfaceOf<INTERFACE>;    // want to keep xxcount private outside this pair of classes
+    friend class CReusableInterfaceOf<INTERFACE>;
 public:
     inline virtual ~CSimpleInterfaceOf() {}
 
@@ -152,7 +180,7 @@ private:
 };
 
 // A more general implementation of IInterface that includes a virtual function beforeDispose().
-// beforeDispose() allows an a fully constructed object to be cleaned up (which means that virtual
+// beforeDispose() allows a fully constructed object to be cleaned up (which means that virtual
 // function calls still work (unlike a virtual destructor).
 // It makes it possible to implement a cache which doesn't link count the object, but is cleared when
 // the object is disposed without a critical section in Release().  (See pattern details below).
@@ -181,6 +209,95 @@ public:
     }
 };
 
+// An extension of CInterfaceOf that allows objects that are being destroyed to be added to a free list instead.
+// Before disposing addToFree() is called.  That function can link the object and return true which will prevent
+// the object being destroyed.
+template <class INTERFACE>
+class CReusableInterfaceOf : public CInterfaceOf<INTERFACE>
+{
+public:
+    //If this function returns true the object is added to a free list - if so it must be linked in the process.
+    //It *must not* link and release the object if it returns false.
+    virtual bool addToFreeList() const { return false; }
+
+    inline bool Release(void) const
+    {
+        if (atomic_dec_and_test(&this->xxcount))
+        {
+            if (addToFreeList())
+            {
+                //It could have been added, reused and disposed between the call and this return, so must return immediately
+                //and in particular cannot check/modify link count
+                return true;
+            }
+
+            //NOTE: This class cannot be used for caches which don't link the pointers => use atomic_set instead of cas
+            atomic_set(&this->xxcount, DEAD_PSEUDO_COUNT);
+            const_cast<CReusableInterfaceOf<INTERFACE> *>(this)->beforeDispose();
+            delete this;
+            return true;
+        }
+        return false;
+    }
+};
+
+//---------------------------------------------------------------------------------------------------------------------
+
+template <class INTERFACE>
+class CSingleThreadInterfaceOf;
+
+//An equivalent to CSimpleInterfaceOf that is not thread safe - but avoids the cost of atomic operations.
+template <class INTERFACE>
+class CSingleThreadSimpleInterfaceOf : public INTERFACE
+{
+    friend class CSingleThreadInterfaceOf<INTERFACE>;    // want to keep xxcount private outside this pair of classes
+public:
+    inline virtual ~CSingleThreadSimpleInterfaceOf() {}
+
+    inline CSingleThreadSimpleInterfaceOf() { xxcount = 1; }
+    inline bool IsShared(void) const    { return xxcount > 1; }
+    inline int getLinkCount(void) const { return xxcount; }
+
+    inline void Link() const            { ++xxcount; }
+
+    inline bool Release(void) const
+    {
+        if (--xxcount == 0)
+        {
+            delete this;
+            return true;
+        }
+        return false;
+    }
+
+private:
+    mutable unsigned xxcount;
+};
+
+//An equivalent to CInterfaceOf that is not thread safe - but avoids the cost of atomic operations.
+template <class INTERFACE>
+class CSingleThreadInterfaceOf : public CSingleThreadSimpleInterfaceOf<INTERFACE>
+{
+public:
+    virtual void beforeDispose() {}
+
+    inline bool isAlive() const         { return this->xxcount < DEAD_PSEUDO_COUNT; }       //only safe if Link() is called first
+
+    inline bool Release(void) const
+    {
+        if (--this->xxcount == 0)
+        {
+            //Because beforeDispose could cause this object to be linked/released or call isAlive(), xxcount is set
+            //to a a high mid-point positive number to avoid poss. of releasing again.
+            this->xxcount = DEAD_PSEUDO_COUNT;
+            const_cast<CSingleThreadInterfaceOf<INTERFACE> *>(this)->beforeDispose();
+            delete this;
+            return true;
+        }
+        return false;
+    }
+};
+
 
 /***
 A pattern for implementing a non-linking cached to a shared object.  Hash table variants are a slightly more complex variant.

system/jlib/jstats.cpp

@@ -338,7 +338,11 @@ const char * queryMeasureName(StatisticMeasure measure)
 StatisticMeasure queryMeasure(const char *  measure)
 {
     //MORE: Use a hash table??
-    return (StatisticMeasure)matchString(measureNames, measure);
+    StatisticMeasure ret = (StatisticMeasure)matchString(measureNames, measure);
+    //Legacy support for an unusual statistic - pretend the sizes are in bytes instead of kb.
+    if ((ret == SMeasureNone) && measure && streq(measure, "kb"))
+        ret = SMeasureSize;
+    return ret;
 }
 
 StatsMergeAction queryMergeMode(StatisticMeasure measure)

thorlcr/master/thmastermain.cpp

@@ -683,12 +683,13 @@ int main( int argc, char *argv[]  )
                 mmemSize = gmemSize;
         }
         bool gmemAllowHugePages = globals->getPropBool("@heapUseHugePages", false);
+        bool gmemAllowTransparentHugePages = globals->getPropBool("@heapUseTransparentHugePages", true);
 
         // if @masterMemorySize and @globalMemorySize unspecified gmemSize will be default based on h/w
         globals->setPropInt("@masterMemorySize", mmemSize);
 
         PROGLOG("Global memory size = %d MB", mmemSize);
-        roxiemem::setTotalMemoryLimit(gmemAllowHugePages, ((memsize_t)mmemSize) * 0x100000, 0, thorAllocSizes, NULL);
+        roxiemem::setTotalMemoryLimit(gmemAllowHugePages, gmemAllowTransparentHugePages, ((memsize_t)mmemSize) * 0x100000, 0, thorAllocSizes, NULL);
 
         const char * overrideBaseDirectory = globals->queryProp("@thorDataDirectory");
         const char * overrideReplicateDirectory = globals->queryProp("@thorReplicateDirectory");

thorlcr/slave/slavmain.cpp

@@ -663,11 +663,12 @@ void slaveMain()
         WARNLOG("Slave has less memory than master node");
     unsigned gmemSize = globals->getPropInt("@globalMemorySize");
     bool gmemAllowHugePages = globals->getPropBool("@heapUseHugePages", false);
+    bool gmemAllowTransparentHugePages = globals->getPropBool("@heapUseTransparentHugePages", true);
     if (gmemSize >= hdwInfo.totalMemory)
     {
         // should prob. error here
     }
-    roxiemem::setTotalMemoryLimit(gmemAllowHugePages, ((memsize_t)gmemSize) * 0x100000, 0, thorAllocSizes, NULL);
+    roxiemem::setTotalMemoryLimit(gmemAllowHugePages, gmemAllowTransparentHugePages, ((memsize_t)gmemSize) * 0x100000, 0, thorAllocSizes, NULL);
 
     CJobListener jobListener;
     CThorResourceSlave slaveResource;