Merge branch 'closedown-5.0.x'

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

dali/server/daserver.cpp

@@ -124,15 +124,6 @@ bool actionOnAbort()
     return true;
 } 
 
-#ifdef _WIN32
-class CReleaseMutex : public CInterface, public Mutex
-{
-public:
-    CReleaseMutex(const char *name) : Mutex(name) { }
-    ~CReleaseMutex() { if (owner) unlock(); }
-}; 
-#endif
-
 USE_JLIB_ALLOC_HOOK;
 
 int main(int argc, char* argv[])
@@ -325,20 +316,17 @@ int main(int argc, char* argv[])
         }
         else
             serverConfig.setown(createPTree());
-#ifdef _WIN32
-        Owned<CReleaseMutex> globalNamedMutex;
+
+        NamedMutex globalNamedMutex("DASERVER");
         if (!serverConfig->getPropBool("allowMultipleDalis"))
         {
             PROGLOG("Checking for existing daserver instances");
-            StringBuffer s("DASERVER");
-            globalNamedMutex.setown(new CReleaseMutex(s.str()));
-            if (!globalNamedMutex->lockWait(10*1000)) // wait for 10 secs
+            if (!globalNamedMutex.lockWait(0))
             {
                 PrintLog("Another DASERVER process is currently running");
                 return 0;
             }
         }
-#endif
 
         SocketEndpoint ep;
         SocketEndpointArray epa;

docs/HPCCClientTools/CT_Mods/CT_ECL_IDE.xml

@@ -2794,7 +2794,7 @@ ENDMACRO;
         <para><graphic fileref="../../images/CT54.jpg" /></para>
       </sect2>
 
-      <sect2 id="Graph_Viewer">
+      <sect2 id="Graph_Viewer" role="brk">
         <title><emphasis>Graph Viewer </emphasis></title>
 
         <para>A workunit’s execution graph is displayed in the Graph
@@ -2819,10 +2819,13 @@ ENDMACRO;
             <para>Use the Ribbon bar Running buttons to watch the progress of
             a running workunit, refreshing the graph as it runs. Options are
             provided so you can Follow Active or Minimise Inactive areas of
-            the graph according to your preference .<graphic
-            fileref="../../images/CT55.jpg" scalefit="1" /></para>
+            the graph according to your preference .</para>
           </listitem>
         </itemizedlist>
+
+        <para></para>
+
+        <graphic fileref="../../images/CT55.jpg" scalefit="1" />
       </sect2>
     </sect1>
 

docs/HPCCSystemAdmin/HPCCSystemAdministratorsGuide.xml

@@ -780,7 +780,7 @@
                 xmlns:xi="http://www.w3.org/2001/XInclude" />
 
     <sect1>
-      <title>Envrionment.conf</title>
+      <title>Environment.conf</title>
 
       <para>Another component of HPCC system configuration is the
       environment.conf file. Environment.conf contains some global definitions

esp/services/ws_smc/ws_smcService.cpp

@@ -676,6 +676,7 @@ ActivityInfo* CWsSMCEx::createActivityInfo(IEspContext &context)
     Owned<ActivityInfo> activityInfo = new ActivityInfo();
     readTargetClusterInfo(context, clusters, serverStatusRoot, activityInfo);
     readRunningWUsAndQueuedWUs(context, envRoot, serverStatusRoot, dfuRecoveryRoot, activityInfo);
+    activityInfo->timeCached.setNow();
     return activityInfo.getClear();
 }
 

esp/services/ws_smc/ws_smcService.hpp

@@ -93,7 +93,7 @@ struct ActivityInfo : public CInterface, implements IInterface
 {
     IMPLEMENT_IINTERFACE;
 
-    ActivityInfo() { timeCached.setNow(); };
+    ActivityInfo() {};
     bool isCachedActivityInfoValid(unsigned timeOutSeconds);
 
     CDateTime timeCached;

esp/src/CMakeLists.txt

@@ -37,8 +37,14 @@ else ()
                 message ( FATAL_ERROR \"Can't find Dojo build tools -- did you initialise submodules? (git submodule update --init --recursive)\" )
             else ()
                 execute_process ( COMMAND \"${CMAKE_CURRENT_SOURCE_DIR}/../build.sh\" \"${ECLWATCH_BUILD_DEST}\" OUTPUT_FILE \"${ECLWATCH_BUILD_OUT}\" ERROR_FILE \"${ECLWATCH_BUILD_ERR}\" )
-                execute_process ( COMMAND \"tail\" \"--lines=4\" \"${ECLWATCH_BUILD_DEST}/build-report.txt\" )
-                message ( \"\" )
+                execute_process ( COMMAND \"sed\" \"-n\" \"/ERROR/p\" \"${CMAKE_CURRENT_BINARY_DIR}/eclwatch_build_err.txt\" OUTPUT_VARIABLE BUILD_ERROR )
+                if ( \"\${BUILD_ERROR}\" STREQUAL \"\" )
+                    execute_process ( COMMAND \"tail\" \"--lines=4\" \"${ECLWATCH_BUILD_DEST}/build-report.txt\" )
+                    message ( \"\" )
+                else ()
+                    message ( \"Process finished with errors:\" )
+                    message ( FATAL_ERROR \"\${BUILD_ERROR}\" )
+                endif ()
             endif ()
         else ()
             message( \"-- ECL Watch:  Reusing (old) Site\" )

plugins/mysql/mysqlembed.cpp

@@ -342,6 +342,8 @@ static bool isInteger(enum_field_types type)
 
 static bool getBooleanResult(const RtlFieldInfo *field, const MYSQL_BIND &bound)
 {
+    if (*bound.is_null)
+        return false;
     if (!isInteger(bound.buffer_type))
         typeError("boolean", field);
     return rtlReadUInt(bound.buffer, *bound.length) != 0;
@@ -349,6 +351,12 @@ static bool getBooleanResult(const RtlFieldInfo *field, const MYSQL_BIND &bound)
 
 static void getDataResult(const RtlFieldInfo *field, const MYSQL_BIND &bound, size32_t &chars, void * &result)
 {
+    if (*bound.is_null)
+    {
+        result = NULL;
+        chars = 0;
+        return;
+    }
     if (bound.buffer_type == MYSQL_TYPE_TINY_BLOB ||
         bound.buffer_type == MYSQL_TYPE_MEDIUM_BLOB ||
         bound.buffer_type == MYSQL_TYPE_LONG_BLOB ||
@@ -366,6 +374,8 @@ static unsigned __int64 getUnsignedResult(const RtlFieldInfo *field, const MYSQL
 
 static double getRealResult(const RtlFieldInfo *field, const MYSQL_BIND &bound)
 {
+    if (*bound.is_null)
+        return 0;
     if (isInteger(bound.buffer_type))
     {
         if (bound.is_unsigned)
@@ -383,6 +393,8 @@ static double getRealResult(const RtlFieldInfo *field, const MYSQL_BIND &bound)
 
 static __int64 getSignedResult(const RtlFieldInfo *field, const MYSQL_BIND &bound)
 {
+    if (*bound.is_null)
+        return 0;
     if (isInteger(bound.buffer_type))
     {
         if (bound.is_unsigned)
@@ -396,6 +408,8 @@ static __int64 getSignedResult(const RtlFieldInfo *field, const MYSQL_BIND &boun
 
 static unsigned __int64 getUnsignedResult(const RtlFieldInfo *field, const MYSQL_BIND &bound)
 {
+    if (*bound.is_null)
+        return 0;
     if (!isInteger(bound.buffer_type))
         typeError("integer", field);
     if (bound.is_unsigned)
@@ -406,6 +420,12 @@ static unsigned __int64 getUnsignedResult(const RtlFieldInfo *field, const MYSQL
 
 static void getStringResult(const RtlFieldInfo *field, const MYSQL_BIND &bound, size32_t &chars, char * &result)
 {
+    if (*bound.is_null)
+    {
+        result = NULL;
+        chars = 0;
+        return;
+    }
     if (bound.buffer_type != MYSQL_TYPE_STRING && bound.buffer_type != MYSQL_TYPE_VAR_STRING)
         typeError("string", field);
     const char *text = (const char *) bound.buffer;
@@ -416,6 +436,12 @@ static void getStringResult(const RtlFieldInfo *field, const MYSQL_BIND &bound,
 
 static void getUTF8Result(const RtlFieldInfo *field, const MYSQL_BIND &bound, size32_t &chars, char * &result)
 {
+    if (*bound.is_null)
+    {
+        result = NULL;
+        chars = 0;
+        return;
+    }
     if (bound.buffer_type != MYSQL_TYPE_STRING && bound.buffer_type != MYSQL_TYPE_VAR_STRING)
         typeError("string", field);
     const char *text = (const char *) bound.buffer;
@@ -426,6 +452,12 @@ static void getUTF8Result(const RtlFieldInfo *field, const MYSQL_BIND &bound, si
 
 static void getUnicodeResult(const RtlFieldInfo *field, const MYSQL_BIND &bound, size32_t &chars, UChar * &result)
 {
+    if (*bound.is_null)
+    {
+        result = NULL;
+        chars = 0;
+        return;
+    }
     if (bound.buffer_type != MYSQL_TYPE_STRING && bound.buffer_type != MYSQL_TYPE_VAR_STRING)
         typeError("string", field);
     const char *text = (const char *) bound.buffer;
@@ -436,6 +468,11 @@ static void getUnicodeResult(const RtlFieldInfo *field, const MYSQL_BIND &bound,
 
 static void getDecimalResult(const RtlFieldInfo *field, const MYSQL_BIND &bound, Decimal &value)
 {
+    if (*bound.is_null)
+    {
+        value.setInt(0);
+        return;
+    }
     size32_t chars;
     rtlDataAttr result;
     mysqlembed::getStringResult(field, bound, chars, result.refstr());

roxie/roxiemem/roxiemem.cpp

@@ -2349,8 +2349,8 @@ public:
     {
         //Possibly over the top, but calculate information so we can do a round robin at various
         //different levels of cost
-        callbackRanges.kill();
-        nextCallbacks.kill();
+        callbackRanges.clear();
+        nextCallbacks.clear();
         nextCallbacks.append(0);
         unsigned prevCost = 0;
         ForEachItemIn(i, rowBufferCallbacks)

system/jlib/jarray.hpp

@@ -136,6 +136,7 @@ public:
     CopyArrayOf() { SELF::_init(); }
     ~CopyArrayOf();
     
+    void clear();
     inline PARAM item(aindex_t pos) const;
     PARAM tos(void) const;
     PARAM tos(aindex_t) const;
@@ -160,6 +161,7 @@ public:
     OwningArrayOf() { SELF::_init(); }
     ~OwningArrayOf();
     
+    void clear(bool nodel = false);                  /* Remove all items, don't free array */
     void kill(bool nodel = false);                   /* Remove all items        */
     void pop(bool nodel = false);
     void popn(aindex_t,bool nodel = false);

system/jlib/jarray.tpp

@@ -124,6 +124,12 @@ void BaseArrayOf<MEMBER, PARAM>::sort(CompareFunc cf)
  ************************************************************************/
 
 template <class MEMBER, class PARAM>
+void CopyArrayOf<MEMBER, PARAM>::clear()
+{
+   SELF::used = 0;
+}
+
+template <class MEMBER, class PARAM>
 PARAM CopyArrayOf<MEMBER, PARAM>::item(aindex_t pos) const
 {
    assertex(SELF::isItem(pos)); return Array__Member2Param(((MEMBER *)AllocatorOf<sizeof(MEMBER)>::_head)[pos]);
@@ -219,6 +225,20 @@ bool CopyArrayOf<MEMBER, PARAM>::zap(PARAM sought)
  ************************************************************************/
 
 template <class MEMBER, class PARAM>
+void OwningArrayOf<MEMBER, PARAM>::clear(bool nodestruct)
+{
+    MEMBER * head= (MEMBER *)SELF::_head;
+    aindex_t count = SELF::used;
+
+    SELF::used = 0;
+    if (!nodestruct)
+    {
+       for (aindex_t i=0; i<count; i++)
+          Array__Destroy(head[i]);
+    }
+}
+
+template <class MEMBER, class PARAM>
 void OwningArrayOf<MEMBER, PARAM>::kill(bool nodestruct)
 {
    MEMBER * head= (MEMBER *)SELF::_head;

testing/regress/ecl/key/mysqlembed.xml

@@ -1,6 +1,7 @@
 <Dataset name='Result 1'>
  <Row><name>name1</name><value>1</value><boolval>true</boolval><r8>1.2</r8><r4>3.400000095367432</r4><d>6161353561613535</d><ddd>1234567.89</ddd><u1>Straße</u1><u2>Straße  </u2></Row>
  <Row><name>name2</name><value>2</value><boolval>false</boolval><r8>5.6</r8><r4>7.800000190734863</r4><d>3030</d><ddd>-1234567.89</ddd><u1>là</u1><u2>là      </u2></Row>
+ <Row><name>nulls</name><value>0</value><boolval>false</boolval><r8>0.0</r8><r4>0.0</r4><d></d><ddd>0</ddd><u1></u1><u2>        </u2></Row>
 </Dataset>
 <Dataset name='Result 2'>
  <Row><Result_2>name1</Result_2></Row>
@@ -9,10 +10,10 @@
  <Row><name>name1</name><value>1</value><boolval>true</boolval><r8>1.2</r8><r4>3.400000095367432</r4><d>6161353561613535</d><ddd>1234567.89</ddd><u1>Straße</u1><u2>Straße  </u2></Row>
 </Dataset>
 <Dataset name='Result 4'>
- <Row><Result_4>name2</Result_4></Row>
+ <Row><Result_4>nulls</Result_4></Row>
 </Dataset>
 <Dataset name='Result 5'>
- <Row><name>name2</name><value>2</value><boolval>false</boolval><r8>5.6</r8><r4>7.800000190734863</r4><d>3030</d><ddd>-1234567.89</ddd><u1>là</u1><u2>là      </u2></Row>
+ <Row><name>nulls</name><value>0</value><boolval>false</boolval><r8>0.0</r8><r4>0.0</r4><d></d><ddd>0</ddd><u1></u1><u2>        </u2></Row>
 </Dataset>
 <Dataset name='Result 6'>
  <Row><name>name1</name><value>1</value><boolval>true</boolval><r8>1.2</r8><r4>3.400000095367432</r4><d>6161353561613535</d><ddd>1234567.89</ddd><u1>Straße</u1><u2>Straße  </u2></Row>

testing/regress/ecl/mysqlembed.ecl

@@ -52,6 +52,10 @@ initialize(dataset(childrec) values) := EMBED(mysql : user('rchapman'),database(
   INSERT INTO tbl1 values (?, ?, ?, ?, ?, ?, ?, ?, ?);
 ENDEMBED;
 
+initializeNulls() := EMBED(mysql : user('rchapman'),database('test'))
+  INSERT INTO tbl1 (name) values ('nulls');
+ENDEMBED;
+
 dataset(childrec) testMySQLDS() := EMBED(mysql : user('rchapman'),database('test'))
   SELECT * from tbl1;
 ENDEMBED;
@@ -116,6 +120,7 @@ sequential (
   drop(),
   create(),
   initialize(init),
+  initializeNulls(),
   OUTPUT(testMySQLDS()),
   OUTPUT(testMySQLRow().name),
   OUTPUT(testMySQLParms('name1', 1, true, 1.2, 3.4, D'aa55aa55', U'Straße', U'Straße')),

thorlcr/activities/hashdistrib/thhashdistribslave.cpp

@@ -2206,7 +2206,6 @@ public:
 #define HASHDEDUP_HT_INC_SIZE 0x10000 // 64k (rows)
 #define HASHDEDUP_BUCKETS_MIN 11 // (NB: prime #)
 #define HASHDEDUP_BUCKETS_MAX 9973 // (NB: prime #)
-#define HASHDEDUP_BUCKET_POSTSPILL_PRIORITY 5 // very high, by this stage it's cheap to dispose of
 
 class HashDedupSlaveActivityBase;
 class CBucket;
@@ -2243,10 +2242,10 @@ public:
         return true;
     }
     void init(rowidx_t sz);
-    const void *allocateNewTable()
+    const void *allocateNewTable(unsigned maxSpillCost)
     {
         rowidx_t newMaxRows = maxRows+HASHDEDUP_HT_INC_SIZE;
-        return allocateRowTable(newMaxRows);
+        return allocateRowTable(newMaxRows, maxSpillCost);
     }
     void rehash(const void **newRows);
     bool lookupRow(unsigned htPos, const void *row) const // return true == match
@@ -2452,7 +2451,7 @@ class CBucketHandler : public CSimpleInterface, implements IInterface, implement
     // IBufferedRowCallback
         virtual unsigned getSpillCost() const
         {
-            return HASHDEDUP_BUCKET_POSTSPILL_PRIORITY;
+            return SPILL_PRIORITY_HASHDEDUP_BUCKET_POSTSPILL;
         }
         virtual bool freeBufferedRows(bool critical)
         {
@@ -2843,11 +2842,13 @@ CBucket::CBucket(HashDedupSlaveActivityBase &_owner, IRowInterfaces *_rowIf, IRo
 
 {
     spilt = false;
-    // ideally want rows in bucket to be contiguous, so when it spills, pages will be released
+    /* Although, using a unique allocator per bucket would mean on a spill event, the pages could be freed,
+     * it is too costly overall, because in effect it means a roxieimem page for each bucket is reserved.
+     * Sharing an allocator, will likely mean that pages are not be freed on spill events, but freed row space will be shared.
+     */
     if (extractKey)
-    {   // use own allocator
-        unsigned flags = owner.allocFlags | roxiemem::RHFunique;
-        _keyAllocator.setown(owner.queryJob().getRowAllocator(keyIf->queryRowMetaData(), owner.queryActivityId(), (roxiemem::RoxieHeapFlags)flags));
+    {
+        _keyAllocator.setown(owner.queryJob().getRowAllocator(keyIf->queryRowMetaData(), owner.queryActivityId(), owner.allocFlags));
         keyAllocator = _keyAllocator;
     }
     else
@@ -2957,7 +2958,7 @@ bool CBucket::rehash()
     OwnedConstThorRow newHtRows;
     {
         CriticalUnblock b(lock); // allocate may cause spill
-        newHtRows.setown(htRows->allocateNewTable());
+        newHtRows.setown(htRows->allocateNewTable(SPILL_PRIORITY_HASHDEDUP_REHASH)); // don't force other hash tables to spill for rehash
     }
     if (!newHtRows)
         return false;
@@ -3169,8 +3170,10 @@ CBucketHandler *CBucketHandler::getNextBucketHandler(Owned<IRowStream> &nextInpu
         if (bucket->isSpilt())
         {
             rowcount_t keyCount, count;
-            // JCSMORE ideally, each key and row stream, would use a unique allocator per destination bucket
-            // thereby keeping rows/keys together in pages, making it easier to free pages on spill requests
+            /* If each key and row stream were to use a unique allocator per destination bucket
+             * thereby keeping rows/keys together in pages, it would make it easier to free pages on spill requests.
+             * However, it would also mean a lot of allocators with at least one page per allocate, which ties up a lot of memory
+             */
             Owned<IRowStream> keyStream = bucket->getKeyStream(&keyCount);
             dbgassertex(keyStream);
             Owned<CBucketHandler> newBucketHandler = new CBucketHandler(owner, rowIf, keyIf, iRowHash, iKeyHash, iCompare, extractKey, depth+1, div*numBuckets);

thorlcr/thorutil/thmem.hpp

@@ -201,6 +201,7 @@ enum {
 graph_decl StringBuffer &getRecordString(const void *key, IOutputRowSerializer *serializer, const char *prefix, StringBuffer &out);
 
 //NB: low priorities are spilt 1st
+#define SPILL_PRIORITY_VERYLOW 50
 #define SPILL_PRIORITY_LOW  100
 #define SPILL_PRIORITY_HIGH 1000000
 #define SPILL_PRIORITY_DEFAULT SPILL_PRIORITY_LOW
@@ -211,7 +212,10 @@ graph_decl StringBuffer &getRecordString(const void *key, IOutputRowSerializer *
 #define SPILL_PRIORITY_RESULT SPILL_PRIORITY_LOW
 
 #define SPILL_PRIORITY_GROUPSORT SPILL_PRIORITY_LOW+1000
+
+#define SPILL_PRIORITY_HASHDEDUP_REHASH SPILL_PRIORITY_LOW+1900
 #define SPILL_PRIORITY_HASHDEDUP SPILL_PRIORITY_LOW+2000
+#define SPILL_PRIORITY_HASHDEDUP_BUCKET_POSTSPILL SPILL_PRIORITY_VERYLOW // very low, by this stage it's cheap to dispose of
 
 #define SPILL_PRIORITY_JOIN SPILL_PRIORITY_HIGH
 #define SPILL_PRIORITY_SELFJOIN SPILL_PRIORITY_HIGH