Browse Source

Merge branch 'candidate-6.0.12' into candidate-6.2.2

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>
Richard Chapman 8 years ago
parent
commit
0926f854ad

+ 8 - 5
docs/ECLLanguageReference/ECLR-includer.xml

@@ -41,7 +41,7 @@
     <xi:include href="common/Version.xml" xpointer="DateVer"
                 xmlns:xi="http://www.w3.org/2001/XInclude" />
 
-    <releaseinfo>© 2015 HPCC Systems<superscript>®</superscript>. All rights
+    <releaseinfo>© 2017 HPCC Systems<superscript>®</superscript>. All rights
     reserved. Except where otherwise noted, ECL Language Reference content
     licensed under Creative Commons public license.</releaseinfo>
 
@@ -549,8 +549,9 @@
     <xi:include href="ECLLanguageReference/ECLR_mods/BltInFunc-FETCH.xml"
                 xpointer="element(/1)"
                 xmlns:xi="http://www.w3.org/2001/XInclude" />
+
     <xi:include href="ECLLanguageReference/ECLR_mods/BltInFunc-FROMJSON.xml"
-		xpointer="element(/1)"
+                xpointer="element(/1)"
                 xmlns:xi="http://www.w3.org/2001/XInclude" />
 
     <xi:include href="ECLLanguageReference/ECLR_mods/BltInFunc-FROMUNICODE.xml"
@@ -724,6 +725,7 @@
     <xi:include href="ECLLanguageReference/ECLR_mods/BltInFunc-ORDERED.xml"
                 xpointer="element(/1)"
                 xmlns:xi="http://www.w3.org/2001/XInclude" />
+
     <xi:include href="ECLLanguageReference/ECLR_mods/BltInFunc-OUTPUT.xml"
                 xpointer="element(/1)"
                 xmlns:xi="http://www.w3.org/2001/XInclude" />
@@ -887,8 +889,9 @@
     <xi:include href="ECLLanguageReference/ECLR_mods/BltInFunc-THISNODE.xml"
                 xpointer="element(/1)"
                 xmlns:xi="http://www.w3.org/2001/XInclude" />
+
     <xi:include href="ECLLanguageReference/ECLR_mods/BltInFunc-TOJSON.xml"
-		xpointer="element(/1)"
+                xpointer="element(/1)"
                 xmlns:xi="http://www.w3.org/2001/XInclude" />
 
     <xi:include href="ECLLanguageReference/ECLR_mods/BltInFunc-TOPN.xml"
@@ -922,6 +925,7 @@
     <xi:include href="ECLLanguageReference/ECLR_mods/BltInFunc-UNICODEORDER.xml"
                 xpointer="element(/1)"
                 xmlns:xi="http://www.w3.org/2001/XInclude" />
+
     <xi:include href="ECLLanguageReference/ECLR_mods/BltInFunc-UNORDERED.xml"
                 xpointer="element(/1)"
                 xmlns:xi="http://www.w3.org/2001/XInclude" />
@@ -1101,12 +1105,11 @@
     <xi:include href="ECLLanguageReference/ECLR_mods/Templ-WARNING.xml"
                 xpointer="element(/1)"
                 xmlns:xi="http://www.w3.org/2001/XInclude" />
- 
 
     <xi:include href="ECLLanguageReference/ECLR_mods/Templ-WEBSERVICE.xml"
                 xpointer="element(/1)"
                 xmlns:xi="http://www.w3.org/2001/XInclude" />
- 
+
     <xi:include href="ECLLanguageReference/ECLR_mods/Templ-WORKUNIT.xml"
                 xpointer="element(/1)"
                 xmlns:xi="http://www.w3.org/2001/XInclude" />

+ 4 - 4
docs/common/Version.xml

@@ -5,11 +5,11 @@
   <chapterinfo>
     <date id="DateVer">DEVELOPER NON-GENERATED VERSION</date>
 
-    <releaseinfo id="FooterInfo">© 2016 HPCC
+    <releaseinfo id="FooterInfo">© 2017 HPCC
     Systems<superscript>®</superscript>. All rights reserved</releaseinfo>
 
     <copyright id="Copyright">
-      <year>2016 HPCC Systems<superscript>®</superscript>. All rights
+      <year>2017 HPCC Systems<superscript>®</superscript>. All rights
       reserved</year>
     </copyright>
   </chapterinfo>
@@ -24,7 +24,7 @@
     and that is to store the chapterinfo the above sections that are being
     used by several other documents.</para>
 
-    <para id="CHMVer">2016 Version X.X</para>
+    <para id="CHMVer">2017 Version X.X</para>
 
     <para>The following line is the code to be put into the document you wish
     to include the above version info in:</para>
@@ -40,6 +40,6 @@
     structure that we can use bookinfo/date and other elements for other
     purposes.</para>
 
-    <para>☺</para>
+    <para>☺ © Γ∏</para>
   </section>
 </chapter>

+ 4 - 4
docs/common/Version.xml.in

@@ -3,13 +3,13 @@
 "http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd">
 <chapter>
   <chapterinfo>
-    <date id="DateVer">2016 Version ${DOC_VERSION}</date>
+    <date id="DateVer">2017 Version ${DOC_VERSION}</date>
 
-    <releaseinfo id="FooterInfo">© 2016 HPCC
+    <releaseinfo id="FooterInfo">© 2017 HPCC
     Systems<superscript>®</superscript>. All rights reserved</releaseinfo>
 
     <copyright id="Copyright">
-      <year>2016 HPCC Systems<superscript>®</superscript>. All rights
+      <year>2017 HPCC Systems<superscript>®</superscript>. All rights
       reserved</year>
     </copyright>
   </chapterinfo>
@@ -24,7 +24,7 @@
     serve one purpose and that is to store the chapterinfo the above sections
     that are being used by several other documents.</para>
 
-    <para id="CHMVer">2016 Version ${DOC_VERSION}</para>
+    <para id="CHMVer">2017 Version ${DOC_VERSION}</para>
 
     <para>The following line is the code to be put into the document you wish
     to include the above version info in:</para>

+ 10 - 1
initfiles/sbin/hpcc_setenv.in

@@ -102,7 +102,16 @@ IFS="${OIFS}"
 umask ${OUMASK}
 
 # use less heap when threaded
-export MALLOC_ARENA_MAX=8
+# but if set too low it can affect performance significantly
+num_arenas=$(getconf _NPROCESSORS_ONLN 2>/dev/null)
+if [ -z "$num_arenas" ] ; then
+    num_arenas = 8
+elif [ $num_arenas -lt 8 ] ; then
+    num_arenas = 8
+elif [ $num_arenas -gt 32 ] ; then
+    num_arenas = 32
+fi
+export MALLOC_ARENA_MAX=$num_arenas
 
 PATH_PREFIX=`cat ${HPCC_CONFIG} | sed -n "/\[${SECTION}\]/,/\[/p" | grep "^path *= *" | sed -e 's/^path *= *//'`
 

+ 89 - 70
system/jlib/jdebug.cpp

@@ -1668,8 +1668,8 @@ class CExtendedStats  // Disk network and cpu stats
     unsigned ncpu;
     bool first;
     char *kbuf;
-    size32_t kbufsz;
     size32_t kbufmax;
+    int kbadcnt;
     unsigned short kbufcrc;
     __uint64 totalcpu;
 
@@ -1858,70 +1858,80 @@ class CExtendedStats  // Disk network and cpu stats
         return true;
     }
 
-    size32_t  getKLog(const char *&data)
+    size32_t getKLog(const char *&data)
     {
 #ifdef __linux__
-        if (kbufmax) {
-            loop {
-                char *newkbuf = (char *)malloc(kbufmax);
-                if (!newkbuf)
-                    break;  // OOM - abort logging
-                size32_t newkbufsz = klogctl(3,newkbuf,kbufmax);
-                if ((int)newkbufsz<0) {
-                    ERRLOG("klogctl error %d",errno);
-                    free(newkbuf);
-                    data = NULL;
-                    return 0;
+        if (kbufmax)
+        {
+            data = nullptr;
+            size32_t ksz = 0;
+            unsigned short lastCRC = 0;
+            // NOTE: allocated 2*kbufmax to work around kernel bug
+            // where klogctl() could sometimes return more than requested
+            size32_t sz = klogctl(3, kbuf, kbufmax);
+            if ((int)sz < 0)
+            {
+                if (kbadcnt < 5)
+                {
+                    ERRLOG("klogctl SYSLOG_ACTION_READ_ALL error %d", errno);
+                    kbadcnt++;
                 }
-                if (newkbufsz<kbufmax) {
-                    unsigned short crc = chksum16(newkbuf,newkbufsz);
-                    if (kbuf) {
-                        if (crc!=kbufcrc) {
-                            unsigned ofs = 0;
-                            if ((newkbufsz>=kbufsz)) {
-                                for (unsigned i=0;i+3<kbufsz;i++) { // not very quick!
-                                    if (memcmp(kbuf+i,newkbuf,kbufsz-i)==0) {
-                                        ofs = kbufsz-i;
-                                        break;
-                                    }
-                                }
-                            }
-                            size32_t ret = newkbufsz-ofs;
-                            if (ret>3) {
-                                kbufcrc = crc;
-                                free(kbuf);
-                                kbuf = newkbuf;
-                                kbufsz = newkbufsz;
-                                data = kbuf+ofs;
-                                return ret;
-                            }
-                        }
+                else
+                    kbufmax = 0;
+                return 0;
+            }
+#if 0
+            kbuf[sz] = '\0';
+#endif
+            if (kbufcrc)
+            {
+                data = kbuf;
+                ksz = sz;
+            }
+            // determine where new info starts ...
+            StringBuffer ln;
+            const char *p = kbuf;
+            const char *e = p+sz;
+            while (p && p!=e)
+            {
+                if (*p=='<')
+                {
+                    ln.clear();
+                    while ((p && p!=e)&&(*p!='\n'))
+                    {
+                        ln.append(*p);
+                        p++;
+                        sz--;
                     }
-                    else  { // first time {
-                        kbuf = newkbuf;
-                        newkbuf = NULL;
-                        kbufsz = newkbufsz;
+                    lastCRC = chksum16(ln.str(), ln.length());
+                    if (kbufcrc && kbufcrc == lastCRC)
+                    {
+                        ksz = sz - 1;
+                        if (ksz && sz)
+                            data = p + 1;
+                        else
+                            data = nullptr;
                     }
-                    free(newkbuf);
-                    data = NULL;
-                    return 0;
                 }
-                if (kbufmax>0x100000) {
-                    // don't believe!
-                    ERRLOG("klogctl buffer too big!");
-                    free(newkbuf);
-                    break;
+                while ((p && p!=e)&&(*p!='\n'))
+                {
+                    p++;
+                    sz--;
+                }
+                if (p && p!=e)
+                {
+                    p++;
+                    sz--;
                 }
-                kbufmax += 0x1000;
-                free(newkbuf);
             }
-            kbufmax = 0;
-            kbufsz = 0;
-            free(kbuf);
-            kbuf = NULL;
+            if (lastCRC)
+                kbufcrc = lastCRC;
+            if (!ksz)
+                data = nullptr;
+            return ksz;
         }
 #endif
-        data = NULL;
+        data = nullptr;
         return 0;
     }
 
@@ -1956,8 +1966,7 @@ public:
         oldblkio = NULL;
         first = true;
         ncpu = 0;
-        kbuf = NULL;
-        kbufsz = 0;
+        kbuf = nullptr;
         kbufcrc = 0;
         memset(&oldcpu, 0, sizeof(oldcpu));
         memset(&newcpu, 0, sizeof(newcpu));
@@ -1967,8 +1976,14 @@ public:
         memset(&oldnet, 0, sizeof(oldnet));
         memset(&newnet, 0, sizeof(newnet));
         ndisks = 0;
+        kbadcnt = 0;
         if (printklog)
+        {
             kbufmax = 0x1000;
+            kbuf = (char *)malloc(kbufmax*2);
+            if (!kbuf)
+                kbufmax = 0;
+        }
         else
             kbufmax = 0;
     }
@@ -1976,9 +1991,10 @@ public:
     ~CExtendedStats()
     {
         free(partition);
-        free(kbuf);
         free(newblkio);
         free(oldblkio);
+        if (kbuf != nullptr)
+            free(kbuf);
     }
 
     bool getLine(StringBuffer &out)
@@ -2042,23 +2058,26 @@ public:
         return true;
     }
 
-#define KERN_EMERG  "<0>"   // system is unusable
-#define KERN_ALERT  "<1>"   // action must be taken immediately
-#define KERN_CRIT   "<2>"   // critical conditions
-#define KERN_ERR    "<3>"   // error conditions
+#define KERN_EMERG   "<0>"   // system is unusable
+#define KERN_ALERT   "<1>"   // action must be taken immediately
+#define KERN_CRIT    "<2>"   // critical conditions
+#define KERN_ERR     "<3>"   // error conditions
 #define KERN_WARNING "<4>"  // warning conditions
-#define KERN_NOTICE "<5>"   // normal but significant condition
-#define KERN_INFO   "<6>"   // informational
-#define KERN_DEBUG  "<7>"   // debug-level messages
+#define KERN_NOTICE  "<5>"   // normal but significant condition
+#define KERN_INFO    "<6>"   // informational
+#define KERN_DEBUG   "<7>"   // debug-level messages
 #define KMSGTEST(S) if (memcmp(p,S,3)==0) { ln.append(#S); level = p[1]-'0'; }
 
     void printKLog(IPerfMonHook *hook)
     {
-        const char *p;
+        const char *p = nullptr;
         size32_t sz = getKLog(p);
+#if 0
+        DBGLOG("getKLog() returns: %u <%s>", sz, p);
+#endif
         StringBuffer ln;
         const char *e = p+sz;
-        while (p!=e) {
+        while (p && (p!=e)) {
             if (*p=='<') {
                 ln.clear();
                 int level = -1;
@@ -2076,16 +2095,16 @@ public:
                 }
                 p += 3;
                 ln.append(": ");
-                while ((p!=e)&&(*p!='\n'))
+                while ((p && p!=e)&&(*p!='\n'))
                     ln.append(*(p++));
                 if (hook)
                     hook->log(level, ln.str());
                 else
                     PROGLOG("%s",ln.str());
             }
-            while ((p!=e)&&(*p!='\n'))
+            while ((p && p!=e)&&(*p!='\n'))
                 p++;
-            if (p!=e)
+            if (p && p!=e)
                 p++;
         }
     }