Merge branch 'candidate-5.2.4' into candidate-5.4.0

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

cmake_modules/commonSetup.cmake

@@ -169,6 +169,14 @@ IF ("${COMMONSETUP_DONE}" STREQUAL "")
   endif ()
   if (CMAKE_COMPILER_IS_GNUCXX OR CMAKE_COMPILER_IS_CLANGXX)
     execute_process(COMMAND ${CMAKE_CXX_COMPILER} -dumpversion OUTPUT_VARIABLE CMAKE_CXX_COMPILER_VERSION OUTPUT_STRIP_TRAILING_WHITESPACE)
+  endif()
+  if (CMAKE_COMPILER_IS_CLANGXX)
+    execute_process( COMMAND ${CMAKE_CXX_COMPILER} --version OUTPUT_VARIABLE clang_full_version_string )
+    if (${clang_full_version_string} MATCHES ".*based on LLVM ([0-9]+\\.[0-9]+).*")
+      string (REGEX REPLACE ".*based on LLVM ([0-9]+\\.[0-9]+).*" "\\1" CLANG_VERSION ${clang_full_version_string})
+    else ()
+      string (REGEX REPLACE ".*clang version ([0-9]+\\.[0-9]+).*" "\\1" CLANG_VERSION ${clang_full_version_string})
+    endif()
   endif ()
 
   if (WIN32)
@@ -215,7 +223,7 @@ IF ("${COMMONSETUP_DONE}" STREQUAL "")
     set ( LIBRARY_OUTPUT_PATH "${CMAKE_BINARY_DIR}/${CMAKE_BUILD_TYPE}/libs" )
 
     if (CMAKE_COMPILER_IS_GNUCXX OR CMAKE_COMPILER_IS_CLANGXX)
-      message ("Using compiler: ${CMAKE_CXX_COMPILER_ID} ${CMAKE_CXX_COMPILER_VERSION}")
+      message ("Using compiler: ${CMAKE_CXX_COMPILER_ID} ${CMAKE_CXX_COMPILER_VERSION} ${CLANG_VERSION}")
       SET (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -frtti -fPIC -fmessage-length=0 -Wformat -Wformat-security -Wformat-nonliteral -pthread -Wuninitialized")
       SET (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -rdynamic")
       SET (CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -g -fno-inline-functions")
@@ -253,7 +261,7 @@ IF ("${COMMONSETUP_DONE}" STREQUAL "")
       SET (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror=logical-op-parentheses -Werror=bool-conversions -Werror=return-type -Werror=comment")
       SET (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}  -Werror=bitwise-op-parentheses -Werror=tautological-compare")
       SET (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}  -Wno-switch-enum -Wno-format-zero-length -Wno-switch")
-      if (CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 4.2.1 OR CMAKE_CXX_COMPILER_VERSION VERSION_EQUAL 4.2.1)
+      if (CLANG_VERSION VERSION_GREATER 3.6 OR CLANG_VERSION VERSION_EQUAL 3.6)
         SET (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-undefined-bool-conversion -Wno-pointer-bool-conversion -Wno-tautological-compare")
       endif()
     endif()

initfiles/componentfiles/configxml/dafilesrv.xsd

@@ -54,6 +54,27 @@
                                 </xs:appinfo>
                             </xs:annotation>
                         </xs:attribute>
+                        <xs:attribute name="parallelRequestLimit" type="xs:nonNegativeInteger" use="optional" default="20">
+                            <xs:annotation>
+                                <xs:appinfo>
+                                    <tooltip>Defines the maximum number of concurrent dafilesrv requests allowed. Requests that exceed the limit will be delayed. A value of 0 disables throttling. Overrides global settting.</tooltip>
+                                </xs:appinfo>
+                            </xs:annotation>
+                        </xs:attribute>
+                        <xs:attribute name="throttleDelayMs" type="xs:nonNegativeInteger" use="optional" default="5000">
+                            <xs:annotation>
+                                <xs:appinfo>
+                                    <tooltip>Defines how many milliseconds delayed requests will be delayed by. Overrides global settting.</tooltip>
+                                </xs:appinfo>
+                            </xs:annotation>
+                        </xs:attribute>
+                        <xs:attribute name="throttleCPULimit" type="nonNegativeInteger" use="optional" default="75">
+                            <xs:annotation>
+                                <xs:appinfo>
+                                    <tooltip>If after the initial delay, the CPU % falls below this setting, the transaction will be allowed to continue, i.e. the limit can be exceeded this way. Overrides global settting.</tooltip>
+                                </xs:appinfo>
+                            </xs:annotation>
+                        </xs:attribute>
                     </xs:complexType>
                 </xs:element>
             </xs:sequence>
@@ -93,6 +114,27 @@
                     </xs:appinfo>
                 </xs:annotation>
             </xs:attribute>
+            <xs:attribute name="parallelRequestLimit" type="xs:nonNegativeInteger" use="optional" default="20">
+                <xs:annotation>
+                    <xs:appinfo>
+                        <tooltip>Defines the maximum number of concurrent dafilesrv requests allowed. Requests that exceed the limit will be delayed. A value of 0 disables throttling. This a global setting.</tooltip>
+                    </xs:appinfo>
+                </xs:annotation>
+            </xs:attribute>
+            <xs:attribute name="throttleDelayMs" type="xs:nonNegativeInteger" use="optional" default="5000">
+                <xs:annotation>
+                    <xs:appinfo>
+                        <tooltip>Defines how many milliseconds delayed requests will be delayed by. This a global setting.</tooltip>
+                    </xs:appinfo>
+                </xs:annotation>
+            </xs:attribute>
+            <xs:attribute name="throttleCPULimit" type="nonNegativeInteger" use="optional" default="75">
+                <xs:annotation>
+                    <xs:appinfo>
+                        <tooltip>If after the initial delay, the CPU % falls below this setting, the transaction will be allowed to continue, i.e. the limit can be exceeded this way. This a global setting.</tooltip>
+                    </xs:appinfo>
+                </xs:annotation>
+            </xs:attribute>
         </xs:complexType>
     </xs:element>
 </xs:schema>

roxie/ccd/ccdserver.cpp

@@ -5322,7 +5322,6 @@ class CRoxieServerWorkUnitReadActivity : public CRoxieServerActivity
 {
     IHThorWorkunitReadArg &helper;
     Owned<IWorkUnitRowReader> wuReader; // MORE - can we use IRoxieInput instead?
-
 public:
     CRoxieServerWorkUnitReadActivity(const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager)
         : CRoxieServerActivity(_factory, _probeManager), helper((IHThorWorkunitReadArg &)basehelper)
@@ -5349,6 +5348,7 @@ public:
 
     virtual void reset() 
     {
+        CriticalBlock b(statecrit);
         wuReader.clear();
         CRoxieServerActivity::reset(); 
     };
@@ -5357,7 +5357,10 @@ public:
 
     virtual const void *nextInGroup()
     {
+        CriticalBlock b(statecrit);
         ActivityTimer t(totalCycles, timeActivities);
+        if (!wuReader)
+            return NULL;
         const void *ret = wuReader->nextInGroup();
         if (ret)
             processed++;
@@ -5772,13 +5775,17 @@ public:
 
     virtual void reset() 
     {
+        CriticalBlock b(statecrit);
         iter.clear();
         CRoxieServerActivity::reset(); 
     };
 
     virtual const void *nextInGroup()
     {
+        CriticalBlock b(statecrit);
         ActivityTimer t(totalCycles, timeActivities);
+        if (!iter)
+            return NULL;
         const void * next = iter->nextInGroup();
         if (next)
         {
@@ -6119,6 +6126,7 @@ public:
 
     virtual void reset() 
     {
+        CriticalBlock b(statecrit);
         if (iter)
             iter->reset();
         iter.clear();
@@ -6127,6 +6135,7 @@ public:
 
     virtual const void *nextInGroup()
     {
+        CriticalBlock b(statecrit);
         ActivityTimer t(totalCycles, timeActivities);
         const void * next = iter ? iter->nextInGroup() : NULL;
         if (next)