Merge branch 'candidate-6.2.0' into candidate-6.4.0

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

cmake_modules/commonSetup.cmake

@@ -172,10 +172,20 @@ IF ("${COMMONSETUP_DONE}" STREQUAL "")
       if(DEFINED SIGN_MODULES_KEYID)
         set(GPG_DEFAULT_KEY_OPTION --default-key)
       endif()
+      execute_process(COMMAND bash "-c" "gpg --version | awk 'NR==1{print $3}'"
+        OUTPUT_VARIABLE GPG_VERSION
+        OUTPUT_STRIP_TRAILING_WHITESPACE
+        ERROR_QUIET)
+    if(${GPG_VERSION} VERSION_GREATER "2.1")
+          set(GPG_PINENTRY_MODE --pinentry-mode loopback --batch --no-tty)
+      else()
+          set(GPG_PINENTRY_MODE --batch --no-tty)
+      endif()
       execute_process(
           COMMAND rm -f sm_keycheck.tmp sm_keycheck.asc
           COMMAND touch sm_keycheck.tmp
-          COMMAND gpg --output sm_keycheck.asc ${GPG_DEFAULT_KEY_OPTION} ${SIGN_MODULES_KEYID}  --clearsign ${GPG_PASSPHRASE_OPTION} ${SIGN_MODULES_PASSPHRASE} --batch --no-tty sm_keycheck.tmp
+          COMMAND gpg --clearsign ${GPG_PINENTRY_MODE} ${GPG_DEFAULT_KEY_OPTION} ${SIGN_MODULES_KEYID} ${GPG_PASSPHRASE_OPTION} ${SIGN_MODULES_PASSPHRASE} --output sm_keycheck.asc sm_keycheck.tmp
+          TIMOUT 120
           WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
           RESULT_VARIABLE rc_var
           ERROR_VARIABLE err_var

dali/base/dautils.cpp

@@ -1668,6 +1668,18 @@ void filterParts(IPropertyTree *file,UnsignedArray &partslist)
 #define SORT_NOCASE  2
 #define SORT_NUMERIC 4
 
+bool ensureSDSPath(const char * sdsPath)
+{
+    if (!sdsPath)
+        return false;
+
+    Owned<IRemoteConnection> conn = querySDS().connect(sdsPath, myProcessSession(), RTM_LOCK_WRITE | RTM_CREATE_QUERY, SDS_LOCK_TIMEOUT);
+    if (!conn)
+        return false;
+
+    return true;
+}
+
 inline void filteredAdd(IArrayOf<IPropertyTree> &results,const char *namefilterlo,const char *namefilterhi,StringArray& unknownAttributes, IPropertyTree *item)
 {
 

dali/base/dautils.hpp

@@ -249,7 +249,7 @@ struct da_decl TransactionLog
 extern da_decl const char * skipScope(const char *lname,const char *scope); // skips a sspecified scope (returns NULL if scope doesn't match)
 extern da_decl const char * querySdsFilesRoot();
 extern da_decl const char * querySdsRelationshipsRoot();
-
+extern da_decl bool ensureSDSPath(const char * sdsPath);
 
 extern da_decl IPropertyTreeIterator *deserializePartAttrIterator(MemoryBuffer &mb);    // clears mb
 extern da_decl MemoryBuffer &serializePartAttr(MemoryBuffer &mb,IPropertyTree *tree); 

docs/ECLLanguageReference/ECLR-includer.xml

@@ -305,6 +305,9 @@
     <xi:include href="ECLLanguageReference/ECLR_mods/ResrvdKywds-LEFT-RIGHT.xml"
                 xpointer="element(/1)"
                 xmlns:xi="http://www.w3.org/2001/XInclude" />
+    <xi:include href="ECLLanguageReference/ECLR_mods/ResrvdKywds-LIKELY.xml"
+                xpointer="element(/1)"
+                xmlns:xi="http://www.w3.org/2001/XInclude" />                
 
     <xi:include href="ECLLanguageReference/ECLR_mods/ResrvdKywds-ROWS--LEFT-RIGHT.xml"
                 xpointer="element(/1)"

docs/ECLLanguageReference/ECLR_mods/ResrvdKywds-LIKELY.xml

@@ -0,0 +1,80 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE sect1 PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN"
+"http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd">
+<sect1 id="likely">
+  <title>LIKELY and UNLIKELY</title>
+
+  <para><emphasis role="bold">[</emphasis><emphasis>attrname</emphasis>
+  :=<emphasis role="bold"> ] LIKELY<indexterm>
+      <primary>LIKELY</primary>
+    </indexterm>(</emphasis><emphasis>filtercondition</emphasis><emphasis
+  role="bold">, [</emphasis><emphasis> likelihood </emphasis><emphasis
+  role="bold">] </emphasis><emphasis role="bold">);</emphasis></para>
+
+  <para><emphasis role="bold">[</emphasis><emphasis>attrname</emphasis>
+  :=<emphasis role="bold"> ] UNLIKELY<indexterm>
+      <primary>UNLIKELY</primary>
+    </indexterm>(</emphasis><emphasis>filtercondition</emphasis><emphasis
+  role="bold">);</emphasis><informaltable colsep="1" frame="all" rowsep="1">
+      <tgroup cols="2">
+        <colspec colwidth="79.05pt" />
+
+        <colspec colwidth="309.75pt" />
+
+        <tbody>
+          <row>
+            <entry><emphasis>filtercondition</emphasis></entry>
+
+            <entry>A filter condition for the hint.</entry>
+          </row>
+
+          <row>
+            <entry><emphasis>likelihood</emphasis></entry>
+
+            <entry>The probability value expressed in a decimal value between
+            0 and 1.</entry>
+          </row>
+        </tbody>
+      </tgroup>
+    </informaltable></para>
+
+  <para>The LIKELY/UNLIKELY hint can be wrapped around a filter condition to
+  indicate to the code generator the likelihood that the filter condition will
+  filter the record.</para>
+
+  <para>LIKELY specifies that the filter condition is likely to match most
+  records. UNLIKELY specifies that very few records are likely to be
+  matched.</para>
+
+  <para>Specific probability value may be provided for LIKELY. The probability
+  value is decimal value greater than 0 and less than 1. The closer this value
+  is to 1.0 the more likely that the filter condition is likely to match a
+  record. The closer the value is to 0.0 the less likely the filter condition
+  is to match records. The code generator makes use of the likelihood
+  information to produce better code.</para>
+
+  <para>The code generator uses the LIKELY/UNLIKELY hint together with the
+  count of usage, to determine the cost of spilling and the cost of
+  re-filtering the dataset every time it is used. Spills are only be generated
+  when the cost of spilling is lower than the cost of re-filtering the dataset
+  every time.</para>
+
+  <para>For example, say there is a dataset of people with millions of
+  records. A filter is created to retain all records where the age is less
+  than 100. The filter is expected to retain 99.9% of records. This filter
+  result is used by 3 different activities. The cost of spilling the results
+  of the filter is likely to be significantly higher than the simply
+  re-filtering the input dataset every time it used. LIKELY can be used to
+  share this likelihood information with the code generator so that it may
+  make sensible decisions regarding when to spill.</para>
+
+  <para>Example:</para>
+
+  <programlisting>PeopleYoungerThan100 := AllPeople( LIKELY(age &lt; 100, 0.999) );
+// Probably not worth spilling PeopleYoungerThan100
+
+PeopleOlderThan100 := AllPeople( UNLIKELY(age&gt;100) );
+// Probably worth spilling even if PeopleOlderThan100 is used by only a couple of activities</programlisting>
+
+  <para></para>
+</sect1>

ecl/hqlcpp/hqlttcpp.cpp

@@ -8338,6 +8338,18 @@ void AutoScopeMigrateTransformer::analyseExpr(IHqlExpression * expr)
     activityDepth = savedDepth;
 }
 
+void AutoScopeMigrateTransformer::doAnalyseConditionalExpr(IHqlExpression * expr, unsigned firstConditional)
+{
+    bool wasConditional = isConditional;
+    ForEachChild(i, expr)
+    {
+        if (i == firstConditional)
+            isConditional = true;
+       analyseExpr(expr->queryChild(i));
+    }
+    isConditional = wasConditional;
+}
+
 void AutoScopeMigrateTransformer::doAnalyseExpr(IHqlExpression * expr)
 {
     AutoScopeMigrateInfo * extra = queryBodyExtra(expr);
@@ -8369,15 +8381,11 @@ void AutoScopeMigrateTransformer::doAnalyseExpr(IHqlExpression * expr)
         return;
     case no_if:
     case no_choose:
+    case no_case:
         {
-            if (expr->isAction())
+            if (expr->isAction() || (graphDepth == 0))
             {
-                bool wasConditional = isConditional;
-                analyseExpr(expr->queryChild(0));
-                isConditional = true;
-                ForEachChildFrom(i, expr, 1)
-                    analyseExpr(expr->queryChild(i));
-                isConditional = wasConditional;
+                doAnalyseConditionalExpr(expr, 1);
                 return;
             }
             break;
@@ -8392,13 +8400,22 @@ void AutoScopeMigrateTransformer::doAnalyseExpr(IHqlExpression * expr)
             return;
         }
         break;
+    case no_map:
+        if (expr->isAction() || (graphDepth == 0))
+        {
+            doAnalyseConditionalExpr(expr, 0);
+            return;
+        }
+        break;
     case no_thor:
         //ignore thor attribute on a dataset..
         if (expr->queryType())
         {
             curGraph++;
+            graphDepth++;
             NewHqlTransformer::analyseExpr(expr);
             curGraph++;     // don't restore - new pseudo graph to aid cse between global branches separated by graphs
+            graphDepth--;
             return;
         }
         break;

ecl/hqlcpp/hqlttcpp.ipp

@@ -744,6 +744,7 @@ protected:
     IHqlExpression * hoist(IHqlExpression * expr, IHqlExpression * hoisted);
     IHqlExpression * transformCond(IHqlExpression * expr);
     void doAnalyseExpr(IHqlExpression * expr);
+    void doAnalyseConditionalExpr(IHqlExpression * expr, unsigned firstConditional);
 
     inline AutoScopeMigrateInfo * queryBodyExtra(IHqlExpression * expr)     { return static_cast<AutoScopeMigrateInfo *>(queryTransformExtra(expr->queryBody())); }
 
@@ -755,6 +756,7 @@ private:
     bool hasCandidate;
     bool isSequential;
     unsigned curGraph;
+    unsigned graphDepth = 0;
     HqlExprArray graphActions;
     unsigned activityDepth;
     HqlExprArray * globalTarget;

esp/services/esdl_svc_engine/CMakeLists.txt

@@ -81,6 +81,7 @@ target_link_libraries ( esdl_svc_engine
          dalibase
          esdllib
          dllserver
+         dalibase
     )
 
 IF (USE_OPENSSL)

esp/services/esdl_svc_engine/esdl_binding.cpp

@@ -37,18 +37,6 @@
 
 #include "loggingagentbase.hpp"
 
-bool EsdlBindingImpl::ensureSDSPath(const char * sdsPath)
-{
-    if (!sdsPath)
-        return false;
-
-    Owned<IRemoteConnection> conn = querySDS().connect(sdsPath, myProcessSession(), RTM_LOCK_WRITE | RTM_CREATE_QUERY, SDS_LOCK_TIMEOUT_DESDL);
-    if (!conn)
-        return false;
-
-    return true;
-}
-
 /*
  * trim xpath at first instance of element
  */

esp/services/esdl_svc_engine/esdl_binding.hpp

@@ -27,6 +27,7 @@
 #include "loggingmanager.h"
 #include "thorplugin.hpp"
 #include "eclrtl.hpp"
+#include "dautils.hpp"
 
 static const char* ESDL_DEFS_ROOT_PATH="/ESDL/Definitions/";
 static const char* ESDL_DEF_PATH="/ESDL/Definitions/Definition";
@@ -370,7 +371,6 @@ public:
     static void splitURLList(const char* urlList, StringBuffer& protocol,StringBuffer& UserName,StringBuffer& Password, StringBuffer& ipportlistbody, StringBuffer& path, StringBuffer& options);
     static void transformGatewaysConfig( IPropertyTree* srvcfg, IPropertyTree* forRoxie );
     static bool makeURL( StringBuffer& url, IPropertyTree& cfg );
-    static bool ensureSDSPath(const char * sdsPath);
 
     bool usesESDLDefinition(const char * name, int version);
     bool usesESDLDefinition(const char * id);

esp/services/ws_esdlconfig/CMakeLists.txt

@@ -68,4 +68,5 @@ target_link_libraries ( ws_esdlconfig
          dllserver
          SMCLib
          esdl_svc_engine
+         dalibase
     )

esp/services/ws_esdlconfig/ws_esdlconfigservice.cpp

@@ -201,10 +201,10 @@ void CWsESDLConfigEx::init(IPropertyTree *cfg, const char *process, const char *
     if(servicecfg == NULL)
         throw MakeStringException(-1, "config not found for service %s/%s",process, service);
 
-    if(!EsdlBindingImpl::ensureSDSPath(ESDL_DEFS_ROOT_PATH))
+    if(!ensureSDSPath(ESDL_DEFS_ROOT_PATH))
         throw MakeStringException(-1, "Could not ensure '%s' element on Dali", ESDL_DEFS_ROOT_PATH);
 
-    if(!EsdlBindingImpl::ensureSDSPath(ESDL_BINDINGS_ROOT_PATH))
+    if(!ensureSDSPath(ESDL_BINDINGS_ROOT_PATH))
         throw MakeStringException(-1, "Could not ensure '%s' element on Dali", ESDL_BINDINGS_ROOT_PATH);
 
 }

esp/services/ws_esdlconfig/ws_esdlconfigservice.hpp

@@ -21,6 +21,7 @@
 #include "ws_esdlconfig_esp.ipp"
 #include "ws_topology_esp.ipp"
 #include "esdlconfig_errors.h"
+#include "dautils.hpp"
 
 static const char* FEATURE_URL="ESDLConfigAccess";
 

testing/unittests/CMakeLists.txt

@@ -41,6 +41,7 @@ include_directories (
          ./../../common/remote
          ./../../dali/base
          ./../../system/security/shared
+         ./../../common/deftype
     )
 
 ADD_DEFINITIONS( -D_CONSOLE )
@@ -52,6 +53,7 @@ target_link_libraries ( unittests
          jlib
          remote
          dalibase
+         deftype
          ${CPPUNIT_LIBRARIES}
     )
 

testing/unittests/unittests.cpp

@@ -20,6 +20,8 @@
 #include "jstats.h"
 #include "jregexp.hpp"
 #include "jfile.hpp"
+#include "deftype.hpp"
+#include "rmtfile.hpp"
 
 /*
  * This is the main unittest driver for HPCC. From here,
@@ -242,9 +244,12 @@ int main(int argc, char* argv[])
         }
         wasSuccessful = list || runner.run( "", false );
     }
+    releaseAtoms();
+    ClearTypeCache();   // Clear this cache before the file hooks are unloaded
+    removeFileHooks();
+
     objects.kill();
     ExitModuleObjects();
-    releaseAtoms();
     return wasSuccessful;
 }