Merge branch 'candidate-5.4.6' into candidate-6.0.0

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

docs/ECLWatch/ECLWa_mods/ECLWatchSrc.xml

@@ -628,7 +628,7 @@
         </sect3>
 
         <sect3 id="ECLWatch_ECLWorkunitsTimersTab" role="brk">
-          <title id="ECLWatch_ECLWorkunits_TimersTab">Timers tab</title>
+          <title id="ECLWatch_ECLWorkUnits_TimersTab">Timers tab</title>
 
           <para>Click on the <emphasis role="bold">Timers</emphasis> tab to
           see the workunit timings. <figure>
@@ -667,7 +667,7 @@
           the graph.</para>
         </sect3>
 
-        <sect3 id="ECLWatch_ECLWorkunitsTimersTab" role="brk">
+        <sect3 id="ECLWatchECLWorkunitsTimersTab" role="brk">
           <title>Timers tab</title>
 
           <para>Click on the <emphasis role="bold">Timers</emphasis> tab to

docs/HPCCMonitoring/MonRep-Mods/MonRep-VM.xml

@@ -89,7 +89,7 @@
   <para>We encourage experienced users to use SSH and log into the VM and
   further examine the configuration of a 1-node monitoring solution.</para>
 
-  <sect3 id="ViewingTheMetrics" role="brk">
+  <sect3 role="brk">
     <title>Viewing the Metrics</title>
 
     <para>To view the metrics page, go to the following page(s) in your

ecl/eclagent/eclgraph.cpp

@@ -894,6 +894,8 @@ void EclSubGraph::doExecute(const byte * parentExtract, bool checkDependencies)
         return;
     }
 
+    if (startGraphTime == 0)
+        startGraphTime = getTimeStampNowValue();
     cycle_t startGraphCycles = get_cycles_now();
     ForEachItemIn(idx, elements)
     {
@@ -939,8 +941,6 @@ void EclSubGraph::execute(const byte * parentExtract)
         return;
     }
 
-    startGraphTime = getTimeStampNowValue();
-    cycle_t startGraphCycles = get_cycles_now();
     createActivities();
     if (debugContext)
         debugContext->checkBreakpoint(DebugStateGraphStart, NULL, parent.queryGraphName() );    //debug probes exist so we can now check breakpoints
@@ -950,7 +950,7 @@ void EclSubGraph::execute(const byte * parentExtract)
 
     if(!owner)
         PROGLOG("Completed subgraph %u", id);
-    if (!parent.queryLibrary())
+    if (!owner && !parent.queryLibrary())
     {
         updateProgress();
         cleanupActivities();

ecl/hql/hqlpmap.hpp

@@ -173,6 +173,7 @@ extern HQL_API IHqlExpression * updateMappedFields(IHqlExpression * expr, IHqlEx
 extern HQL_API void replaceSelectors(HqlExprArray & out, unsigned first, IHqlExpression * oldDataset, IHqlExpression * newDataset);
 extern HQL_API IHqlExpression * scopedReplaceSelector(IHqlExpression * expr, IHqlExpression * oldDataset, IHqlExpression * newDataset);
 extern HQL_API IHqlExpression * replaceSelfRefSelector(IHqlExpression * expr, IHqlExpression * newDataset);
+extern HQL_API IHqlExpression * updateActiveSelectorFields(IHqlExpression * expr, IHqlExpression * oldRecord, IHqlExpression * newRecord, unsigned firstChild);
 
 extern HQL_API bool isNullProject(IHqlExpression * expr, bool canIgnorePayload, bool canLoseFieldsFromEnd);
 extern HQL_API bool isSimpleProject(IHqlExpression * expr);                             // Restriction or rearrangement only

ecl/hql/hqltrans.cpp

@@ -3214,6 +3214,11 @@ void NewSelectorReplacingTransformer::setNestedMapping(IHqlExpression * oldSel,
     }
 }
 
+void NewSelectorReplacingTransformer::setActiveSelectorMapping(IHqlExpression * oldRecord, IHqlExpression * newRecord)
+{
+    setNestedMapping(queryActiveTableSelector(), queryActiveTableSelector(), oldRecord->querySimpleScope(), newRecord, false);
+}
+
 void NewSelectorReplacingTransformer::setRootMapping(IHqlExpression * oldSel, IHqlExpression * newSel, IHqlExpression * oldRecord, bool isSelector)
 {
     if (isSelector)
@@ -3357,6 +3362,34 @@ IHqlExpression * updateMappedFields(IHqlExpression * expr, IHqlExpression * oldS
     return expr->clone(args);
 }
 
+IHqlExpression * updateActiveSelectorFields(IHqlExpression * expr, IHqlExpression * oldRecord, IHqlExpression * newRecord, unsigned firstChild)
+{
+    if (oldRecord == newRecord)
+        return LINK(expr);
+
+    unsigned max = expr->numChildren();
+    unsigned i;
+    HqlExprArray args;
+    args.ensure(max);
+    for (i = 0; i < firstChild; i++)
+        args.append(*LINK(expr->queryChild(i)));
+
+    NewSelectorReplacingTransformer transformer;
+    transformer.setActiveSelectorMapping(oldRecord, newRecord);
+    bool same = true;
+    for (; i < max; i++)
+    {
+        IHqlExpression * cur = expr->queryChild(i);
+        IHqlExpression * transformed = transformer.transformRoot(cur);
+        args.append(*transformed);
+        if (cur != transformed)
+            same = false;
+    }
+    if (same)
+        return LINK(expr);
+    return expr->clone(args);
+}
+
 //---------------------------------------------------------------------------
 
 /*

ecl/hql/hqltrans.ipp

@@ -991,6 +991,7 @@ public:
     inline bool foundAmbiguity() const { return introducesAmbiguity; }
 
     void setRootMapping(IHqlExpression * oldSel, IHqlExpression * newSel, IHqlExpression * record, bool isSelector);
+    void setActiveSelectorMapping(IHqlExpression * oldRecord, IHqlExpression * newRecord);
 
 protected:
     void setNestedMapping(IHqlExpression * oldSel, IHqlExpression * newSel, IHqlSimpleScope * oldScope, IHqlExpression * newRecord, bool isSelector);

ecl/hqlcpp/hqliproj.cpp

@@ -2942,6 +2942,7 @@ IHqlExpression * ImplicitProjectTransformer::createTransformed(IHqlExpression *
                 args.append(*next.getClear());
             }
             transformed.setown(expr->clone(args));
+            transformed.setown(updateSelectors(transformed, expr));
             logChange("Passthrough modified", expr, complexExtra->outputFields);
         }
         else
@@ -3213,7 +3214,6 @@ IHqlExpression * ImplicitProjectTransformer::updateSelectors(IHqlExpression * ne
     {
     case childdataset_none: 
     case childdataset_many_noscope:
-    case childdataset_many:
     case childdataset_if:
     case childdataset_case:
     case childdataset_map:
@@ -3221,6 +3221,9 @@ IHqlExpression * ImplicitProjectTransformer::updateSelectors(IHqlExpression * ne
         return LINK(newExpr);
         //None of these have any scoped arguments, so no need to remove them
         break;
+    case childdataset_many:
+        //The selectors listed in the sorted list may need updating if nested records have changed
+        return updateActiveSelectorFields(newExpr, oldExpr->queryRecord(), newExpr->queryRecord(), getNumChildTables(newExpr));
     case childdataset_dataset:
         {
             return updateMappedFields(newExpr, oldDs->queryNormalizedSelector(), newDs->queryNormalizedSelector(), 1);

ecl/hthor/hthorkey.cpp

@@ -3429,6 +3429,12 @@ public:
         }
     }
 
+    virtual void done()
+    {
+        ldFile.clear();
+        CHThorThreadedActivityBase::done();
+    }
+
     virtual void initializeThreadPool()
     {
         threadPool.setown(createThreadPool("hthor keyed join fetch thread pool", &threadFactory));

ecl/regress/issue14052.ecl

@@ -0,0 +1,54 @@
+layout1 := {
+  unsigned4 doc_id;
+  unicode uni_val{maxlength(30)}
+};
+
+derived1 := {
+  layout1 ref;
+  layout1 def;
+  unsigned4 doc_id;
+};
+
+inline := dataset([
+   {1,u'one',3,u'three',3}
+  ,{2,u'two',4,u'four',4}
+  ,{3,u'three',5,u'five',5}
+  ,{4,u'four',6,u'six',6}
+],derived1);
+
+getStuff(dataset(derived1) ds1) := function
+
+  ds1s := sort(ds1,ref.doc_id,def.doc_id,local);
+
+  ds1sb := ds1s(doc_id % 2 = 1);
+
+  ds1m := merge(ds1s(doc_id % 2 = 0),
+                ds1sb,
+                sorted(ref.doc_id,def.doc_id),
+                // dedup,   // fails to compile with or without...leaving out for simplicity
+                local);
+  // This will work
+  // ds1m := dedup(sort(ds1s(doc_id % 2 = 0)+ds1sb,ref.doc_id,def.doc_id,local),ref.doc_id,def.doc_id,local);
+
+  ds2 := project(ds1m,transform(layout1,
+                                self.doc_id:= left.def.doc_id;
+                                self.uni_val := left.def.uni_val));
+
+  debug() := parallel(
+     evaluate('')
+    ,output(ds1m,,named('ds1m'))
+  );
+
+  ret := module
+    export debugger := debug();
+    export finalOut := ds2;
+    // export finalOut := ds1m;  // works
+  end;
+
+  return ret;
+end;
+
+example := getStuff(inline);
+
+// example.debugger;  // works if not commented
+output(example.finalout,all);

system/security/LdapSecurity/ldapconnection.cpp

@@ -4441,6 +4441,11 @@ private:
         StringBuffer filter;
         filter.append("distinguishedName=").append(dn);
 
+        filter.replaceString("\\", "\\5c");//Replace special characters with valid UTF-8 string (see valueencoding rule in RFC 4515)
+        filter.replaceString("*", "\\2a");
+        filter.replaceString("(", "\\28");
+        filter.replaceString(")", "\\29");
+
         char        *attribute;
         LDAPMessage *message;
 
@@ -5014,6 +5019,17 @@ private:
         }
     }
 
+    virtual const bool organizationalUnitExists(const char * ou) const
+    {
+        Owned<ILdapConnection> lconn = m_connections->getConnection();
+        LDAP* sys_ld = ((CLdapConnection*)lconn.get())->getLd();
+        char* attrs[] = {"ou", NULL};
+        CLDAPMessage searchResult;
+        TIMEVAL timeOut = {LDAPTIMEOUT,0};
+        int rc = ldap_search_ext_s(sys_ld,const_cast <char*>(ou),LDAP_SCOPE_ONELEVEL,NULL,attrs,0,NULL,NULL,&timeOut,LDAP_NO_LIMIT,&searchResult.msg);
+        return rc == LDAP_SUCCESS;
+    }
+
     virtual void createLdapBasedn(ISecUser* user, const char* basedn, SecPermissionType ptype)
     {
         if(basedn == NULL || basedn[0] == '\0')
@@ -5037,7 +5053,7 @@ private:
             ptr = comma + 1;
         }
 
-        if(ptr != NULL)
+        if (ptr && strstr(ptr,"ou=") && !organizationalUnitExists(ptr))
             createLdapBasedn(user, ptr, ptype);
 
         addOrganizationalUnit(user, oubuf.str(), ptr, ptype);
@@ -5055,6 +5071,9 @@ private:
         StringBuffer dn;
         dn.append("ou=").append(name).append(",").append(basedn);
 
+        if (organizationalUnitExists(dn.str()))
+            return true;
+
         char *ou_values[] = {(char*)name, NULL };
         LDAPMod ou_attr = 
         {