Merge remote-tracking branch 'origin/closedown-4.0.x'

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

docs/ECLLanguageReference/ECLR_mods/BltInFunc-JOIN.xml

@@ -14,9 +14,8 @@
   jointype</emphasis><emphasis role="bold">] [</emphasis><emphasis>,
   joinflags</emphasis><emphasis role="bold">] )</emphasis></para>
 
-  <para><emphasis role="bold"></emphasis><emphasis
-  role="bold">JOIN(</emphasis><emphasis>setofdatasets, joincondition,
-  transform</emphasis>,<emphasis role="bold"> SORTED<indexterm>
+  <para><emphasis role="bold">JOIN(</emphasis><emphasis>setofdatasets,
+  joincondition, transform</emphasis>,<emphasis role="bold"> SORTED<indexterm>
       <primary>SORTED</primary>
     </indexterm>( </emphasis><emphasis>fields</emphasis><emphasis
   role="bold">) [</emphasis><emphasis>, jointype</emphasis><emphasis
@@ -166,7 +165,7 @@
 
         <tbody>
           <row>
-            <entry>"equality"</entry>
+            <entry>"equality" (hard match)</entry>
 
             <entry>All the simple "LEFT.field = RIGHT.field" logic that
             defines matching records. For JOINs that use keys, all these must
@@ -177,7 +176,7 @@
           </row>
 
           <row>
-            <entry>"non-equality"</entry>
+            <entry>"non-equality" (soft match)</entry>
 
             <entry>All other matching criteria in the
             <emphasis>joincondition</emphasis> logic, such as "LEFT.field &gt;
@@ -461,13 +460,18 @@
             whereas LIMIT would either fail the job entirely, or SKIP the
             record (eliminating the <emphasis>leftrecset</emphasis> record
             entirely from the output). If omitted, the default is
-            LIMIT(10000); LIMIT(0) is unlimited.</entry>
+            LIMIT(10000). The LIMIT is applied to the set of records that meet
+            the the hard match ("equality") portion of the
+            <emphasis>joincondition</emphasis> but before the soft match
+            ("non-equality") portion of the <emphasis>joincondition</emphasis>
+            is evaluated. </entry>
           </row>
 
           <row>
             <entry><emphasis>value</emphasis></entry>
 
-            <entry>The maximum number of matches allowed.</entry>
+            <entry>The maximum number of matches allowed; LIMIT(0) is
+            unlimited.</entry>
           </row>
 
           <row>
@@ -481,16 +485,17 @@
           <row>
             <entry><emphasis>transform</emphasis></entry>
 
-            <entry>Optional. Specifies outputting a single record produced 
-            by the <emphasis>transform</emphasis> instead of failing the workunit 
+            <entry>Optional. Specifies outputting a single record produced by
+            the <emphasis>transform</emphasis> instead of failing the workunit
             (similar to the ONFAIL option of the LIMIT function).</entry>
           </row>
 
           <row>
             <entry><emphasis role="bold">FAIL</emphasis></entry>
 
-            <entry>Optional. Specifies using the <link linkend="FAIL">FAIL</link> action to configure the error 
-            message when the job fails.</entry>
+            <entry>Optional. Specifies using the <link
+            linkend="FAIL">FAIL</link> action to configure the error message
+            when the job fails.</entry>
           </row>
 
           <row>
@@ -681,7 +686,7 @@
 
         <tbody>
           <row>
-            <entry>INNER (default)</entry>
+            <entry>inner (default)</entry>
 
             <entry>Only those records that exist in both the
             <emphasis>leftrecset</emphasis> and

ecl/hqlcpp/hqlresource.cpp

@@ -999,6 +999,7 @@ ResourcerInfo::ResourcerInfo(IHqlExpression * _original, CResourceOptions * _opt
     currentSource = 0;
     linkedFromChild = false;
     neverSplit = false;
+    isConditionalFilter = false;
 }
 
 void ResourcerInfo::setConditionSource(IHqlExpression * condition, bool isFirst)            
@@ -2709,6 +2710,13 @@ bool EclResourcer::findSplitPoints(IHqlExpression * expr)
         case no_filter:
             if (options.preventKeyedSplit && filterIsKeyed(expr))
                 insideNeverSplit = true;
+            else
+            {
+                LinkedHqlExpr invariant;
+                OwnedHqlExpr cond = extractFilterConditions(invariant, expr, expr->queryNormalizedSelector(), false);
+                if (invariant)
+                    info->isConditionalFilter = true;
+            }
             break;
         case no_hqlproject:
         case no_newusertable:
@@ -2911,6 +2919,14 @@ void EclResourcer::createInitialGraph(IHqlExpression * expr, IHqlExpression * ow
             if (!options.noConditionalLinks || expr->isAction())
                 forceNewChildGraph = true;
             break;
+        case no_filter:
+            if (info->isConditionalFilter)
+            {
+                thisGraph->mergedConditionSource = true;
+                if (!options.noConditionalLinks)
+                    forceNewChildGraph = true;
+            }
+            break;
 //      case no_nonempty:
         case no_sequential:
             {
@@ -3110,6 +3126,20 @@ void EclResourcer::markAsUnconditional(IHqlExpression * expr, ResourceGraphInfo
         }
         markChildDependentsAsUnconditional(info, condition);
         return;
+    case no_filter:
+        if (!info->isConditionalFilter || options.noConditionalLinks)
+            break;
+
+        if (condition)
+            markCondition(expr, condition, wasConditional);
+        else
+        {
+            //This list is processed in a second phase.
+            if (rootConditions.find(*expr) == NotFound)
+                rootConditions.append(*LINK(expr));
+        }
+        markChildDependentsAsUnconditional(info, condition);
+        return;
     case no_sequential:
 //  case no_nonempty:
         if (!options.isChildQuery)
@@ -3165,8 +3195,15 @@ void EclResourcer::markConditionBranch(unsigned childIndex, IHqlExpression * exp
 
 void EclResourcer::markCondition(IHqlExpression * expr, IHqlExpression * condition, bool wasConditional)
 {
-    ForEachChildFrom(i, expr, 1)
-        markConditionBranch(i, expr, condition, wasConditional);
+    if (expr->getOperator() == no_filter)
+    {
+        markConditionBranch(0, expr, condition, wasConditional);
+    }
+    else
+    {
+        ForEachChildFrom(i, expr, 1)
+            markConditionBranch(i, expr, condition, wasConditional);
+    }
 }
 
 void EclResourcer::markConditions(HqlExprArray & exprs)
@@ -3868,8 +3905,8 @@ bool EclResourcer::queryMergeGraphLink(ResourceGraphLink & link)
                         ResourcerInfo * sinkInfo = queryResourceInfo(cur.sinkNode);
 
                         //If this is conditional, don't merge if there is a link to another graph
-                        if ((!cur.isDependency() && sinkInfo->isConditionExpr()) || 
-                        //if (sinkInfo->isConditionExpr() || 
+                        if ((!cur.isDependency() && sinkInfo->isConditionExpr()) ||
+                        //if (sinkInfo->isConditionExpr() ||
                             (!sinkInfo->isUnconditional() && sinkInfo->conditions.ordinality()))
                             isConditionalInSinkGraph = true;
                     }

ecl/hqlcpp/hqlresource.ipp

@@ -238,10 +238,14 @@ public:
         switch (original->getOperator())
         {
         case no_if:
+        case no_choose:
+        case no_chooseds:
             return true;
         case no_output:
         case no_buildindex:
             return isUpdatedConditionally(original);
+        case no_filter:
+            return isConditionalFilter;
         }
         return false;
     }
@@ -289,6 +293,7 @@ public:
     bool linkedFromChild;
     bool neverSplit;
     byte pathToExpr;
+    bool isConditionalFilter;
 };
 
 struct DependencySourceInfo

ecl/regress/issue9076.ecl

@@ -0,0 +1,8 @@
+#option ('targetClusterType', 'hthor');
+
+boolean b := false : stored('b');
+
+d := nofold(sort(nofold(dataset([{'hello'}], {string a})),a));
+
+d(b);
+d(a = 'fred');

plugins/stringlib/stringlib.cpp

@@ -69,8 +69,8 @@ static const char * EclDefinition =
 "  boolean StringWildExactMatch(const string src, const string _pattern, boolean _noCase) : c, pure,entrypoint='slStringWildExactMatch'; \n"
 "  boolean StringContains(const string src, const string _pattern, boolean _noCase) : c, pure,entrypoint='slStringContains'; \n"
 "  string StringExtractMultiple(const string src, unsigned8 mask) : c,pure,entrypoint='slStringExtractMultiple'; \n"
-"  unsigned integer4 EditDistance(const string l, const string r) : c, pure,entrypoint='slEditDistance'; \n"
-"  boolean EditDistanceWithinRadius(const string l, const string r, unsigned4 radius) : c,pure,entrypoint='slEditDistanceWithinRadius'; \n"
+"  unsigned integer4 EditDistance(const string l, const string r) : c, pure,entrypoint='slEditDistanceV2'; \n"
+"  boolean EditDistanceWithinRadius(const string l, const string r, unsigned4 radius) : c,pure,entrypoint='slEditDistanceWithinRadiusV2'; \n"
 "  unsigned integer4 EditDistanceV2(const string l, const string r) : c, pure,entrypoint='slEditDistanceV2'; \n"
 "  boolean EditDistanceWithinRadiusV2(const string l, const string r, unsigned4 radius) : c,pure,entrypoint='slEditDistanceWithinRadiusV2'; \n"
 "  string StringGetNthWord(const string src, unsigned4 n) : c, pure,entrypoint='slStringGetNthWord'; \n"
@@ -158,134 +158,10 @@ inline unsigned min3(unsigned a, unsigned b, unsigned c)
     return mi;
 }
 
-class CEditDistance
-{
-private:
-    unsigned char da[256][256];
-
-public:
-    unsigned editDistance(unsigned leftLen, const char * left, unsigned rightLen, const char * right)
-    {
-        unsigned i, j, cost;
-        char l_i, r_j;
-
-        clip(leftLen, left);
-        clip(rightLen, right);
-
-        if (leftLen > 255)
-        {
-            leftLen = 255;
-        }
-        if (rightLen > 255)
-        {
-            rightLen = 255;
-        }
-
-        if (leftLen == 0)
-        {
-            return rightLen;
-        }
-        if (rightLen == 0)
-        {
-            return leftLen;
-        }
-
-        for (i = 0; i <= leftLen; i++)
-        {
-            da[i][0] = i;
-        }
-
-        for (j = 0; j <= rightLen; j++)
-        {
-            da[0][j] = j;
-        }
-
-        for (i = 1; i <= leftLen; i++)
-        {
-            l_i = left[i - 1];
-
-            for (j = 1; j <= rightLen; j++)
-            {
-                    r_j = right[j - 1];
-                    cost = (l_i == r_j) ? 0 : 1;
-                    da[i][j] = min3(da[i-1][j]+1, da[i][j-1]+1, da[i-1][j-1] + cost);
-            }
-        }
-
-        return da[leftLen][rightLen];
-    }
-
-    unsigned editDistance(unsigned leftLen, const char * left, unsigned rightLen, const char * right, unsigned radius)
-    {
-        unsigned i, j, cost;
-        char l_i, r_j;
-
-        clip(leftLen, left);
-        clip(rightLen, right);
-
-        if (leftLen > 255)
-        {
-            leftLen = 255;
-        }
-        if (rightLen > 255)
-        {
-            rightLen = 255;
-        }
-
-        if (leftLen == 0)
-        {
-            return rightLen;
-        }
-        if (rightLen == 0)
-        {
-            return leftLen;
-        }
-
-        if (leftLen > rightLen)
-        {
-            const char *tstr = left;
-            left = right;
-            right = tstr;
-
-            unsigned tlen = leftLen;
-            leftLen = rightLen;
-            rightLen = tlen;
-        }
-
-        for (i = 0; i <= leftLen; i++)
-        {
-            da[i][0] = i;
-        }
-
-        for (j = 0; j <= rightLen; j++)
-        {
-            da[0][j] = j;
-        }
-
-        for (i = 1; i <= leftLen; i++)
-        {
-            l_i = left[i - 1];
-
-            for (j = 1; j <= rightLen; j++)
-            {
-                    r_j = right[j - 1];
-                    cost = (l_i == r_j) ? 0 : 1;
-                    da[i][j] = min3(da[i-1][j]+1, da[i][j-1]+1, da[i-1][j-1] + cost);
-            }
-
-            // bail out early if ed can't possibly be <= radius
-            if ((da[i][rightLen] - (leftLen - i)) > radius)
-                return da[i][rightLen];
-        }
-
-        return da[leftLen][rightLen];
-    }
-};
-
-//--- Optimized versions of the edit distance functions above.
+//--- Optimized versions of the edit distance functions
 inline unsigned mask(unsigned x) { return x & 1; }
 
-unsigned editDistanceV2(unsigned leftLen, const char * left, unsigned rightLen, const char * right)
+unsigned editDistance(unsigned leftLen, const char * left, unsigned rightLen, const char * right)
 {
     unsigned i, j;
 
@@ -340,89 +216,13 @@ unsigned editDistanceV2(unsigned leftLen, const char * left, unsigned rightLen,
     return da[mask(leftLen-1)][rightLen-1];
 }
 
-unsigned editDistanceV2(unsigned leftLen, const char * left, unsigned rightLen, const char * right, unsigned radius)
-{
-    unsigned i, j;
-
-    clip(leftLen, left);
-    clip(rightLen, right);
-
-    unsigned minED = (leftLen < rightLen)? rightLen - leftLen: leftLen - rightLen;
-    if (minED > radius)
-        return minED;
-
-    if (leftLen > 255)
-        leftLen = 255;
-
-    if (rightLen > 255)
-        rightLen = 255;
-
-    //Checking for leading common substrings actually slows the function down.
-    if (leftLen == 0)
-        return rightLen;
-
-    if (rightLen == 0)
-        return leftLen;
-
-    /*
-    This function applies two optimizations over the function above.
-    a) Adding a charcter (next row) can at most decrease the edit distance by 1, so short circuit when
-       we there is no possiblity of getting within the distance.
-    b) We only need to evaluate the martix da[i-radius..i+radius][j-radius..j+radius]
-       not taking into account values outside that range [can use max value to prevent access]
-    */
-
-    //Optimize the storage requirements by
-    //i) Only storing two stripes
-    //ii) Calculate, but don't store the row comparing against the null string
-    //NB: A byte array is ok because the +1 is added after the minimum, and that will always include 254 as an option.
-    unsigned char da[2][255];
-    char r_0 = right[0];
-    char l_0 = left[0];
-    bool matched_l0 = false;
-    for (j = 0; j < rightLen; j++)
-    {
-        if (right[j] == l_0) matched_l0 = true;
-        da[0][j] = (matched_l0) ? j : j+1;
-    }
-
-    bool matched_r0 = (l_0 == r_0);
-    for (i = 1; i < leftLen; i++)
-    {
-        char l_i = left[i];
-        if (l_i == r_0)
-            matched_r0 = true;
-
-        byte da_i_0 = matched_r0 ? i : i+1;
-        da[mask(i)][0] = da_i_0;
-        byte da_i_prevj = da_i_0;
-
-        unsigned first = (i > radius) ? i-radius : 1;
-        unsigned last = (i + radius > rightLen) ? rightLen : i + radius;
-        for (j = 1; j < rightLen; j++)
-        {
-            char r_j = right[j];
-            unsigned char next = (l_i == r_j) ? da[mask(i-1)][j-1] :
-                        min3(da[mask(i-1)][j], da_i_prevj, da[mask(i-1)][j-1]) + 1;
-            da[mask(i)][j] = next;
-            da_i_prevj = next;
-        }
-
-        // bail out early if ed can't possibly be <= radius
-        unsigned maxdelta = (leftLen - (i+1));
-        if (da_i_prevj > radius + maxdelta)         // if da_i_prvj - maxdelta > radius can't ever get low enough
-            return da_i_prevj;
-    }
-
-    return da[mask(leftLen-1)][rightLen-1];
-}
 
 //This could be further improved in the following ways:
 // * Only use 2*radius bytes of temporary storage - I doubt it is worth it.
 // * special case edit1 - you could use variables for the 6 interesting array elements, and get
 //   rid of the array completely.  You could also unwind the first (and last iterations).
 // * I suspect the early exit condition could be improved depending the lengths of the strings.
-extern STRINGLIB_API unsigned editDistanceV3(unsigned leftLen, const char * left, unsigned rightLen, const char * right, unsigned radius)
+extern STRINGLIB_API unsigned editDistanceWithinRadius(unsigned leftLen, const char * left, unsigned rightLen, const char * right, unsigned radius)
 {
     if (radius >= 255)
         return 255;
@@ -1127,41 +927,15 @@ STRINGLIB_API bool STRINGLIB_CALL slStringContains(unsigned srcLen, const char *
     return true;
 }
 
-STRINGLIB_API unsigned STRINGLIB_CALL slEditDistance(unsigned leftLen, const char * left, unsigned rightLen, const char * right)
-{
-    CEditDistance * ed = new CEditDistance();
-    unsigned rval = ed->editDistance(leftLen, left, rightLen, right);
-    delete ed;
-    return rval;
-}
-
-
-STRINGLIB_API bool STRINGLIB_CALL slEditDistanceWithinRadius(unsigned leftLen, const char * left, unsigned rightLen, const char * right, unsigned radius)
-{
-    unsigned minED = (leftLen < rightLen)? rightLen - leftLen: leftLen - rightLen;
-
-    if (minED > radius)
-    {
-        return false;
-    }
-    else
-    {
-        CEditDistance *ed = new CEditDistance();
-        unsigned rval = ed->editDistance(leftLen, left, rightLen, right, radius);
-        delete ed;
-        return (rval <= radius);
-    }
-}
-
 STRINGLIB_API unsigned STRINGLIB_CALL slEditDistanceV2(unsigned leftLen, const char * left, unsigned rightLen, const char * right)
 {
-    return nsStringlib::editDistanceV2(leftLen, left, rightLen, right);
+    return nsStringlib::editDistance(leftLen, left, rightLen, right);
 }
 
 
 STRINGLIB_API bool STRINGLIB_CALL slEditDistanceWithinRadiusV2(unsigned leftLen, const char * left, unsigned rightLen, const char * right, unsigned radius)
 {
-    return nsStringlib::editDistanceV3(leftLen, left, rightLen, right, radius) <= radius;
+    return nsStringlib::editDistanceWithinRadius(leftLen, left, rightLen, right, radius) <= radius;
 }
 
 inline bool isWordSeparator(char x)

plugins/stringlib/stringlib.hpp

@@ -75,8 +75,6 @@ STRINGLIB_API bool STRINGLIB_CALL slStringWildMatch(unsigned srcLen, const char
 STRINGLIB_API bool STRINGLIB_CALL slStringWildExactMatch(unsigned srcLen, const char * src, unsigned patLen, const char * pat, bool noCase);
 STRINGLIB_API bool STRINGLIB_CALL slStringContains(unsigned srcLen, const char * src, unsigned sampleLen, const char * sample, bool noCase);
 STRINGLIB_API void STRINGLIB_CALL slStringExtractMultiple(unsigned & tgtLen, char * & tgt, unsigned srcLen, const char * src, unsigned __int64 mask);
-STRINGLIB_API unsigned STRINGLIB_CALL slEditDistance(unsigned leftLen, const char * left, unsigned rightLen, const char * right);
-STRINGLIB_API bool STRINGLIB_CALL slEditDistanceWithinRadius(unsigned leftLen, const char * left, unsigned rightLen, const char * right, unsigned radius);
 STRINGLIB_API unsigned STRINGLIB_CALL slEditDistanceV2(unsigned leftLen, const char * left, unsigned rightLen, const char * right);
 STRINGLIB_API bool STRINGLIB_CALL slEditDistanceWithinRadiusV2(unsigned leftLen, const char * left, unsigned rightLen, const char * right, unsigned radius);
 STRINGLIB_API void STRINGLIB_CALL slStringGetNthWord(unsigned & tgtLen, char * & tgt, unsigned srcLen, const char * src, unsigned n);

system/jlib/jutil.cpp

@@ -1536,26 +1536,18 @@ bool callExternalProgram(const char *progname, const StringBuffer &input, String
         pipe2.CloseRead();
         pipe2.SetStdout();
 
+        const char *cmd[] = { progname, NULL };
         if (env_in)
         {
-            const char *cmd[] = { progname, (char *)0 };
-            const char *envp[256]={0};
+            const char **envp = (const char **) alloca((env_in->ordinality()+1) * sizeof(const char *));
             ForEachItemIn(index, *env_in)
                 envp[index]=env_in->item(index);
-            if(execve(progname, (char * const *)cmd, (char * const *)envp)<0)
-            {
-                ERRLOG("Exec failed %s %d",progname,errno);
-                exit(EXIT_FAILURE);
-            }
+            envp[env_in->ordinality()] = NULL;
+            execvpe(progname, (char * const *)cmd, (char * const *)envp);  // will not return, on success
         }
         else
-        {
-            if(exec(progname)<0)
-            {
-                ERRLOG("Exec failed %s %d",progname,errno);
-                exit(EXIT_FAILURE);
-            }
-        }
+            execvp(progname, (char * const *)cmd);  // will not return, on success
+        _exit(EXIT_FAILURE); // must be _exit!!
     }
     else
     {