Merge branch 'candidate-6.2.2' into candidate-6.4.0

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

.gitmodules

@@ -40,7 +40,7 @@
 [submodule "plugins/couchbase/libcouchbase-cxx"]
 	path = plugins/couchbase/libcouchbase-cxx
 	url = https://github.com/hpcc-systems/libcouchbase-cxx.git
-[submodule "libcouchbase"]
+[submodule "plugins/couchbase/libcouchbase"]
 	path = plugins/couchbase/libcouchbase
 	url = https://github.com/hpcc-systems/libcouchbase.git
 [submodule "plugins/redis/hiredis"]

ecl/eclcc/eclcc.cpp

@@ -1181,7 +1181,7 @@ void EclCC::processSingleQuery(EclCompileInstance & instance,
 
             gatherParseWarnings(ctx.errs, instance.query, parseCtx.orphanedWarnings);
 
-            if (instance.query && !syntaxChecking && !optGenerateMeta && !optEvaluateResult)
+            if (instance.query && !optGenerateMeta && !optEvaluateResult)
                 instance.query.setown(convertAttributeToQuery(instance.query, ctx));
 
             unsigned __int64 parseTimeNs = cycle_to_nanosec(get_cycles_now() - startCycles);

ecl/hql/hqlexpr.cpp

@@ -4204,6 +4204,8 @@ void CHqlRealExpression::updateFlagsAfterOperands()
                 infoFlags &= ~(HEFonFailDependent|HEFcontainsSkip); // ONFAIL(SKIP) - skip shouldn't extend any further
             else if (name == _volatileId_Atom)
                 infoFlags |= (HEFnoduplicate|HEFcontextDependentException);
+            else if (name == _selectorSequence_Atom)
+                infoFlags2 &= ~(HEF2containsSelf);
             infoFlags &= ~(HEFthrowscalar|HEFthrowds|HEFoldthrows);
             break;
         }
@@ -4289,6 +4291,19 @@ void CHqlRealExpression::updateFlagsAfterOperands()
                 infoFlags |= (HEFnoduplicate|HEFcontextDependentException);
             break;
         }
+    case no_funcdef:
+        {
+            IHqlExpression * body = queryChild(0);
+            if (body->getOperator() == no_outofline)
+            {
+                //This test could be done for all funcdefs, but the cost of calculating often outweighs
+                //the savings when binding functions.
+                if (!containsExternalParameter(body, queryChild(1)))
+                    infoFlags &= ~HEFunbound;
+            }
+            break;
+        }
+
     }
 
 #ifdef VERIFY_EXPR_INTEGRITY
@@ -8186,7 +8201,6 @@ IHqlExpression * createFunctionDefinition(IIdAtom * id, HqlExprArray & args)
         attrs.set(cachedContextAttribute);
 
     //This is a bit of a waste of time, but need to improve assignableFrom for a function type to ignore the uids.
-    QuickExpressionReplacer defaultReplacer;
     HqlExprArray normalized;
     ForEachChild(i, formals)
     {
@@ -8195,11 +8209,25 @@ IHqlExpression * createFunctionDefinition(IIdAtom * id, HqlExprArray & args)
         unwindChildren(attrs, formal);
         IHqlExpression * normal = createParameter(formal->queryId(), UnadornedParameterIndex, formal->getType(), attrs);
         normalized.append(*normal);
-        defaultReplacer.setMapping(formal->queryBody(), normal);
     }
 
+    OwnedHqlExpr newDefaults;
+    if (defaults && !defaults->isFullyBound())
+    {
+        QuickExpressionReplacer defaultReplacer;
+        ForEachChild(i, formals)
+        {
+            IHqlExpression * formal = formals->queryChild(i);
+            IHqlExpression * normal = &normalized.item(i);
+            defaultReplacer.setMapping(formal->queryBody(), normal);
+        }
+
+        newDefaults.setown(defaultReplacer.transform(defaults));
+    }
+    else
+        newDefaults.set(defaults);
+
     OwnedHqlExpr newFormals = formals->clone(normalized);
-    OwnedHqlExpr newDefaults = defaults ? defaultReplacer.transform(defaults) : NULL;
 
     ITypeInfo * type = makeFunctionType(body->getType(), newFormals.getClear(), newDefaults.getClear(), attrs.getClear());
     return createNamedValue(no_funcdef, type, id, args);
@@ -9512,7 +9540,7 @@ IHqlExpression *CHqlVirtualScope::lookupSymbol(IIdAtom * searchName, unsigned lo
             if (matchOp == no_unboundselect || matchOp == no_purevirtual)
                 throwError1(HQLERR_MemberXContainsVirtualRef, str(searchName));
 
-            if (containsInternalSelect(match))
+            if ((this != definitionModule) && containsInternalSelect(match))
             {
                 //throwError1(HQLERR_MemberXContainsVirtualRef, searchName->str());
                 //All internal references need to be recursively replaced with no_internalselect
@@ -11900,110 +11928,113 @@ static void normalizeCallParameters(HqlExprArray & resolvedActuals, IHqlExpressi
     if (isExternalMethodDefinition(funcdef))
         resolvedActuals.append(OLINK(actuals.item(curActual++)));
 
-    QuickExpressionReplacer defaultReplacer;
-    for (unsigned i = 0; i < maxFormal; i++)
+    if (maxFormal)
     {
-        //NOTE: For functional parameters, formal is a no_funcdef, not a no_param.  I suspect something better should be implemented..
-        IHqlExpression * formal = formals->queryChild(i);
-        LinkedHqlExpr actual;
-        if (actuals.isItem(curActual))
-            actual.set(&actuals.item(curActual++));
-        if (!actual || actual->getOperator()==no_omitted || actual->isAttribute())
+        QuickExpressionReplacer defaultReplacer;
+        for (unsigned i = 0; i < maxFormal; i++)
         {
-            actual.set(queryDefaultValue(defaults, i));
-            if (!actual)
-                actual.setown(createNullExpr(formal));
-            else if (actual->getOperator() == no_sequence)
-                actual.setown(createSequenceExpr());
-
-            //implicit parameters added to out of line function definitions, may reference the function parameters (e..g, global(myParameter * 2)
-            //so they need substituting first.  May also occur if defaults are based on other parameter values.
-            //MORE: Should probably lazily create the mapping etc since 95% of the time it won't be used
-            if (!actual->isFullyBound() && !actual->isFunction())
-                actual.setown(defaultReplacer.transform(actual));
-        }
+            //NOTE: For functional parameters, formal is a no_funcdef, not a no_param.  I suspect something better should be implemented..
+            IHqlExpression * formal = formals->queryChild(i);
+            LinkedHqlExpr actual;
+            if (actuals.isItem(curActual))
+                actual.set(&actuals.item(curActual++));
+            if (!actual || actual->getOperator()==no_omitted || actual->isAttribute())
+            {
+                actual.set(queryDefaultValue(defaults, i));
+                if (!actual)
+                    actual.setown(createNullExpr(formal));
+                else if (actual->getOperator() == no_sequence)
+                    actual.setown(createSequenceExpr());
+
+                //implicit parameters added to out of line function definitions, may reference the function parameters (e..g, global(myParameter * 2)
+                //so they need substituting first.  May also occur if defaults are based on other parameter values.
+                //MORE: Should probably lazily create the mapping etc since 95% of the time it won't be used
+                if (!actual->isFullyBound() && !actual->isFunction())
+                    actual.setown(defaultReplacer.transform(actual));
+            }
 
-        ITypeInfo * type = formal->queryType();
-        if (type)
-        {
-            switch (type->getTypeCode())
+            ITypeInfo * type = formal->queryType();
+            if (type)
             {
-            case type_record:
-                // MORE - should be more exact?  Don't add a cast when binding parameters into a transform...
-                break;
-            case type_table:
-            case type_groupedtable:
-                if (isAbstractDataset(formal))
+                switch (type->getTypeCode())
                 {
-#ifdef NEW_VIRTUAL_DATASETS
-                    HqlExprArray abstractSelects;
-                    gatherAbstractSelects(abstractSelects, funcdef->queryChild(0), formal);
-
-                    if (actual->getOperator()==no_fieldmap)
+                case type_record:
+                    // MORE - should be more exact?  Don't add a cast when binding parameters into a transform...
+                    break;
+                case type_table:
+                case type_groupedtable:
+                    if (isAbstractDataset(formal))
                     {
-                        LinkedHqlExpr map = actual->queryChild(1);
-                        actual.set(actual->queryChild(0));
-                        associateBindMap(abstractSelects, formal, actual, map);
-                    }
+    #ifdef NEW_VIRTUAL_DATASETS
+                        HqlExprArray abstractSelects;
+                        gatherAbstractSelects(abstractSelects, funcdef->queryChild(0), formal);
 
-                    associateBindByName(abstractSelects, formal, actual);
-#else
-                    if (actual->getOperator()==no_fieldmap)
-                        actual.set(actual->queryChild(0));
-#endif
-                }
-                else
-                {
-                    OwnedHqlExpr actualRecord = getUnadornedRecordOrField(actual->queryRecord());
-                    IHqlExpression * formalRecord = ::queryOriginalRecord(type);
-                    OwnedHqlExpr normalFormalRecord = getUnadornedRecordOrField(formalRecord);
-                    if (actualRecord && normalFormalRecord && normalFormalRecord->numChildren() && (normalFormalRecord->queryBody() != actualRecord->queryBody()))
+                        if (actual->getOperator()==no_fieldmap)
+                        {
+                            LinkedHqlExpr map = actual->queryChild(1);
+                            actual.set(actual->queryChild(0));
+                            associateBindMap(abstractSelects, formal, actual, map);
+                        }
+
+                        associateBindByName(abstractSelects, formal, actual);
+    #else
+                        if (actual->getOperator()==no_fieldmap)
+                            actual.set(actual->queryChild(0));
+    #endif
+                    }
+                    else
                     {
-                        //If the actual dataset is derived from the input dataset, then insert a project so types remain correct
-                        //otherwise x+y will change meaning.
-                        OwnedHqlExpr seqAttr = createSelectorSequence();
-                        OwnedHqlExpr self = createSelector(no_self, formalRecord, NULL);
-                        OwnedHqlExpr left = createSelector(no_left, actual, seqAttr);
-                        HqlExprArray assigns;
-                        createAssignAll(assigns, self, left, formalRecord);
-                        OwnedHqlExpr transform = createValue(no_transform, makeTransformType(formalRecord->getType()), assigns);
-                        actual.setown(createDataset(no_hqlproject, actual.getClear(), createComma(transform.getClear(), LINK(seqAttr))));
+                        OwnedHqlExpr actualRecord = getUnadornedRecordOrField(actual->queryRecord());
+                        IHqlExpression * formalRecord = ::queryOriginalRecord(type);
+                        OwnedHqlExpr normalFormalRecord = getUnadornedRecordOrField(formalRecord);
+                        if (actualRecord && normalFormalRecord && normalFormalRecord->numChildren() && (normalFormalRecord->queryBody() != actualRecord->queryBody()))
+                        {
+                            //If the actual dataset is derived from the input dataset, then insert a project so types remain correct
+                            //otherwise x+y will change meaning.
+                            OwnedHqlExpr seqAttr = createSelectorSequence();
+                            OwnedHqlExpr self = createSelector(no_self, formalRecord, NULL);
+                            OwnedHqlExpr left = createSelector(no_left, actual, seqAttr);
+                            HqlExprArray assigns;
+                            createAssignAll(assigns, self, left, formalRecord);
+                            OwnedHqlExpr transform = createValue(no_transform, makeTransformType(formalRecord->getType()), assigns);
+                            actual.setown(createDataset(no_hqlproject, actual.getClear(), createComma(transform.getClear(), LINK(seqAttr))));
+                        }
                     }
-                }
-                break;
-            case type_dictionary:
-                // MORE - needs some code
-                // For now, never cast
-                break;
-            case type_row:
-            case type_transform:
-            case type_function:
-                break;
-            case type_unicode:
-                if ((type->getSize() == UNKNOWN_LENGTH) && (actual->queryType()->getTypeCode() == type_varunicode))
                     break;
-                actual.setown(ensureExprType(actual, type));
-                break;
-#if 0
-            case type_string:
-                if (type->getSize() == UNKNOWN_LENGTH)
-                {
-                    ITypeInfo * actualType = actual->queryType();
-                    if ((actualType->getTypeCode() == type_varstring) && (actualType->queryCharset() == type->queryCharset())))
+                case type_dictionary:
+                    // MORE - needs some code
+                    // For now, never cast
+                    break;
+                case type_row:
+                case type_transform:
+                case type_function:
+                    break;
+                case type_unicode:
+                    if ((type->getSize() == UNKNOWN_LENGTH) && (actual->queryType()->getTypeCode() == type_varunicode))
                         break;
-                }
-                actual.setown(ensureExprType(actual, type));
-                break;
-#endif
-            default:
-                if (type != actual->queryType())
                     actual.setown(ensureExprType(actual, type));
-                break;
+                    break;
+    #if 0
+                case type_string:
+                    if (type->getSize() == UNKNOWN_LENGTH)
+                    {
+                        ITypeInfo * actualType = actual->queryType();
+                        if ((actualType->getTypeCode() == type_varstring) && (actualType->queryCharset() == type->queryCharset())))
+                            break;
+                    }
+                    actual.setown(ensureExprType(actual, type));
+                    break;
+    #endif
+                default:
+                    if (type != actual->queryType())
+                        actual.setown(ensureExprType(actual, type));
+                    break;
+                }
             }
+
+            defaultReplacer.setMapping(formal->queryBody(), actual);
+            resolvedActuals.append(*LINK(actual));
         }
-    
-        defaultReplacer.setMapping(formal->queryBody(), actual);
-        resolvedActuals.append(*LINK(actual));
     }
     while (actuals.isItem(curActual))
     {

ecl/hql/hqlgram2.cpp

@@ -1591,9 +1591,12 @@ protected:
         case no_assign:
             {
                 //The following optimization would be required if we ever supported recursive records.
-                IHqlExpression * lhs = expr->queryChild(0);
                 IHqlExpression * rhs = expr->queryChild(1);
-                return createAssign(LINK(lhs), recursiveReplaceExpression(rhs));
+                OwnedHqlExpr newRhs = recursiveReplaceExpression(rhs);
+                if (rhs == newRhs)
+                    return LINK(expr);
+                IHqlExpression * lhs = expr->queryChild(0);
+                return createAssign(LINK(lhs), newRhs.getClear());
             }
         }
 
@@ -1615,6 +1618,9 @@ protected:
 
     IHqlExpression * recursiveReplaceExpression(IHqlExpression * expr)
     {
+        if (!containsSelf(expr))
+            return LINK(expr);
+
         IHqlExpression * mapped = (IHqlExpression *)expr->queryTransformExtra();
         if (mapped)
             return LINK(mapped);

ecl/hql/hqltrans.cpp

@@ -4774,6 +4774,43 @@ void * transformerAlloc(size32_t size)
 }
 #endif
 
+//---------------------------------------------------------------------------------------------------------------------
+
+static HqlTransformerInfo containsExternalTransformerInfo("ContainsExternalParamSpotter");
+ContainsExternalParamSpotter::ContainsExternalParamSpotter(IHqlExpression * params)
+: QuickHqlTransformer(containsExternalTransformerInfo, NULL)
+{
+    ForEachChild(i, params)
+        internal.append(*params->queryChild(i));
+}
+
+//NB: This cannot be short circuited, because it is also gathering information about whether or
+void ContainsExternalParamSpotter::doAnalyseBody(IHqlExpression * expr)
+{
+    if (seenExternal)
+        return;
+    if (expr->isFullyBound())
+        return;
+
+    switch (expr->getOperator())
+    {
+    case no_param:
+        if (!internal.contains(*expr))
+            seenExternal = true;
+        return;
+    }
+
+    QuickHqlTransformer::doAnalyseBody(expr);
+}
+
+
+bool containsExternalParameter(IHqlExpression * expr, IHqlExpression * params)
+{
+    ContainsExternalParamSpotter spotter(params);
+    spotter.analyse(expr);
+    return spotter.containsExternal();
+}
+
 //------------------------------------------------------------------------------------------------
 
 /*

ecl/hql/hqltrans.ipp

@@ -1177,6 +1177,24 @@ protected:
     inline SplitterVerifierInfo * queryExtra(IHqlExpression * expr)     { return static_cast<SplitterVerifierInfo *>(queryTransformExtra(expr)); }
 };
 
+//---------------------------------------------------------------------------
+
+class ContainsExternalParamSpotter : public QuickHqlTransformer
+{
+public:
+    ContainsExternalParamSpotter(IHqlExpression * params);
+
+    bool containsExternal() const { return seenExternal; }
+
+protected:
+    virtual void doAnalyseBody(IHqlExpression * expr);
+
+private:
+    bool seenExternal = false;
+    HqlExprCopyArray internal;
+};
+bool containsExternalParameter(IHqlExpression * expr, IHqlExpression * params);
+
 /*
 
 If something can be transformed more than one way depending on the context then must either

ecl/hqlcpp/hqlttcpp.cpp

@@ -855,6 +855,7 @@ YesNoOption HqlThorBoundaryTransformer::calcNormalizeThor(IHqlExpression * expr)
     case no_all:
     case no_self:
     case no_activerow:
+    case no_param:
         return OptionMaybe;
     case no_evaluate:
         throwUnexpected();
@@ -869,6 +870,7 @@ YesNoOption HqlThorBoundaryTransformer::calcNormalizeThor(IHqlExpression * expr)
             case no_externalcall:
                 return normalizeThor(ds);
             case no_self:
+            case no_param:
                 return OptionMaybe;
             }
             return isNew ? OptionYes : OptionMaybe;
@@ -9747,6 +9749,28 @@ IHqlExpression * HqlScopeTagger::transformAmbiguousChildren(IHqlExpression * exp
 }
 
 
+IHqlExpression * HqlScopeTagger::transformCall(IHqlExpression * expr)
+{
+    unsigned max = expr->numChildren();
+    bool same = true;
+    HqlExprArray args;
+    args.ensure(max);
+    for(unsigned i=0; i < max; i++)
+    {
+        IHqlExpression * cur = expr->queryChild(i);
+        IHqlExpression * tr = transformAmbiguous(cur, false);
+        args.append(*tr);
+        if (cur != tr)
+            same = false;
+    }
+    IHqlExpression * funcdef = expr->queryFunctionDefinition();
+    OwnedHqlExpr newFuncDef = transform(funcdef);
+    if (same && funcdef == newFuncDef)
+        return LINK(expr);
+    return createReboundFunction(newFuncDef, args);
+}
+
+
 IHqlExpression * HqlScopeTagger::transformSizeof(IHqlExpression * expr)
 {
     IHqlExpression * arg = expr->queryChild(0)->queryNormalizedSelector();
@@ -9866,12 +9890,13 @@ IHqlExpression * HqlScopeTagger::createTransformed(IHqlExpression * expr)
         break;
     case no_select:
         return transformSelect(expr);
-    case no_call:
     case no_externalcall:
     case no_rowvalue:
 //  case no_addfiles:
 //  case no_libraryscopeinstance:??
         return transformAmbiguousChildren(expr);
+    case no_call:
+        return transformCall(expr);
     case no_offsetof:
     case no_sizeof:
         return transformSizeof(expr);
@@ -10699,6 +10724,8 @@ bool containsCompound(IHqlExpression * expr)
     return spotter.containsCompound;
 }
 
+//---------------------------------------------------------------------------------------------------------------------
+
 static HqlTransformerInfo nestedCompoundTransformerInfo("NestedCompoundTransformer");
 NestedCompoundTransformer::NestedCompoundTransformer(HqlCppTranslator & _translator)
 : HoistingHqlTransformer(nestedCompoundTransformerInfo, CTFnoteifactions), translator(_translator), translatorOptions(_translator.queryOptions())

ecl/hqlcpp/hqlttcpp.ipp

@@ -1065,6 +1065,7 @@ protected:
 
     IHqlExpression * transformAmbiguous(IHqlExpression * expr, bool isActiveOk);
     IHqlExpression * transformAmbiguousChildren(IHqlExpression * expr);
+    IHqlExpression * transformCall(IHqlExpression * expr);
     IHqlExpression * transformNewDataset(IHqlExpression * expr, bool isActiveOk);
     IHqlExpression * transformRelated(IHqlExpression * expr);
     IHqlExpression * transformSelectorsAttr(IHqlExpression * expr);

plugins/cassandra/CMakeLists.txt

@@ -83,6 +83,7 @@ if(USE_CASSANDRA)
     install(
       FILES ${PROJECT_BINARY_DIR}/libuv.so.0.10
       DESTINATION ${LIB_DIR}
+      PERMISSIONS OWNER_WRITE OWNER_READ OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE
       COMPONENT Runtime)
   endif()
 

plugins/couchbase/couchbaseembed.cpp

@@ -32,7 +32,7 @@ static const char *g_version = "Couchbase Embed Helper 1.0.0";
 static const char *g_compatibleVersions[] = { g_version, nullptr };
 static const NullFieldProcessor NULLFIELD(NULL);
 
-COUCHBASEEMBED_PLUGIN_API bool getECLPluginDefinition(ECLPluginDefinitionBlock *pb)
+extern "C" COUCHBASEEMBED_PLUGIN_API bool getECLPluginDefinition(ECLPluginDefinitionBlock *pb)
 {
     if (pb->size == sizeof(ECLPluginDefinitionBlockEx))
     {