HPCC-17651 Add --fastsyntax to delay expansion of functions

Signed-off-by: Gavin Halliday <gavin.halliday@lexisnexis.com>

ecl/eclcc/eclcc.cpp

@@ -397,6 +397,7 @@ protected:
     bool optGenerateHeader = false;
     bool optShowPaths = false;
     bool optNoSourcePath = false;
+    bool optFastSyntax = false;
     mutable bool daliConnected = false;
     mutable bool disconnectReported = false;
     int argc;
@@ -1139,6 +1140,8 @@ void EclCC::processSingleQuery(EclCompileInstance & instance,
     {
         //Minimize the scope of the parse context to reduce lifetime of cached items.
         HqlParseContext parseCtx(instance.dataServer, this, instance.archive);
+        if (optFastSyntax)
+            parseCtx.setFastSyntax();
         if (optMaxErrors > 0)
             parseCtx.maxErrors = optMaxErrors;
         parseCtx.unsuppressImmediateSyntaxErrors = optUnsuppressImmediateSyntaxErrors;
@@ -2183,6 +2186,9 @@ int EclCC::parseCommandLineOptions(int argc, const char* argv[])
         {
             debugOptions.append(tempArg);
         }
+        else if (iter.matchFlag(optFastSyntax, "--fastsyntax"))
+        {
+        }
         else if (iter.matchFlag(tempBool, "-g") || iter.matchFlag(tempBool, "--debug"))
         {
             if (tempBool)

ecl/hql/hqlexpr.cpp

@@ -9095,7 +9095,7 @@ IHqlExpression * createDelayedReference(node_operator op, IHqlExpression * modul
     else
         ret.setown(createValue(op, attr->getType(), args));
 
-    if (attr->isScope())
+    if (attr->isScope() || attr->getOperator() == no_enum)
     {
         if (attr->getOperator() != no_funcdef)
             ret.setown(createDelayedScope(ret.getClear()));
@@ -9298,6 +9298,26 @@ no_purevirtual - a member that has no associated definition.
 
 */
 
+bool definesMacro(IHqlExpression * expr)
+{
+    IHqlScope * scope = expr->queryScope();
+    HqlExprArray syms;
+    scope->getSymbols(syms);
+    ForEachItemIn(i, syms)
+    {
+        //HACK - needs more work
+        IHqlExpression & symbol = syms.item(i);
+        if (symbol.isMacro())
+            return true;
+        if (symbol.isScope())
+        {
+            //MORE: Need to ensure that this symbol is defined, and walk the children
+        }
+
+    }
+    return false;
+}
+
 bool canBeDelayed(IHqlExpression * expr)
 {
     switch (expr->getOperator())
@@ -9309,7 +9329,18 @@ bool canBeDelayed(IHqlExpression * expr)
     case no_record:             // LEFT has problems because queryRecord() is the unbound funcdef
     case no_keyindex:           // BUILD() and other functions a reliant on this being expanded out
     case no_newkeyindex:
+    case no_forwardscope:
+    case no_type:
         return false;
+    case no_remotescope:
+    case no_scope:
+    case no_privatescope:
+    case no_virtualscope:
+    {
+        if (definesMacro(expr))
+            return false;
+        return true;
+    }
     case no_funcdef:
         return canBeDelayed(expr->queryChild(0));
     }
@@ -10045,13 +10076,21 @@ CHqlDelayedScope::CHqlDelayedScope(HqlExprArray &_ownedOperands)
  : CHqlExpressionWithTables(no_delayedscope), type(nullptr)
 {
     setOperands(_ownedOperands); // after type is initialized
-    type = queryChild(0)->queryType();
+    IHqlExpression * arg0 = queryChild(0);
+    if (arg0->getOperator() == no_delayedselect)
+        arg0 = arg0->queryChild(2);
+    type = arg0->queryType();
 
     ITypeInfo * scopeType = type;
     if (scopeType->getTypeCode() == type_function)
         scopeType = scopeType->queryChildType();
 
     typeScope = ::queryScope(scopeType);
+    if (!typeScope)
+    {
+        typeScope = arg0->queryScope();
+        type = typeScope->queryExpression()->queryType();
+    }
     assertex(typeScope);
 
     if (!hasAttribute(_virtualSeq_Atom))
@@ -10600,6 +10639,9 @@ IHqlExpression * CHqlDelayedScopeCall::lookupSymbol(IIdAtom * searchName, unsign
     if (lookupFlags & LSFignoreBase)
         return match.getClear();
 
+    if (!canBeVirtual(match) && match->isFullyBound())
+        return match.getClear();
+
     return createDelayedReference(no_delayedselect, this, match, lookupFlags, ctx);
 }
 
@@ -12208,6 +12250,16 @@ void expandDelayedFunctionCalls(IErrorReceiver * errors, HqlExprArray & exprs)
     replaceArray(exprs, target);
 }
 
+IHqlExpression * expandDelayedFunctionCalls(IErrorReceiver * errors, IHqlExpression * expr)
+{
+    HqlExprArray functionCache;
+    CallExpansionContext ctx;
+    ctx.functionCache = &functionCache;
+    ctx.errors = errors;
+    CallExpandTransformer binder(ctx);
+
+    return binder.transform(expr);
+}
 
 extern IHqlExpression * createReboundFunction(IHqlExpression *func, HqlExprArray &actuals)
 {

ecl/hql/hqlexpr.hpp

@@ -33,14 +33,6 @@
 //It nearly works - but there are still some examples which have problems - primarily libraries, old parameter syntax, enums and other issues.
 //There may also problems with queryRecord() which needs to really be replaced with recordof(x), especially if "templates" are delayed expanded.
 //To work properly it may require many of the transformations in hqlgram2.cpp to be moved to after the expansion.  (E.g., BUILD)
-//#define DELAY_CALL_EXPANSION
-
-#ifdef DELAY_CALL_EXPANSION
-#define DEFAULT_EXPAND_CALL false
-#else
-#define DEFAULT_EXPAND_CALL true
-#endif
-
 
 #define GATHER_HIDDEN_SELECTORS
 
@@ -345,7 +337,7 @@ enum node_operator : unsigned short {
         no_datasetfromdictionary,
         no_delayedscope,
         no_assertconcrete,
-        no_unboundselect,
+        no_unboundselect,               // A symbol selected from a module derived from a parameter
         no_id,
         no_orderedactionlist,
         no_dataset_from_transform,
@@ -870,7 +862,7 @@ public:
     HqlParseContext(IEclRepository * _eclRepository, ICodegenContextCallback *_codegenCtx, IPropertyTree * _archive)
     : archive(_archive), eclRepository(_eclRepository), codegenCtx(_codegenCtx)
     {
-        expandCallsWhenBound = DEFAULT_EXPAND_CALL;
+        expandCallsWhenBound = true;
         ignoreUnknownImport = false;
         ignoreSignatures = false;
         _clear(metaState);
@@ -895,6 +887,7 @@ public:
     inline IEclRepository * queryRepository() const { return eclRepository; }
     inline bool isAborting() const { return aborting; }
     inline void setAborting() { aborting = true; }
+    inline void setFastSyntax() { expandCallsWhenBound = false; }
 
 public:
     Linked<IPropertyTree> archive;
@@ -1285,6 +1278,7 @@ extern HQL_API IHqlExpression * createTypeTransfer(IHqlExpression * expr, ITypeI
 extern HQL_API IHqlExpression * cloneFieldMangleName(IHqlExpression * expr);
 extern HQL_API IHqlExpression * expandOutOfLineFunctionCall(IHqlExpression * expr);
 extern HQL_API void expandDelayedFunctionCalls(IErrorReceiver * errors, HqlExprArray & exprs);
+extern HQL_API IHqlExpression * expandDelayedFunctionCalls(IErrorReceiver * errors, IHqlExpression * expr);
 
 extern HQL_API IHqlExpression *createQuoted(const char * name, ITypeInfo *type);
 extern HQL_API IHqlExpression *createVariable(const char * name, ITypeInfo *type);

ecl/hql/hqlfold.cpp

@@ -3756,6 +3756,12 @@ IHqlExpression * foldConstantOperator(IHqlExpression * expr, unsigned foldOption
             OwnedHqlExpr folded = expandOutOfLineFunctionCall(expr);
             if ((folded != expr) && folded->isConstant())
                 return folded.getClear();
+            if (foldOptions & HFOforcefold)
+            {
+                OwnedHqlExpr transformed = expandDelayedFunctionCalls(nullptr, expr);
+                if (expr != transformed)
+                    return transformed.getClear();
+            }
             break;
         }
     case no_trim:

ecl/hql/hqlgram.y

@@ -4256,14 +4256,18 @@ recordDef
     | RECORDOF '(' goodObject ')'
                         {
                             OwnedHqlExpr ds = $3.getExpr();
-                            IHqlExpression * record = queryOriginalRecord(ds);
+                            LinkedHqlExpr record = queryOriginalRecord(ds);
                             if (!record)
                             {
                                 parser->reportError(ERR_EXPECTED, $3, "The argument does not have a associated record");
-                                record = queryNullRecord();
+                                record.set(queryNullRecord());
                             }
                             else if (ds->isFunction() && !record->isFullyBound())
-                                parser->reportError(ERR_EXPECTED, $1, "RECORDOF(function-definition), result record depends on the function parameters");
+                            {
+                                record.setown(getUnadornedRecordOrField(record));
+                                if (!record->isFullyBound())
+                                    parser->reportError(ERR_EXPECTED, $1, "RECORDOF(function-definition), result record depends on the function parameters");
+                            }
 
                             $$.setExpr(LINK(record));
                             $$.setPosition($1);
@@ -9282,7 +9286,7 @@ simpleDataSet
                                 {
                                     if (compareDs)
                                     {
-                                        if (isGrouped(cur) != isGrouped(compareDs))
+                                        if (parser->lookupCtx.queryParseContext().expandCallsWhenBound && (isGrouped(cur) != isGrouped(compareDs)))
                                             parser->reportError(ERR_GROUPING_MISMATCH, $1, "Branches of the condition have different grouping");
                                         OwnedHqlExpr mapped = parser->checkEnsureRecordsMatch(compareDs, cur, $5.pos, false);
                                         if (mapped != cur)

ecl/hql/hqlgram2.cpp

@@ -2616,6 +2616,9 @@ void HqlGram::addField(const attribute &errpos, IIdAtom * name, ITypeInfo *_type
             fieldType.set(defaultIntegralType);
         }
         break;
+    case type_enumerated:
+        fieldType.set(fieldType->queryChildType());
+        break;
     case type_decimal:
         if (fieldType->getSize() == UNKNOWN_LENGTH)
         {
@@ -3520,9 +3523,10 @@ IHqlExpression *HqlGram::lookupSymbol(IIdAtom * searchName, const attribute& err
         if (dotScope) 
         {
             IHqlExpression *ret = NULL;
-            if (dotScope->getOperator() == no_enum)
+            IHqlScope * scope = dotScope->queryScope();
+            if (scope)
             {
-                ret = dotScope->queryScope()->lookupSymbol(searchName, LSFrequired, lookupCtx);
+                ret = scope->lookupSymbol(searchName, LSFrequired, lookupCtx);
             }
             else
             {
@@ -6244,7 +6248,13 @@ void HqlGram::checkFormals(IIdAtom * name, HqlExprArray& parms, HqlExprArray& de
         // check default value
         if (isMacro)
         {
-            IHqlExpression* def = &defaults.item(idx);
+            LinkedHqlExpr def = &defaults.item(idx);
+            if (!def->isConstant() && !lookupCtx.queryParseContext().expandCallsWhenBound)
+            {
+                OwnedHqlExpr expanded = expandDelayedFunctionCalls(nullptr, def);
+                defaults.replace(*LINK(expanded), idx);
+                def.set(expanded);
+            }
             
             if ((def->getOperator() != no_omitted) && !def->isConstant()) 
             {
@@ -6327,6 +6337,7 @@ IHqlExpression *HqlGram::bindParameters(const attribute & errpos, IHqlExpression
                 if (requireLateBind(function, actuals))
                 {
                     IHqlExpression * ret = NULL;
+                    const bool expandCallsWhenBound = true;
                     if (!expandCallsWhenBound)
                     {
                         HqlExprArray args;
@@ -6598,6 +6609,7 @@ IHqlExpression * HqlGram::checkParameter(const attribute * errpos, IHqlExpressio
             formalType = formalType->queryChildType();
         if (!formalType->assignableFrom(actualType->queryPromotedType()))
         {
+
             if (errpos)
             {
                 StringBuffer s,tp1,tp2;
@@ -7194,8 +7206,8 @@ void HqlGram::checkOutputRecord(attribute & errpos, bool outerLevel)
     OwnedHqlExpr record = errpos.getExpr();
     bool allConstant = true;
     errpos.setExpr(checkOutputRecord(record, errpos, allConstant, outerLevel));
-    if (allConstant && (record->getOperator() != no_null) && (record->numChildren() != 0))
-        reportWarning(CategoryUnusual, WRN_OUTPUT_ALL_CONSTANT,errpos.pos,"All values for OUTPUT are constant - is this the intention?");
+    if (allConstant && (record->getOperator() != no_null) && (record->numChildren() != 0) && record->isFullyBound())
+        reportWarning(CategoryUnusual, WRN_OUTPUT_ALL_CONSTANT, errpos.pos, "All values for OUTPUT are constant - is this the intention?");
 }
 
 void HqlGram::checkSoapRecord(attribute & errpos)
@@ -8834,7 +8846,7 @@ void HqlGram::ensureMapToRecordsMatch(OwnedHqlExpr & defaultExpr, HqlExprArray &
         }
     }
 
-    if (groupingDiffers)
+    if (lookupCtx.queryParseContext().expandCallsWhenBound && groupingDiffers)
         reportError(ERR_GROUPING_MISMATCH, errpos, "Branches of the condition have different grouping");
 }
 
@@ -8951,7 +8963,7 @@ void HqlGram::checkMergeInputSorted(attribute &atr, bool isLocal)
     
     if (isGrouped(expr) && appearsToBeSorted(expr, false, false))
         reportWarning(CategoryUnexpected, WRN_MERGE_NOT_SORTED, atr.pos, "Input to MERGE is only sorted with the group");
-    else
+    else if (expr->isFullyBound())
         reportWarning(CategoryUnexpected, WRN_MERGE_NOT_SORTED, atr.pos, "Input to MERGE doesn't appear to be sorted");
 }
 
@@ -9145,7 +9157,7 @@ IHqlExpression * HqlGram::processIfProduction(attribute & condAttr, attribute &
     if (left->queryRecord() && falseAttr)
         right.setown(checkEnsureRecordsMatch(left, right, falseAttr->pos, false));
 
-    if (isGrouped(left) != isGrouped(right))
+    if (lookupCtx.queryParseContext().expandCallsWhenBound && (isGrouped(left) != isGrouped(right)))
         reportError(ERR_GROUPING_MISMATCH, trueAttr, "Branches of the condition have different grouping");
 
     if (cond->isConstant())

ecl/hql/hqlthql.cpp

@@ -23,6 +23,7 @@
 #include "hqlutil.hpp"
 #include "hqlmeta.hpp"
 #include "workunit.hpp"
+#include "hqlerrors.hpp"
 
 //The following constants can be uncommented to increase the level of detail which is added to the processed graphs
 //E.g. generated when -fl used in hqltest
@@ -448,7 +449,16 @@ StringBuffer &HqltHql::callEclFunction(StringBuffer &s, IHqlExpression * expr, b
 {
     assertex(expr->isNamedSymbol());
     IHqlExpression * funcdef = expr->queryFunctionDefinition();
-    assertex(funcdef->getOperator() == no_funcdef || funcdef->getOperator() == no_internalselect);
+    switch (funcdef->getOperator())
+    {
+    case no_funcdef:
+    case no_internalselect:
+    case no_delayedselect:
+        break;
+    default:
+        throw makeStringExceptionV(ERR_INTERNALEXCEPTION, "Internal: Unexpected function definition %s", getOpString(funcdef->getOperator()));
+    }
+
     IHqlExpression * formals = queryFunctionParameters(funcdef);
 
     s.append('(');

ecl/hql/hqltrans.cpp

@@ -1060,6 +1060,7 @@ QuickExpressionReplacer::QuickExpressionReplacer()
 
 void QuickExpressionReplacer::setMapping(IHqlExpression * oldValue, IHqlExpression * newValue)
 {
+    assertex(oldValue);
     for (;;)
     {
         oldValue->setTransformExtra(newValue);
@@ -1970,7 +1971,7 @@ IHqlExpression * NewHqlTransformer::transformCall(IHqlExpression * expr)
     bool same = transformChildren(body, args);
     if (same && (oldFuncdef == newFuncdef))
         return LINK(expr);
-    OwnedHqlExpr newCall = createBoundFunction(NULL, newFuncdef, args, NULL, DEFAULT_EXPAND_CALL);
+    OwnedHqlExpr newCall = createBoundFunction(NULL, newFuncdef, args, NULL, false);
     return expr->cloneAllAnnotations(newCall);
 }
 

ecl/hqlcpp/hqlhtcpp.cpp

@@ -10813,6 +10813,7 @@ void HqlCppTranslator::addSchemaField(IHqlExpression *field, MemoryBuffer &schem
     switch(schemaType->getTypeCode())
     {
     case type_alien:
+    case type_enumerated:
         schemaType.set(schemaType->queryChildType());
         break;
     case type_bitfield:

ecl/hqlcpp/hqliproj.cpp

@@ -2911,7 +2911,7 @@ IHqlExpression * ImplicitProjectTransformer::createTransformed(IHqlExpression *
                         OwnedHqlExpr newBody = replaceChild(body, 0, newBodyCode);
                         OwnedHqlExpr newFuncdef = replaceChild(funcdef, 0, newBody);
                         unwindChildren(args, expr, 0);
-                        transformed.setown(createBoundFunction(NULL, newFuncdef, args, NULL, DEFAULT_EXPAND_CALL));
+                        transformed.setown(createBoundFunction(NULL, newFuncdef, args, NULL, false));
 
                         logChange("Auto project embed", expr, complexExtra->outputFields);
                         return transformed.getClear();

ecl/hqlcpp/hqlttcpp.cpp

@@ -11596,6 +11596,15 @@ void HqlTreeNormalizer::analyseExpr(IHqlExpression * expr)
             analyseChildren(body);
             break;
         }
+    case no_if:
+        if (expr->isDataset())
+        {
+            IHqlExpression * left = expr->queryChild(1);
+            IHqlExpression * right = expr->queryChild(2);
+            if (right && (isGrouped(left) != isGrouped(right)))
+                translator.reportError(expr, ERR_GROUPING_MISMATCH, "Branches of the condition have different grouping");
+            break;
+        }
     }
 
     Parent::analyseExpr(body);

ecl/regress/ifdataset_err.ecl

@@ -0,0 +1,29 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+
+namesRecord :=
+            RECORD
+string20        surname;
+string10        forename;
+integer2        age := 25;
+            END;
+
+namesTable1 := dataset('x',namesRecord,FLAT);
+namesTable2 := group(namesTable1, surname);
+p := IF(count(namesTable1) > 100, namesTable1, namesTable2);
+output(p);