HPCC-9172 Fix mismatching records from datasets in MAP/CASE

Signed-off-by: Gavin Halliday <gavin.halliday@lexisnexis.com>

ecl/hql/hqlexpr.cpp

@@ -2527,6 +2527,7 @@ bool definesColumnList(IHqlExpression * dataset)
     case no_cogroup:
     case no_dataset_alias:
     case no_dataset_from_transform:
+    case no_mapto:
         return true;
     case no_select:
     case no_field:
@@ -9999,6 +10000,7 @@ IHqlExpression *createDictionary(node_operator op, HqlExprArray & parms)
         type.setown(makeDictionaryType(makeRowType(createRecordType(&parms.item(0)))));
         break;
     case no_select:
+    case no_mapto:
         type.set(parms.item(1).queryType());
         break;
     case no_addfiles:
@@ -11648,6 +11650,7 @@ IHqlExpression *createDataset(node_operator op, HqlExprArray & parms)
             break;
         }
     case no_case:
+    case no_mapto:
         //following is wrong, but they get removed pretty quickly so I don't really care
         type.set(parms.item(1).queryType());
         break;
@@ -11962,10 +11965,7 @@ extern IHqlExpression *createRow(node_operator op, HqlExprArray & args)
             ITypeInfo * fieldType = field.queryType();
             type_t fieldTC = fieldType->getTypeCode();
             assertex(fieldTC == type_row);
-            if (fieldTC == type_row)
-                type = LINK(fieldType);
-            else
-                type = makeRowType(LINK(fieldType));
+            type = LINK(fieldType);
             break;
         }
     case no_embedbody:
@@ -11987,6 +11987,7 @@ extern IHqlExpression *createRow(node_operator op, HqlExprArray & args)
         }
     case no_compound:
     case no_case:
+    case no_mapto:
         type = args.item(1).getType();
         break;
     case no_translated:

ecl/hql/hqlgram.hpp

@@ -655,9 +655,11 @@ public:
 
     void enterType(const attribute &errpos, bool isParameteried);
     void leaveType(const YYSTYPE & errpos);
-    int checkRecordTypes(IHqlExpression *left, IHqlExpression *right, attribute &atr, unsigned maxFields = (unsigned)-1);
-    bool checkRecordCreateTransform(HqlExprArray & assigns, IHqlExpression *leftExpr, IHqlExpression *leftSelect, IHqlExpression *rightExpr, IHqlExpression *rightSelect, attribute &atr);
-    IHqlExpression * checkEnsureRecordsMatch(IHqlExpression * left, IHqlExpression * right, attribute & errpos, bool rightIsRow);
+    void checkRecordTypesMatch(IHqlExpression *ds1, IHqlExpression *ds2, const attribute & errpos);
+    int checkRecordTypesSimilar(IHqlExpression *left, IHqlExpression *right, const attribute &atr, unsigned maxFields = (unsigned)-1);
+    bool checkRecordCreateTransform(HqlExprArray & assigns, IHqlExpression *leftExpr, IHqlExpression *leftSelect, IHqlExpression *rightExpr, IHqlExpression *rightSelect, const attribute &atr);
+    IHqlExpression * checkEnsureRecordsMatch(IHqlExpression * left, IHqlExpression * right, const attribute & errpos, bool rightIsRow);
+    void ensureMapToRecordsMatch(OwnedHqlExpr & recordExpr, HqlExprArray & args, const attribute & errpos, bool isRow);
     void checkRecordIsValid(const attribute &atr, IHqlExpression *record);
     void checkValidRecordMode(IHqlExpression * dataset, attribute & atr, attribute & modeatr);
     void checkValidCsvRecord(const attribute & errpos, IHqlExpression * record);

ecl/hql/hqlgram.y

@@ -7223,7 +7223,7 @@ simpleDictionary
                             ForEachItemIn(idx, args)
                             {
                                 IHqlExpression * cur = args.item(idx).queryChild(1);
-                                parser->checkRecordTypes(cur, elseDict, $5);
+                                parser->checkRecordTypesMatch(cur, elseDict, $5);
                             }
                             args.append(*elseDict);
                             $$.setExpr(::createDictionary(no_map, args));
@@ -7241,7 +7241,7 @@ simpleDictionary
                             ForEachItemIn(idx, args)
                             {
                                 IHqlExpression * cur = args.item(idx).queryChild(1);
-                                parser->checkRecordTypes(cur, elseDict, $1);
+                                parser->checkRecordTypesMatch(cur, elseDict, $1);
                             }
                             args.append(*elseDict);
                             $$.setExpr(::createDictionary(no_map, args));
@@ -7257,7 +7257,7 @@ simpleDictionary
                             ForEachItemIn(idx, args)
                             {
                                 IHqlExpression * cur = args.item(idx).queryChild(1);
-                                parser->checkRecordTypes(cur, elseDict, $8);
+                                parser->checkRecordTypesMatch(cur, elseDict, $8);
                             }
                             args.add(*$3.getExpr(),0);
                             args.append(*elseDict);
@@ -7278,7 +7278,7 @@ simpleDictionary
                             ForEachItemIn(idx, args)
                             {
                                 IHqlExpression * cur = args.item(idx).queryChild(1);
-                                parser->checkRecordTypes(cur, elseDict, $6);
+                                parser->checkRecordTypesMatch(cur, elseDict, $6);
                             }
                             args.add(*$3.getExpr(),0);
                             args.append(*elseDict);
@@ -7324,7 +7324,7 @@ simpleDictionary
                                 {
                                     if (compareDict)
                                     {
-                                        parser->checkRecordTypes(cur, compareDict, $5);
+                                        parser->checkRecordTypesMatch(cur, compareDict, $5);
                                     }
                                     else
                                         compareDict = cur;
@@ -7409,7 +7409,7 @@ dataSet
                         {
                             OwnedHqlExpr left = $1.getExpr();
                             OwnedHqlExpr right = $3.getExpr();
-                            parser->checkRecordTypes(left, right, $3);
+                            parser->checkRecordTypesSimilar(left, right, $3);
 
                             OwnedHqlExpr seq = parser->createActiveSelectorSequence(left, right);
                             OwnedHqlExpr leftSelect = createSelector(no_left, left, seq);
@@ -7425,7 +7425,7 @@ dataSet
                         {
                             OwnedHqlExpr left = $1.getExpr();
                             OwnedHqlExpr right = $3.getExpr();
-                            parser->checkRecordTypes(left, right, $3);
+                            parser->checkRecordTypesSimilar(left, right, $3);
 
                             OwnedHqlExpr seq = parser->createActiveSelectorSequence(left, right);
                             OwnedHqlExpr leftSelect = createSelector(no_left, left, seq);
@@ -8700,19 +8700,10 @@ simpleDataSet
     | MAP '(' mapDatasetSpec ',' dataSet ')'
                         {
                             HqlExprArray args;
-                            IHqlExpression * elseDs = $5.getExpr();
+                            OwnedHqlExpr elseExpr = $5.getExpr();
                             $3.unwindCommaList(args);
-                            bool groupingDiffers = false;
-                            ForEachItemIn(idx, args)
-                            {
-                                IHqlExpression * cur = args.item(idx).queryChild(1);
-                                if (isGrouped(cur) != isGrouped(elseDs))
-                                    groupingDiffers = true;
-                                parser->checkRecordTypes(cur, elseDs, $5);
-                            }
-                            if (groupingDiffers)
-                                parser->reportError(ERR_GROUPING_MISMATCH, $1, "Branches of the condition have different grouping");
-                            args.append(*elseDs);
+                            parser->ensureMapToRecordsMatch(elseExpr, args, $5, false);
+                            args.append(*elseExpr.getClear());
                             $$.setExpr(::createDataset(no_map, args));
                             $$.setPosition($1);
                         }
@@ -8720,22 +8711,14 @@ simpleDataSet
                         {
                             HqlExprArray args;
                             $3.unwindCommaList(args);
-                            IHqlExpression * elseDs;
+                            OwnedHqlExpr elseExpr;
                             if (args.ordinality())
-                                elseDs = createNullExpr(&args.item(0));
+                                elseExpr.setown(createNullExpr(&args.item(0)));
                             else
-                                elseDs = createDataset(no_null, LINK(queryNullRecord()));
-                            bool groupingDiffers = false;
-                            ForEachItemIn(idx, args)
-                            {
-                                IHqlExpression * cur = args.item(idx).queryChild(1);
-                                if (isGrouped(cur) != isGrouped(elseDs))
-                                    groupingDiffers = true;
-                                parser->checkRecordTypes(cur, elseDs, $1);
-                            }
-                            if (groupingDiffers)
-                                parser->reportError(ERR_GROUPING_MISMATCH, $1, "Branches of the condition have different grouping");
-                            args.append(*elseDs);
+                                elseExpr.setown(createDataset(no_null, LINK(queryNullRecord())));
+
+                            parser->ensureMapToRecordsMatch(elseExpr, args, $3, false);
+                            args.append(*elseExpr.getClear());
                             $$.setExpr(::createDataset(no_map, args));
                             $$.setPosition($1);
                         }
@@ -8743,21 +8726,14 @@ simpleDataSet
                         {
                             parser->normalizeExpression($3, type_scalar, false);
                             HqlExprArray args;
-                            IHqlExpression * elseDs = $8.getExpr();
+                            OwnedHqlExpr elseExpr = $8.getExpr();
                             parser->endList(args);
                             parser->checkCaseForDuplicates(args, $6);
-                            bool groupingDiffers = false;
-                            ForEachItemIn(idx, args)
-                            {
-                                IHqlExpression * cur = args.item(idx).queryChild(1);
-                                if (isGrouped(cur) != isGrouped(elseDs))
-                                    groupingDiffers = true;
-                                parser->checkRecordTypes(cur, elseDs, $8);
-                            }
-                            if (groupingDiffers)
-                                parser->reportError(ERR_GROUPING_MISMATCH, $1, "Branches of the condition have different grouping");
+
+                            parser->ensureMapToRecordsMatch(elseExpr, args, $8, false);
+
                             args.add(*$3.getExpr(),0);
-                            args.append(*elseDs);
+                            args.append(*elseExpr.getClear());
                             $$.setExpr(::createDataset(no_case, args));
                             $$.setPosition($1);
                         }
@@ -8766,24 +8742,17 @@ simpleDataSet
                             parser->normalizeExpression($3, type_scalar, false);
                             HqlExprArray args;
                             parser->endList(args);
-                            IHqlExpression * elseDs;
+                            OwnedHqlExpr elseDs;
                             if (args.ordinality())
-                                elseDs = createNullExpr(&args.item(0));
+                                elseDs.setown(createNullExpr(&args.item(0)));
                             else
-                                elseDs = createDataset(no_null, LINK(queryNullRecord()));
+                                elseDs.setown(createDataset(no_null, LINK(queryNullRecord())));
                             parser->checkCaseForDuplicates(args, $6);
-                            bool groupingDiffers = false;
-                            ForEachItemIn(idx, args)
-                            {
-                                IHqlExpression * cur = args.item(idx).queryChild(1);
-                                if (isGrouped(cur) != isGrouped(elseDs))
-                                    groupingDiffers = true;
-                                parser->checkRecordTypes(cur, elseDs, $6);
-                            }
-                            if (groupingDiffers)
-                                parser->reportError(ERR_GROUPING_MISMATCH, $1, "Branches of the condition have different grouping");
+
+                            parser->ensureMapToRecordsMatch(elseDs, args, $6, false);
+
                             args.add(*$3.getExpr(),0);
-                            args.append(*elseDs);
+                            args.append(*elseDs.getClear());
                             $$.setExpr(::createDataset(no_case, args));
                             $$.setPosition($1);
                         }
@@ -8817,7 +8786,9 @@ simpleDataSet
                                     {
                                         if (isGrouped(cur) != isGrouped(compareDs))
                                             parser->reportError(ERR_GROUPING_MISMATCH, $1, "Branches of the condition have different grouping");
-                                        parser->checkRecordTypes(cur, compareDs, $5);
+                                        OwnedHqlExpr mapped = parser->checkEnsureRecordsMatch(compareDs, cur, $5, false);
+                                        if (mapped != cur)
+                                            args.replace(*mapped.getClear(), idx);
                                     }
                                     else
                                         compareDs = cur;
@@ -9069,14 +9040,12 @@ simpleDataSet
     | MAP '(' mapDatarowSpec ',' dataRow ')'
                         {
                             HqlExprArray args;
-                            IHqlExpression * elseDs = $5.getExpr();
+                            OwnedHqlExpr elseExpr = $5.getExpr();
                             $3.unwindCommaList(args);
-                            ForEachItemIn(idx, args)
-                            {
-                                IHqlExpression * cur = args.item(idx).queryChild(1);
-                                parser->checkRecordTypes(cur, elseDs, $5);
-                            }
-                            args.append(*elseDs);
+
+                            parser->ensureMapToRecordsMatch(elseExpr, args, $5, true);
+
+                            args.append(*elseExpr.getClear());
                             $$.setExpr(::createRow(no_map, args));
                             $$.setPosition($1);
                         }
@@ -9084,16 +9053,14 @@ simpleDataSet
                         {
                             parser->normalizeExpression($3, type_scalar, false);
                             HqlExprArray args;
-                            IHqlExpression * elseDs = $8.getExpr();
+                            OwnedHqlExpr elseExpr = $8.getExpr();
                             parser->endList(args);
                             parser->checkCaseForDuplicates(args, $6);
-                            ForEachItemIn(idx, args)
-                            {
-                                IHqlExpression * cur = args.item(idx).queryChild(1);
-                                parser->checkRecordTypes(cur, elseDs, $8);
-                            }
+
+                            parser->ensureMapToRecordsMatch(elseExpr, args, $8, true);
+
                             args.add(*$3.getExpr(),0);
-                            args.append(*elseDs);
+                            args.append(*elseExpr.getClear());
                             $$.setExpr(::createRow(no_case, args), $1);
                         }
     | WHEN '(' dataSet ',' action sideEffectOptions ')'
@@ -11291,7 +11258,7 @@ mapDatasetItem
     : booleanExpr GOESTO dataSet
                         {
                             IHqlExpression *e3 = $3.getExpr();
-                            $$.setExpr(createValue(no_mapto, e3->getType(), $1.getExpr(), e3));
+                            $$.setExpr(createDataset(no_mapto, $1.getExpr(), e3));
                             $$.setPosition($3);
                         }
     ;
@@ -11309,7 +11276,7 @@ mapDictionaryItem
     : booleanExpr GOESTO dictionary
                         {
                             IHqlExpression *e3 = $3.getExpr();
-                            $$.setExpr(createValue(no_mapto, e3->getType(), $1.getExpr(), e3));
+                            $$.setExpr(createDictionary(no_mapto, $1.getExpr(), e3));
                             $$.setPosition($3);
                         }
     ;
@@ -11325,7 +11292,7 @@ caseDatasetItem
                             parser->normalizeExpression($1);
                             parser->applyDefaultPromotions($1, true);
                             IHqlExpression *e3 = $3.getExpr();
-                            parser->addListElement(createValue(no_mapto, e3->getType(), $1.getExpr(), e3));
+                            parser->addListElement(createDataset(no_mapto, $1.getExpr(), e3));
                             $$.clear();
                             $$.setPosition($3);
                         }
@@ -11342,7 +11309,7 @@ caseDictionaryItem
                             parser->normalizeExpression($1);
                             parser->applyDefaultPromotions($1, true);
                             IHqlExpression *e3 = $3.getExpr();
-                            parser->addListElement(createValue(no_mapto, e3->getType(), $1.getExpr(), e3));
+                            parser->addListElement(createDictionary(no_mapto, $1.getExpr(), e3));
                             $$.clear();
                             $$.setPosition($3);
                         }
@@ -11361,7 +11328,7 @@ mapDatarowItem
     : booleanExpr GOESTO dataRow
                         {
                             IHqlExpression *e3 = $3.getExpr();
-                            $$.setExpr(createValue(no_mapto, e3->getType(), $1.getExpr(), e3));
+                            $$.setExpr(createRow(no_mapto, $1.getExpr(), e3));
                             $$.setPosition($3);
                         }
     ;
@@ -11377,7 +11344,7 @@ caseDatarowItem
                             parser->normalizeExpression($1);
                             parser->applyDefaultPromotions($1, true);
                             IHqlExpression *e3 = $3.getExpr();
-                            parser->addListElement(createValue(no_mapto, e3->getType(), $1.getExpr(), e3));
+                            parser->addListElement(createRow(no_mapto, $1.getExpr(), e3));
                             $$.clear();
                             $$.setPosition($3);
                         }

ecl/hql/hqlgram2.cpp

@@ -8071,7 +8071,7 @@ void HqlGram::checkDistributer(attribute & err, HqlExprArray & args)
             unsigned inputKeyedFields = firstPayloadField(input->queryRecord(), inputPayload ? (unsigned)getIntValue(inputPayload->queryChild(0)) : 1);
             if (numKeyedFields != inputKeyedFields)
                 reportError(ERR_DISTRIBUTED_MISSING, err, "Index and DISTRIBUTE(index) have different numbers of keyed fields");
-            checkRecordTypes(args.item(0).queryRecord(), cur.queryChild(0)->queryRecord(), err, numKeyedFields);
+            checkRecordTypesSimilar(args.item(0).queryRecord(), cur.queryChild(0)->queryRecord(), err, numKeyedFields);
         }
     }
 }
@@ -8118,7 +8118,7 @@ void HqlGram::checkValidPipeRecord(const attribute & errpos, IHqlExpression * re
         checkValidCsvRecord(errpos, record);
 }
 
-int HqlGram::checkRecordTypes(IHqlExpression *left, IHqlExpression *right, attribute &atr, unsigned maxFields)
+int HqlGram::checkRecordTypesSimilar(IHqlExpression *left, IHqlExpression *right, const attribute &atr, unsigned maxFields)
 {
     if (recordTypesMatch(left, right)) 
         return 0;
@@ -8187,15 +8187,15 @@ int HqlGram::checkRecordTypes(IHqlExpression *left, IHqlExpression *right, attri
         }
         
         // recursive call to check sub fields.
-        if(lchildrectype && rchildrectype && checkRecordTypes(lfield, rfield, atr) != 0) 
-            return -1;
+        if(lchildrectype && rchildrectype)
+            return checkRecordTypesSimilar(lfield, rfield, atr);
     }
 
     return 0;
 }
 
 
-bool HqlGram::checkRecordCreateTransform(HqlExprArray & assigns, IHqlExpression *leftExpr, IHqlExpression *leftSelect, IHqlExpression *rightExpr, IHqlExpression *rightSelect, attribute &atr)
+bool HqlGram::checkRecordCreateTransform(HqlExprArray & assigns, IHqlExpression *leftExpr, IHqlExpression *leftSelect, IHqlExpression *rightExpr, IHqlExpression *rightSelect, const attribute &atr)
 {
     if (leftExpr->getOperator() != rightExpr->getOperator())
     {
@@ -8274,15 +8274,16 @@ bool HqlGram::checkRecordCreateTransform(HqlExprArray & assigns, IHqlExpression
 }
 
 
-IHqlExpression * HqlGram::checkEnsureRecordsMatch(IHqlExpression * left, IHqlExpression * right, attribute & errpos, bool rightIsRow)
+IHqlExpression * HqlGram::checkEnsureRecordsMatch(IHqlExpression * left, IHqlExpression * right, const attribute & errpos, bool rightIsRow)
 {
-    checkRecordTypes(left, right, errpos);
-
     //Need to add a project to make the field names correct, otherwise problems occur if one the left side is optimized away,
     //because that causes the record type and fields to change.
     if (recordTypesMatch(left, right)) 
         return LINK(right);
-    
+
+    if (checkRecordTypesSimilar(left, right, errpos) != 0)
+        return LINK(left); // error conditional - return something compatible with left
+
     HqlExprArray assigns;
     OwnedHqlExpr seq = createActiveSelectorSequence(right, NULL);
     OwnedHqlExpr rightSelect = createSelector(no_left, right, seq);
@@ -8302,14 +8303,53 @@ IHqlExpression * HqlGram::checkEnsureRecordsMatch(IHqlExpression * left, IHqlExp
         return createDataset(no_hqlproject, args);
 }
 
+void HqlGram::ensureMapToRecordsMatch(OwnedHqlExpr & defaultExpr, HqlExprArray & args, const attribute & errpos, bool isRow)
+{
+    //The record of the final result should match the record of the first argument.
+    IHqlExpression * expected = (args.ordinality() != 0) ? &args.item(0) : defaultExpr.get();
+    bool groupingDiffers = false;
+    ForEachItemIn(i, args)
+    {
+        IHqlExpression & mapTo = args.item(i);
+        IHqlExpression * value = mapTo.queryChild(1);
+        if (isGrouped(value) != isGrouped(expected))
+            groupingDiffers = true;
+        OwnedHqlExpr checked = checkEnsureRecordsMatch(expected, value, errpos, isRow);
+        if (value != checked)
+        {
+            args.replace(*replaceChild(&mapTo, 1, checked), i);
+            reportWarning(ERR_TYPE_INCOMPATIBLE, errpos.pos, "Datasets in list have slightly different records");
+        }
+    }
+
+    if (defaultExpr)
+    {
+        if (isGrouped(defaultExpr) != isGrouped(expected))
+            groupingDiffers = true;
+        OwnedHqlExpr checked = checkEnsureRecordsMatch(expected, defaultExpr, errpos, isRow);
+        if (defaultExpr != checked)
+        {
+            defaultExpr.set(checked);
+            reportWarning(ERR_TYPE_INCOMPATIBLE, errpos.pos, "Default value has a slightly different record");
+        }
+    }
+
+    if (groupingDiffers)
+        reportError(ERR_GROUPING_MISMATCH, errpos, "Branches of the condition have different grouping");
+}
+
 void HqlGram::checkMergeSortOrder(attribute &atr, IHqlExpression *ds1, IHqlExpression *ds2, IHqlExpression * sortorder)
 {
     if (!recordTypesMatch(ds1, ds2)) 
         reportError(ERR_TYPE_INCOMPATIBLE, atr, "Datasets in list must have identical records");
     return;
-    checkRecordTypes(ds1, ds2, atr);
-    // MORE - should check that sort orders match
-    // but tricky because they don't have to apply to the same records...
+}
+
+void HqlGram::checkRecordTypesMatch(IHqlExpression *ds1, IHqlExpression *ds2, const attribute &errpos)
+{
+    if (!recordTypesMatch(ds1, ds2))
+        reportError(ERR_TYPE_INCOMPATIBLE, errpos, "Arguments must have the same record type");
+    return;
 }
 
 IHqlExpression * HqlGram::createScopedSequenceExpr()

ecl/hql/hqlutil.cpp

@@ -2332,6 +2332,57 @@ void checkDependencyConsistency(const HqlExprArray & exprs)
 
 //---------------------------------------------------------------------------
 
+static HqlTransformerInfo selectConsistencyCheckerInfo("SelectConsistencyChecker");
+class SelectConsistencyChecker  : public NewHqlTransformer
+{
+public:
+    SelectConsistencyChecker() : NewHqlTransformer(selectConsistencyCheckerInfo)
+    {
+    }
+
+    virtual void analyseExpr(IHqlExpression * expr)
+    {
+        if (alreadyVisited(expr))
+            return;
+
+        if (expr->getOperator() == no_select)
+            checkSelect(expr);
+
+        NewHqlTransformer::analyseExpr(expr);
+    }
+
+    virtual void analyseSelector(IHqlExpression * expr)
+    {
+        if (expr->getOperator() == no_select)
+            checkSelect(expr);
+
+        NewHqlTransformer::analyseSelector(expr);
+    }
+
+protected:
+    void checkSelect(IHqlExpression * expr)
+    {
+        IHqlExpression * ds = expr->queryChild(0);
+        IHqlExpression * field = expr->queryChild(1);
+        IHqlSimpleScope * scope = ds->queryRecord()->querySimpleScope();
+        OwnedHqlExpr match = scope->lookupSymbol(field->queryName());
+        if (match != field)
+        {
+            EclIR::dbglogIR(2, field, match.get());
+            throw MakeStringException(ERR_RECURSIVE_DEPENDENCY, "Inconsistent select - field doesn't match parent record's field");
+        }
+    }
+};
+
+
+void checkSelectConsistency(IHqlExpression * expr)
+{
+    SelectConsistencyChecker checker;
+    checker.analyse(expr, 0);
+}
+
+//---------------------------------------------------------------------------
+
 void DependencyGatherer::doGatherDependencies(IHqlExpression * expr)
 {
     if (expr->queryTransformExtra())

ecl/hql/hqlutil.hpp

@@ -310,6 +310,7 @@ public:
 
 extern HQL_API void checkDependencyConsistency(IHqlExpression * expr);
 extern HQL_API void checkDependencyConsistency(const HqlExprArray & exprs);
+extern HQL_API void checkSelectConsistency(IHqlExpression * expr);
 extern HQL_API bool isUngroup(IHqlExpression * expr);
 extern HQL_API bool containsExpression(IHqlExpression * expr, IHqlExpression * search);
 extern HQL_API bool containsOperator(IHqlExpression * expr, node_operator search);

ecl/hqlcpp/hqlcpp.cpp

@@ -1569,6 +1569,7 @@ void HqlCppTranslator::cacheOptions()
         DebugOption(options.moveUnconditionalActions,"moveUnconditionalActions", false),
         DebugOption(options.paranoidCheckNormalized, "paranoidCheckNormalized", paranoid),
         DebugOption(options.paranoidCheckDependencies, "paranoidCheckDependencies", paranoid),
+        DebugOption(options.paranoidCheckSelects, "paranoidCheckSelects", paranoid),
         DebugOption(options.preventKeyedSplit,"preventKeyedSplit", true),
         DebugOption(options.preventSteppedSplit,"preventSteppedSplit", true),
         DebugOption(options.canGenerateSimpleAction,"canGenerateSimpleAction", true),

ecl/hqlcpp/hqlcpp.ipp

@@ -715,6 +715,7 @@ struct HqlCppOptions
     bool                removeXpathFromOutput;
     bool                canLinkConstantRows;
     bool                checkAmbiguousRollupCondition;
+    bool                paranoidCheckSelects;
 };
 
 //Any information gathered while processing the query should be moved into here, rather than cluttering up the translator class
@@ -2018,7 +2019,6 @@ extern bool filterIsTableInvariant(IHqlExpression * expr);
 extern bool mustEvaluateInContext(BuildCtx & ctx, IHqlExpression * expr);
 extern const char * boolToText(bool value);
 extern GraphLocalisation queryActivityLocalisation(IHqlExpression * expr);
-extern void checkDependencyConsistency(WorkflowArray & workflow);
 extern bool isGraphIndependent(IHqlExpression * expr, IHqlExpression * graph);
 extern IHqlExpression * adjustBoundIntegerValues(IHqlExpression * left, IHqlExpression * right, bool subtract);
 extern bool isNullAssign(const CHqlBoundTarget & target, IHqlExpression * expr);

ecl/hqlcpp/hqlhtcpp.cpp

@@ -17817,16 +17817,9 @@ void HqlCppTranslator::traceExpressions(const char * title, WorkflowArray & work
 
 void HqlCppTranslator::checkNormalized(WorkflowArray & workflow)
 {
-    if (options.paranoidCheckDependencies)
-        checkDependencyConsistency(workflow);
-
-    if (options.paranoidCheckNormalized)
+    ForEachItemIn(i, workflow)
     {
-        ForEachItemIn(i, workflow)
-        {
-            OwnedHqlExpr compound = createActionList(workflow.item(i).queryExprs());
-            ::checkNormalized(compound);
-        }
+        checkNormalized(workflow.item(i).queryExprs());
     }
 }
 
@@ -17839,6 +17832,9 @@ void HqlCppTranslator::checkNormalized(IHqlExpression * expr)
     {
         ::checkNormalized(expr);
     }
+
+    if (options.paranoidCheckSelects)
+        checkSelectConsistency(expr);
 }
 
 void HqlCppTranslator::checkNormalized(HqlExprArray & exprs)
@@ -17846,10 +17842,12 @@ void HqlCppTranslator::checkNormalized(HqlExprArray & exprs)
     if (options.paranoidCheckDependencies)
         checkDependencyConsistency(exprs);
 
-    if (options.paranoidCheckNormalized)
+    ForEachItemIn(i, exprs)
     {
-        ForEachItemIn(i, exprs)
+        if (options.paranoidCheckNormalized)
             ::checkNormalized(&exprs.item(i));
+        if (options.paranoidCheckSelects)
+            checkSelectConsistency(&exprs.item(i));
     }
 }
 
@@ -17873,6 +17871,9 @@ void HqlCppTranslator::checkNormalized(BuildCtx & ctx, IHqlExpression * expr)
 
         ::checkNormalized(expr, activeTables);
     }
+
+    if (options.paranoidCheckSelects)
+        checkSelectConsistency(expr);
 }
 
 

ecl/hqlcpp/hqlttcpp.cpp

@@ -234,12 +234,6 @@ static IHqlExpression * mergeLimitIntoDataset(IHqlExpression * dataset, IHqlExpr
     return addAttrOwnToDataset(dataset, createAttribute(limitAtom, LINK(limit->queryChild(1)), LINK(limit->queryChild(2))));
 }
 
-void checkDependencyConsistency(WorkflowArray & workflow)
-{
-    ForEachItemIn(icheck, workflow)
-        checkDependencyConsistency(workflow.item(icheck).queryExprs());
-}
-
 //---------------------------------------------------------------------------
 
 static bool isOptionTooLate(const char * name)