HPCC-15682 Add auto project for unprojected indexes

Signed-off-by: Gavin Halliday <gavin.halliday@lexisnexis.com>

ecl/hql/hqlpmap.cpp

@@ -524,8 +524,15 @@ IHqlExpression * NewProjectMapper2::recursiveExpandSelect(IHqlExpression * expr,
         unsigned match = targets.find(*plain);
         if (match == NotFound)
         {
-            if (ignoreMissingFields)
-                return LINK(expr);
+            /*
+             * Sometimes the implicit project code will remove fields that are used in WILD() expressions.  This creates an inconsistency
+             * in the graph - a reference to a field that doesn't exist in the output of the previous expression.  If we are expanding
+             * a selector reference, and it isn't found in the input's dataset, assume it has been projected out, and remap it to the
+             * parent dataset.  (If this is wrong it will trigger an internal error later on.)
+             */
+            if (insideSelectorReference)
+                return createSelectExpr(LINK(newDataset), LINK(expr->queryChild(1)));
+
 #if 0
             dbglogExpr(plain.get());
             ForEachItemIn(i, targets)
@@ -589,6 +596,19 @@ IHqlExpression * NewProjectMapper2::doExpandFields(IHqlExpression * expr, IHqlEx
     case no_record:
         ret = LINK(expr);
         break;
+    case no_attr_expr:
+        if (expr->queryName() == _selectors_Atom)
+        {
+            if (!insideSelectorReference)
+            {
+                //Process any child no_selects so that missing entries are not fatal.
+                insideSelectorReference = true;
+                ret = doExpandFields(expr, oldDataset, newDataset, oldParent);
+                insideSelectorReference = false;
+                return ret;
+            }
+        }
+        break;
     case no_activerow:
         {
             IHqlExpression * ds = expr->queryChild(0);

ecl/hql/hqlpmap.hpp

@@ -32,7 +32,7 @@ class TableProjectMapper;
 class HQL_API NewProjectMapper2 : public CInterface
 {
 public:
-    NewProjectMapper2() { ignoreMissingFields = false; mapping = NULL; matchedAll = true; expandCallback = NULL; }
+    NewProjectMapper2() { mapping = NULL; matchedAll = true; expandCallback = NULL; }
 
     //if newDataset is NULL, the mapping stored in the transform is returned unchanged (useful for seeing if filters can move over joins)
     IHqlExpression * expandFields(IHqlExpression * expr, IHqlExpression * oldDataset, IHqlExpression * newDataset, IHqlExpression * mapperParent, IExpandCallback * _expandCallback = NULL);
@@ -44,7 +44,6 @@ public:
     void initSelf(IHqlExpression * dataset);
     bool isMappingKnown();
     inline IHqlExpression * querySelf() { return self; }
-    void setIgnoreMissing() { ignoreMissingFields = true; }
     void setMapping(IHqlExpression * mapping);
     void setUnknownMapping();
 
@@ -83,7 +82,7 @@ private:
     CIArrayOf<NewProjectMapper2> children;
     OwnedHqlExpr self;
     IHqlExpression * mapping;
-    bool ignoreMissingFields;
+    bool insideSelectorReference = false;
     //Following are only set when expanding/contracting
     bool matchedAll;
     IExpandCallback * expandCallback;

ecl/hqlcpp/hqliproj.cpp

@@ -2066,8 +2066,7 @@ ProjectExprKind ImplicitProjectTransformer::getProjectExprKind(IHqlExpression *
             return SourceActivity;
         }
     case no_newkeyindex:
-        //Not compoundable for the moment - because it causes problems with assertwild(all, <old-expression>)
-        return SourceActivity;//CompoundableActivity;
+        return CompoundableActivity;
     case no_compound_diskread:
     case no_compound_disknormalize:
     case no_compound_indexread: