HPCC-9589 Remove colocal/parent extract from some child activities

If the child activity doesn't need to get anything from the parent activity
then it doesn't need a colocal or parent extract.  This can significantly
cut down the size of some queries (and simplify the classes).

Signed-off-by: Gavin Halliday <gavin.halliday@lexisnexis.com>

ecl/hql/hqlexpr.cpp

@@ -3243,6 +3243,10 @@ void CHqlExpression::initFlagsBeforeOperands()
     case no_decimalstack:
         infoFlags |= HEFaction;
         break;
+    case no_getresult:
+    case no_workunit_dataset:
+        infoFlags |= HEFaccessRuntimeContext;
+        break;
     }
 }
 
@@ -4935,19 +4939,41 @@ void CHqlExpressionWithTables::cachePotentialTablesUsed(CUsedTablesBuilder & use
 }
 
 
-void CHqlExpressionWithTables::cacheTablesProcessChildScope(CUsedTablesBuilder & used)
+void CHqlExpressionWithTables::cacheTablesProcessChildScope(CUsedTablesBuilder & used, bool ignoreInputs)
 {
     unsigned max = numChildren();
     switch (getChildDatasetType(this))
     {
     case childdataset_none: 
+        cacheChildrenTablesUsed(used, 0, max);
+        break;
     case childdataset_many_noscope:
+        if (ignoreInputs)
+        {
+            unsigned first = getFirstActivityArgument(this);
+            unsigned last = first + getNumActivityArguments(this);
+            cacheChildrenTablesUsed(used, 0, first);
+            cacheChildrenTablesUsed(used, last, max);
+        }
+        else
+            cacheChildrenTablesUsed(used, 0, max);
+        break;
+    case childdataset_dataset_noscope:
+        if (ignoreInputs)
+            cacheChildrenTablesUsed(used, 1, max);
+        else
+            cacheChildrenTablesUsed(used, 0, max);
+        break;
     case childdataset_if:
+        if (ignoreInputs)
+            cacheChildrenTablesUsed(used, 0, 1);
+        else
+            cacheChildrenTablesUsed(used, 0, max);
+        break;
     case childdataset_case:
     case childdataset_map:
-    case childdataset_dataset_noscope: 
+        //Assume the worst
         cacheChildrenTablesUsed(used, 0, max);
-        //None of these have any scoped arguments, so no need to remove them
         break;
     case childdataset_many:
         {
@@ -4955,7 +4981,8 @@ void CHqlExpressionWithTables::cacheTablesProcessChildScope(CUsedTablesBuilder &
             unsigned firstAttr = getNumChildTables(this);
             cacheChildrenTablesUsed(used, firstAttr, max);
             used.removeActive(queryActiveTableSelector());
-            cacheChildrenTablesUsed(used, 0, firstAttr);
+            if (!ignoreInputs)
+                cacheChildrenTablesUsed(used, 0, firstAttr);
             break;
         }
     case childdataset_dataset:
@@ -4963,7 +4990,8 @@ void CHqlExpressionWithTables::cacheTablesProcessChildScope(CUsedTablesBuilder &
             IHqlExpression * ds = queryChild(0);
             cacheChildrenTablesUsed(used, 1, max);
             used.removeParent(ds);
-            cacheChildrenTablesUsed(used, 0, 1);
+            if (!ignoreInputs)
+                cacheChildrenTablesUsed(used, 0, 1);
         }
         break;
     case childdataset_datasetleft:
@@ -4987,7 +5015,8 @@ void CHqlExpressionWithTables::cacheTablesProcessChildScope(CUsedTablesBuilder &
                 used.removeActive(queryXmlParsePseudoTable());
                 break;
             }
-            cacheChildrenTablesUsed(used, 0, 1);
+            if (!ignoreInputs)
+                cacheChildrenTablesUsed(used, 0, 1);
 #ifdef GATHER_HIDDEN_SELECTORS
             used.addHiddenTable(left, selSeq);
 #endif
@@ -5001,7 +5030,9 @@ void CHqlExpressionWithTables::cacheTablesProcessChildScope(CUsedTablesBuilder &
             cacheChildrenTablesUsed(used, 1, max);
             used.removeActive(left);
             used.removeRows(this, left, NULL);
-            cacheChildrenTablesUsed(used, 0, 1);
+
+            if (!ignoreInputs)
+                cacheChildrenTablesUsed(used, 0, 1);
 #ifdef GATHER_HIDDEN_SELECTORS
             used.addHiddenTable(left, selSeq);
 #endif
@@ -5018,7 +5049,9 @@ void CHqlExpressionWithTables::cacheTablesProcessChildScope(CUsedTablesBuilder &
             used.removeActive(left);
             used.removeActive(right);
             used.removeRows(this, left, right);
-            cacheChildrenTablesUsed(used, 0, 1);
+
+            if (!ignoreInputs)
+                cacheChildrenTablesUsed(used, 0, 1);
 #ifdef GATHER_HIDDEN_SELECTORS
             used.addHiddenTable(left, selSeq);
             used.addHiddenTable(right, selSeq);
@@ -5058,12 +5091,14 @@ void CHqlExpressionWithTables::cacheTablesProcessChildScope(CUsedTablesBuilder &
                 //two datasets form of normalize is weird because right dataset is based on left
                 cacheChildrenTablesUsed(used, 1, 2);
                 used.removeActive(left);
-                cacheChildrenTablesUsed(used, 0, 1);
+                if (!ignoreInputs)
+                    cacheChildrenTablesUsed(used, 0, 1);
             }
             else
             {
                 used.removeActive(left);
-                cacheChildrenTablesUsed(used, 0, 2);
+                if (!ignoreInputs)
+                    cacheChildrenTablesUsed(used, 0, 2);
             }
 #ifdef GATHER_HIDDEN_SELECTORS
             used.addHiddenTable(left, selSeq);
@@ -5080,6 +5115,177 @@ void CHqlExpressionWithTables::cacheTablesProcessChildScope(CUsedTablesBuilder &
 }
 
 
+void CHqlExpressionWithTables::calcTablesUsed(CUsedTablesBuilder & used, bool ignoreInputs)
+{
+    switch (op)
+    {
+    case no_attr:
+    case no_attr_link:
+    case no_keyed:
+    case no_colon:
+    case no_cluster:
+    case no_nameof:
+    case no_translated:
+    case no_constant:
+        break;
+    case no_select:
+        {
+            IHqlExpression * ds = queryChild(0);
+            if (isSelectRootAndActive())
+            {
+                used.addActiveTable(ds);
+            }
+            else
+            {
+                ds->gatherTablesUsed(used);
+            }
+            break;
+        }
+    case no_activerow:
+        used.addActiveTable(queryChild(0));
+        break;
+    case no_rows:
+    case no_rowset:
+        //MORE: This is a bit strange!
+        used.addActiveTable(queryChild(0));
+        break;
+    case no_left:
+    case no_right:
+        used.addActiveTable(this);
+        break;
+    case no_counter:
+        //NB: Counter is added as a pseudo table, because it is too hard to keep track of nested counters otherwise
+        used.addActiveTable(this);
+        break;
+    case no_filepos:
+    case no_file_logicalname:
+        used.addActiveTable(queryChild(0));
+        break;
+    case NO_AGGREGATE:
+    case no_createset:
+        {
+#ifdef GATHER_HIDDEN_SELECTORS
+            cachePotentialTablesUsed(used);
+            used.removeParent(queryChild(0));
+#else
+            HqlExprCopyArray childInScopeTables;
+            ForEachChild(idx, this)
+                queryChild(idx)->gatherTablesUsed(NULL, &childInScopeTables);
+
+            //The argument to the operator is a new table, don't inherit grandchildren
+            IHqlExpression * ds = queryChild(0);
+            cacheInheritChildTablesUsed(ds, used, childInScopeTables);
+#endif
+        }
+        break;
+    case no_sizeof:
+        cachePotentialTablesUsed(used);
+        used.removeActive(queryActiveTableSelector());
+        used.removeActiveRecords();
+        break;
+    case no_externalcall:
+    case no_rowvalue:
+    case no_offsetof:
+    case no_eq:
+    case no_ne:
+    case no_lt:
+    case no_le:
+    case no_gt:
+    case no_ge:
+    case no_order:
+    case no_assign:
+    case no_call:
+    case no_libraryscopeinstance:
+        //MORE: Should check this doesn't make the comparison invalid.
+        cachePotentialTablesUsed(used);
+        break;
+    case no_keyindex:
+    case no_newkeyindex:
+        cacheChildrenTablesUsed(used, 1, numChildren());
+        used.removeParent(queryChild(0));
+        //Distributed attribute might contain references to the no_activetable
+        used.removeActive(queryActiveTableSelector());
+        break;
+    case no_evaluate:
+        cacheTableUseage(used, queryChild(0));
+        queryChild(1)->gatherTablesUsed(used);
+        break;
+    case no_table:
+        {
+            cacheChildrenTablesUsed(used, 0, numChildren());
+            IHqlExpression * parent = queryChild(3);
+            if (parent)
+                used.removeParent(parent);
+            break;
+        }
+    case no_pat_production:
+        {
+            cacheChildrenTablesUsed(used, 0, numChildren());
+            used.cleanupProduction();
+            break;
+        }
+    default:
+        {
+            ITypeInfo * thisType = queryType();
+            unsigned max = numChildren();
+            if (max)
+            {
+                if (thisType)
+                {
+                    switch (thisType->getTypeCode())
+                    {
+                    case type_void:
+                    case type_dictionary:
+                    case type_table:
+                    case type_groupedtable:
+                    case type_row:
+                    case type_transform:
+                        {
+                            cacheTablesProcessChildScope(used, ignoreInputs);
+                            IHqlExpression * counter = queryAttribute(_countProject_Atom);
+                            if (counter)
+                                used.removeActive(counter->queryChild(0));
+                            break;
+                        }
+                    default:
+                        cacheChildrenTablesUsed(used, 0, max);
+                        break;
+                    }
+                }
+                else
+                    cacheChildrenTablesUsed(used, 0, max);
+            }
+            break;
+        }
+    }
+
+    switch (op)
+    {
+    case no_xmltext:
+    case no_xmlunicode:
+    case no_xmlproject:
+        used.addActiveTable(queryXmlParsePseudoTable());
+        break;
+    case no_matched:
+    case no_matchtext:
+    case no_matchunicode:
+    case no_matchlength:
+    case no_matchposition:
+    case no_matchrow:
+    case no_matchutf8:
+    case no_matchattr:
+        used.addActiveTable(queryNlpParsePseudoTable());
+        break;
+    case no_externalcall:
+        {
+            IHqlExpression * def = queryExternalDefinition()->queryChild(0);
+            if (def->hasAttribute(userMatchFunctionAtom))
+                used.addActiveTable(queryNlpParsePseudoTable());
+            break;
+        }
+    }
+}
+
 void CHqlExpressionWithTables::cacheTablesUsed()
 {
     if (!(infoFlags & HEFgatheredNew))
@@ -5089,7 +5295,8 @@ void CHqlExpressionWithTables::cacheTablesUsed()
         //So need an ensureCached() function surrounding it that is cs protected.
 
         //Special case some common operators that can avoid going through the general code
-        bool specialCased = true;
+        bool specialCased = false;
+        if (false)
         switch (op)
         {
         case no_attr:
@@ -5144,135 +5351,7 @@ void CHqlExpressionWithTables::cacheTablesUsed()
         if (!specialCased)
         {
             CUsedTablesBuilder used;
-            if (numChildren())
-            {
-                switch (op)
-                {
-                case no_select:
-                    {
-                        IHqlExpression * ds = queryChild(0);
-                        dbgassertex(!isSelectRootAndActive());
-                        ds->gatherTablesUsed(used);
-                        break;
-                    }
-                case NO_AGGREGATE:
-                case no_createset:
-                    {
-    #ifdef GATHER_HIDDEN_SELECTORS
-                        cachePotentialTablesUsed(used);
-                        used.removeParent(queryChild(0));
-    #else
-                        HqlExprCopyArray childInScopeTables;
-                        ForEachChild(idx, this)
-                            queryChild(idx)->gatherTablesUsed(NULL, &childInScopeTables);
-
-                        //The argument to the operator is a new table, don't inherit grandchildren
-                        IHqlExpression * ds = queryChild(0);
-                        cacheInheritChildTablesUsed(ds, used, childInScopeTables);
-    #endif
-                    }
-                    break;
-                case no_sizeof:
-                    cachePotentialTablesUsed(used);
-                    used.removeActive(queryActiveTableSelector());
-                    used.removeActiveRecords();
-                    break;
-                case no_externalcall:
-                case no_rowvalue:
-                case no_offsetof:
-                case no_eq:
-                case no_ne:
-                case no_lt:
-                case no_le:
-                case no_gt:
-                case no_ge:
-                case no_order:
-                case no_assign:
-                case no_call:
-                case no_libraryscopeinstance:
-                    //MORE: Should check this doesn't make the comparison invalid.
-                    cachePotentialTablesUsed(used);
-                    break;
-                case no_keyindex:
-                case no_newkeyindex:
-                    cacheChildrenTablesUsed(used, 1, numChildren());
-                    used.removeParent(queryChild(0));
-                    //Distributed attribute might contain references to the no_activetable
-                    used.removeActive(queryActiveTableSelector());
-                    break;
-                case no_evaluate:
-                    cacheTableUseage(used, queryChild(0));
-                    queryChild(1)->gatherTablesUsed(used);
-                    break;
-                case no_table:
-                    {
-                        cacheChildrenTablesUsed(used, 0, numChildren());
-                        IHqlExpression * parent = queryChild(3);
-                        if (parent)
-                            used.removeParent(parent);
-                        break;
-                    }
-                case no_pat_production:
-                    {
-                        cacheChildrenTablesUsed(used, 0, numChildren());
-                        used.cleanupProduction();
-                        break;
-                    }
-                default:
-                    {
-                        ITypeInfo * thisType = queryType();
-                        if (thisType)
-                        {
-                            switch (thisType->getTypeCode())
-                            {
-                            case type_void:
-                            case type_dictionary:
-                            case type_table:
-                            case type_groupedtable:
-                            case type_row:
-                            case type_transform:
-                                {
-                                    cacheTablesProcessChildScope(used);
-                                    IHqlExpression * counter = queryAttribute(_countProject_Atom);
-                                    if (counter)
-                                        used.removeActive(counter->queryChild(0));
-                                    break;
-                                }
-                            default:
-                                cacheChildrenTablesUsed(used, 0, numChildren());
-                            }
-                        }
-                        else
-                            cacheChildrenTablesUsed(used, 0, numChildren());
-                        break;
-                    }
-                }
-            }
-            switch (op)
-            {
-            case no_xmltext:
-            case no_xmlunicode:
-            case no_xmlproject:
-                used.addActiveTable(queryXmlParsePseudoTable());
-                break;
-            case no_matched:
-            case no_matchtext:
-            case no_matchunicode:
-            case no_matchlength:
-            case no_matchposition:
-            case no_matchrow:
-            case no_matchutf8:
-            case no_matchattr:
-                used.addActiveTable(queryNlpParsePseudoTable());
-                break;
-            case no_externalcall:
-                {
-                    IHqlExpression * def = queryExternalDefinition()->queryChild(0);
-                    if (def->hasAttribute(userMatchFunctionAtom))
-                        used.addActiveTable(queryNlpParsePseudoTable());
-                    break;
-                }
-            }
+            calcTablesUsed(used, false);
             used.set(usedTables);
         }
         infoFlags |= HEFgatheredNew;
@@ -5285,6 +5364,14 @@ bool CHqlExpressionWithTables::isIndependentOfScope()
     return usedTables.isIndependentOfScope();
 }
 
+bool CHqlExpressionWithTables::isIndependentOfScopeIgnoringInputs()
+{
+    CUsedTablesBuilder used;
+    //MORE: We could try using a flag set by cacheTablesUsed() instead
+    calcTablesUsed(used, true);
+    return used.isIndependentOfScope();
+}
+
 bool CHqlExpressionWithTables::usesSelector(IHqlExpression * selector)
 {
     cacheTablesUsed();
@@ -5402,6 +5489,20 @@ bool CHqlSelectBaseExpression::isIndependentOfScope()
     }
 }
 
+bool CHqlSelectBaseExpression::isIndependentOfScopeIgnoringInputs()
+{
+    if (isSelectRootAndActive())
+    {
+        return false;
+    }
+    else
+    {
+        IHqlExpression * ds = queryChild(0);
+        return ds->isIndependentOfScopeIgnoringInputs();
+    }
+}
+
+
 bool CHqlSelectBaseExpression::usesSelector(IHqlExpression * selector)
 {
     IHqlExpression * ds = queryChild(0);
@@ -6441,6 +6542,11 @@ bool CHqlAnnotation::isIndependentOfScope()
     return body->isIndependentOfScope();
 }
 
+bool CHqlAnnotation::isIndependentOfScopeIgnoringInputs()
+{
+    return body->isIndependentOfScopeIgnoringInputs();
+}
+
 bool CHqlAnnotation::usesSelector(IHqlExpression * selector)
 {
     return body->usesSelector(selector);
@@ -9697,6 +9803,9 @@ CHqlExternalCall::CHqlExternalCall(IHqlExpression * _funcdef, ITypeInfo * _type,
 
     if (def->hasAttribute(contextSensitiveAtom))
         infoFlags |= HEFcontextDependentException;
+
+    if (def->hasAttribute(ctxmethodAtom) || def->hasAttribute(gctxmethodAtom) || def->hasAttribute(globalContextAtom) || def->hasAttribute(contextAtom))
+        infoFlags |= HEFaccessRuntimeContext;
 }
 
 bool CHqlExternalCall::equals(const IHqlExpression & other) const

ecl/hql/hqlexpr.hpp

@@ -126,7 +126,7 @@ enum
     HEFfunctionOfGroupAggregate = 0x00000200,
     HEFvolatile                 = 0x00000400,           // value changes each time called - e.g., random()
     HEFaction                   = 0x00000800,           // an action, or something that can have a side-effect
-  HEF____unused4____          = 0x00000100,
+    HEFaccessRuntimeContext     = 0x00000100,
     HEFthrowscalar              = 0x00002000,           // scalar/action that can throw an exception
     HEFthrowds                  = 0x00004000,           // dataset that can throw an exception
     HEFoldthrows                = 0x00008000,           // old throws flag, which I should remove asap
@@ -166,7 +166,7 @@ enum
                                    HEFonFailDependent|HEFcontainsActiveDataset|HEFcontainsActiveNonSelector|HEFcontainsDataset|
                                    HEFtranslated|HEFgraphDependent|HEFcontainsNlpText|HEFcontainsXmlText|HEFtransformDependent|
                                    HEFcontainsSkip|HEFcontainsCounter|HEFassertkeyed|HEFcontextDependentException|HEFcontainsAlias|HEFcontainsAliasLocally|
-                                   HEFinternalSelect|HEFcontainsThisNode|HEFcontainsDatasetAliasLocally),
+                                   HEFinternalSelect|HEFcontainsThisNode|HEFcontainsDatasetAliasLocally|HEFaccessRuntimeContext),
 
     HEFcontextDependentNoThrow  = (HEFcontextDependent & ~(HEFthrowscalar|HEFthrowds|HEFoldthrows)),
     HEFcontextDependentDataset  = (HEFcontextDependent & ~(HEFthrowscalar)),
@@ -1119,6 +1119,7 @@ interface IHqlExpression : public IInterface
     virtual unsigned numChildren() const = 0;
     virtual bool isIndependentOfScope() = 0;
     virtual bool usesSelector(IHqlExpression * selector) = 0;
+    virtual bool isIndependentOfScopeIgnoringInputs() = 0;
     virtual void gatherTablesUsed(HqlExprCopyArray * newScope, HqlExprCopyArray * inScope) = 0;
     virtual IValue *queryValue() const = 0;
     virtual IInterface *queryUnknownExtra() = 0;
@@ -1696,6 +1697,7 @@ inline bool containsTranslated(IHqlExpression * expr)   { return (expr->getInfoF
 inline bool containsSideEffects(IHqlExpression * expr)  { return (expr->getInfoFlags() & (HEFaction|HEFthrowscalar|HEFthrowds)) != 0; }
 inline bool containsInternalSelect(IHqlExpression * expr)  { return (expr->getInfoFlags() & (HEFinternalSelect)) != 0; }
 inline bool containsThisNode(IHqlExpression * expr)     { return (expr->getInfoFlags() & (HEFcontainsThisNode)) != 0; }
+inline bool usesRuntimeContext(IHqlExpression * expr)   { return (expr->getInfoFlags() & (HEFaccessRuntimeContext)) != 0; }
 
 inline bool containsWorkflow(IHqlExpression * expr)     { return (expr->getInfoFlags2() & (HEF2workflow)) != 0; }
 inline bool containsMustHoist(IHqlExpression * expr)    { return (expr->getInfoFlags2() & (HEF2mustHoist)) != 0; }

ecl/hql/hqlexpr.ipp

@@ -101,6 +101,8 @@ public:
     void removeRows(IHqlExpression * expr, IHqlExpression * left, IHqlExpression * right);
     void set(CUsedTables & tables) { tables.set(inScopeTables, newScopeTables); }
 
+    inline bool isIndependentOfScope() const { return inScopeTables.ordinality() == 0; }
+
 protected:
     HqlExprCopyArray inScopeTables;     // may need to rename, since use has changed.
     HqlExprCopyArray newScopeTables;
@@ -285,6 +287,7 @@ public:
     inline CHqlExpressionWithTables(node_operator op) : CHqlExpression(op) {}
 
     virtual bool isIndependentOfScope();
+    virtual bool isIndependentOfScopeIgnoringInputs();
     virtual bool usesSelector(IHqlExpression * selector);
     virtual void gatherTablesUsed(CUsedTablesBuilder & used);
     virtual void gatherTablesUsed(HqlExprCopyArray * newScope, HqlExprCopyArray * inScope);
@@ -293,10 +296,12 @@ protected:
     void cacheChildrenTablesUsed(CUsedTablesBuilder & used, unsigned from, unsigned to);
     void cacheInheritChildTablesUsed(IHqlExpression * ds, CUsedTablesBuilder & used, const HqlExprCopyArray & childInScopeTables);
     void cachePotentialTablesUsed(CUsedTablesBuilder & used);
-    void cacheTablesProcessChildScope(CUsedTablesBuilder & used);
+    void cacheTablesProcessChildScope(CUsedTablesBuilder & used, bool ignoreInputs);
     void cacheTablesUsed();
     void cacheTableUseage(CUsedTablesBuilder & used, IHqlExpression * expr);
 
+    void calcTablesUsed(CUsedTablesBuilder & used, bool ignoreInputs);
+
 protected:
     CUsedTables usedTables;
 };
@@ -355,6 +360,7 @@ public:
     virtual ITypeInfo *getType();
 
     virtual bool isIndependentOfScope();
+    virtual bool isIndependentOfScopeIgnoringInputs();
     virtual bool usesSelector(IHqlExpression * selector);
     virtual void gatherTablesUsed(CUsedTablesBuilder & used);
     virtual void gatherTablesUsed(HqlExprCopyArray * newScope, HqlExprCopyArray * inScope);
@@ -493,6 +499,7 @@ public:
     virtual IHqlExpression *queryChild(unsigned idx) const;
     virtual unsigned numChildren() const;
     virtual bool isIndependentOfScope();
+    virtual bool isIndependentOfScopeIgnoringInputs();
     virtual bool usesSelector(IHqlExpression * selector);
     virtual void gatherTablesUsed(CUsedTablesBuilder & used);
     virtual void gatherTablesUsed(HqlExprCopyArray * newScope, HqlExprCopyArray * inScope);
@@ -1278,6 +1285,7 @@ public:
     virtual ITypeInfo *getType();
 
     virtual bool isIndependentOfScope() { return true; }
+    virtual bool isIndependentOfScopeIgnoringInputs() { return true; }
     virtual bool usesSelector(IHqlExpression * selector) { return false; }
     virtual void gatherTablesUsed(CUsedTablesBuilder & used) {}
     virtual void gatherTablesUsed(HqlExprCopyArray * newScope, HqlExprCopyArray * inScope) {}

ecl/hqlcpp/hqlcpp.cpp

@@ -1704,6 +1704,7 @@ void HqlCppTranslator::cacheOptions()
         DebugOption(options.expandHashJoin,"expandHashJoin",false),
         DebugOption(options.traceIR,"traceIR",false),
         DebugOption(options.preserveCaseExternalParameter,"preserveCaseExternalParameter",true),
+        DebugOption(options.optimizeParentAccess,"optimizeParentAccess",false),
     };
 
     //get options values from workunit

ecl/hqlcpp/hqlcpp.ipp

@@ -41,11 +41,12 @@
 
 enum { HintSpeed = 1, HintSize = 2 };
 enum GraphLocalisation {
-    GraphNoAccess = 0,
-    GraphCoLocal = 1,
-    GraphNonLocal = 2,
-    GraphCoNonLocal = 3,
-    GraphRemote = 4
+    GraphNeverAccess,  // This variant of an activity never accesses the parent
+    GraphNoAccess,  // This activity would normally access the parent, but doesn't here
+    GraphCoLocal,  // activity runs on the same node as the
+    GraphNonLocal,
+    GraphCoNonLocal,
+    GraphRemote
 };
 
 enum { 
@@ -718,6 +719,7 @@ struct HqlCppOptions
     bool                expandHashJoin;
     bool                traceIR;
     bool                preserveCaseExternalParameter;
+    bool                optimizeParentAccess;
 };
 
 //Any information gathered while processing the query should be moved into here, rather than cluttering up the translator class
@@ -2003,7 +2005,7 @@ extern bool canIterateInline(BuildCtx * ctx, IHqlExpression * expr);
 extern bool canAssignInline(BuildCtx * ctx, IHqlExpression * expr);
 extern bool canEvaluateInline(BuildCtx * ctx, IHqlExpression * expr);
 extern bool canAssignNotEvaluateInline(BuildCtx * ctx, IHqlExpression * expr);
-extern bool isNonLocal(IHqlExpression * expr);
+extern bool isNonLocal(IHqlExpression * expr, bool optimizeParentAccess);
 extern bool alwaysEvaluatesToBound(IHqlExpression * expr);
 
 extern void buildClearPointer(BuildCtx & ctx, IHqlExpression * expr);
@@ -2021,7 +2023,8 @@ extern bool filterIsTableInvariant(IHqlExpression * expr);
 // e.g., if same LEFT selector is in current context and parent.
 extern bool mustEvaluateInContext(BuildCtx & ctx, IHqlExpression * expr);
 extern const char * boolToText(bool value);
-extern GraphLocalisation queryActivityLocalisation(IHqlExpression * expr);
+extern bool activityNeedsParent(IHqlExpression * expr);
+extern GraphLocalisation queryActivityLocalisation(IHqlExpression * expr, bool optimizeParentAccess);
 extern bool isGraphIndependent(IHqlExpression * expr, IHqlExpression * graph);
 extern IHqlExpression * adjustBoundIntegerValues(IHqlExpression * left, IHqlExpression * right, bool subtract);
 extern bool isNullAssign(const CHqlBoundTarget & target, IHqlExpression * expr);

ecl/hqlcpp/hqlhtcpp.cpp

@@ -1802,6 +1802,7 @@ ActivityInstance::ActivityInstance(HqlCppTranslator & _translator, BuildCtx & ct
 
     includedInHeader = false;
     isCoLocal = false;
+    isNoAccess = false;
     executedRemotely = translator.targetThor();// && !translator.isNeverDistributed(dataset);
     containerActivity = NULL;
     subgraph = queryActiveSubGraph(ctx);
@@ -1815,17 +1816,28 @@ ActivityInstance::ActivityInstance(HqlCppTranslator & _translator, BuildCtx & ct
     parentEvalContext.set(translator.queryEvalContext(ctx));
     parentExtract.set(static_cast<ParentExtract*>(ctx.queryFirstAssociation(AssocExtract)));
 
+    bool optimizeParentAccess = translator.queryOptions().optimizeParentAccess;
     if (parentExtract)
     {
         GraphLocalisation localisation = parentExtract->queryLocalisation();
-        activityLocalisation = translator.isAlwaysCoLocal() ? GraphCoLocal : queryActivityLocalisation(dataset);
+        activityLocalisation = translator.isAlwaysCoLocal() ? GraphCoLocal : queryActivityLocalisation(dataset, optimizeParentAccess);
+        if (activityLocalisation == GraphNoAccess)
+            isNoAccess = true;
+        else if (activityLocalisation == GraphNeverAccess)
+            activityLocalisation = GraphNoAccess;
 
         if (translator.targetThor() && !translator.insideChildQuery(ctx))
             executedRemotely = true;
         else
             executedRemotely = ((activityLocalisation == GraphNonLocal) || (localisation == GraphRemote));
 
-        isCoLocal = containerActivity && !executedRemotely && (localisation != GraphNonLocal) && (activityLocalisation != GraphNoAccess);    // if we supported GraphNonCoLocal the last test would not be needed
+        isCoLocal = false;
+        if (containerActivity && !executedRemotely && (localisation != GraphNonLocal))
+        {
+            // if we supported GraphNonCoLocal this test would not be needed
+            if (activityLocalisation != GraphNoAccess)
+                isCoLocal = true;
+        }
 
         //if top level activity within a query library then need to force access to the parent extract
         if (!containerActivity && translator.insideLibrary())
@@ -1844,14 +1856,14 @@ ActivityInstance::ActivityInstance(HqlCppTranslator & _translator, BuildCtx & ct
     {
         if (executedRemotely)
         {
-            GraphLocalisation localisation = queryActivityLocalisation(dataset);
-            if ((kind == TAKsimpleaction) || (localisation == GraphNoAccess))
+            GraphLocalisation localisation = queryActivityLocalisation(dataset, optimizeParentAccess);
+            if ((kind == TAKsimpleaction) || (localisation == GraphNeverAccess) || (localisation == GraphNoAccess))
                 executedRemotely = false;
         }
     }
 
     if (!parentExtract && (translator.getTargetClusterType() == RoxieCluster))
-        executedRemotely = isNonLocal(dataset);
+        executedRemotely = isNonLocal(dataset, false);
 
 
     unsigned containerId = 0;
@@ -2148,6 +2160,8 @@ void ActivityInstance::createGraphNode(IPropertyTree * defaultSubGraph, bool alw
         addAttributeInt("_parentActivity", containerActivity->activityId);
     if (parentExtract && isCoLocal)
         addAttributeBool("coLocal", true);
+    if (isNoAccess)
+        addAttributeBool("noAccess", true);
 
     if (graphEclText.length() == 0)
         toECL(dataset->queryBody(), graphEclText, false, true);
@@ -5019,7 +5033,7 @@ void HqlCppTranslator::buildSetResultInfo(BuildCtx & ctx, IHqlExpression * origi
     ITypeInfo * resultType = type ? type->queryPromotedType() : value->queryType()->queryPromotedType();
     Linked<ITypeInfo> schemaType = resultType;
     type_t retType = schemaType->getTypeCode();
-    IIdAtom * func;
+    IIdAtom * func = NULL;
     CHqlBoundExpr valueToSave;
     LinkedHqlExpr castValue = value;
     switch(retType)
@@ -5125,6 +5139,7 @@ void HqlCppTranslator::buildSetResultInfo(BuildCtx & ctx, IHqlExpression * origi
         }
     }
 
+    assertex(func);
     OwnedHqlExpr nameText = createResultName(name, isPersist);
     if (retType == type_decimal)
     {

ecl/hqlcpp/hqlhtcpp.ipp

@@ -196,6 +196,7 @@ public:
     bool         isMember;
     bool         instanceIsLocal;
     bool         isCoLocal;
+    bool         isNoAccess;
     bool         executedRemotely;
     bool         includedInHeader;
     bool         isLocal;

ecl/hqlcpp/hqlinline.cpp

@@ -494,6 +494,10 @@ bool alwaysEvaluatesToBound(IHqlExpression * expr)
 inline GraphLocalisation mergeLocalisation(GraphLocalisation l, GraphLocalisation r)
 {
     //options are GraphCoLocal, GraphNonLocal, GraphCoNonLocal, GraphNoAccess
+    if (l == GraphNeverAccess)
+        l = GraphNoAccess;
+    if (r == GraphNeverAccess)
+        r = GraphNoAccess;
     if (l == GraphNoAccess)
         return r;
     if (r == GraphNoAccess)
@@ -505,7 +509,122 @@ inline GraphLocalisation mergeLocalisation(GraphLocalisation l, GraphLocalisatio
     return l;
 }
 
-// A minimal function to catch the obviously context invarient items.
+bool exprExcludingInputNeedsParent(IHqlExpression * expr)
+{
+    unsigned first = getFirstActivityArgument(expr);
+    unsigned last = first + getNumActivityArguments(expr);
+
+    ForEachChild(i, expr)
+    {
+        if (i >= first && i < last)
+            continue;
+
+        IHqlExpression * cur = expr->queryChild(i);
+        if (isGraphDependent(cur) || usesRuntimeContext(cur))
+            return true;
+    }
+
+    if (!expr->isIndependentOfScopeIgnoringInputs())
+        return true;
+
+    return false;
+}
+
+bool exprIncludingInputNeedsParent(IHqlExpression * expr)
+{
+    if (isGraphDependent(expr) || usesRuntimeContext(expr))
+        return true;
+    if (!expr->isIndependentOfScope())
+        return true;
+    return false;
+}
+
+bool activityNeedsParent(IHqlExpression * expr)
+{
+    //This should always err on the side of yes...
+    switch (expr->getOperator())
+    {
+    case no_select:
+    case no_compound_diskread:
+    case no_compound_disknormalize:
+    case no_compound_diskaggregate:
+    case no_compound_diskcount:
+    case no_compound_diskgroupaggregate:
+    case no_compound_indexread:
+    case no_compound_indexnormalize:
+    case no_compound_indexaggregate:
+    case no_compound_indexcount:
+    case no_compound_indexgroupaggregate:
+        if (exprIncludingInputNeedsParent(expr))
+            return true;
+        break;
+    case no_libraryinput:
+        return true;
+    case no_hqlproject:
+    case no_newusertable:
+        //Filters might be merged into projects, so need to walk the inputs.
+        loop
+        {
+            if (activityNeedsParent(expr))
+                return true;
+            expr = expr->queryChild(0);
+            switch (expr->getOperator())
+            {
+            case no_sorted:
+            case no_filter:
+                break;
+            default:
+                return false;
+            }
+        }
+    case no_libraryscopeinstance:
+        {
+            //Obscure way to get at the name of the library being called.
+            IHqlExpression * moduleFunction = expr->queryBody()->queryDefinition();
+            IHqlExpression * module = moduleFunction->queryChild(0);
+            assertex(module->getOperator() == no_libraryscope);
+            IHqlExpression * nameAttr = module->queryAttribute(nameAtom);
+            if (activityNeedsParent(nameAttr))
+                return true;
+            return exprExcludingInputNeedsParent(expr);
+        }
+    default:
+        if (expr->isDatarow())
+        {
+            switch (expr->getOperator())
+            {
+            case no_split:
+            case no_spill:
+            case no_selectnth:
+                if (exprExcludingInputNeedsParent(expr))
+                    return true;
+                break;
+            default:
+                if (exprIncludingInputNeedsParent(expr))
+                    return true;
+                break;
+            }
+        }
+        else
+        {
+            if (exprExcludingInputNeedsParent(expr))
+                return true;
+        }
+        break;
+    }
+
+    //Assume the worse for queries based on ds.childdataset
+    if (getChildDatasetType(expr) & childdataset_hasdataset)
+    {
+        IHqlExpression * ds = queryRoot(expr);
+        if (ds && ds->getOperator() == no_select)
+            return true;
+    }
+    return false;
+}
+
+
+// A minimal function to catch the obviously context invariant items.
 static bool accessesData(IHqlExpression * expr)
 {
     switch (expr->getOperator())
@@ -535,9 +654,10 @@ static bool accessesData(IHqlExpression * expr)
     }
 }
 
-GraphLocalisation queryActivityLocalisation(IHqlExpression * expr)
+GraphLocalisation queryActivityLocalisation(IHqlExpression * expr, bool optimizeParentAccess)
 {
-    switch (expr->getOperator())
+    node_operator op = expr->getOperator();
+    switch (op)
     {
     case no_compound_diskread:
         {
@@ -546,12 +666,12 @@ GraphLocalisation queryActivityLocalisation(IHqlExpression * expr)
             //If a compound operation has been added, but with no other effect then don't allow that to change the localisation
             IHqlExpression * ds = expr->queryChild(0);
             if (ds->getOperator() == no_table)
-                return queryActivityLocalisation(ds);
+                return queryActivityLocalisation(ds, optimizeParentAccess);
             return GraphNonLocal;
         }
     case no_table:
         if (expr->hasAttribute(_noAccess_Atom))
-            return GraphNoAccess;
+            return GraphNeverAccess;
         //fallthrough
     case no_keyindex:
     case no_newkeyindex:
@@ -581,7 +701,7 @@ GraphLocalisation queryActivityLocalisation(IHqlExpression * expr)
         break;
     case no_output:
         if (expr->hasAttribute(_spill_Atom))
-            return GraphNoAccess;
+            return GraphNeverAccess;
         break;
     case no_setgraphresult:
     case no_spillgraphresult:
@@ -609,7 +729,7 @@ GraphLocalisation queryActivityLocalisation(IHqlExpression * expr)
     case no_actionlist:
     case no_definesideeffect:
     case no_dataset_alias:
-        return GraphNoAccess;               // Will never access any data values from anywhere
+        return GraphNeverAccess;               // Will never access any data values from anywhere
     case no_hqlproject:
     case no_newusertable:
         //Many more of these could return GraphNoAccess if I determined that only constants and the input
@@ -620,17 +740,20 @@ GraphLocalisation queryActivityLocalisation(IHqlExpression * expr)
     case no_datasetfromrow:
         {
             if (getNumActivityArguments(expr) != 0)
-                return GraphNoAccess;
+                return GraphNeverAccess;
 
             IHqlExpression * row = expr->queryChild(0);
             switch (row->getOperator())
             {
             case no_createrow:
             case no_null:
-                return queryActivityLocalisation(row);
+                return queryActivityLocalisation(row, optimizeParentAccess);
             }
             break;
         }
+    case no_workunit_dataset:
+        return GraphCoLocal; // weird exception in roxie
+    case no_getgraphresult:
     case no_datasetfromdictionary:
         return GraphCoLocal;
     case no_createrow:
@@ -642,7 +765,7 @@ GraphLocalisation queryActivityLocalisation(IHqlExpression * expr)
                 if (accessesData(expr->queryChild(i)))
                     return GraphCoLocal;
             }
-            return GraphNoAccess;
+            return GraphNeverAccess;
         }
     case no_group:
     case no_choosen:
@@ -656,31 +779,45 @@ GraphLocalisation queryActivityLocalisation(IHqlExpression * expr)
                 if (accessesData(expr->queryChild(i)))
                     return GraphCoLocal;
             }
-            return GraphNoAccess;
+            return GraphNeverAccess;
         }
     case no_newaggregate:
         if (!queryRealChild(expr, 3))
         {
             node_operator op = querySimpleAggregate(expr, false, false);
             if (op == no_existsgroup || op == no_countgroup)
-                return GraphNoAccess;
+                return GraphNeverAccess;
             //Need to check if it accesses anything in the context!
         }
         break;
     }
+
+    if (optimizeParentAccess && !activityNeedsParent(expr))
+    {
+        switch (op)
+        {
+        case no_if:
+        case no_case:
+        case no_map:
+            //May be combined into a switch.
+            break;
+        default:
+            return GraphNoAccess;
+        }
+    }
     return GraphCoLocal;
 }
 
-bool isNonLocal(IHqlExpression * expr)
+bool isNonLocal(IHqlExpression * expr, bool optimizeParentAccess)
 {
-    return (queryActivityLocalisation(expr) == GraphNonLocal);
+    return (queryActivityLocalisation(expr, optimizeParentAccess) == GraphNonLocal);
 }
 
 
-static GraphLocalisation doGetGraphLocalisation(IHqlExpression * expr);
-static GraphLocalisation queryGraphLocalisation(IHqlExpression * expr)
+static GraphLocalisation doGetGraphLocalisation(IHqlExpression * expr, bool optimizeParentAccess);
+static GraphLocalisation queryGraphLocalisation(IHqlExpression * expr, bool optimizeParentAccess)
 {
-    GraphLocalisation localisation = queryActivityLocalisation(expr);
+    GraphLocalisation localisation = queryActivityLocalisation(expr, optimizeParentAccess);
 
     if (isSourceActivity(expr))
         return localisation;
@@ -700,7 +837,7 @@ static GraphLocalisation queryGraphLocalisation(IHqlExpression * expr)
         break;
     case no_attr:
     case no_attr_expr:
-        return GraphNoAccess;
+        return GraphNeverAccess;
     default:
         numChildren = getNumActivityArguments(expr);
         break;
@@ -708,19 +845,21 @@ static GraphLocalisation queryGraphLocalisation(IHqlExpression * expr)
 
     for (unsigned i = firstChild; i < numChildren; i++)
     {
-        localisation = mergeLocalisation(localisation, doGetGraphLocalisation(expr->queryChild(i)));
+        localisation = mergeLocalisation(localisation, doGetGraphLocalisation(expr->queryChild(i), optimizeParentAccess));
         if (localisation == GraphCoNonLocal)
             return localisation;
     }
     return localisation;
 }
 
-static GraphLocalisation doGetGraphLocalisation(IHqlExpression * expr)
+static GraphLocalisation doGetGraphLocalisation(IHqlExpression * expr, bool optimizeParentAccess)
 {
     IHqlExpression * cached = (IHqlExpression *)expr->queryTransformExtra();
     if (cached)
         return (GraphLocalisation)cached->queryValue()->getIntValue();
-    GraphLocalisation ret = queryGraphLocalisation(expr);
+    GraphLocalisation ret = queryGraphLocalisation(expr, optimizeParentAccess);
+    if (ret == GraphNeverAccess)
+        ret = GraphNoAccess;
     expr->setTransformExtraOwned(getSizetConstant((unsigned)ret));
     return ret;
 }
@@ -737,7 +876,7 @@ GraphLocalisation HqlCppTranslator::getGraphLocalisation(IHqlExpression * expr,
         return GraphNonLocal;
 
     TransformMutexBlock lock;
-    return doGetGraphLocalisation(expr);
+    return doGetGraphLocalisation(expr, options.optimizeParentAccess);
 }
 
 bool HqlCppTranslator::isNeverDistributed(IHqlExpression * expr)
@@ -787,7 +926,7 @@ ParentExtract::~ParentExtract()
 
 void ParentExtract::associateCursors(BuildCtx & declarectx, BuildCtx & evalctx, GraphLocalisation childLocalisation)
 {
-    const CursorArray * boundCursors;
+    const CursorArray * boundCursors = NULL;
     switch (childLocalisation)
     {
     case GraphCoLocal:
@@ -1369,7 +1508,7 @@ bool EvalContext::evaluateInParent(BuildCtx & ctx, IHqlExpression * expr, bool h
     if (parent->isRowInvariant(expr))
         return true;
 
-    if (parentExtract->canEvaluate(expr))
+    if (parentExtract && parentExtract->canEvaluate(expr))
     {
         if (!isColocal() || !hasOnStart)
             return true;