Merge pull request #4715 from ghalliday/issue9589

HPCC-9589 Remove colocal/parent extract from some child activities

Reviewed-By: Richard Chapman <rchapman@hpccsystems.com>

ecl/hql/hqlatoms.cpp

@@ -231,7 +231,6 @@ IAtom * lzwAtom;
 IAtom * macroAtom;
 IAtom * manyAtom;
 IAtom * markerAtom;
-IAtom * matchxxxPseudoFileAtom;
 IAtom * maxAtom;
 IAtom * maxCountAtom;
 IAtom * maxLengthAtom;
@@ -250,6 +249,7 @@ IAtom * namedAtom;
 IAtom * namespaceAtom;
 IAtom * newAtom;
 IAtom * newSetAtom;
+IAtom * _nlpParse_Atom;
 IAtom * noBoundCheckAtom;
 IAtom * noCaseAtom;
 IAtom * _noHoist_Atom;
@@ -410,6 +410,7 @@ IAtom * workunitAtom;
 IAtom * wuidAtom;
 IAtom * xmlAtom;
 IAtom * xmlDefaultAtom;
+IAtom * _xmlParse_Atom;
 IAtom * xpathAtom;
 
 #define MAKEID(x)   x##Id = createIdAtom(#x)
@@ -634,7 +635,6 @@ MODULE_INIT(INIT_PRIORITY_HQLATOM)
     MAKEATOM(macro);
     MAKEATOM(many);
     MAKEATOM(marker);
-    MAKEATOM(matchxxxPseudoFile);
     MAKEATOM(max);
     MAKEATOM(maxCount);
     MAKEATOM(maxLength);
@@ -653,6 +653,7 @@ MODULE_INIT(INIT_PRIORITY_HQLATOM)
     MAKEATOM(namespace);
     MAKEATOM(new);
     MAKEATOM(newSet);
+    MAKESYSATOM(nlpParse);
     MAKEATOM(noBoundCheck);
     MAKEATOM(noCase);
     MAKESYSATOM(noHoist);
@@ -812,6 +813,7 @@ MODULE_INIT(INIT_PRIORITY_HQLATOM)
     MAKEATOM(wuid);
     MAKEATOM(xml);
     MAKEATOM(xmlDefault);
+    MAKESYSATOM(xmlParse);
     MAKEATOM(xpath);
 
     return true;

ecl/hql/hqlatoms.hpp

@@ -234,7 +234,6 @@ extern HQL_API IAtom * lzwAtom;
 extern HQL_API IAtom * macroAtom;
 extern HQL_API IAtom * manyAtom;
 extern HQL_API IAtom * markerAtom;
-extern HQL_API IAtom * matchxxxPseudoFileAtom;
 extern HQL_API IAtom * maxAtom;
 extern HQL_API IAtom * maxCountAtom;
 extern HQL_API IAtom * maxLengthAtom;
@@ -253,6 +252,7 @@ extern HQL_API IAtom * namedAtom;
 extern HQL_API IAtom * namespaceAtom;
 extern HQL_API IAtom * newAtom;
 extern HQL_API IAtom * newSetAtom;
+extern HQL_API IAtom * _nlpParse_Atom;
 extern HQL_API IAtom * noBoundCheckAtom;
 extern HQL_API IAtom * noCaseAtom;
 extern HQL_API IAtom * _noHoist_Atom;
@@ -414,6 +414,7 @@ extern HQL_API IAtom * workunitAtom;
 extern HQL_API IAtom * wuidAtom;
 extern HQL_API IAtom * xmlAtom;
 extern HQL_API IAtom * xmlDefaultAtom;
+extern HQL_API IAtom * _xmlParse_Atom;
 extern HQL_API IAtom * xpathAtom;
 
 inline bool isInternalAttributeName(IAtom * name) { return (name->str()[0] == '$'); }

ecl/hql/hqlattr.cpp

@@ -3424,7 +3424,11 @@ IHqlExpression * HqlLocationIndependentNormalizer::doCreateTransformed(IHqlExpre
         {
             //Original attributes cause chaos => remove all children from attributes
             if (expr->numChildren() != 0)
-                return createAttribute(expr->queryName());
+            {
+                IAtom * name = expr->queryName();
+                if (name != _countProject_Atom)
+                    return createAttribute(expr->queryName());
+            }
             return LINK(expr);
         }
     case no_field:

ecl/hql/hqlexpr.cpp

@@ -71,8 +71,8 @@
 //#define DEBUG_SCOPE
 //#define CHECK_RECORD_CONSISTENCY
 //#define PARANOID
-//#define SEARCH_NAME1   "vL6R"
-//#define SEARCH_NAME2   "v19"
+//#define SEARCH_NAME1   "v1"
+//#define SEARCH_NAME2   "v2"
 //#define SEARCH_IEXPR 0x0681cb0
 //#define CHECK_SELSEQ_CONSISTENCY
 //#define GATHER_COMMON_STATS
@@ -3243,6 +3243,10 @@ void CHqlExpression::initFlagsBeforeOperands()
     case no_decimalstack:
         infoFlags |= HEFaction;
         break;
+    case no_getresult:
+    case no_workunit_dataset:
+        infoFlags |= HEFaccessRuntimeContext;
+        break;
     }
 }
 
@@ -4816,8 +4820,13 @@ void CUsedTablesBuilder::cleanupProduction()
 {
     ForEachItemInRev(i, inScopeTables)
     {
-        if (inScopeTables.item(i).getOperator() == no_matchattr)
+        switch (inScopeTables.item(i).getOperator())
+        {
+        case no_matchattr:
+        case no_matchrow:
             inScopeTables.remove(i);
+            break;
+        }
     }
 }
 
@@ -4930,19 +4939,41 @@ void CHqlExpressionWithTables::cachePotentialTablesUsed(CUsedTablesBuilder & use
 }
 
 
-void CHqlExpressionWithTables::cacheTablesProcessChildScope(CUsedTablesBuilder & used)
+void CHqlExpressionWithTables::cacheTablesProcessChildScope(CUsedTablesBuilder & used, bool ignoreInputs)
 {
     unsigned max = numChildren();
     switch (getChildDatasetType(this))
     {
     case childdataset_none: 
+        cacheChildrenTablesUsed(used, 0, max);
+        break;
     case childdataset_many_noscope:
+        if (ignoreInputs)
+        {
+            unsigned first = getFirstActivityArgument(this);
+            unsigned last = first + getNumActivityArguments(this);
+            cacheChildrenTablesUsed(used, 0, first);
+            cacheChildrenTablesUsed(used, last, max);
+        }
+        else
+            cacheChildrenTablesUsed(used, 0, max);
+        break;
+    case childdataset_dataset_noscope:
+        if (ignoreInputs)
+            cacheChildrenTablesUsed(used, 1, max);
+        else
+            cacheChildrenTablesUsed(used, 0, max);
+        break;
     case childdataset_if:
+        if (ignoreInputs)
+            cacheChildrenTablesUsed(used, 0, 1);
+        else
+            cacheChildrenTablesUsed(used, 0, max);
+        break;
     case childdataset_case:
     case childdataset_map:
-    case childdataset_dataset_noscope: 
+        //Assume the worst
         cacheChildrenTablesUsed(used, 0, max);
-        //None of these have any scoped arguments, so no need to remove them
         break;
     case childdataset_many:
         {
@@ -4950,7 +4981,8 @@ void CHqlExpressionWithTables::cacheTablesProcessChildScope(CUsedTablesBuilder &
             unsigned firstAttr = getNumChildTables(this);
             cacheChildrenTablesUsed(used, firstAttr, max);
             used.removeActive(queryActiveTableSelector());
-            cacheChildrenTablesUsed(used, 0, firstAttr);
+            if (!ignoreInputs)
+                cacheChildrenTablesUsed(used, 0, firstAttr);
             break;
         }
     case childdataset_dataset:
@@ -4958,7 +4990,8 @@ void CHqlExpressionWithTables::cacheTablesProcessChildScope(CUsedTablesBuilder &
             IHqlExpression * ds = queryChild(0);
             cacheChildrenTablesUsed(used, 1, max);
             used.removeParent(ds);
-            cacheChildrenTablesUsed(used, 0, 1);
+            if (!ignoreInputs)
+                cacheChildrenTablesUsed(used, 0, 1);
         }
         break;
     case childdataset_datasetleft:
@@ -4970,7 +5003,20 @@ void CHqlExpressionWithTables::cacheTablesProcessChildScope(CUsedTablesBuilder &
             used.removeActive(left);
             used.removeParent(ds);
             used.removeRows(this, left, NULL);
-            cacheChildrenTablesUsed(used, 0, 1);
+            switch (op)
+            {
+            case no_parse:
+            case no_newparse:
+                used.cleanupProduction();
+                used.removeActive(queryNlpParsePseudoTable());
+                break;
+            case no_xmlparse:
+            case no_newxmlparse:
+                used.removeActive(queryXmlParsePseudoTable());
+                break;
+            }
+            if (!ignoreInputs)
+                cacheChildrenTablesUsed(used, 0, 1);
 #ifdef GATHER_HIDDEN_SELECTORS
             used.addHiddenTable(left, selSeq);
 #endif
@@ -4984,7 +5030,9 @@ void CHqlExpressionWithTables::cacheTablesProcessChildScope(CUsedTablesBuilder &
             cacheChildrenTablesUsed(used, 1, max);
             used.removeActive(left);
             used.removeRows(this, left, NULL);
-            cacheChildrenTablesUsed(used, 0, 1);
+
+            if (!ignoreInputs)
+                cacheChildrenTablesUsed(used, 0, 1);
 #ifdef GATHER_HIDDEN_SELECTORS
             used.addHiddenTable(left, selSeq);
 #endif
@@ -5001,7 +5049,9 @@ void CHqlExpressionWithTables::cacheTablesProcessChildScope(CUsedTablesBuilder &
             used.removeActive(left);
             used.removeActive(right);
             used.removeRows(this, left, right);
-            cacheChildrenTablesUsed(used, 0, 1);
+
+            if (!ignoreInputs)
+                cacheChildrenTablesUsed(used, 0, 1);
 #ifdef GATHER_HIDDEN_SELECTORS
             used.addHiddenTable(left, selSeq);
             used.addHiddenTable(right, selSeq);
@@ -5041,12 +5091,14 @@ void CHqlExpressionWithTables::cacheTablesProcessChildScope(CUsedTablesBuilder &
                 //two datasets form of normalize is weird because right dataset is based on left
                 cacheChildrenTablesUsed(used, 1, 2);
                 used.removeActive(left);
-                cacheChildrenTablesUsed(used, 0, 1);
+                if (!ignoreInputs)
+                    cacheChildrenTablesUsed(used, 0, 1);
             }
             else
             {
                 used.removeActive(left);
-                cacheChildrenTablesUsed(used, 0, 2);
+                if (!ignoreInputs)
+                    cacheChildrenTablesUsed(used, 0, 2);
             }
 #ifdef GATHER_HIDDEN_SELECTORS
             used.addHiddenTable(left, selSeq);
@@ -5063,6 +5115,177 @@ void CHqlExpressionWithTables::cacheTablesProcessChildScope(CUsedTablesBuilder &
 }
 
 
+void CHqlExpressionWithTables::calcTablesUsed(CUsedTablesBuilder & used, bool ignoreInputs)
+{
+    switch (op)
+    {
+    case no_attr:
+    case no_attr_link:
+    case no_keyed:
+    case no_colon:
+    case no_cluster:
+    case no_nameof:
+    case no_translated:
+    case no_constant:
+        break;
+    case no_select:
+        {
+            IHqlExpression * ds = queryChild(0);
+            if (isSelectRootAndActive())
+            {
+                used.addActiveTable(ds);
+            }
+            else
+            {
+                ds->gatherTablesUsed(used);
+            }
+            break;
+        }
+    case no_activerow:
+        used.addActiveTable(queryChild(0));
+        break;
+    case no_rows:
+    case no_rowset:
+        //MORE: This is a bit strange!
+        used.addActiveTable(queryChild(0));
+        break;
+    case no_left:
+    case no_right:
+        used.addActiveTable(this);
+        break;
+    case no_counter:
+        //NB: Counter is added as a pseudo table, because it is too hard to keep track of nested counters otherwise
+        used.addActiveTable(this);
+        break;
+    case no_filepos:
+    case no_file_logicalname:
+        used.addActiveTable(queryChild(0));
+        break;
+    case NO_AGGREGATE:
+    case no_createset:
+        {
+#ifdef GATHER_HIDDEN_SELECTORS
+            cachePotentialTablesUsed(used);
+            used.removeParent(queryChild(0));
+#else
+            HqlExprCopyArray childInScopeTables;
+            ForEachChild(idx, this)
+                queryChild(idx)->gatherTablesUsed(NULL, &childInScopeTables);
+
+            //The argument to the operator is a new table, don't inherit grandchildren
+            IHqlExpression * ds = queryChild(0);
+            cacheInheritChildTablesUsed(ds, used, childInScopeTables);
+#endif
+        }
+        break;
+    case no_sizeof:
+        cachePotentialTablesUsed(used);
+        used.removeActive(queryActiveTableSelector());
+        used.removeActiveRecords();
+        break;
+    case no_externalcall:
+    case no_rowvalue:
+    case no_offsetof:
+    case no_eq:
+    case no_ne:
+    case no_lt:
+    case no_le:
+    case no_gt:
+    case no_ge:
+    case no_order:
+    case no_assign:
+    case no_call:
+    case no_libraryscopeinstance:
+        //MORE: Should check this doesn't make the comparison invalid.
+        cachePotentialTablesUsed(used);
+        break;
+    case no_keyindex:
+    case no_newkeyindex:
+        cacheChildrenTablesUsed(used, 1, numChildren());
+        used.removeParent(queryChild(0));
+        //Distributed attribute might contain references to the no_activetable
+        used.removeActive(queryActiveTableSelector());
+        break;
+    case no_evaluate:
+        cacheTableUseage(used, queryChild(0));
+        queryChild(1)->gatherTablesUsed(used);
+        break;
+    case no_table:
+        {
+            cacheChildrenTablesUsed(used, 0, numChildren());
+            IHqlExpression * parent = queryChild(3);
+            if (parent)
+                used.removeParent(parent);
+            break;
+        }
+    case no_pat_production:
+        {
+            cacheChildrenTablesUsed(used, 0, numChildren());
+            used.cleanupProduction();
+            break;
+        }
+    default:
+        {
+            ITypeInfo * thisType = queryType();
+            unsigned max = numChildren();
+            if (max)
+            {
+                if (thisType)
+                {
+                    switch (thisType->getTypeCode())
+                    {
+                    case type_void:
+                    case type_dictionary:
+                    case type_table:
+                    case type_groupedtable:
+                    case type_row:
+                    case type_transform:
+                        {
+                            cacheTablesProcessChildScope(used, ignoreInputs);
+                            IHqlExpression * counter = queryAttribute(_countProject_Atom);
+                            if (counter)
+                                used.removeActive(counter->queryChild(0));
+                            break;
+                        }
+                    default:
+                        cacheChildrenTablesUsed(used, 0, max);
+                        break;
+                    }
+                }
+                else
+                    cacheChildrenTablesUsed(used, 0, max);
+            }
+            break;
+        }
+    }
+
+    switch (op)
+    {
+    case no_xmltext:
+    case no_xmlunicode:
+    case no_xmlproject:
+        used.addActiveTable(queryXmlParsePseudoTable());
+        break;
+    case no_matched:
+    case no_matchtext:
+    case no_matchunicode:
+    case no_matchlength:
+    case no_matchposition:
+    case no_matchrow:
+    case no_matchutf8:
+    case no_matchattr:
+        used.addActiveTable(queryNlpParsePseudoTable());
+        break;
+    case no_externalcall:
+        {
+            IHqlExpression * def = queryExternalDefinition()->queryChild(0);
+            if (def->hasAttribute(userMatchFunctionAtom))
+                used.addActiveTable(queryNlpParsePseudoTable());
+            break;
+        }
+    }
+}
+
 void CHqlExpressionWithTables::cacheTablesUsed()
 {
     if (!(infoFlags & HEFgatheredNew))
@@ -5072,7 +5295,8 @@ void CHqlExpressionWithTables::cacheTablesUsed()
         //So need an ensureCached() function surrounding it that is cs protected.
 
         //Special case some common operators that can avoid going through the general code
-        bool specialCased = true;
+        bool specialCased = false;
+        if (false)
         switch (op)
         {
         case no_attr:
@@ -5127,131 +5351,7 @@ void CHqlExpressionWithTables::cacheTablesUsed()
         if (!specialCased)
         {
             CUsedTablesBuilder used;
-            if (numChildren())
-            {
-                switch (op)
-                {
-                case no_select:
-                    {
-                        IHqlExpression * ds = queryChild(0);
-                        dbgassertex(!isSelectRootAndActive());
-                        ds->gatherTablesUsed(used);
-                        break;
-                    }
-                case NO_AGGREGATE:
-                case no_createset:
-                    {
-    #ifdef GATHER_HIDDEN_SELECTORS
-                        cachePotentialTablesUsed(used);
-                        used.removeParent(queryChild(0));
-    #else
-                        HqlExprCopyArray childInScopeTables;
-                        ForEachChild(idx, this)
-                            queryChild(idx)->gatherTablesUsed(NULL, &childInScopeTables);
-
-                        //The argument to the operator is a new table, don't inherit grandchildren
-                        IHqlExpression * ds = queryChild(0);
-                        cacheInheritChildTablesUsed(ds, used, childInScopeTables);
-    #endif
-                    }
-                    break;
-                case no_sizeof:
-                    cachePotentialTablesUsed(used);
-                    used.removeActive(queryActiveTableSelector());
-                    used.removeActiveRecords();
-                    break;
-                case no_externalcall:
-                case no_rowvalue:
-                case no_offsetof:
-                case no_eq:
-                case no_ne:
-                case no_lt:
-                case no_le:
-                case no_gt:
-                case no_ge:
-                case no_order:
-                case no_assign:
-                case no_call:
-                case no_libraryscopeinstance:
-                    //MORE: Should check this doesn't make the comparison invalid.
-                    cachePotentialTablesUsed(used);
-                    break;
-                case no_keyindex:
-                case no_newkeyindex:
-                    cacheChildrenTablesUsed(used, 1, numChildren());
-                    used.removeParent(queryChild(0));
-                    //Distributed attribute might contain references to the no_activetable
-                    used.removeActive(queryActiveTableSelector());
-                    break;
-                case no_evaluate:
-                    cacheTableUseage(used, queryChild(0));
-                    queryChild(1)->gatherTablesUsed(used);
-                    break;
-                case no_table:
-                    {
-                        cacheChildrenTablesUsed(used, 0, numChildren());
-                        IHqlExpression * parent = queryChild(3);
-                        if (parent)
-                            used.removeParent(parent);
-                        break;
-                    }
-                case no_pat_production:
-                    {
-                        cacheChildrenTablesUsed(used, 0, numChildren());
-                        used.cleanupProduction();
-                        break;
-                    }
-                case no_parse:
-                case no_newparse:
-                    {
-                        cacheTablesProcessChildScope(used);
-                        used.cleanupProduction();
-                        //Not strictly true - need to inherit from arg(0) if present.
-                        used.removeActive(queryMatchxxxPseudoFile());
-                        break;
-                    }
-                default:
-                    {
-                        ITypeInfo * thisType = queryType();
-                        if (thisType)
-                        {
-                            switch (thisType->getTypeCode())
-                            {
-                            case type_void:
-                            case type_dictionary:
-                            case type_table:
-                            case type_groupedtable:
-                            case type_row:
-                            case type_transform:
-                                {
-                                    cacheTablesProcessChildScope(used);
-                                    IHqlExpression * counter = queryAttribute(_countProject_Atom);
-                                    if (counter)
-                                        used.removeActive(counter->queryChild(0));
-                                    break;
-                                }
-                            default:
-                                cacheChildrenTablesUsed(used, 0, numChildren());
-                            }
-                        }
-                        else
-                            cacheChildrenTablesUsed(used, 0, numChildren());
-                        break;
-                    }
-                }
-            }
-            switch (op)
-            {
-            case no_matched:
-            case no_matchtext:
-            case no_matchunicode:
-            case no_matchlength:
-            case no_matchposition:
-            case no_matchrow:
-            case no_matchutf8:
-                used.addActiveTable(queryMatchxxxPseudoFile());
-                break;
-            }
+            calcTablesUsed(used, false);
             used.set(usedTables);
         }
         infoFlags |= HEFgatheredNew;
@@ -5264,6 +5364,14 @@ bool CHqlExpressionWithTables::isIndependentOfScope()
     return usedTables.isIndependentOfScope();
 }
 
+bool CHqlExpressionWithTables::isIndependentOfScopeIgnoringInputs()
+{
+    CUsedTablesBuilder used;
+    //MORE: We could try using a flag set by cacheTablesUsed() instead
+    calcTablesUsed(used, true);
+    return used.isIndependentOfScope();
+}
+
 bool CHqlExpressionWithTables::usesSelector(IHqlExpression * selector)
 {
     cacheTablesUsed();
@@ -5381,6 +5489,20 @@ bool CHqlSelectBaseExpression::isIndependentOfScope()
     }
 }
 
+bool CHqlSelectBaseExpression::isIndependentOfScopeIgnoringInputs()
+{
+    if (isSelectRootAndActive())
+    {
+        return false;
+    }
+    else
+    {
+        IHqlExpression * ds = queryChild(0);
+        return ds->isIndependentOfScopeIgnoringInputs();
+    }
+}
+
+
 bool CHqlSelectBaseExpression::usesSelector(IHqlExpression * selector)
 {
     IHqlExpression * ds = queryChild(0);
@@ -6420,6 +6542,11 @@ bool CHqlAnnotation::isIndependentOfScope()
     return body->isIndependentOfScope();
 }
 
+bool CHqlAnnotation::isIndependentOfScopeIgnoringInputs()
+{
+    return body->isIndependentOfScopeIgnoringInputs();
+}
+
 bool CHqlAnnotation::usesSelector(IHqlExpression * selector)
 {
     return body->usesSelector(selector);
@@ -9676,6 +9803,9 @@ CHqlExternalCall::CHqlExternalCall(IHqlExpression * _funcdef, ITypeInfo * _type,
 
     if (def->hasAttribute(contextSensitiveAtom))
         infoFlags |= HEFcontextDependentException;
+
+    if (def->hasAttribute(ctxmethodAtom) || def->hasAttribute(gctxmethodAtom) || def->hasAttribute(globalContextAtom) || def->hasAttribute(contextAtom))
+        infoFlags |= HEFaccessRuntimeContext;
 }
 
 bool CHqlExternalCall::equals(const IHqlExpression & other) const

ecl/hql/hqlexpr.hpp

@@ -126,7 +126,7 @@ enum
     HEFfunctionOfGroupAggregate = 0x00000200,
     HEFvolatile                 = 0x00000400,           // value changes each time called - e.g., random()
     HEFaction                   = 0x00000800,           // an action, or something that can have a side-effect
-  HEF____unused4____          = 0x00000100,
+    HEFaccessRuntimeContext     = 0x00000100,
     HEFthrowscalar              = 0x00002000,           // scalar/action that can throw an exception
     HEFthrowds                  = 0x00004000,           // dataset that can throw an exception
     HEFoldthrows                = 0x00008000,           // old throws flag, which I should remove asap
@@ -166,7 +166,7 @@ enum
                                    HEFonFailDependent|HEFcontainsActiveDataset|HEFcontainsActiveNonSelector|HEFcontainsDataset|
                                    HEFtranslated|HEFgraphDependent|HEFcontainsNlpText|HEFcontainsXmlText|HEFtransformDependent|
                                    HEFcontainsSkip|HEFcontainsCounter|HEFassertkeyed|HEFcontextDependentException|HEFcontainsAlias|HEFcontainsAliasLocally|
-                                   HEFinternalSelect|HEFcontainsThisNode|HEFcontainsDatasetAliasLocally),
+                                   HEFinternalSelect|HEFcontainsThisNode|HEFcontainsDatasetAliasLocally|HEFaccessRuntimeContext),
 
     HEFcontextDependentNoThrow  = (HEFcontextDependent & ~(HEFthrowscalar|HEFthrowds|HEFoldthrows)),
     HEFcontextDependentDataset  = (HEFcontextDependent & ~(HEFthrowscalar)),
@@ -1119,6 +1119,7 @@ interface IHqlExpression : public IInterface
     virtual unsigned numChildren() const = 0;
     virtual bool isIndependentOfScope() = 0;
     virtual bool usesSelector(IHqlExpression * selector) = 0;
+    virtual bool isIndependentOfScopeIgnoringInputs() = 0;
     virtual void gatherTablesUsed(HqlExprCopyArray * newScope, HqlExprCopyArray * inScope) = 0;
     virtual IValue *queryValue() const = 0;
     virtual IInterface *queryUnknownExtra() = 0;
@@ -1696,6 +1697,7 @@ inline bool containsTranslated(IHqlExpression * expr)   { return (expr->getInfoF
 inline bool containsSideEffects(IHqlExpression * expr)  { return (expr->getInfoFlags() & (HEFaction|HEFthrowscalar|HEFthrowds)) != 0; }
 inline bool containsInternalSelect(IHqlExpression * expr)  { return (expr->getInfoFlags() & (HEFinternalSelect)) != 0; }
 inline bool containsThisNode(IHqlExpression * expr)     { return (expr->getInfoFlags() & (HEFcontainsThisNode)) != 0; }
+inline bool usesRuntimeContext(IHqlExpression * expr)   { return (expr->getInfoFlags() & (HEFaccessRuntimeContext)) != 0; }
 
 inline bool containsWorkflow(IHqlExpression * expr)     { return (expr->getInfoFlags2() & (HEF2workflow)) != 0; }
 inline bool containsMustHoist(IHqlExpression * expr)    { return (expr->getInfoFlags2() & (HEF2mustHoist)) != 0; }

ecl/hql/hqlexpr.ipp

@@ -101,6 +101,8 @@ public:
     void removeRows(IHqlExpression * expr, IHqlExpression * left, IHqlExpression * right);
     void set(CUsedTables & tables) { tables.set(inScopeTables, newScopeTables); }
 
+    inline bool isIndependentOfScope() const { return inScopeTables.ordinality() == 0; }
+
 protected:
     HqlExprCopyArray inScopeTables;     // may need to rename, since use has changed.
     HqlExprCopyArray newScopeTables;
@@ -285,6 +287,7 @@ public:
     inline CHqlExpressionWithTables(node_operator op) : CHqlExpression(op) {}
 
     virtual bool isIndependentOfScope();
+    virtual bool isIndependentOfScopeIgnoringInputs();
     virtual bool usesSelector(IHqlExpression * selector);
     virtual void gatherTablesUsed(CUsedTablesBuilder & used);
     virtual void gatherTablesUsed(HqlExprCopyArray * newScope, HqlExprCopyArray * inScope);
@@ -293,10 +296,12 @@ protected:
     void cacheChildrenTablesUsed(CUsedTablesBuilder & used, unsigned from, unsigned to);
     void cacheInheritChildTablesUsed(IHqlExpression * ds, CUsedTablesBuilder & used, const HqlExprCopyArray & childInScopeTables);
     void cachePotentialTablesUsed(CUsedTablesBuilder & used);
-    void cacheTablesProcessChildScope(CUsedTablesBuilder & used);
+    void cacheTablesProcessChildScope(CUsedTablesBuilder & used, bool ignoreInputs);
     void cacheTablesUsed();
     void cacheTableUseage(CUsedTablesBuilder & used, IHqlExpression * expr);
 
+    void calcTablesUsed(CUsedTablesBuilder & used, bool ignoreInputs);
+
 protected:
     CUsedTables usedTables;
 };
@@ -355,6 +360,7 @@ public:
     virtual ITypeInfo *getType();
 
     virtual bool isIndependentOfScope();
+    virtual bool isIndependentOfScopeIgnoringInputs();
     virtual bool usesSelector(IHqlExpression * selector);
     virtual void gatherTablesUsed(CUsedTablesBuilder & used);
     virtual void gatherTablesUsed(HqlExprCopyArray * newScope, HqlExprCopyArray * inScope);
@@ -493,6 +499,7 @@ public:
     virtual IHqlExpression *queryChild(unsigned idx) const;
     virtual unsigned numChildren() const;
     virtual bool isIndependentOfScope();
+    virtual bool isIndependentOfScopeIgnoringInputs();
     virtual bool usesSelector(IHqlExpression * selector);
     virtual void gatherTablesUsed(CUsedTablesBuilder & used);
     virtual void gatherTablesUsed(HqlExprCopyArray * newScope, HqlExprCopyArray * inScope);
@@ -1278,6 +1285,7 @@ public:
     virtual ITypeInfo *getType();
 
     virtual bool isIndependentOfScope() { return true; }
+    virtual bool isIndependentOfScopeIgnoringInputs() { return true; }
     virtual bool usesSelector(IHqlExpression * selector) { return false; }
     virtual void gatherTablesUsed(CUsedTablesBuilder & used) {}
     virtual void gatherTablesUsed(HqlExprCopyArray * newScope, HqlExprCopyArray * inScope) {}

ecl/hql/hqlopt.cpp

@@ -1782,9 +1782,15 @@ IHqlExpression * CTreeOptimizer::moveProjectionOverSimple(IHqlExpression * trans
     {
         if (idx != 0)
         {
-            bool ok = false;
+            bool ok = true;
             IHqlExpression * cur = child->queryChild(idx);
-            IHqlExpression * collapsed = mapper->collapseFields(cur, grandchild, newProject, &ok);
+            IHqlExpression * collapsed;
+            //NB: Attributes are generally independent of the input dataset, so they shouldn't be reverse mapped,
+            //otherwise if a input-invariant expression is projected it can cause problems (jholt44.eclxml)
+            if (cur->isAttribute())
+                collapsed = LINK(cur);
+            else
+                collapsed = mapper->collapseFields(cur, grandchild, newProject, &ok);
             if (!ok)
             {
                 ::Release(collapsed);

ecl/hql/hqlpmap.cpp

@@ -315,28 +315,28 @@ void NewProjectMapper2::setUnknownMapping()
     mapping = queryUnknownAttribute();
 }
 
-void NewProjectMapper2::initMapping()
+bool NewProjectMapper2::ensureMapping()
 {
     if (targets.ordinality())
-        return;
+        return true;
 
     switch (mapping->getOperator())
     {
     case no_record:
         setRecord(mapping);
-        break;
+        return true;
     case no_newtransform:
         setTransform(mapping);
-        break;
+        return true;
     case no_transform:
         setTransform(mapping);
-        break;
+        return true;
     case no_alias_scope:
     case no_none:
     case no_externalcall:
     case no_outofline:
     case no_attr:
-        break;              // avoid internal error when values not provided for a record structure
+        return false;              // avoid internal error when values not provided for a record structure
     default:
         UNIMPLEMENTED_XY("mapping", getOpString(mapping->getOperator()));
         break;
@@ -351,8 +351,7 @@ void NewProjectMapper2::initSelf(IHqlExpression * dataset)
 
 bool NewProjectMapper2::isMappingKnown()
 {
-    initMapping();
-    return targets.ordinality() != 0;
+    return ensureMapping();
 }
 
 void NewProjectMapper2::setRecord(IHqlExpression * record, IHqlExpression * selector)

ecl/hql/hqlpmap.hpp

@@ -51,8 +51,7 @@ public:
 private:
     void addMapping(IHqlExpression * field, IHqlExpression * expr);
 
-    inline void ensureMapping()                                 { if (targets.ordinality() == 0) initMapping(); }
-    void initMapping();
+    bool ensureMapping();
 
     void setRecord(IHqlExpression * record, IHqlExpression * selector);
     void setRecord(IHqlExpression * record);

ecl/hql/hqlthql.cpp

@@ -681,8 +681,6 @@ void HqltHql::toECL(IHqlExpression *expr, StringBuffer &s, bool paren, bool inTy
             s.append('[');
             unsigned flags = expr->getInfoFlags();
             if (flags & HEFgraphDependent) s.append('G');
-            if (flags & HEFcontainsNlpText) s.append('N');
-            if (flags & HEFcontainsXmlText) s.append('X');
             if (flags & HEFcontainsSkip) s.append('S');
             if (flags & HEFcontainsCounter) s.append('C');
             if (flags & HEFtransformDependent) s.append('D');

ecl/hql/hqlutil.cpp

@@ -56,7 +56,8 @@ static IHqlExpression * cacheLinkCountedAttr;
 static IHqlExpression * cacheReferenceAttr;
 static IHqlExpression * cacheStreamedAttr;
 static IHqlExpression * cacheUnadornedAttr;
-static IHqlExpression * matchxxxPseudoFile;
+static IHqlExpression * nlpParsePsuedoTable;
+static IHqlExpression * xmlParsePsuedoTable;
 static IHqlExpression * cachedQuotedNullExpr;
 static IHqlExpression * cachedGlobalSequenceNumber;
 static IHqlExpression * cachedLocalSequenceNumber;
@@ -84,7 +85,8 @@ MODULE_INIT(INIT_PRIORITY_STANDARD)
     cacheReferenceAttr = createAttribute(referenceAtom);
     cacheStreamedAttr = createAttribute(streamedAtom);
     cacheUnadornedAttr = createAttribute(_propUnadorned_Atom);
-    matchxxxPseudoFile = createDataset(no_pseudods, createRecord()->closeExpr(), createAttribute(matchxxxPseudoFileAtom));
+    nlpParsePsuedoTable = createDataset(no_pseudods, createRecord()->closeExpr(), createAttribute(_nlpParse_Atom));
+    xmlParsePsuedoTable = createDataset(no_pseudods, createRecord()->closeExpr(), createAttribute(_xmlParse_Atom));
     cachedQuotedNullExpr = createValue(no_nullptr, makeBoolType());
     cachedOmittedValueExpr = createValue(no_omitted, makeAnyType());
 
@@ -110,7 +112,8 @@ MODULE_EXIT()
     cacheReferenceAttr->Release();
     cacheStreamedAttr->Release();
     cacheUnadornedAttr->Release();
-    matchxxxPseudoFile->Release();
+    xmlParsePsuedoTable->Release();
+    nlpParsePsuedoTable->Release();
     cachedQuotedNullExpr->Release();
     cachedGlobalSequenceNumber->Release();
     cachedLocalSequenceNumber->Release();
@@ -230,9 +233,14 @@ extern HQL_API IHqlExpression * getReferenceAttr()
     return LINK(cacheReferenceAttr);
 }
 
-extern HQL_API IHqlExpression * queryMatchxxxPseudoFile()
+extern HQL_API IHqlExpression * queryNlpParsePseudoTable()
 {
-    return matchxxxPseudoFile;
+    return nlpParsePsuedoTable;
+}
+
+extern HQL_API IHqlExpression * queryXmlParsePseudoTable()
+{
+    return xmlParsePsuedoTable;
 }
 
 IHqlExpression * getGlobalSequenceNumber()      { return LINK(cachedGlobalSequenceNumber); }

ecl/hql/hqlutil.hpp

@@ -585,7 +585,8 @@ extern HQL_API IHqlExpression * getFixedSizeAttr(unsigned size);
 extern HQL_API IHqlExpression * queryAlignedAttr();
 extern HQL_API IHqlExpression * queryLinkCountedAttr();
 extern HQL_API IHqlExpression * queryUnadornedAttr();
-extern HQL_API IHqlExpression * queryMatchxxxPseudoFile();
+extern HQL_API IHqlExpression * queryNlpParsePseudoTable();
+extern HQL_API IHqlExpression * queryXmlParsePseudoTable();
 extern HQL_API IHqlExpression * queryQuotedNullExpr();
 
 extern HQL_API IHqlExpression * getEmbeddedAttr();

ecl/hqlcpp/hqlcpp.cpp

@@ -1704,6 +1704,7 @@ void HqlCppTranslator::cacheOptions()
         DebugOption(options.expandHashJoin,"expandHashJoin",false),
         DebugOption(options.traceIR,"traceIR",false),
         DebugOption(options.preserveCaseExternalParameter,"preserveCaseExternalParameter",true),
+        DebugOption(options.optimizeParentAccess,"optimizeParentAccess",false),
     };
 
     //get options values from workunit

ecl/hqlcpp/hqlcpp.ipp

@@ -41,11 +41,12 @@
 
 enum { HintSpeed = 1, HintSize = 2 };
 enum GraphLocalisation {
-    GraphNoAccess = 0,
-    GraphCoLocal = 1,
-    GraphNonLocal = 2,
-    GraphCoNonLocal = 3,
-    GraphRemote = 4
+    GraphNeverAccess,  // This variant of an activity never accesses the parent
+    GraphNoAccess,  // This activity would normally access the parent, but doesn't here
+    GraphCoLocal,  // activity runs on the same node as the
+    GraphNonLocal,
+    GraphCoNonLocal,
+    GraphRemote
 };
 
 enum { 
@@ -718,6 +719,7 @@ struct HqlCppOptions
     bool                expandHashJoin;
     bool                traceIR;
     bool                preserveCaseExternalParameter;
+    bool                optimizeParentAccess;
 };
 
 //Any information gathered while processing the query should be moved into here, rather than cluttering up the translator class
@@ -2003,7 +2005,7 @@ extern bool canIterateInline(BuildCtx * ctx, IHqlExpression * expr);
 extern bool canAssignInline(BuildCtx * ctx, IHqlExpression * expr);
 extern bool canEvaluateInline(BuildCtx * ctx, IHqlExpression * expr);
 extern bool canAssignNotEvaluateInline(BuildCtx * ctx, IHqlExpression * expr);
-extern bool isNonLocal(IHqlExpression * expr);
+extern bool isNonLocal(IHqlExpression * expr, bool optimizeParentAccess);
 extern bool alwaysEvaluatesToBound(IHqlExpression * expr);
 
 extern void buildClearPointer(BuildCtx & ctx, IHqlExpression * expr);
@@ -2021,7 +2023,8 @@ extern bool filterIsTableInvariant(IHqlExpression * expr);
 // e.g., if same LEFT selector is in current context and parent.
 extern bool mustEvaluateInContext(BuildCtx & ctx, IHqlExpression * expr);
 extern const char * boolToText(bool value);
-extern GraphLocalisation queryActivityLocalisation(IHqlExpression * expr);
+extern bool activityNeedsParent(IHqlExpression * expr);
+extern GraphLocalisation queryActivityLocalisation(IHqlExpression * expr, bool optimizeParentAccess);
 extern bool isGraphIndependent(IHqlExpression * expr, IHqlExpression * graph);
 extern IHqlExpression * adjustBoundIntegerValues(IHqlExpression * left, IHqlExpression * right, bool subtract);
 extern bool isNullAssign(const CHqlBoundTarget & target, IHqlExpression * expr);

ecl/hqlcpp/hqlcse.cpp

@@ -116,6 +116,9 @@ bool canCreateTemporary(IHqlExpression * expr)
     case type_transform:
     case type_null:
     case type_void:
+    case type_rule:
+    case type_pattern:
+    case type_token:
         return false;
     default:
         return true;
@@ -1318,6 +1321,9 @@ bool TableInvariantTransformer::isInvariant(IHqlExpression * expr)
         break;
     default:
         if (!isContextDependent(expr))
+        //MORE: The following line is needed if the xml/parse flags are removed from the context, but it causes problems
+        //preventing counts from being hoisted as aliases.  That is really correct - but it makes code worse for some examples.
+        //if (!isContextDependent(expr) && expr->isIndependentOfScope())
         {
             if (!expr->isAction())// && !expr->isDataset() && !expr->isDatarow())
             {

ecl/hqlcpp/hqlhtcpp.cpp

@@ -1802,6 +1802,7 @@ ActivityInstance::ActivityInstance(HqlCppTranslator & _translator, BuildCtx & ct
 
     includedInHeader = false;
     isCoLocal = false;
+    isNoAccess = false;
     executedRemotely = translator.targetThor();// && !translator.isNeverDistributed(dataset);
     containerActivity = NULL;
     subgraph = queryActiveSubGraph(ctx);
@@ -1815,17 +1816,28 @@ ActivityInstance::ActivityInstance(HqlCppTranslator & _translator, BuildCtx & ct
     parentEvalContext.set(translator.queryEvalContext(ctx));
     parentExtract.set(static_cast<ParentExtract*>(ctx.queryFirstAssociation(AssocExtract)));
 
+    bool optimizeParentAccess = translator.queryOptions().optimizeParentAccess;
     if (parentExtract)
     {
         GraphLocalisation localisation = parentExtract->queryLocalisation();
-        activityLocalisation = translator.isAlwaysCoLocal() ? GraphCoLocal : queryActivityLocalisation(dataset);
+        activityLocalisation = translator.isAlwaysCoLocal() ? GraphCoLocal : queryActivityLocalisation(dataset, optimizeParentAccess);
+        if (activityLocalisation == GraphNoAccess)
+            isNoAccess = true;
+        else if (activityLocalisation == GraphNeverAccess)
+            activityLocalisation = GraphNoAccess;
 
         if (translator.targetThor() && !translator.insideChildQuery(ctx))
             executedRemotely = true;
         else
             executedRemotely = ((activityLocalisation == GraphNonLocal) || (localisation == GraphRemote));
 
-        isCoLocal = containerActivity && !executedRemotely && (localisation != GraphNonLocal) && (activityLocalisation != GraphNoAccess);    // if we supported GraphNonCoLocal the last test would not be needed
+        isCoLocal = false;
+        if (containerActivity && !executedRemotely && (localisation != GraphNonLocal))
+        {
+            // if we supported GraphNonCoLocal this test would not be needed
+            if (activityLocalisation != GraphNoAccess)
+                isCoLocal = true;
+        }
 
         //if top level activity within a query library then need to force access to the parent extract
         if (!containerActivity && translator.insideLibrary())
@@ -1844,14 +1856,14 @@ ActivityInstance::ActivityInstance(HqlCppTranslator & _translator, BuildCtx & ct
     {
         if (executedRemotely)
         {
-            GraphLocalisation localisation = queryActivityLocalisation(dataset);
-            if ((kind == TAKsimpleaction) || (localisation == GraphNoAccess))
+            GraphLocalisation localisation = queryActivityLocalisation(dataset, optimizeParentAccess);
+            if ((kind == TAKsimpleaction) || (localisation == GraphNeverAccess) || (localisation == GraphNoAccess))
                 executedRemotely = false;
         }
     }
 
     if (!parentExtract && (translator.getTargetClusterType() == RoxieCluster))
-        executedRemotely = isNonLocal(dataset);
+        executedRemotely = isNonLocal(dataset, false);
 
 
     unsigned containerId = 0;
@@ -2148,6 +2160,8 @@ void ActivityInstance::createGraphNode(IPropertyTree * defaultSubGraph, bool alw
         addAttributeInt("_parentActivity", containerActivity->activityId);
     if (parentExtract && isCoLocal)
         addAttributeBool("coLocal", true);
+    if (isNoAccess)
+        addAttributeBool("noAccess", true);
 
     if (graphEclText.length() == 0)
         toECL(dataset->queryBody(), graphEclText, false, true);
@@ -5019,7 +5033,7 @@ void HqlCppTranslator::buildSetResultInfo(BuildCtx & ctx, IHqlExpression * origi
     ITypeInfo * resultType = type ? type->queryPromotedType() : value->queryType()->queryPromotedType();
     Linked<ITypeInfo> schemaType = resultType;
     type_t retType = schemaType->getTypeCode();
-    IIdAtom * func;
+    IIdAtom * func = NULL;
     CHqlBoundExpr valueToSave;
     LinkedHqlExpr castValue = value;
     switch(retType)
@@ -5125,6 +5139,7 @@ void HqlCppTranslator::buildSetResultInfo(BuildCtx & ctx, IHqlExpression * origi
         }
     }
 
+    assertex(func);
     OwnedHqlExpr nameText = createResultName(name, isPersist);
     if (retType == type_decimal)
     {
@@ -16184,6 +16199,7 @@ ABoundActivity * HqlCppTranslator::doBuildActivityXmlParse(BuildCtx & ctx, IHqlE
 
     OwnedHqlExpr helperName = createQuoted("parsed", makeBoolType());
     funcctx.associateExpr(xmlColumnProviderMarkerExpr, helperName);
+    bindTableCursor(funcctx, queryXmlParsePseudoTable(), queryXmlParsePseudoTable());
     xmlUsesContents = false;
     doTransform(funcctx, transform, selfCursor);
     buildReturnRecordSize(funcctx, selfCursor);
@@ -17834,6 +17850,9 @@ static void logECL(const LogMsgCategory & category, size32_t len, const char * e
 
 void HqlCppTranslator::traceExpression(const char * title, IHqlExpression * expr, unsigned level)
 {
+    if (!expr)
+        return;
+
     checkAbort();
 
     LOG(MCdebugInfo(200), unknownJob, "Tracing expressions: %s", title);

ecl/hqlcpp/hqlhtcpp.ipp

@@ -196,6 +196,7 @@ public:
     bool         isMember;
     bool         instanceIsLocal;
     bool         isCoLocal;
+    bool         isNoAccess;
     bool         executedRemotely;
     bool         includedInHeader;
     bool         isLocal;

ecl/hqlcpp/hqlinline.cpp

@@ -494,6 +494,10 @@ bool alwaysEvaluatesToBound(IHqlExpression * expr)
 inline GraphLocalisation mergeLocalisation(GraphLocalisation l, GraphLocalisation r)
 {
     //options are GraphCoLocal, GraphNonLocal, GraphCoNonLocal, GraphNoAccess
+    if (l == GraphNeverAccess)
+        l = GraphNoAccess;
+    if (r == GraphNeverAccess)
+        r = GraphNoAccess;
     if (l == GraphNoAccess)
         return r;
     if (r == GraphNoAccess)
@@ -505,7 +509,122 @@ inline GraphLocalisation mergeLocalisation(GraphLocalisation l, GraphLocalisatio
     return l;
 }
 
-// A minimal function to catch the obviously context invarient items.
+bool exprExcludingInputNeedsParent(IHqlExpression * expr)
+{
+    unsigned first = getFirstActivityArgument(expr);
+    unsigned last = first + getNumActivityArguments(expr);
+
+    ForEachChild(i, expr)
+    {
+        if (i >= first && i < last)
+            continue;
+
+        IHqlExpression * cur = expr->queryChild(i);
+        if (isGraphDependent(cur) || usesRuntimeContext(cur))
+            return true;
+    }
+
+    if (!expr->isIndependentOfScopeIgnoringInputs())
+        return true;
+
+    return false;
+}
+
+bool exprIncludingInputNeedsParent(IHqlExpression * expr)
+{
+    if (isGraphDependent(expr) || usesRuntimeContext(expr))
+        return true;
+    if (!expr->isIndependentOfScope())
+        return true;
+    return false;
+}
+
+bool activityNeedsParent(IHqlExpression * expr)
+{
+    //This should always err on the side of yes...
+    switch (expr->getOperator())
+    {
+    case no_select:
+    case no_compound_diskread:
+    case no_compound_disknormalize:
+    case no_compound_diskaggregate:
+    case no_compound_diskcount:
+    case no_compound_diskgroupaggregate:
+    case no_compound_indexread:
+    case no_compound_indexnormalize:
+    case no_compound_indexaggregate:
+    case no_compound_indexcount:
+    case no_compound_indexgroupaggregate:
+        if (exprIncludingInputNeedsParent(expr))
+            return true;
+        break;
+    case no_libraryinput:
+        return true;
+    case no_hqlproject:
+    case no_newusertable:
+        //Filters might be merged into projects, so need to walk the inputs.
+        loop
+        {
+            if (activityNeedsParent(expr))
+                return true;
+            expr = expr->queryChild(0);
+            switch (expr->getOperator())
+            {
+            case no_sorted:
+            case no_filter:
+                break;
+            default:
+                return false;
+            }
+        }
+    case no_libraryscopeinstance:
+        {
+            //Obscure way to get at the name of the library being called.
+            IHqlExpression * moduleFunction = expr->queryBody()->queryDefinition();
+            IHqlExpression * module = moduleFunction->queryChild(0);
+            assertex(module->getOperator() == no_libraryscope);
+            IHqlExpression * nameAttr = module->queryAttribute(nameAtom);
+            if (activityNeedsParent(nameAttr))
+                return true;
+            return exprExcludingInputNeedsParent(expr);
+        }
+    default:
+        if (expr->isDatarow())
+        {
+            switch (expr->getOperator())
+            {
+            case no_split:
+            case no_spill:
+            case no_selectnth:
+                if (exprExcludingInputNeedsParent(expr))
+                    return true;
+                break;
+            default:
+                if (exprIncludingInputNeedsParent(expr))
+                    return true;
+                break;
+            }
+        }
+        else
+        {
+            if (exprExcludingInputNeedsParent(expr))
+                return true;
+        }
+        break;
+    }
+
+    //Assume the worse for queries based on ds.childdataset
+    if (getChildDatasetType(expr) & childdataset_hasdataset)
+    {
+        IHqlExpression * ds = queryRoot(expr);
+        if (ds && ds->getOperator() == no_select)
+            return true;
+    }
+    return false;
+}
+
+
+// A minimal function to catch the obviously context invariant items.
 static bool accessesData(IHqlExpression * expr)
 {
     switch (expr->getOperator())
@@ -535,9 +654,10 @@ static bool accessesData(IHqlExpression * expr)
     }
 }
 
-GraphLocalisation queryActivityLocalisation(IHqlExpression * expr)
+GraphLocalisation queryActivityLocalisation(IHqlExpression * expr, bool optimizeParentAccess)
 {
-    switch (expr->getOperator())
+    node_operator op = expr->getOperator();
+    switch (op)
     {
     case no_compound_diskread:
         {
@@ -546,12 +666,12 @@ GraphLocalisation queryActivityLocalisation(IHqlExpression * expr)
             //If a compound operation has been added, but with no other effect then don't allow that to change the localisation
             IHqlExpression * ds = expr->queryChild(0);
             if (ds->getOperator() == no_table)
-                return queryActivityLocalisation(ds);
+                return queryActivityLocalisation(ds, optimizeParentAccess);
             return GraphNonLocal;
         }
     case no_table:
         if (expr->hasAttribute(_noAccess_Atom))
-            return GraphNoAccess;
+            return GraphNeverAccess;
         //fallthrough
     case no_keyindex:
     case no_newkeyindex:
@@ -581,7 +701,7 @@ GraphLocalisation queryActivityLocalisation(IHqlExpression * expr)
         break;
     case no_output:
         if (expr->hasAttribute(_spill_Atom))
-            return GraphNoAccess;
+            return GraphNeverAccess;
         break;
     case no_setgraphresult:
     case no_spillgraphresult:
@@ -609,7 +729,7 @@ GraphLocalisation queryActivityLocalisation(IHqlExpression * expr)
     case no_actionlist:
     case no_definesideeffect:
     case no_dataset_alias:
-        return GraphNoAccess;               // Will never access any data values from anywhere
+        return GraphNeverAccess;               // Will never access any data values from anywhere
     case no_hqlproject:
     case no_newusertable:
         //Many more of these could return GraphNoAccess if I determined that only constants and the input
@@ -620,17 +740,20 @@ GraphLocalisation queryActivityLocalisation(IHqlExpression * expr)
     case no_datasetfromrow:
         {
             if (getNumActivityArguments(expr) != 0)
-                return GraphNoAccess;
+                return GraphNeverAccess;
 
             IHqlExpression * row = expr->queryChild(0);
             switch (row->getOperator())
             {
             case no_createrow:
             case no_null:
-                return queryActivityLocalisation(row);
+                return queryActivityLocalisation(row, optimizeParentAccess);
             }
             break;
         }
+    case no_workunit_dataset:
+        return GraphCoLocal; // weird exception in roxie
+    case no_getgraphresult:
     case no_datasetfromdictionary:
         return GraphCoLocal;
     case no_createrow:
@@ -642,7 +765,7 @@ GraphLocalisation queryActivityLocalisation(IHqlExpression * expr)
                 if (accessesData(expr->queryChild(i)))
                     return GraphCoLocal;
             }
-            return GraphNoAccess;
+            return GraphNeverAccess;
         }
     case no_group:
     case no_choosen:
@@ -656,31 +779,45 @@ GraphLocalisation queryActivityLocalisation(IHqlExpression * expr)
                 if (accessesData(expr->queryChild(i)))
                     return GraphCoLocal;
             }
-            return GraphNoAccess;
+            return GraphNeverAccess;
         }
     case no_newaggregate:
         if (!queryRealChild(expr, 3))
         {
             node_operator op = querySimpleAggregate(expr, false, false);
             if (op == no_existsgroup || op == no_countgroup)
-                return GraphNoAccess;
+                return GraphNeverAccess;
             //Need to check if it accesses anything in the context!
         }
         break;
     }
+
+    if (optimizeParentAccess && !activityNeedsParent(expr))
+    {
+        switch (op)
+        {
+        case no_if:
+        case no_case:
+        case no_map:
+            //May be combined into a switch.
+            break;
+        default:
+            return GraphNoAccess;
+        }
+    }
     return GraphCoLocal;
 }
 
-bool isNonLocal(IHqlExpression * expr)
+bool isNonLocal(IHqlExpression * expr, bool optimizeParentAccess)
 {
-    return (queryActivityLocalisation(expr) == GraphNonLocal);
+    return (queryActivityLocalisation(expr, optimizeParentAccess) == GraphNonLocal);
 }
 
 
-static GraphLocalisation doGetGraphLocalisation(IHqlExpression * expr);
-static GraphLocalisation queryGraphLocalisation(IHqlExpression * expr)
+static GraphLocalisation doGetGraphLocalisation(IHqlExpression * expr, bool optimizeParentAccess);
+static GraphLocalisation queryGraphLocalisation(IHqlExpression * expr, bool optimizeParentAccess)
 {
-    GraphLocalisation localisation = queryActivityLocalisation(expr);
+    GraphLocalisation localisation = queryActivityLocalisation(expr, optimizeParentAccess);
 
     if (isSourceActivity(expr))
         return localisation;
@@ -700,7 +837,7 @@ static GraphLocalisation queryGraphLocalisation(IHqlExpression * expr)
         break;
     case no_attr:
     case no_attr_expr:
-        return GraphNoAccess;
+        return GraphNeverAccess;
     default:
         numChildren = getNumActivityArguments(expr);
         break;
@@ -708,19 +845,21 @@ static GraphLocalisation queryGraphLocalisation(IHqlExpression * expr)
 
     for (unsigned i = firstChild; i < numChildren; i++)
     {
-        localisation = mergeLocalisation(localisation, doGetGraphLocalisation(expr->queryChild(i)));
+        localisation = mergeLocalisation(localisation, doGetGraphLocalisation(expr->queryChild(i), optimizeParentAccess));
         if (localisation == GraphCoNonLocal)
             return localisation;
     }
     return localisation;
 }
 
-static GraphLocalisation doGetGraphLocalisation(IHqlExpression * expr)
+static GraphLocalisation doGetGraphLocalisation(IHqlExpression * expr, bool optimizeParentAccess)
 {
     IHqlExpression * cached = (IHqlExpression *)expr->queryTransformExtra();
     if (cached)
         return (GraphLocalisation)cached->queryValue()->getIntValue();
-    GraphLocalisation ret = queryGraphLocalisation(expr);
+    GraphLocalisation ret = queryGraphLocalisation(expr, optimizeParentAccess);
+    if (ret == GraphNeverAccess)
+        ret = GraphNoAccess;
     expr->setTransformExtraOwned(getSizetConstant((unsigned)ret));
     return ret;
 }
@@ -737,7 +876,7 @@ GraphLocalisation HqlCppTranslator::getGraphLocalisation(IHqlExpression * expr,
         return GraphNonLocal;
 
     TransformMutexBlock lock;
-    return doGetGraphLocalisation(expr);
+    return doGetGraphLocalisation(expr, options.optimizeParentAccess);
 }
 
 bool HqlCppTranslator::isNeverDistributed(IHqlExpression * expr)
@@ -787,7 +926,7 @@ ParentExtract::~ParentExtract()
 
 void ParentExtract::associateCursors(BuildCtx & declarectx, BuildCtx & evalctx, GraphLocalisation childLocalisation)
 {
-    const CursorArray * boundCursors;
+    const CursorArray * boundCursors = NULL;
     switch (childLocalisation)
     {
     case GraphCoLocal:
@@ -1362,14 +1501,14 @@ bool EvalContext::evaluateInParent(BuildCtx & ctx, IHqlExpression * expr, bool h
     if (isContextDependent(expr))
         return false;
 
-    if (!containsActiveDataset(expr))
+    if (isIndependentOfScope(expr))
         return true;//isColocal();
 
     //If can evaluate in parent's start context then always worth doing there.
     if (parent->isRowInvariant(expr))
         return true;
 
-    if (parentExtract->canEvaluate(expr))
+    if (parentExtract && parentExtract->canEvaluate(expr))
     {
         if (!isColocal() || !hasOnStart)
             return true;
@@ -1589,7 +1728,7 @@ AliasKind ClassEvalContext::evaluateExpression(BuildCtx & ctx, IHqlExpression *
 
         if (!isContextDependentExceptGraph(value))
         {
-            if (!isContextDependent(value) && !containsActiveDataset(value))
+            if (!isContextDependent(value) && !containsActiveDataset(value) && value->isIndependentOfScope())
             {
                 createMemberAlias(onCreate, ctx, value, tgt);
                 return CreateTimeAlias;

ecl/hqlcpp/hqlnlp.cpp

@@ -257,7 +257,7 @@ void NlpParseContext::buildValidators(HqlCppTranslator & translator, BuildCtx &
             }
             validctx.associateExpr(activeNlpMarkerExpr, activeNlpMarkerExpr);
             validctx.associateExpr(activeValidateMarkerExpr, activeValidateMarkerExpr);
-            translator.bindTableCursor(validctx, queryMatchxxxPseudoFile(), queryMatchxxxPseudoFile());
+            translator.bindTableCursor(validctx, queryNlpParsePseudoTable(), queryNlpParsePseudoTable());
             if (translator.queryOptions().spotCSE)
                 validateExpr.setown(spotScalarCSE(validateExpr));
             translator.buildReturn(validctx, validateExpr);
@@ -515,7 +515,7 @@ void HqlCppTranslator::doBuildParseTransform(BuildCtx & classctx, IHqlExpression
     ensureRowAllocated(funcctx, "crSelf");
     funcctx.addQuoted("const unsigned char * left = (const unsigned char *) _left;");
     funcctx.associateExpr(activeNlpMarkerExpr, activeNlpMarkerExpr);
-    bindTableCursor(funcctx, queryMatchxxxPseudoFile(), queryMatchxxxPseudoFile());
+    bindTableCursor(funcctx, queryNlpParsePseudoTable(), queryNlpParsePseudoTable());
 
     // Bind left to "left" and right to RIGHT
     IHqlExpression * dataset = expr->queryChild(0);

ecl/hqlcpp/hqlresource.cpp

@@ -2127,7 +2127,7 @@ protected:
     void findSplitPoints(IHqlExpression * expr)
     {
         //containsNonActiveDataset() would be nice - but that isn't percolated outside assigns etc.
-        if (containsAnyDataset(expr) || containsMustHoist(expr))
+        if (containsAnyDataset(expr) || containsMustHoist(expr) || !expr->isIndependentOfScope())
         {
             if (!gathered)
             {

ecl/hqlcpp/hqlttcpp.cpp

@@ -6959,7 +6959,7 @@ void ScalarGlobalTransformer::analyseExpr(IHqlExpression * expr)
             if (extra->createGlobal)
                 return;
             //Allow a global to be created inside a global marked from somewhere else.
-            if (containsAnyDataset(expr) || expr->isConstant() || isContextDependent(expr))
+            if (containsAnyDataset(expr) || expr->isConstant() || isContextDependent(expr) || !expr->isIndependentOfScope())
                 return;
         }
     }
@@ -7012,7 +7012,7 @@ void ScalarGlobalTransformer::doAnalyseExpr(IHqlExpression * expr)
 #ifndef NEW_SCALAR_CODE
 //  Commented line has problems with SELF used in HOLE definition, and explosion in thumphrey7 etc.
 //  if (okToHoist && isIndependentOfScope(expr) && !expr->isConstant() && !isContextDependent(expr) && expr->isPure())
-    if (okToHoist && !containsAnyDataset(expr) && !expr->isConstant() && !isContextDependent(expr) && expr->isPure())
+    if (okToHoist && !containsAnyDataset(expr) && !expr->isConstant() && !isContextDependent(expr) && expr->isPure() && expr->isIndependentOfScope())
     {
         ITypeInfo * type = expr->queryType();
         if (isTypeToHoist(type))