Change the way depth and dependencies are calculated

- Previously when resourcing the graphs the dependencies were expanded
  and inherited from the child graphs into the parent graph.  On very
  complex queries (e.g., 4000 subgraphs) this became prohibitively
  expensive.  The new code calculates depths and isDependency() on
  demand.

- Also fixes a minor issue where IF() on row was being marked as the
  wrong time in the implicit project code. (Could cause internal error)

Signed-off-by: Gavin Halliday <gavin.halliday@lexisnexis.com>

ecl/hql/hqlexpr.cpp

@@ -3269,7 +3269,7 @@ void CHqlExpression::updateFlagsAfterOperands()
             infoFlags = (infoFlags &~HEFthrowds)|HEFthrowscalar;
         break;
     case no_select:
-        if (!hasProperty(newAtom))
+        if (!isNewSelector(this))
         {
             IHqlExpression * left = queryChild(0);
             node_operator lOp = left->getOperator();

ecl/hqlcpp/hqliproj.cpp

@@ -1544,7 +1544,7 @@ ProjectExprKind ImplicitProjectTransformer::getProjectExprKind(IHqlExpression *
         if (expr->isDataset())
             return PassThroughActivity;
         if (expr->isDatarow())
-            return ComplexNonActivity;
+            return PassThroughActivity;
         return NonActivity;
     case no_addfiles:
     case no_merge:

ecl/hqlcpp/hqlresource.cpp

@@ -653,14 +653,15 @@ ResourceGraphInfo::ResourceGraphInfo(CResourceOptions * _options) : resources(_o
 {
     options = _options;
     depth = 0;
+    depthSequence = -1;
     beenResourced = false;
     isUnconditional = false;
     mergedConditionSource = false;
     hasConditionSource = false;
     isDead = false;
-    gatheredDependants = false;
     startedGeneratingResourced = false;
     inheritedExpandedDependencies = false;
+    cachedDependent.other = NULL;
 }
 
 ResourceGraphInfo::~ResourceGraphInfo()
@@ -768,72 +769,31 @@ void ResourceGraphInfo::display()
     DBGLOG("%s", s.str());
 }
 
-void ResourceGraphInfo::expandDependants(ResourceGraphArray & target)
+void ResourceGraphInfo::getMergeFailReason(StringBuffer & reasonText, ResourceGraphInfo * otherGraph, const CResources & limit)
 {
-    ForEachItemIn(idx, dependsOn)
-    {
-        ResourceGraphInfo * source = dependsOn.item(idx).sourceGraph;
-        if (target.find(*source) == NotFound)
-            target.append(*LINK(source));
-    }
-    ForEachItemIn(idx2, sources)
-    {
-        ResourceGraphLink & cur = sources.item(idx2);
-//      if (!queryResourceInfo(cur.sourceNode)->expandRatherThanSpill())
-        {
-            ResourceGraphInfo * source = cur.sourceGraph;
-            if (target.find(*source) == NotFound)
-                target.append(*LINK(source));
-        }
-    }
-    ForEachItemIn(idx3, indirectSources)
-    {
-        ResourceGraphInfo & cur = indirectSources.item(idx3);
-        if (target.find(cur) == NotFound)
-            target.append(OLINK(cur));
-    }
+    resources.getExceedsReason(reasonText, otherGraph->resources, limit);
 }
 
-void ResourceGraphInfo::gatherDependants(bool recalculate)
+unsigned ResourceGraphInfo::getDepth()
 {
-    if (gatheredDependants && !recalculate)
-        return;
-    gatheredDependants = true;
-    depth = 0;
-    indirectSources.kill();
+    //If no graphs have been merged since this was last called it is still valid.
+    if (depthSequence == options->state.updateSequence)
+        return depth;
 
+    depthSequence = options->state.updateSequence;
+    depth = 0;
     ForEachItemIn(idx, dependsOn)
     {
         ResourceGraphInfo * source = dependsOn.item(idx).sourceGraph;
-        source->gatherDependants(false);
-        source->expandDependants(indirectSources);
-        if (source->depth >= depth)
-            depth = source->depth + 1;
+        if (source->getDepth() >= depth)
+            depth = source->getDepth() + 1;
     }
     ForEachItemIn(idx2, sources)
     {
         ResourceGraphInfo * source = sources.item(idx2).sourceGraph;
-        source->gatherDependants(false);
-        source->expandDependants(indirectSources);
         if (source->getDepth() >= depth)
             depth = source->getDepth() + 1;
     }
-}
-
-void ResourceGraphInfo::getMergeFailReason(StringBuffer & reasonText, ResourceGraphInfo * otherGraph, const CResources & limit)
-{
-    resources.getExceedsReason(reasonText, otherGraph->resources, limit);
-}
-
-void ResourceGraphInfo::replaceReferences(ResourceGraphInfo * oldGraph, ResourceGraphInfo * newGraph)
-{
-    if (indirectSources.find(*oldGraph) != NotFound)
-        gatheredDependants = false;
-}
-
-unsigned ResourceGraphInfo::getDepth()
-{
-    gatherDependants(false);
     return depth;
 }
 
@@ -847,33 +807,46 @@ bool ResourceGraphInfo::hasSameConditions(ResourceGraphInfo & other)
     return true;
 }
 
-bool ResourceGraphInfo::isDependentOn(ResourceGraphInfo & other, bool allowDirect)
+bool ResourceGraphInfo::evalDependentOn(ResourceGraphInfo & other, bool ignoreSources)
 {
-    gatherDependants(false);
     ForEachItemIn(idx, dependsOn)
     {
         ResourceGraphInfo * cur = dependsOn.item(idx).sourceGraph;
         if (cur == &other)
             return true;
-    }
-    ForEachItemIn(idx2, indirectSources)
-    {
-        ResourceGraphInfo & cur = indirectSources.item(idx2);
-        if (&cur == &other)
+        if (cur->isDependentOn(other, false))
             return true;
     }
-    if (!allowDirect)
+    ForEachItemIn(idx2, sources)
     {
-        ForEachItemIn(idx3, sources)
-        {
-            if (sources.item(idx3).sourceGraph == &other)
-                return true;
-        }
+        ResourceGraphInfo * cur = sources.item(idx2).sourceGraph;
+        if ((cur == &other) && !ignoreSources)
+            return true;
+        if (cur->isDependentOn(other, false))
+            return true;
     }
     return false;
 }
 
 
+bool ResourceGraphInfo::isDependentOn(ResourceGraphInfo & other, bool ignoreSources)
+{
+    //Cache the last query so that traversal doesn't convert a graph into a tree walk
+    if ((cachedDependent.other == &other) && (cachedDependent.ignoreSources == ignoreSources) &&
+        (cachedDependent.updateSequence == options->state.updateSequence))
+        return cachedDependent.value;
+
+    if (getDepth() <= other.getDepth())
+        return false;
+
+    cachedDependent.other = &other;
+    cachedDependent.ignoreSources = ignoreSources;
+    cachedDependent.updateSequence = options->state.updateSequence;
+    cachedDependent.value = evalDependentOn(other, ignoreSources);
+    return cachedDependent.value;
+}
+
+
 bool ResourceGraphInfo::isVeryCheap()
 {
     if (sinks.ordinality() != 1)
@@ -950,16 +923,8 @@ void ResourceGraphInfo::mergeGraph(ResourceGraphInfo & other, bool isConditional
 
     //Recalculate the dependents, because sources of the source merged in may no longer be indirect
     //although they may be via another path.  
-    gatheredDependants = false;
+    options->noteGraphsChanged();
 
-    //Any of my sources will already have the correct dependencies, however, 
-    //anything that is a sink of the source has just gained some dependencies.
-    ForEachItemIn(idx, other.sinks)
-    {
-        ResourceGraphInfo * graph = other.sinks.item(idx).sinkGraph;
-        if (graph)
-            graph->gatheredDependants = false;
-    }
     //If was very cheap and merged into an unconditional graph, make sure this is now tagged as
     //unconditional...
     if (other.isUnconditional)
@@ -1965,9 +1930,6 @@ void EclResourcer::replaceGraphReferences(ResourceGraphInfo * oldGraph, Resource
                 cur.changeSinkGraph(newGraph);
         }
     }
-
-    ForEachItemIn(idx3, graphs)
-        graphs.item(idx3).replaceReferences(oldGraph, newGraph);
 }
 
 //------------------------------------------------------------------------------------------

ecl/hqlcpp/hqlresource.ipp

@@ -37,7 +37,10 @@ enum ResourceType {
 class CResourceOptions
 {
 public:
+    CResourceOptions() { state.updateSequence = 0; }
+
     IHqlExpression * createSpillName(bool isGraphResult);
+    void noteGraphsChanged() { state.updateSequence++; }
 
 public:
     unsigned filteredSpillThreshold;
@@ -72,6 +75,12 @@ public:
     StringAttr filenameMangler;
     ClusterType targetClusterType;
 
+    //Used
+    struct
+    {
+        unsigned updateSequence;
+    } state;
+
     inline bool canSplit() const            { return targetClusterType != HThorCluster; }
     inline bool checkResources() const      { return isThorCluster(targetClusterType) && !isChildQuery; }
     inline bool targetRoxie() const         { return targetClusterType == RoxieCluster; }
@@ -159,7 +168,6 @@ public:
     bool mergeInSibling(ResourceGraphInfo & other, const CResources & limit);
     bool mergeInSource(ResourceGraphInfo & other, const CResources & limit, bool isConditionalLink);
     void removeResources(const CResources & value);
-    void replaceReferences(ResourceGraphInfo * oldGraph, ResourceGraphInfo * newGraph);
 
     bool isSharedInput(IHqlExpression * expr);
     void addSharedInput(IHqlExpression * expr, IHqlExpression * mapped);
@@ -167,9 +175,8 @@ public:
 
 protected:
     void display();
-    void expandDependants(ResourceGraphArray & indirectSources);
-    void gatherDependants(bool recalculate);
     void mergeGraph(ResourceGraphInfo & other, bool isConditionalLink, bool mergeConditions);
+    bool evalDependentOn(ResourceGraphInfo & other, bool ignoreSources);
 
 public:
     OwnedHqlExpr createdGraph;
@@ -177,21 +184,27 @@ public:
     GraphLinkArray dependsOn;           // NB: These do no link....
     GraphLinkArray sources;
     GraphLinkArray sinks;
-    ResourceGraphArray indirectSources;
     HqlExprArray conditions;
     HqlExprArray sharedInputs;
     HqlExprArray unbalancedExternalSources;
     HqlExprArray balancedExternalSources;
     CResources resources;
     unsigned depth;
+    unsigned depthSequence;
     bool beenResourced;
     bool isUnconditional;
     bool mergedConditionSource;
     bool hasConditionSource;
     bool isDead;
-    bool gatheredDependants;
     bool startedGeneratingResourced;
     bool inheritedExpandedDependencies;
+    struct
+    {
+        ResourceGraphInfo * other;
+        unsigned updateSequence;
+        bool ignoreSources;
+        bool value;
+    } cachedDependent;
 };