Merge pull request #2392 from ghalliday/independent

Significant speed up for complex SEQUENTIALS

Reviewed-By: Renato Golin <rengolin@hpccsystems.com>
Reviewed-By: Richard Chapman <rchapman@hpccsystems.com>

ecl/hql/hqltrans.cpp

@@ -2101,6 +2101,14 @@ HoistingHqlTransformer::HoistingHqlTransformer(HqlTransformerInfo & _info, unsig
     containsUnknownIndependentContents = false;
 }
 
+void HoistingHqlTransformer::setParent(const HoistingHqlTransformer * parent)
+{
+    assertex(parent->independentCache);
+    independentCache.set(parent->independentCache);
+}
+
+
+
 void HoistingHqlTransformer::transformRoot(const HqlExprArray & in, HqlExprArray & out)
 {
     HqlExprArray * savedTarget = target;
@@ -2310,6 +2318,39 @@ IHqlExpression * HoistingHqlTransformer::transformEnsureResult(IHqlExpression *
     return replaceChild(expr, 0, value);
 }
 
+IHqlExpression * HoistingHqlTransformer::IndependentTransformMap::getTransformed(IHqlExpression * expr)
+{
+    for (unsigned i=0; i < cache.ordinality(); i+=2)
+    {
+        if (expr == &cache.item(i))
+            return LINK(&cache.item(i+1));
+    }
+    return NULL;
+}
+
+
+void HoistingHqlTransformer::IndependentTransformMap::setTransformed(IHqlExpression * expr, IHqlExpression * transformed)
+{
+    cache.append(*LINK(expr));
+    cache.append(*LINK(transformed));
+}
+
+IHqlExpression * HoistingHqlTransformer::transformIndependent(IHqlExpression * expr)
+{
+    if (!independentCache)
+        independentCache.setown(new IndependentTransformMap);
+
+    //Separately cache all transformations of independent expressions.  Otherwise highly nested independent
+    //expressions can cause a combinatorial explosion in the number of times the leaves are transformed.
+    IHqlExpression * prev = independentCache->getTransformed(expr);
+    if (prev)
+        return prev;
+
+    OwnedHqlExpr transformed = doTransformIndependent(expr);
+    independentCache->setTransformed(expr, transformed);
+    return transformed.getClear();
+}
+
 //---------------------------------------------------------------------------
 
 void NestedHqlTransformer::beginNestedScope()

ecl/hql/hqltrans.ipp

@@ -602,6 +602,14 @@ private:
 //Useful for processing 
 class HQL_API HoistingHqlTransformer : public NewHqlTransformer
 {
+    class IndependentTransformMap : public CInterface
+    {
+    public:
+        IHqlExpression * getTransformed(IHqlExpression * expr);
+        void setTransformed(IHqlExpression * expr, IHqlExpression * transformed);
+    protected:
+        HqlExprArray cache;
+    };
 public:
     enum { HTFnoteconditionalactions = 0x01, 
            HTFnoteconditionaldatasets= 0x02,
@@ -613,6 +621,7 @@ public:
 
     void appendToTarget(IHqlExpression & curOwned);
     void setFlags(unsigned _flags) { flags = _flags; }
+    void setParent(const HoistingHqlTransformer * parent);
     void transformRoot(const HqlExprArray & in, HqlExprArray & out);
     IHqlExpression * transformRoot(IHqlExpression * expr);
 
@@ -627,7 +636,8 @@ protected:
 
     virtual void analyseExpr(IHqlExpression * expr);
     virtual IHqlExpression * createTransformed(IHqlExpression * expr);
-    virtual IHqlExpression * transformIndependent(IHqlExpression * expr) = 0;
+    IHqlExpression * transformIndependent(IHqlExpression * expr);
+    virtual IHqlExpression * doTransformIndependent(IHqlExpression * expr) = 0;
 
     void transformArray(const HqlExprArray & in, HqlExprArray & out);
     IHqlExpression * transformEnsureResult(IHqlExpression * expr);
@@ -640,10 +650,10 @@ protected:
             ((flags & HTFnoteconditionaldatarows) && expr->isDatarow()));
     }
 
-
 private:
     HqlExprArray *      target;
     unsigned flags;
+    Owned<IndependentTransformMap> independentCache;
 
 protected:
     unsigned conditionDepth;

ecl/hqlcpp/hqlhtcpp.cpp

@@ -619,7 +619,7 @@ public:
         return transformed.getClear();
     }
 
-    virtual IHqlExpression * transformIndependent(IHqlExpression * expr) { return LINK(expr); }
+    virtual IHqlExpression * doTransformIndependent(IHqlExpression * expr) { return LINK(expr); }
 
     bool hasCandidate() const { return candidate; }
     ChildGraphBuilder * getBuilder() { return builder.getClear(); };

ecl/hqlcpp/hqlttcpp.ipp

@@ -119,9 +119,10 @@ protected:
     virtual void setTransformed(IHqlExpression * expr, IHqlExpression * transformed);
     virtual void setTransformedSelector(IHqlExpression * expr, IHqlExpression * transformed);
 
-    virtual IHqlExpression * transformIndependent(IHqlExpression * expr)
+    virtual IHqlExpression * doTransformIndependent(IHqlExpression * expr)
     {
         ThorScalarTransformer nested(options);
+        nested.setParent(this);
         nested.analyse(expr, 0);
         if (nested.needToTransform())
             return nested.transformRoot(expr);
@@ -550,11 +551,12 @@ protected:
 
     virtual void doAnalyseExpr(IHqlExpression * expr);
 
-    virtual IHqlExpression * transformIndependent(IHqlExpression * expr)
+    virtual IHqlExpression * doTransformIndependent(IHqlExpression * expr)
     {
         if (containsUnknownIndependentContents || seenLocalGlobalScope)
         {
             ExplicitGlobalTransformer nested(wu, translator);
+            nested.setParent(this);
             nested.analyse(expr, 0);
             if (nested.needToTransform())
                 return nested.transformRoot(expr);
@@ -597,9 +599,10 @@ protected:
     virtual void doAnalyseExpr(IHqlExpression * expr);
     bool isComplex(IHqlExpression * expr, bool checkGlobal);
 
-    virtual IHqlExpression * transformIndependent(IHqlExpression * expr)
+    virtual IHqlExpression * doTransformIndependent(IHqlExpression * expr)
     {
         ScalarGlobalTransformer nested(translator);
+        nested.setParent(this);
         nested.analyse(expr, 0);
         return nested.transformRoot(expr);
     }
@@ -634,9 +637,10 @@ protected:
     virtual IHqlExpression * createTransformed(IHqlExpression * expr);
     virtual ANewTransformInfo * createTransformInfo(IHqlExpression * expr) { return CREATE_NEWTRANSFORMINFO(ScopeMigrateInfo, expr); }
 
-    virtual IHqlExpression * transformIndependent(IHqlExpression * expr)
+    virtual IHqlExpression * doTransformIndependent(IHqlExpression * expr)
     {
         NewScopeMigrateTransformer nested(wu, translator);
+        nested.setParent(this);
         nested.analyse(expr, 0);
         return nested.transformRoot(expr);
     }
@@ -1039,11 +1043,12 @@ public:
 protected:
     virtual IHqlExpression * createTransformed(IHqlExpression * expr);
 
-    virtual IHqlExpression * transformIndependent(IHqlExpression * expr)
+    virtual IHqlExpression * doTransformIndependent(IHqlExpression * expr)
     {
         if (!containsCompound(expr))
             return LINK(expr);
         NestedCompoundTransformer nested(translator);
+        nested.setParent(this);
         nested.analyse(expr, 0);                    // analyse called so set up whether expressions are unconditional etc.
         return nested.transformRoot(expr);
     }