浏览代码

New option -freportFieldUsage to track fields used

This adds a new option for tracking which fields are actually used
by a query for each dataset and index that is used by a query.
If enabled the code generator creates an extra xml output file
containing the details.

Signed-off-by: Gavin Halliday <gavin.halliday@lexisnexis.com>
Gavin Halliday 13 年之前
父节点
当前提交
25d7368832

+ 1 - 0
common/workunit/workunit.cpp

@@ -6132,6 +6132,7 @@ mapEnums queryFileTypes[] = {
    { FileTypeDll, "dll" },
    { FileTypeResText, "res" },
    { FileTypeHintXml, "hint" },
+   { FileTypeXml, "xml" },
    { FileTypeSize,  NULL },
 };
 

+ 2 - 1
common/workunit/workunit.hpp

@@ -331,7 +331,8 @@ enum WUFileType
     FileTypeDll = 1,
     FileTypeResText = 2,
     FileTypeHintXml = 3,
-    FileTypeSize = 4
+    FileTypeXml = 4,
+    FileTypeSize = 5
 };
 
 

+ 2 - 0
ecl/hql/CMakeLists.txt

@@ -50,6 +50,7 @@ set (   SRCS
         hqlstack.cpp 
         hqlthql.cpp 
         hqltrans.cpp 
+        hqlusage.cpp
         hqlutil.cpp 
         hqlvalid.cpp 
         hqlwuerr.cpp 
@@ -80,6 +81,7 @@ set (   SRCS
         hqlstack.hpp
         hqlthor.hpp
         hqlthql.hpp
+        hqlusage.hpp
         hqlutil.hpp
         hqlvalid.hpp
         hqlwuerr.hpp

+ 1 - 0
ecl/hql/hqlgram2.cpp

@@ -45,6 +45,7 @@
 #include "hqlpregex.hpp"
 #include "hqlutil.hpp"
 #include "hqltrans.ipp"
+#include "hqlusage.hpp"
 #include "hqlvalid.hpp"
 #include "hqlrepository.hpp"
 

+ 0 - 281
ecl/hql/hqltrans.cpp

@@ -4434,51 +4434,6 @@ IHqlExpression * expandCreateRowSelectors(IHqlExpression * expr)
     return expander.createTransformed(expr);
 }
 
-HQL_API bool containsSelector(IHqlExpression * expr, IHqlExpression * selector)
-{
-    return exprReferencesDataset(expr, selector);
-}
-
-//---------------------------------------------------------------------------
-
-static HqlTransformerInfo hqlSelectorAnywhereLocatorInfo("HqlSelectorAnywhereLocator");
-HqlSelectorAnywhereLocator::HqlSelectorAnywhereLocator(IHqlExpression * _selector) : NewHqlTransformer(hqlSelectorAnywhereLocatorInfo)
-{ 
-    selector.set(_selector); 
-    foundSelector = false; 
-}
-
-
-void HqlSelectorAnywhereLocator::analyseExpr(IHqlExpression * expr)
-{
-    if (foundSelector || alreadyVisited(expr))
-        return;
-    NewHqlTransformer::analyseExpr(expr);
-}
-
-void HqlSelectorAnywhereLocator::analyseSelector(IHqlExpression * expr)
-{
-    if (expr == selector)
-    {
-        foundSelector = true;
-        return;
-    }
-    NewHqlTransformer::analyseSelector(expr);
-}
-
-bool HqlSelectorAnywhereLocator::containsSelector(IHqlExpression * expr)
-{
-    foundSelector = false;
-    analyse(expr, 0);
-    return foundSelector;
-}
-
-HQL_API bool containsSelectorAnywhere(IHqlExpression * expr, IHqlExpression * selector)
-{
-    HqlSelectorAnywhereLocator locator(selector);
-    return locator.containsSelector(expr);
-}
-
 //---------------------------------------------------------------------------
 
 /*
@@ -4546,242 +4501,6 @@ void * transformerAlloc(size32_t size)
 }
 #endif
 
-class ExpressionStatsInfo
-{
-public:
-    enum { MaxOperands = 17 };
-public:
-    ExpressionStatsInfo() { count = 0; _clear(numOperands); countMax = 0; sumMax = 0; }
-
-    void trace()
-    {
-        DBGLOG("numUnique %u", count);
-        for (unsigned i=0; i < MaxOperands; i++)
-            DBGLOG("  %u operands: %u", i, numOperands[i]);
-        DBGLOG("  %u experessions total %u operands", countMax, sumMax);
-    }
-
-    unsigned count;
-    unsigned numOperands[MaxOperands];
-    unsigned countMax;
-    unsigned sumMax;
-};
-
-static void calcNumUniqueExpressions(IHqlExpression * expr, ExpressionStatsInfo & info)
-{
-    if (expr->queryTransformExtra())
-        return;
-    expr->setTransformExtraUnlinked(expr);
-
-    //use head recursion
-    loop
-    {
-        info.count++;
-        unsigned max = expr->numChildren();
-        if (max >= ExpressionStatsInfo::MaxOperands)
-        {
-            info.countMax++;
-            info.sumMax += max;
-        }
-        else
-            info.numOperands[max]++;
-
-        if (max == 0)
-            return;
-
-        for (unsigned idx=1; idx < max; idx++)
-            calcNumUniqueExpressions(expr->queryChild(idx), info);
-        expr = expr->queryChild(0);
-    }
-}
-
-unsigned getNumUniqueExpressions(IHqlExpression * expr)
-{
-    TransformMutexBlock block;
-    ExpressionStatsInfo info;
-    calcNumUniqueExpressions(expr, info);
-    return info.count;
-}
-
-
-unsigned getNumUniqueExpressions(const HqlExprArray & exprs)
-{
-    TransformMutexBlock block;
-    ExpressionStatsInfo info;
-    ForEachItemIn(i, exprs)
-        calcNumUniqueExpressions(&exprs.item(i),info);
-    return info.count;
-}
-
-//------------------------------------------------------------------------------------------------
-
-static HqlTransformerInfo quickExpressionCounterInfo("QuickExpressionCounter");
-class HQL_API QuickExpressionCounter : public QuickHqlTransformer
-{
-public:
-    QuickExpressionCounter(IHqlExpression * _search, unsigned _limit) 
-    : QuickHqlTransformer(quickExpressionCounterInfo, NULL), search(_search), limit(_limit)
-    {
-        matches = 0;
-    }
-
-    void analyse(IHqlExpression * expr)
-    {
-        if (expr == search)
-            matches++;
-        if (matches >= limit)
-            return;
-        QuickHqlTransformer::analyse(expr);
-    }
-
-    bool limitReached() const { return matches >= limit; }
-    unsigned numMatches() const { return matches; }
-
-protected:
-    HqlExprAttr search;
-    unsigned matches;
-    unsigned limit;
-};
-
-
-
-extern HQL_API unsigned getNumOccurences(HqlExprArray & exprs, IHqlExpression * search, unsigned limit)
-{
-    QuickExpressionCounter counter(search, limit);
-    ForEachItemIn(i, exprs)
-        counter.analyse(&exprs.item(i));
-    return counter.numMatches();
-}
-
-
-extern HQL_API void logTreeStats(IHqlExpression * expr)
-{
-    TransformMutexBlock block;
-    ExpressionStatsInfo info;
-    calcNumUniqueExpressions(expr,info);
-    info.trace();
-}
-
-extern HQL_API void logTreeStats(const HqlExprArray & exprs)
-{
-    TransformMutexBlock block;
-    ExpressionStatsInfo info;
-    ForEachItemIn(i, exprs)
-        calcNumUniqueExpressions(&exprs.item(i),info);
-    info.trace();
-}
-
-//------------------------------------------------------------------------------------------------
-
-static HqlTransformerInfo selectCollectingTransformerInfo("SelectCollectingTransformer");
-class SelectCollectingTransformer : public NewHqlTransformer
-{
-public:
-    SelectCollectingTransformer(HqlExprArray & _found)
-    : NewHqlTransformer(selectCollectingTransformerInfo), found(_found)
-    {
-    }
-
-    virtual void analyseExpr(IHqlExpression * expr)
-    {
-        if (alreadyVisited(expr))
-            return;
-        if (expr->getOperator() == no_select)
-        {
-            if (!found.contains(*expr))
-                found.append(*LINK(expr));
-            return;
-        }
-        NewHqlTransformer::analyseExpr(expr);
-    }
-
-protected:
-    HqlExprArray & found;
-};
-
-
-void gatherSelectExprs(HqlExprArray & target, IHqlExpression * expr)
-{
-    SelectCollectingTransformer collector(target);
-    collector.analyse(expr, 0);
-}
-
-//------------------------------------------------------------------------------------------------
-
-static HqlTransformerInfo fieldAccessAnalyserInfo("FieldAccessAnalyser");
-FieldAccessAnalyser::FieldAccessAnalyser(IHqlExpression * _selector) : NewHqlTransformer(fieldAccessAnalyserInfo), selector(_selector)
-{
-    unwindFields(fields, selector->queryRecord());
-    numAccessed = 0;
-    accessed.setown(createBitSet());
-}
-
-IHqlExpression * FieldAccessAnalyser::queryLastFieldAccessed() const
-{
-    if (numAccessed == 0)
-        return NULL;
-    if (accessedAll())
-        return &fields.tos();
-    ForEachItemInRev(i, fields)
-    {
-        if (accessed->test(i))
-            return &fields.item(i);
-    }
-    throwUnexpected();
-}
-
-void FieldAccessAnalyser::analyseExpr(IHqlExpression * expr)
-{
-    if (accessedAll() || alreadyVisited(expr))
-        return;
-    if (expr == selector)
-    {
-        setAccessedAll();
-        return;
-    }
-    if (expr->getOperator() == no_select)
-    {
-        if (expr->queryChild(0) == selector)
-        {
-            unsigned match = fields.find(*expr->queryChild(1));
-            assertex(match != NotFound);
-            if (!accessed->test(match))
-            {
-                accessed->set(match);
-                numAccessed++;
-            }
-            return;
-        }
-    }
-    NewHqlTransformer::analyseExpr(expr);
-}
-
-
-void FieldAccessAnalyser::analyseSelector(IHqlExpression * expr)
-{
-    if (expr == selector)
-    {
-        setAccessedAll();
-        return;
-    }
-    if (expr->getOperator() == no_select)
-    {
-        if (expr->queryChild(0) == selector)
-        {
-            unsigned match = fields.find(*expr->queryChild(1));
-            assertex(match != NotFound);
-            if (!accessed->test(match))
-            {
-                accessed->set(match);
-                numAccessed++;
-            }
-            return;
-        }
-    }
-    NewHqlTransformer::analyseSelector(expr);
-}
-
-
 //------------------------------------------------------------------------------------------------
 
 /*

+ 0 - 45
ecl/hql/hqltrans.ipp

@@ -1158,22 +1158,6 @@ inline bool activityHidesSelector(IHqlExpression * expr, IHqlExpression * select
     return activityHidesSelectorGetNumNonHidden(expr, selector) != 0;
 }
 
-
-class HQL_API HqlSelectorAnywhereLocator : public NewHqlTransformer
-{
-public:
-    HqlSelectorAnywhereLocator(IHqlExpression * _selector);
-
-    virtual void analyseExpr(IHqlExpression * expr);
-    virtual void analyseSelector(IHqlExpression * expr);
-
-    bool containsSelector(IHqlExpression * expr);
-
-protected:
-    bool foundSelector;
-    OwnedHqlExpr selector;
-};
-
 class SplitterVerifierInfo : public NewTransformInfo
 {
 public:
@@ -1194,29 +1178,6 @@ protected:
     inline SplitterVerifierInfo * queryExtra(IHqlExpression * expr)     { return static_cast<SplitterVerifierInfo *>(queryTransformExtra(expr)); }
 };
 
-extern HQL_API bool containsSelector(IHqlExpression * expr, IHqlExpression * selector);
-extern HQL_API bool containsSelectorAnywhere(IHqlExpression * expr, IHqlExpression * selector);         // searches through nested "hidden" definitions
-
-class HQL_API FieldAccessAnalyser : public NewHqlTransformer
-{
-public:
-    FieldAccessAnalyser(IHqlExpression * selector);
-
-    inline bool accessedAll() const { return numAccessed == fields.ordinality(); }
-    IHqlExpression * queryLastFieldAccessed() const;
-
-protected:
-    virtual void analyseExpr(IHqlExpression * expr);
-    virtual void analyseSelector(IHqlExpression * expr);
-
-    inline void setAccessedAll() { numAccessed = fields.ordinality(); }
-
-protected:
-    LinkedHqlExpr selector;
-    HqlExprCopyArray fields;
-    Owned<IBitSet> accessed;
-    unsigned numAccessed;
-};
 /*
 
 If something can be transformed more than one way depending on the context then must either
@@ -1236,14 +1197,8 @@ extern HQL_API IHqlExpression * queryNewReplaceSelector(IHqlExpression * expr, I
 extern HQL_API IHqlExpression * expandCreateRowSelectors(IHqlExpression * expr);
 extern HQL_API void verifySplitConsistency(IHqlExpression * expr);
 extern HQL_API IHqlExpression * convertWorkflowToImplicitParmeters(HqlExprArray & parameters, HqlExprArray & defaults, IHqlExpression * expr);
-extern HQL_API unsigned getNumUniqueExpressions(IHqlExpression * expr);
-extern HQL_API unsigned getNumUniqueExpressions(const HqlExprArray & exprs);
-extern HQL_API unsigned getNumOccurences(HqlExprArray & exprs, IHqlExpression * search, unsigned limit);
-extern HQL_API void logTreeStats(IHqlExpression * expr);
-extern HQL_API void logTreeStats(const HqlExprArray & exprs);
 extern HQL_API IHqlExpression * quickFullReplaceExpression(IHqlExpression * expr, IHqlExpression * oldValue, IHqlExpression * newValue);
 extern HQL_API IHqlExpression * quickFullReplaceExpressions(IHqlExpression * expr, const HqlExprArray & oldValues, const HqlExprArray & newValues);
-extern HQL_API void gatherSelectExprs(HqlExprArray & target, IHqlExpression * expr);
 extern HQL_API void dbglogTransformStats(bool reset);
 
 #ifdef OPTIMIZE_TRANSFORM_ALLOCATOR

+ 594 - 0
ecl/hql/hqlusage.cpp

@@ -0,0 +1,594 @@
+/*##############################################################################
+
+    Copyright (C) 2011 HPCC Systems.
+
+    All rights reserved. This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as
+    published by the Free Software Foundation, either version 3 of the
+    License, or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+############################################################################## */
+
+#include "jlib.hpp"
+#include "hqltrans.ipp"
+#include "hqlusage.hpp"
+
+#include "hqlutil.hpp"
+#include "hqlpmap.hpp"
+#include "hqlthql.hpp"
+
+//#define SEARCH_FILENAME "xxx"
+//#define SEARCH_FIELD "field1"
+
+#ifdef SEARCH_FIELD
+static void breakOnMatchField()
+{
+    strlen("");
+}
+#endif
+
+class ExpressionStatsInfo
+{
+public:
+    enum { MaxOperands = 17 };
+public:
+    ExpressionStatsInfo() { count = 0; _clear(numOperands); countMax = 0; sumMax = 0; }
+
+    void trace()
+    {
+        DBGLOG("numUnique %u", count);
+        for (unsigned i=0; i < MaxOperands; i++)
+            DBGLOG("  %u operands: %u", i, numOperands[i]);
+        DBGLOG("  %u experessions total %u operands", countMax, sumMax);
+    }
+
+    unsigned count;
+    unsigned numOperands[MaxOperands];
+    unsigned countMax;
+    unsigned sumMax;
+};
+
+static void calcNumUniqueExpressions(IHqlExpression * expr, ExpressionStatsInfo & info)
+{
+    if (expr->queryTransformExtra())
+        return;
+    expr->setTransformExtraUnlinked(expr);
+
+    //use head recursion
+    loop
+    {
+        info.count++;
+        unsigned max = expr->numChildren();
+        if (max >= ExpressionStatsInfo::MaxOperands)
+        {
+            info.countMax++;
+            info.sumMax += max;
+        }
+        else
+            info.numOperands[max]++;
+
+        if (max == 0)
+            return;
+
+        for (unsigned idx=1; idx < max; idx++)
+            calcNumUniqueExpressions(expr->queryChild(idx), info);
+        expr = expr->queryChild(0);
+    }
+}
+
+unsigned getNumUniqueExpressions(IHqlExpression * expr)
+{
+    TransformMutexBlock block;
+    ExpressionStatsInfo info;
+    calcNumUniqueExpressions(expr, info);
+    return info.count;
+}
+
+
+unsigned getNumUniqueExpressions(const HqlExprArray & exprs)
+{
+    TransformMutexBlock block;
+    ExpressionStatsInfo info;
+    ForEachItemIn(i, exprs)
+        calcNumUniqueExpressions(&exprs.item(i),info);
+    return info.count;
+}
+
+//------------------------------------------------------------------------------------------------
+
+static HqlTransformerInfo quickExpressionCounterInfo("QuickExpressionCounter");
+class HQL_API QuickExpressionCounter : public QuickHqlTransformer
+{
+public:
+    QuickExpressionCounter(IHqlExpression * _search, unsigned _limit)
+    : QuickHqlTransformer(quickExpressionCounterInfo, NULL), search(_search), limit(_limit)
+    {
+        matches = 0;
+    }
+
+    void analyse(IHqlExpression * expr)
+    {
+        if (expr == search)
+            matches++;
+        if (matches >= limit)
+            return;
+        QuickHqlTransformer::analyse(expr);
+    }
+
+    bool limitReached() const { return matches >= limit; }
+    unsigned numMatches() const { return matches; }
+
+protected:
+    HqlExprAttr search;
+    unsigned matches;
+    unsigned limit;
+};
+
+
+
+extern HQL_API unsigned getNumOccurences(HqlExprArray & exprs, IHqlExpression * search, unsigned limit)
+{
+    QuickExpressionCounter counter(search, limit);
+    ForEachItemIn(i, exprs)
+        counter.analyse(&exprs.item(i));
+    return counter.numMatches();
+}
+
+
+extern HQL_API void logTreeStats(IHqlExpression * expr)
+{
+    TransformMutexBlock block;
+    ExpressionStatsInfo info;
+    calcNumUniqueExpressions(expr,info);
+    info.trace();
+}
+
+extern HQL_API void logTreeStats(const HqlExprArray & exprs)
+{
+    TransformMutexBlock block;
+    ExpressionStatsInfo info;
+    ForEachItemIn(i, exprs)
+        calcNumUniqueExpressions(&exprs.item(i),info);
+    info.trace();
+}
+
+//---------------------------------------------------------------------------
+
+HQL_API bool containsSelector(IHqlExpression * expr, IHqlExpression * selector)
+{
+    return exprReferencesDataset(expr, selector);
+}
+
+//---------------------------------------------------------------------------
+
+static HqlTransformerInfo hqlSelectorAnywhereLocatorInfo("HqlSelectorAnywhereLocator");
+HqlSelectorAnywhereLocator::HqlSelectorAnywhereLocator(IHqlExpression * _selector) : NewHqlTransformer(hqlSelectorAnywhereLocatorInfo)
+{
+    selector.set(_selector);
+    foundSelector = false;
+}
+
+
+void HqlSelectorAnywhereLocator::analyseExpr(IHqlExpression * expr)
+{
+    if (foundSelector || alreadyVisited(expr))
+        return;
+    NewHqlTransformer::analyseExpr(expr);
+}
+
+void HqlSelectorAnywhereLocator::analyseSelector(IHqlExpression * expr)
+{
+    if (expr == selector)
+    {
+        foundSelector = true;
+        return;
+    }
+    NewHqlTransformer::analyseSelector(expr);
+}
+
+bool HqlSelectorAnywhereLocator::containsSelector(IHqlExpression * expr)
+{
+    foundSelector = false;
+    analyse(expr, 0);
+    return foundSelector;
+}
+
+HQL_API bool containsSelectorAnywhere(IHqlExpression * expr, IHqlExpression * selector)
+{
+    HqlSelectorAnywhereLocator locator(selector);
+    return locator.containsSelector(expr);
+}
+
+//------------------------------------------------------------------------------------------------
+
+static HqlTransformerInfo selectCollectingTransformerInfo("SelectCollectingTransformer");
+class SelectCollectingTransformer : public NewHqlTransformer
+{
+public:
+    SelectCollectingTransformer(HqlExprArray & _found)
+    : NewHqlTransformer(selectCollectingTransformerInfo), found(_found)
+    {
+    }
+
+    virtual void analyseExpr(IHqlExpression * expr)
+    {
+        if (alreadyVisited(expr))
+            return;
+        if (expr->getOperator() == no_select)
+        {
+            if (!found.contains(*expr))
+                found.append(*LINK(expr));
+            return;
+        }
+        NewHqlTransformer::analyseExpr(expr);
+    }
+
+protected:
+    HqlExprArray & found;
+};
+
+
+void gatherSelectExprs(HqlExprArray & target, IHqlExpression * expr)
+{
+    SelectCollectingTransformer collector(target);
+    collector.analyse(expr, 0);
+}
+
+//------------------------------------------------------------------------------------------------
+
+static HqlTransformerInfo fieldAccessAnalyserInfo("FieldAccessAnalyser");
+FieldAccessAnalyser::FieldAccessAnalyser(IHqlExpression * _selector) : NewHqlTransformer(fieldAccessAnalyserInfo), selector(_selector)
+{
+    unwindFields(fields, selector->queryRecord());
+    numAccessed = 0;
+    accessed.setown(createBitSet());
+}
+
+IHqlExpression * FieldAccessAnalyser::queryLastFieldAccessed() const
+{
+    if (numAccessed == 0)
+        return NULL;
+    if (accessedAll())
+        return &fields.tos();
+    ForEachItemInRev(i, fields)
+    {
+        if (accessed->test(i))
+            return &fields.item(i);
+    }
+    throwUnexpected();
+}
+
+void FieldAccessAnalyser::analyseExpr(IHqlExpression * expr)
+{
+    if (accessedAll() || alreadyVisited(expr))
+        return;
+    if (expr == selector)
+    {
+        setAccessedAll();
+        return;
+    }
+    if (expr->getOperator() == no_select)
+    {
+        if (expr->queryChild(0) == selector)
+        {
+            unsigned match = fields.find(*expr->queryChild(1));
+            assertex(match != NotFound);
+            if (!accessed->test(match))
+            {
+                accessed->set(match);
+                numAccessed++;
+            }
+            return;
+        }
+    }
+    NewHqlTransformer::analyseExpr(expr);
+}
+
+
+void FieldAccessAnalyser::analyseSelector(IHqlExpression * expr)
+{
+    if (expr == selector)
+    {
+        setAccessedAll();
+        return;
+    }
+    if (expr->getOperator() == no_select)
+    {
+        if (expr->queryChild(0) == selector)
+        {
+            unsigned match = fields.find(*expr->queryChild(1));
+            assertex(match != NotFound);
+            if (!accessed->test(match))
+            {
+                accessed->set(match);
+                numAccessed++;
+            }
+            return;
+        }
+    }
+    NewHqlTransformer::analyseSelector(expr);
+}
+
+
+//------------------------------------------------------------------------------------------------
+
+static void expandSelectText(StringBuffer & s, IHqlExpression * expr)
+{
+    if (expr->getOperator() != no_field)
+    {
+        assertex(expr->getOperator() == no_select);
+        IHqlExpression * ds = expr->queryChild(0);
+        IHqlExpression * field = expr->queryChild(1);
+        if (ds != queryActiveTableSelector())
+        {
+            expandSelectText(s, ds);
+            s.append(".");
+        }
+        s.append(field->queryName());
+    }
+    else
+        s.append(expr->queryName());
+}
+
+
+static IPropertyTree * addSelect(IPropertyTree * xml, IHqlExpression * expr, bool isUsed)
+{
+    StringBuffer text;
+    expandSelectText(text, expr);
+    Owned<IPropertyTree> field = createPTree(isUsed ? "field" : "unused");
+    field->setProp("@name", text.str());
+    return xml->addPropTree(field->queryName(), field.getClear());
+}
+
+SourceFieldUsage::SourceFieldUsage(IHqlExpression * _source)
+: source(_source)
+{
+    usedAll = false;
+    usedFilepos = false;
+}
+
+void SourceFieldUsage::noteSelect(IHqlExpression * select, IHqlExpression * selector)
+{
+#ifdef SEARCH_FIELD
+    if (select->queryChild(1)->queryName() == createAtom(SEARCH_FIELD))
+    {
+        if (matchesConstantString(queryFilename(), SEARCH_FILENAME, true))
+        {
+            breakOnMatchField();
+        }
+    }
+#endif
+
+    //MORE: For simple selectors may be more efficient to search before replacing the selector.
+    OwnedHqlExpr mapped = replaceSelector(select, selector, queryActiveTableSelector());
+    //MORE: May need to use a hash table.
+    if (!selects.contains(*mapped))
+        selects.append(*mapped.getClear());
+}
+
+IHqlExpression * SourceFieldUsage::queryFilename() const
+{
+    switch (source->getOperator())
+    {
+    case no_newkeyindex:
+        return source->queryChild(3);
+    case no_table:
+        return source->queryChild(0);
+    }
+    UNIMPLEMENTED;
+    return NULL;
+}
+
+IPropertyTree * SourceFieldUsage::createReport() const
+{
+    bool sourceIsKey = isKey(source);
+    Owned<IPropertyTree> entry = createPTree(sourceIsKey ? "index" : "dataset");
+    IHqlExpression * filename = queryFilename();
+    if (filename)
+    {
+        StringBuffer nameText;
+        getExprECL(filename, nameText);
+        entry->setProp("@name", nameText);
+    }
+
+    unsigned numFields = 0;
+    unsigned numFieldsUsed = 0;
+    expandSelects(entry, source->queryRecord(), queryActiveTableSelector(), usedAll, numFields, numFieldsUsed);
+    if (isKey(source))
+    {
+        IHqlExpression * original = queryPropertyChild(source, _original_Atom, 0);
+        if (!original)
+            original = source;
+        IHqlExpression * lastField = queryLastField(original->queryRecord());
+        if (usedFilepos || !lastField->hasProperty(_implicitFpos_Atom))
+        {
+            numFields++;
+            if (usedFilepos || usedAll)
+            {
+                addSelect(entry, lastField, true);
+                numFieldsUsed++;
+            }
+        }
+    }
+
+    entry->setPropInt("@numFields", numFields);
+    entry->setPropInt("@numFieldsUsed", numFieldsUsed);
+    return entry.getClear();
+}
+
+
+void SourceFieldUsage::expandSelects(IPropertyTree * xml, IHqlExpression * record, IHqlExpression * selector, bool allUsed, unsigned & numFields, unsigned & numFieldsUsed) const
+{
+    bool seenAll = true;
+    ForEachChild(i, record)
+    {
+        IHqlExpression * cur = record->queryChild(i);
+        switch (cur->getOperator())
+        {
+        case no_field:
+            {
+                OwnedHqlExpr selected = createSelectExpr(LINK(selector), LINK(cur));
+                bool thisUsed = allUsed || selects.contains(*selected);
+                if (cur->isDatarow())
+                {
+                    expandSelects(xml, cur->queryRecord(), selected, thisUsed, numFields, numFieldsUsed);
+                }
+                else
+                {
+                    numFields++;
+                    if (thisUsed)
+                    {
+                        addSelect(xml, selected, thisUsed);
+                        numFieldsUsed++;
+                    }
+                    else
+                    {
+                        //could have an option to output unused fields, with code like the following:
+                        //addSelect(xml, selected, thisUsed);
+                        seenAll = false;
+                    }
+                }
+                break;
+            }
+        case no_record:
+            expandSelects(xml, cur, selector, allUsed, numFields, numFieldsUsed);
+            break;
+        case no_ifblock:
+            //MORE: Theoretically if any of the fields within the ifblock are used, then the fields
+            //used in the ifblock condition are also used.  Needs to be handled by a preprocessing step.
+            expandSelects(xml, cur->queryChild(1), selector, allUsed, numFields, numFieldsUsed);
+            break;
+        }
+    }
+}
+
+
+//------------------------------------------------------------------------------------------------
+
+static HqlTransformerInfo sourceFieldTrackerInfo("SourceFieldTracker");
+class SourceFieldTracker : public NewHqlTransformer
+{
+public:
+    SourceFieldTracker(SourceFieldUsage * _fieldUsage, IHqlExpression * _selector)
+        : NewHqlTransformer(sourceFieldTrackerInfo), fieldUsage(_fieldUsage), selector(_selector)
+    {
+    }
+
+    virtual void analyseExpr(IHqlExpression * expr);
+
+protected:
+    bool isSelected(IHqlExpression * expr) const;
+
+protected:
+    SourceFieldUsage * fieldUsage;
+    IHqlExpression * selector;
+};
+
+void SourceFieldTracker::analyseExpr(IHqlExpression * expr)
+{
+    if (fieldUsage->seenAll() || alreadyVisited(expr))
+        return;
+    if (expr == selector)
+    {
+        fieldUsage->noteAll();
+        return;
+    }
+
+    if (isSelected(expr))
+    {
+        fieldUsage->noteSelect(expr->queryNormalizedSelector(), selector);
+        return;
+    }
+
+    switch (expr->getOperator())
+    {
+    case no_filepos:
+        if (expr->queryChild(0) == selector)
+        {
+            fieldUsage->noteFilepos();
+            return;
+        }
+        break;
+    }
+
+    NewHqlTransformer::analyseExpr(expr);
+}
+
+
+bool SourceFieldTracker::isSelected(IHqlExpression * expr) const
+{
+    loop
+    {
+        if (expr->getOperator() != no_select)
+            return false;
+        IHqlExpression * ds = expr->queryChild(0);
+        if (ds->queryNormalizedSelector() == selector)
+            return true;
+        expr = ds;
+    }
+}
+
+//------------------------------------------------------------------------------------------------
+
+void gatherFieldUsage(SourceFieldUsage * fieldUsage, IHqlExpression * expr, IHqlExpression * selector)
+{
+    unsigned first = getNumChildTables(expr);
+    SourceFieldTracker tracker(fieldUsage, selector);
+    ForEachChildFrom(i, expr, first)
+        tracker.analyse(expr->queryChild(i), 0);
+}
+
+void gatherParentFieldUsage(SourceFieldUsage * fieldUsage, IHqlExpression * expr)
+{
+    bool hasDs = false;
+    bool hasLeft = false;
+    bool hasRight = false;
+    switch (getChildDatasetType(expr))
+    {
+    case childdataset_dataset:
+        hasDs = true;
+        break;
+    case childdataset_datasetleft:
+        hasDs = true;
+        hasLeft = true;
+        break;
+    case childdataset_left:
+        hasLeft = true;
+        break;
+    case childdataset_same_left_right:
+        hasLeft = true;
+        hasRight = true;
+        break;
+    case childdataset_top_left_right:
+        hasDs = true;
+        hasLeft = true;
+        hasRight = true;
+        break;
+    case childdataset_leftright: // e.g., no_aggregate
+        hasLeft = true;
+        break;
+    }
+
+    IHqlExpression * ds = expr->queryChild(0);
+    IHqlExpression * selSeq = querySelSeq(expr);
+    //MORE: Do all this in a single pass
+    if (hasDs)
+        gatherFieldUsage(fieldUsage, expr, ds->queryNormalizedSelector());
+    if (hasLeft)
+    {
+        OwnedHqlExpr left = createSelector(no_left, ds, selSeq);
+        gatherFieldUsage(fieldUsage, expr, left);
+    }
+    if (hasRight)
+    {
+        OwnedHqlExpr right = createSelector(no_right, ds, selSeq);
+        gatherFieldUsage(fieldUsage, expr, right);
+    }
+}

+ 95 - 0
ecl/hql/hqlusage.hpp

@@ -0,0 +1,95 @@
+/*##############################################################################
+
+    Copyright (C) 2011 HPCC Systems.
+
+    All rights reserved. This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as
+    published by the Free Software Foundation, either version 3 of the
+    License, or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+############################################################################## */
+
+#ifndef __HQLUSAGE_HPP_
+#define __HQLUSAGE_HPP_
+
+class HQL_API HqlSelectorAnywhereLocator : public NewHqlTransformer
+{
+public:
+    HqlSelectorAnywhereLocator(IHqlExpression * _selector);
+
+    virtual void analyseExpr(IHqlExpression * expr);
+    virtual void analyseSelector(IHqlExpression * expr);
+
+    bool containsSelector(IHqlExpression * expr);
+
+protected:
+    bool foundSelector;
+    OwnedHqlExpr selector;
+};
+
+class HQL_API FieldAccessAnalyser : public NewHqlTransformer
+{
+public:
+    FieldAccessAnalyser(IHqlExpression * selector);
+
+    inline bool accessedAll() const { return numAccessed == fields.ordinality(); }
+    IHqlExpression * queryLastFieldAccessed() const;
+
+protected:
+    virtual void analyseExpr(IHqlExpression * expr);
+    virtual void analyseSelector(IHqlExpression * expr);
+
+    inline void setAccessedAll() { numAccessed = fields.ordinality(); }
+
+protected:
+    LinkedHqlExpr selector;
+    HqlExprCopyArray fields;
+    Owned<IBitSet> accessed;
+    unsigned numAccessed;
+};
+
+class HQL_API SourceFieldUsage : public CInterface
+{
+public:
+    SourceFieldUsage(IHqlExpression * _source);
+
+    void noteSelect(IHqlExpression * select, IHqlExpression * selector);
+    inline void noteAll() { usedAll = true; }
+    inline void noteFilepos() { usedFilepos = true; }
+
+    IHqlExpression * queryFilename() const;
+    inline bool matches(IHqlExpression * search) const { return source == search; }
+    inline bool seenAll() const { return usedAll; }
+
+    IPropertyTree * createReport() const;
+
+protected:
+    void expandSelects(IPropertyTree * xml, IHqlExpression * record, IHqlExpression * selector, bool allUsed, unsigned & numFields, unsigned & numFieldsUsed) const;
+
+protected:
+    LinkedHqlExpr source;
+    HqlExprArray selects;
+    bool usedAll;
+    bool usedFilepos;
+};
+
+
+extern HQL_API unsigned getNumUniqueExpressions(IHqlExpression * expr);
+extern HQL_API unsigned getNumUniqueExpressions(const HqlExprArray & exprs);
+extern HQL_API unsigned getNumOccurences(HqlExprArray & exprs, IHqlExpression * search, unsigned limit);
+extern HQL_API void logTreeStats(IHqlExpression * expr);
+extern HQL_API void logTreeStats(const HqlExprArray & exprs);
+extern HQL_API void gatherSelectExprs(HqlExprArray & target, IHqlExpression * expr);
+extern HQL_API bool containsSelector(IHqlExpression * expr, IHqlExpression * selector);
+extern HQL_API bool containsSelectorAnywhere(IHqlExpression * expr, IHqlExpression * selector);         // searches through nested "hidden" definitions
+extern HQL_API void gatherFieldUsage(SourceFieldUsage * fieldUsage, IHqlExpression * expr, IHqlExpression * selector);
+extern HQL_API void gatherParentFieldUsage(SourceFieldUsage * fieldUsage, IHqlExpression * expr);
+
+#endif

+ 1 - 0
ecl/hql/hqlutil.cpp

@@ -30,6 +30,7 @@
 #include "hqlfold.hpp"
 #include "hqlerrors.hpp"
 #include "hqltrans.ipp"
+#include "hqlusage.hpp"
 #include "hqlthql.hpp"
 #include "deffield.hpp"
 #include "workunit.hpp"

+ 56 - 6
ecl/hqlcpp/hqlckey.cpp

@@ -276,6 +276,7 @@ protected:
     HqlExprAttr     expandedKey;
     HqlExprAttr     file;
     HqlExprAttr     expandedFile;
+    HqlExprAttr     rawFile;
     HqlExprAttr     keyAccessDataset;
     HqlExprAttr     keyAccessTransform;
     HqlExprAttr     fileAccessDataset;
@@ -311,6 +312,7 @@ KeyedJoinInfo::KeyedJoinInfo(HqlCppTranslator & _translator, IHqlExpression * _e
         if (!rightTable || rightTable->queryNormalizedSelector() != right->queryNormalizedSelector())
             translator.throwError(HQLERR_FullKeyedNeedsFile);
         expandedFile.setown(convertToPhysicalTable(rightTable, true));
+        rawFile.set(queryPhysicalRootTable(expandedFile));
         keyedMapper.setDataset(key);
     }
     else if (right->getOperator() == no_newkeyindex)
@@ -333,7 +335,7 @@ KeyedJoinInfo::KeyedJoinInfo(HqlCppTranslator & _translator, IHqlExpression * _e
     counter.set(queryPropertyChild(expr, _countProject_Atom, 0));
 
     if (isFullJoin())
-        rawRhs.set(queryPhysicalRootTable(expandedFile));
+        rawRhs.set(rawFile);
     else
         rawRhs.set(rawKey);
 }
@@ -595,9 +597,17 @@ static IHqlExpression * expandDatasetReferences(IHqlExpression * expr, IHqlExpre
 
 IHqlExpression * KeyedJoinInfo::expandDatasetReferences(IHqlExpression * transform, IHqlExpression * ds)
 {
+    switch (ds->getOperator())
+    {
+    case no_newusertable:
+    case no_hqlproject:
+        break;
+    default:
+        return LINK(transform);
+    }
+
     OwnedHqlExpr oldRight = createSelector(no_right, ds, joinSeq);
     OwnedHqlExpr newRight = createSelector(no_right, ds->queryChild(0), joinSeq);
-
     return ::expandDatasetReferences(transform, ds, oldRight, newRight);
 }
 
@@ -634,10 +644,7 @@ void KeyedJoinInfo::buildTransformBody(BuildCtx & ctx, IHqlExpression * transfor
     else
     {
         if (isFullJoin())
-        {
-            if (expandedFile != queryPhysicalRootTable(file))
-                newTransform.setown(expandDatasetReferences(newTransform, expandedFile));
-        }
+            newTransform.setown(expandDatasetReferences(newTransform, expandedFile));
         else
             newTransform.setown(expandDatasetReferences(newTransform, expandedKey));
     }
@@ -656,6 +663,27 @@ void KeyedJoinInfo::buildTransformBody(BuildCtx & ctx, IHqlExpression * transfor
 
     ctx.associateExpr(fileposExpr, fileposVar);
     translator.doBuildTransformBody(ctx, newTransform, selfCursor);
+
+    if (isFullJoin())
+    {
+        SourceFieldUsage * fileUsage = translator.querySourceFieldUsage(rawFile);
+        if (fileUsage)
+        {
+            OwnedHqlExpr rawTransform = expandDatasetReferences(transform, expandedFile);
+            OwnedHqlExpr right = createSelector(no_right, rawFile, joinSeq);
+            ::gatherFieldUsage(fileUsage, rawTransform, right);
+        }
+    }
+    else
+    {
+        SourceFieldUsage * keyUsage = translator.querySourceFieldUsage(rawKey);
+        if (keyUsage)
+        {
+            OwnedHqlExpr rawTransform = expandDatasetReferences(transform, expandedKey);
+            OwnedHqlExpr right = createSelector(no_right, rawKey, joinSeq);
+            ::gatherFieldUsage(keyUsage, rawTransform, right);
+        }
+    }
 }
 
 void KeyedJoinInfo::buildFailureTransform(BuildCtx & ctx, IHqlExpression * onFailTransform)
@@ -1091,6 +1119,25 @@ bool KeyedJoinInfo::processFilter()
     if (expr->getOperator() != no_keyeddistribute)
         optimizeExtractJoinFields();
 
+    SourceFieldUsage * keyUsage = translator.querySourceFieldUsage(rawKey);
+    if (keyUsage)
+    {
+        gatherFieldUsage(keyUsage, newFilter, rawKey->queryNormalizedSelector());
+        if (isFullJoin())
+            keyUsage->noteFilepos();
+    }
+
+    if (file && fileFilter)
+    {
+        SourceFieldUsage * fileUsage = translator.querySourceFieldUsage(rawFile);
+        if (fileUsage)
+        {
+            OwnedHqlExpr rawFilter = expandDatasetReferences(fileFilter, expandedFile);
+            OwnedHqlExpr fileRight = createSelector(no_right, rawFile, joinSeq);
+            gatherFieldUsage(fileUsage, rawFilter, fileRight);
+        }
+    }
+
     return monitors->isKeyed();
 }
 
@@ -1503,9 +1550,11 @@ ABoundActivity * HqlCppTranslator::doBuildActivityKeyDiff(BuildCtx & ctx, IHqlEx
 
     //virtual const char * queryOriginalName() = 0;         // may be null
     buildRefFilenameFunction(*instance, instance->startctx, "queryOriginalName", original);
+    noteAllFieldsUsed(original);
 
     //virtual const char * queryPatchName() = 0;
     buildRefFilenameFunction(*instance, instance->startctx, "queryUpdatedName", updated);
+    noteAllFieldsUsed(updated);
 
     //virtual const char * queryOutputName() = 0;
     buildFilenameFunction(*instance, instance->startctx, "queryOutputName", output, hasDynamicFilename(expr));
@@ -1545,6 +1594,7 @@ ABoundActivity * HqlCppTranslator::doBuildActivityKeyPatch(BuildCtx & ctx, IHqlE
 
     //virtual const char * queryOriginalName() = 0;
     buildRefFilenameFunction(*instance, instance->startctx, "queryOriginalName", original);
+    noteAllFieldsUsed(original);
 
     //virtual const char * queryPatchName() = 0;
     buildFilenameFunction(*instance, instance->startctx, "queryPatchName", patch, true);

+ 2 - 0
ecl/hqlcpp/hqlcpp.cpp

@@ -53,6 +53,7 @@
 #include "hqlcse.ipp"
 #include "thorplugin.hpp"
 #include "hqlinline.hpp"
+#include "hqlusage.hpp"
 
 #ifdef _DEBUG
 //#define ADD_ASSIGNMENT_COMMENTS
@@ -1679,6 +1680,7 @@ void HqlCppTranslator::cacheOptions()
         DebugOption(options.implicitGroupShuffle,"implicitGroupShuffle",false),
         DebugOption(options.implicitGroupHashAggregate,"implicitGroupHashAggregate",false),
         DebugOption(options.implicitGroupHashDedup,"implicitGroupHashDedup",false),
+        DebugOption(options.reportFieldUsage,"reportFieldUsage",false),
         DebugOption(options.shuffleLocalJoinConditions,"shuffleLocalJoinConditions",false),
     };
 

+ 7 - 0
ecl/hqlcpp/hqlcpp.ipp

@@ -28,6 +28,7 @@
 #include "hqlutil.hpp"
 #include "hqlwcpp.hpp"
 #include "hqltrans.ipp"
+#include "hqlusage.hpp"
 
 #define DEBUG_TIMER(name, time)                     if (options.addTimingToWorkunit) { timeReporter->addTiming(name, time); }
 #define DEBUG_TIMERX(timeReporter, name, time)      if (timeReporter) { timeReporter->addTiming(name, time); }
@@ -708,6 +709,7 @@ struct HqlCppOptions
     bool                implicitGroupShuffle;  // use shuffle if some sort conditions match when grouping
     bool                implicitGroupHashAggregate;  // convert aggreate(sort(x,a),{..},a,d) to aggregate(group(sort(x,a),a_,{},d))
     bool                implicitGroupHashDedup;
+    bool                reportFieldUsage;
     bool                shuffleLocalJoinConditions;
 };
 
@@ -892,6 +894,7 @@ public:
     void useFunction(IHqlExpression * funcdef);
     void useLibrary(const char * libname);
     void finalizeResources();
+    void generateStatistics(const char * targetDir);
 
             unsigned getHints()                             { return hints; }
     inline  bool checkForRowOverflow() const                { return options.checkRowOverflow; }
@@ -1798,6 +1801,9 @@ protected:
     bool getDebugFlag(const char * name, bool defValue);
     void initOptions();
     void postProcessOptions();
+    SourceFieldUsage * querySourceFieldUsage(IHqlExpression * expr);
+    void reportFieldUsage(const char * filename);
+    void noteAllFieldsUsed(IHqlExpression * expr);
 
 public:
     IHqlExpression * convertToPhysicalIndex(IHqlExpression * tableExpr);
@@ -1881,6 +1887,7 @@ protected:
     PointerArray recordIndexCache;
     Owned<ITimeReporter> timeReporter;
     WarningProcessor warningProcessor;
+    CIArrayOf<SourceFieldUsage> trackedSources;
 };
 
 

+ 1 - 0
ecl/hqlcpp/hqlcppds.cpp

@@ -48,6 +48,7 @@
 #include "hqlccommon.hpp"
 #include "hqliter.ipp"
 #include "hqlinline.hpp"
+#include "hqlusage.hpp"
 
 #define MAX_FIXED_SIZE_RAW 1024
 #define INLINE_TABLE_EXPAND_LIMIT 4

+ 1 - 0
ecl/hqlcpp/hqlecl.cpp

@@ -266,6 +266,7 @@ bool HqlDllGenerator::generateCode(IHqlExpression * exprs)
                 wu->setState(WUStateCompleted);
                 return true;
             }
+            translator.generateStatistics(targetDir);
             translator.finalizeResources();
             translator.expandFunctions(true);
         }

+ 68 - 0
ecl/hqlcpp/hqlhtcpp.cpp

@@ -54,6 +54,7 @@
 #include "hqlccommon.hpp"
 #include "deffield.hpp"
 #include "hqlinline.hpp"
+#include "hqlusage.hpp"
 
 //The following are include to ensure they call compile...
 #include "eclhelper.hpp"
@@ -5702,6 +5703,73 @@ double HqlCppTranslator::getComplexity(HqlExprArray & exprs)
     return complexity;
 }
 
+//---------------------------------------------------------------------------------------------------------------------
+
+void HqlCppTranslator::reportFieldUsage(const char * filename)
+{
+    Owned<IPropertyTree> sources = createPTree("usedsources");
+    ForEachItemIn(i, trackedSources)
+    {
+        IPropertyTree * next = trackedSources.item(i).createReport();
+        sources->addPropTree(next->queryName(), next);
+    }
+
+    saveXML(filename, sources);
+}
+
+SourceFieldUsage * HqlCppTranslator::querySourceFieldUsage(IHqlExpression * expr)
+{
+    if (!options.reportFieldUsage || !expr)
+        return NULL;
+
+    if (expr->hasProperty(_spill_Atom) || expr->hasProperty(jobTempAtom))
+        return NULL;
+
+    OwnedHqlExpr normalized = removeProperty(expr, _uid_Atom);
+    IHqlExpression * original = normalized->queryProperty(_original_Atom);
+    if (original)
+    {
+        OwnedHqlExpr normalTable = removeProperty(original->queryChild(0), _uid_Atom);
+        OwnedHqlExpr normalOriginal = replaceChild(original, 0, normalTable);
+        normalized.setown(replaceOwnedProperty(normalized, normalOriginal.getClear()));
+    }
+
+    ForEachItemIn(i, trackedSources)
+    {
+        SourceFieldUsage & cur = trackedSources.item(i);
+        if (cur.matches(normalized))
+            return &cur;
+    }
+    SourceFieldUsage * next = new SourceFieldUsage(normalized);
+    trackedSources.append(*next);
+    return next;
+}
+
+void HqlCppTranslator::noteAllFieldsUsed(IHqlExpression * expr)
+{
+    SourceFieldUsage * match = querySourceFieldUsage(expr);
+    if (match)
+        match->noteAll();
+}
+
+void HqlCppTranslator::generateStatistics(const char * targetDir)
+{
+    if (options.reportFieldUsage && trackedSources.ordinality())
+    {
+        StringBuffer fullname;
+        addDirectoryPrefix(fullname, targetDir).append(soName).append("_fieldusage.xml");
+
+        reportFieldUsage(fullname);
+
+        Owned<IWUQuery> query = wu()->updateQuery();
+        associateLocalFile(query, FileTypeXml, fullname, "FieldUsage", 0);
+
+        ctxCallback->registerFile(fullname.str(), "FieldUsage");
+    }
+}
+
+//---------------------------------------------------------------------------------------------------------------------
+
 BoundRow * HqlCppTranslator::resolveDatasetRequired(BuildCtx & ctx, IHqlExpression * expr)
 {
     BoundRow * cursor = resolveSelectorDataset(ctx, expr);

+ 0 - 11
ecl/hqlcpp/hqlresource.cpp

@@ -544,17 +544,6 @@ IHqlExpression * CResourceOptions::createSpillName(bool isGraphResult)
 
 //---------------------------------------------------------------------------
 
-static bool isIndex(IHqlExpression * expr)
-{
-    switch (expr->getOperator())
-    {
-    case no_keyindex:
-    case no_newkeyindex:
-        return true;
-    }
-    return false;
-}
-
 IHqlExpression * appendUniqueAttr(IHqlExpression * expr)
 {
     return replaceOwnedProperty(expr, createUniqueId());

+ 52 - 13
ecl/hqlcpp/hqlsource.cpp

@@ -742,6 +742,7 @@ protected:
     IHqlExpression * ensureAggregateGroupingAliased(IHqlExpression * aggregate);
     void gatherSteppingMeta(IHqlExpression * expr, SourceSteppingInfo & info);
     void gatherSteppingMeta(IHqlExpression * expr, SteppingFieldSelection & outputStepping, SteppingFieldSelection & rawStepping);
+    void gatherFieldUsage(SourceFieldUsage * fieldUsage, IHqlExpression * expr);
     void rebindFilepositons(BuildCtx & ctx, IHqlExpression * dataset, node_operator side, IHqlExpression * selSeq, bool isLocal);
     void rebindFilepositons(BuildCtx & ctx, IHqlExpression * dataset, node_operator side, IHqlExpression * selSeq);
 
@@ -1686,6 +1687,44 @@ void SourceBuilder::extractMonitors(IHqlExpression * ds, SharedHqlExpr & unkeyed
 void SourceBuilder::analyseGraph(IHqlExpression * expr)
 {
     analyse(expr);
+    SourceFieldUsage * fieldUsage = translator.querySourceFieldUsage(tableExpr);
+    if (fieldUsage && !fieldUsage->seenAll())
+    {
+        if (expr->queryNormalizedSelector() == tableExpr->queryNormalizedSelector())
+            fieldUsage->noteAll();
+        else
+            gatherFieldUsage(fieldUsage, expr);
+    }
+}
+
+void SourceBuilder::gatherFieldUsage(SourceFieldUsage * fieldUsage, IHqlExpression * expr)
+{
+    loop
+    {
+        if (expr->queryBody() == tableExpr->queryBody())
+            return;
+        if (fieldUsage->seenAll())
+            return;
+
+        IHqlExpression * ds = expr->queryChild(0);
+        switch (expr->getOperator())
+        {
+        case no_fetch:
+            {
+                assertex(ds->queryBody() == tableExpr->queryBody());
+                IHqlExpression * selSeq = querySelSeq(expr);
+                OwnedHqlExpr left = createSelector(no_left, ds, selSeq);
+                ::gatherFieldUsage(fieldUsage, expr, left);
+                return;
+            }
+        }
+
+        assertex(getNumChildTables(expr) == 1);
+        if (ds->queryNormalizedSelector() == tableExpr->queryNormalizedSelector())
+            gatherParentFieldUsage(fieldUsage, expr);
+
+        expr = ds;
+    }
 }
 
 inline bool useDescriptiveGraphLabel(ThorActivityKind kind)
@@ -2859,7 +2898,7 @@ protected:
 
 void DiskNormalizeBuilder::analyseGraph(IHqlExpression * expr)
 {
-    analyse(expr);
+    DiskReadBuilderBase::analyseGraph(expr);
     needDefaultTransform = (expr->queryNormalizedSelector()->getOperator() == no_select);
 }
 
@@ -2926,7 +2965,7 @@ protected:
     }
     virtual void analyseGraph(IHqlExpression * expr)
     {
-        analyse(expr);
+        DiskReadBuilderBase::analyseGraph(expr);
         returnIfFilterFails = !isNormalize;
     }
 };
@@ -2995,7 +3034,7 @@ protected:
     }
     virtual void analyseGraph(IHqlExpression * expr)
     {
-        analyse(expr);
+        DiskReadBuilderBase::analyseGraph(expr);
         returnIfFilterFails = !isNormalize;
         if (aggOp == no_existsgroup)
             choosenValue.setown(getSizetConstant(1));
@@ -3091,7 +3130,7 @@ protected:
     }
     virtual void analyseGraph(IHqlExpression * expr)
     {
-        analyse(expr);
+        DiskReadBuilderBase::analyseGraph(expr);
         returnIfFilterFails = !isNormalize;
     }
 };
@@ -3179,7 +3218,7 @@ protected:
 
 void ChildNormalizeBuilder::analyseGraph(IHqlExpression * expr)
 {
-    analyse(expr);
+    ChildBuilderBase::analyseGraph(expr);
     needDefaultTransform = (expr->queryNormalizedSelector()->getOperator() == no_select);
 }
 
@@ -3234,7 +3273,7 @@ protected:
     }
     virtual void analyseGraph(IHqlExpression * expr)
     {
-        analyse(expr);
+        ChildBuilderBase::analyseGraph(expr);
         returnIfFilterFails = false;
     }
 };
@@ -3290,7 +3329,7 @@ protected:
     }
     virtual void analyseGraph(IHqlExpression * expr)
     {
-        analyse(expr);
+        ChildBuilderBase::analyseGraph(expr);
         returnIfFilterFails = false;
     }
 };
@@ -3359,7 +3398,7 @@ protected:
 
 void ChildThroughNormalizeBuilder::analyseGraph(IHqlExpression * expr)
 {
-    analyse(expr);
+    ChildBuilderBase::analyseGraph(expr);
     needDefaultTransform = (expr->queryNormalizedSelector()->getOperator() == no_select);
 }
 
@@ -6256,7 +6295,7 @@ public:
 
     virtual void analyseGraph(IHqlExpression * expr)
     {
-        analyse(expr);
+        IndexReadBuilderBase::analyseGraph(expr);
         gatherSteppingMeta(expr, steppingInfo);
         if (steppedExpr && transformCanFilter && translator.queryOptions().optimizeSteppingPostfilter)
         {
@@ -6358,7 +6397,7 @@ protected:
 
 void IndexNormalizeBuilder::analyseGraph(IHqlExpression * expr)
 {
-    analyse(expr);
+    IndexReadBuilderBase::analyseGraph(expr);
     needDefaultTransform = (expr->queryNormalizedSelector()->getOperator() == no_select);
 }
 
@@ -6425,7 +6464,7 @@ protected:
     }
     virtual void analyseGraph(IHqlExpression * expr)
     {
-        analyse(expr);
+        IndexReadBuilderBase::analyseGraph(expr);
         returnIfFilterFails = !isNormalize;
     }
 };
@@ -6495,7 +6534,7 @@ protected:
     }
     virtual void analyseGraph(IHqlExpression * expr)
     {
-        analyse(expr);
+        IndexReadBuilderBase::analyseGraph(expr);
         returnIfFilterFails = !isNormalize;
         IHqlExpression * aggregate = expr->queryChild(0);
         if (isKeyedCountAggregate(aggregate))
@@ -6604,7 +6643,7 @@ protected:
     }
     virtual void analyseGraph(IHqlExpression * expr)
     {
-        analyse(expr);
+        IndexReadBuilderBase::analyseGraph(expr);
         returnIfFilterFails = !isNormalize;
     }
 

+ 1 - 0
ecl/hqlcpp/hqltcppc.cpp

@@ -42,6 +42,7 @@
 #include "hqlpmap.hpp"
 #include "hqlutil.hpp"
 #include "hqlinline.hpp"
+#include "hqlusage.hpp"
 
 #define LIMIT_FOR_GET       (NULL)
 

+ 1 - 0
ecl/hqlcpp/hqltcppc2.cpp

@@ -40,6 +40,7 @@
 #include "hqlpmap.hpp"
 #include "hqlthql.hpp"
 #include "hqlattr.hpp"
+#include "hqlusage.hpp"
 
 //#define TraceExprPrintLog(x, expr)                PrintLog(x ": %s", expr->toString(StringBuffer()).str());