HPCC-12408 Various code generator optimizations

Signed-off-by: Gavin Halliday <gavin.halliday@lexisnexis.com>

common/deftype/deftype.hpp

@@ -39,6 +39,8 @@ typedef unsigned short UChar;
 
 interface ITypeInfo;
 interface IValue;
+interface IHqlExpression;
+interface IHqlScope;
 
 #if __BYTE_ORDER == __LITTLE_ENDIAN
 #define type_bigendianint       type_swapint
@@ -199,6 +201,8 @@ public:
     virtual unsigned getCrc() = 0;      // must be run independant.
     virtual typemod_t queryModifier() = 0;
     virtual IInterface * queryModifierExtra() = 0;
+    virtual IHqlExpression * castToExpression() = 0; // Here to avoid dynamic casts
+    virtual IHqlScope * castToScope() = 0;
 
     inline bool isBoolean() const { return getTypeCode() == type_boolean; }
 

common/deftype/deftype.ipp

@@ -64,6 +64,8 @@ public:
     virtual unsigned getCrc();
     virtual typemod_t queryModifier()           { return typemod_none; }
     virtual IInterface * queryModifierExtra()   { return NULL; }
+    virtual IHqlExpression * castToExpression() { return NULL; }
+    virtual IHqlScope * castToScope()           { return NULL; }
 
     virtual void serialize(MemoryBuffer &tgt)   { tgt.append((unsigned char) getTypeCode()); }
     virtual void deserialize(MemoryBuffer &tgt) { UNIMPLEMENTED; }

ecl/hql/hqlattr.cpp

@@ -802,6 +802,14 @@ public:
         {
         case no_field:
             {
+                //Remove the default value before transforming to avoid waste when processing before the expression tree is normalized
+                if (queryRealChild(expr, 0))
+                {
+                    HqlExprArray children;
+                    unwindChildren(children, expr, 1);
+                    OwnedHqlExpr plain = expr->clone(children);
+                    return transform(plain);
+                }
                 if (expr->hasAttribute(_linkCounted_Atom))
                 {
                     OwnedHqlExpr transformed = QuickHqlTransformer::createTransformedBody(expr);

ecl/hql/hqlexpr.cpp

@@ -250,6 +250,9 @@ MODULE_INIT(INIT_PRIORITY_HQLINTERNAL)
 }
 MODULE_EXIT()
 {
+#ifdef TRACE_HASH
+    exprCache->dumpStats();
+#endif
     for (unsigned i=0; i<=8; i++)
     {
         ::Release(nullIntValue[i][0]);
@@ -3945,7 +3948,7 @@ IHqlSimpleScope * CHqlExpression::querySimpleScope()
     return NULL; 
 }
 
-#define HASHFIELD(p) hashcode = hashc((unsigned char *) &p, sizeof(p), hashcode)
+#define HASHFIELD(p) hashcode = hashvalue(p, hashcode)
 
 void CHqlExpression::setInitialHash(unsigned typeHash)
 {
@@ -4375,6 +4378,7 @@ IHqlExpression *CHqlExpression::queryRecord()
 }
 
 //== Commoning up code.... ==
+#if (defined(GATHER_LINK_STATS) || defined(DEBUG_TRACK_INSTANCEID))
 void CHqlExpression::Link(void) const
 { 
 #ifdef GATHER_LINK_STATS
@@ -4396,6 +4400,7 @@ bool CHqlExpression::Release(void) const
     CHECK_EXPR_SEQID(2);
     return Parent::Release();
 }
+#endif
 
 
 void CHqlExpression::beforeDispose()
@@ -4706,7 +4711,7 @@ inline void addUniqueTable(HqlExprCopyArray & array, IHqlExpression * ds)
         array.append(*ds);
 }
 
-inline void addHiddenTable(HqlExprCopyArray & array, IHqlExpression * ds, IHqlExpression * selSeq)
+inline void addHiddenTable(HqlExprCopyArray & array, IHqlExpression * ds)
 {
 #if defined(GATHER_HIDDEN_SELECTORS)
     if (array.find(*ds) == NotFound)
@@ -4727,6 +4732,24 @@ inline void addActiveTable(HqlExprCopyArray & array, IHqlExpression * ds)
 }
 
 
+inline void addUniqueTable(UsedExpressionHashTable & array, IHqlExpression * ds)
+{
+    array.addOrFindExact(*ds);
+}
+
+inline void addActiveTable(UsedExpressionHashTable & array, IHqlExpression * ds)
+{
+    //left.subfield  should be reduced the base cursor
+    ds = queryDatasetCursor(ds);
+
+//  This test is valid once the tree is normalized, but now this can be called on a parse tree.
+//  assertex(ds == ds->queryNormalizedSelector());
+    node_operator dsOp = ds->getOperator();
+    if (dsOp != no_self && dsOp != no_selfref)
+        addUniqueTable(array, ds->queryNormalizedSelector());
+}
+
+
 //---------------------------------------------------------------------------------------------------------------------
 
 CUsedTables::CUsedTables()
@@ -4738,8 +4761,17 @@ CUsedTables::CUsedTables()
 
 CUsedTables::~CUsedTables()
 {
-    if (numTables > 1)
+    if (numTables == 1)
+    {
+        if (numActiveTables == 0)
+            tables.single->Release();
+    }
+    else if (numTables != 0)
+    {
+        for (unsigned i2=numActiveTables; i2 < numTables; i2++)
+            tables.multi[i2]->Release();
         delete [] tables.multi;
+    }
 }
 
 
@@ -4825,6 +4857,7 @@ void CUsedTables::set(HqlExprCopyArray & activeTables, HqlExprCopyArray & newTab
         else
         {
             tables.single = &newTables.item(0);
+            tables.single->Link();
         }
     }
     else if (numTables != 0)
@@ -4833,7 +4866,10 @@ void CUsedTables::set(HqlExprCopyArray & activeTables, HqlExprCopyArray & newTab
         for (unsigned i1=0; i1 < numActiveTables; i1++)
             multi[i1] = &activeTables.item(i1);
         for (unsigned i2=numActiveTables; i2 < numTables; i2++)
+        {
             multi[i2] = &newTables.item(i2-numActiveTables);
+            multi[i2]->Link();
+        }
         tables.multi = multi;
     }
 }
@@ -4852,9 +4888,16 @@ void CUsedTablesBuilder::addNewTable(IHqlExpression * expr)
     addUniqueTable(newScopeTables, expr);
 }
 
-void CUsedTablesBuilder::addHiddenTable(IHqlExpression * expr, IHqlExpression * selSeq)
+void CUsedTablesBuilder::addHiddenSelector(IHqlExpression * expr)
 {
-    ::addHiddenTable(newScopeTables, expr, selSeq);
+#if defined(GATHER_HIDDEN_SELECTORS)
+    //expr is always a newly created selector.  If this expression isn't shared, then it will not be used anywhere else
+    //in the expression tree, so don't add it.
+    if (!static_cast<CHqlExpression *>(expr)->IsShared())
+        return;
+
+    addUniqueTable(newScopeTables, expr);
+#endif
 }
 
 void CUsedTablesBuilder::addActiveTable(IHqlExpression * expr)
@@ -4864,16 +4907,22 @@ void CUsedTablesBuilder::addActiveTable(IHqlExpression * expr)
 
 void CUsedTablesBuilder::cleanupProduction()
 {
-    ForEachItemInRev(i, inScopeTables)
+    HqlExprCopyArray toRemove;
+    SuperHashIteratorOf<IHqlExpression> iter(inScopeTables);
+    ForEach(iter)
     {
-        switch (inScopeTables.item(i).getOperator())
+        IHqlExpression & cur = iter.query();
+        switch (cur.getOperator())
         {
         case no_matchattr:
         case no_matchrow:
-            inScopeTables.remove(i);
+            toRemove.append(cur);
             break;
         }
     }
+
+    ForEachItemIn(i, toRemove)
+        inScopeTables.remove(&toRemove.item(i));
 }
 
 void CUsedTablesBuilder::removeParent(IHqlExpression * expr)
@@ -4893,15 +4942,6 @@ void CUsedTablesBuilder::removeParent(IHqlExpression * expr)
     }
 }
 
-void CUsedTablesBuilder::removeActiveRecords()
-{
-    ForEachItemInRev(i, inScopeTables)
-    {
-        if (inScopeTables.item(i).isRecord())
-            inScopeTables.remove(i);
-    }
-}
-
 void CUsedTablesBuilder::removeRows(IHqlExpression * expr, IHqlExpression * left, IHqlExpression * right)
 {
     node_operator rowsSide = queryHasRows(expr);
@@ -4928,6 +4968,40 @@ void CUsedTablesBuilder::removeRows(IHqlExpression * expr, IHqlExpression * left
     }
 }
 
+void CUsedTablesBuilder::removeActiveRecords()
+{
+    HqlExprCopyArray toRemove;
+    SuperHashIteratorOf<IHqlExpression> iter(inScopeTables);
+    ForEach(iter)
+    {
+        IHqlExpression & cur = iter.query();
+        if (cur.isRecord())
+            toRemove.append(cur);
+    }
+
+    ForEachItemIn(i, toRemove)
+        inScopeTables.remove(&toRemove.item(i));
+}
+
+inline void expand(HqlExprCopyArray & target, const UsedExpressionHashTable & source)
+{
+    SuperHashIteratorOf<IHqlExpression> iter(source);
+    ForEach(iter)
+    {
+        IHqlExpression & cur = iter.query();
+        target.append(cur);
+    }
+}
+
+void CUsedTablesBuilder::set(CUsedTables & tables)
+{
+    HqlExprCopyArray inTables;
+    HqlExprCopyArray newTables;
+    expand(inTables, inScopeTables);
+    expand(newTables, newScopeTables);
+    tables.set(inTables, newTables);
+}
+
 //---------------------------------------------------------------------------------------------------------------------
 
 //Don't need to check if already visited, because the information is cached in the expression itself.
@@ -5042,10 +5116,10 @@ void CHqlExpressionWithTables::cacheTablesProcessChildScope(CUsedTablesBuilder &
         break;
     case childdataset_datasetleft:
         {
+            cacheChildrenTablesUsed(used, 1, max);
             IHqlExpression * ds = queryChild(0);
             IHqlExpression * selSeq = querySelSeq(this);
             OwnedHqlExpr left = createSelector(no_left, ds, selSeq);
-            cacheChildrenTablesUsed(used, 1, max);
             used.removeActive(left);
             used.removeParent(ds);
             used.removeRows(this, left, NULL);
@@ -5063,73 +5137,72 @@ void CHqlExpressionWithTables::cacheTablesProcessChildScope(CUsedTablesBuilder &
             }
             if (!ignoreInputs)
                 cacheChildrenTablesUsed(used, 0, 1);
-#ifdef GATHER_HIDDEN_SELECTORS
-            used.addHiddenTable(left, selSeq);
-#endif
+
+            used.addHiddenSelector(left);
             break;
         }
     case childdataset_left:
         { 
+            cacheChildrenTablesUsed(used, 1, max);
             IHqlExpression * ds = queryChild(0);
             IHqlExpression * selSeq = querySelSeq(this);
             OwnedHqlExpr left = createSelector(no_left, ds, selSeq);
-            cacheChildrenTablesUsed(used, 1, max);
             used.removeActive(left);
             used.removeRows(this, left, NULL);
 
             if (!ignoreInputs)
                 cacheChildrenTablesUsed(used, 0, 1);
-#ifdef GATHER_HIDDEN_SELECTORS
-            used.addHiddenTable(left, selSeq);
-#endif
+
+            used.addHiddenSelector(left);
             break;
         }
     case childdataset_same_left_right:
     case childdataset_nway_left_right:
         {
+            cacheChildrenTablesUsed(used, 1, max);
+
             IHqlExpression * ds = queryChild(0);
             IHqlExpression * selSeq = querySelSeq(this);
             OwnedHqlExpr left = createSelector(no_left, ds, selSeq);
             OwnedHqlExpr right = createSelector(no_right, ds, selSeq);
-            cacheChildrenTablesUsed(used, 1, max);
             used.removeActive(left);
             used.removeActive(right);
             used.removeRows(this, left, right);
 
             if (!ignoreInputs)
                 cacheChildrenTablesUsed(used, 0, 1);
-#ifdef GATHER_HIDDEN_SELECTORS
-            used.addHiddenTable(left, selSeq);
-            used.addHiddenTable(right, selSeq);
-#endif
+
+            used.addHiddenSelector(left);
+            used.addHiddenSelector(right);
             break;
         }
     case childdataset_top_left_right:
         {
+            cacheChildrenTablesUsed(used, 1, max);
+
             IHqlExpression * ds = queryChild(0);
             IHqlExpression * selSeq = querySelSeq(this);
             OwnedHqlExpr left = createSelector(no_left, ds, selSeq);
             OwnedHqlExpr right = createSelector(no_right, ds, selSeq);
-            cacheChildrenTablesUsed(used, 1, max);
             used.removeParent(ds);
             used.removeActive(left);
             used.removeActive(right);
             used.removeRows(this, left, right);
             cacheChildrenTablesUsed(used, 0, 1);
-#ifdef GATHER_HIDDEN_SELECTORS
-            used.addHiddenTable(left, selSeq);
-            used.addHiddenTable(right, selSeq);
-#endif
+
+            used.addHiddenSelector(left);
+            used.addHiddenSelector(right);
             break;
         }
     case childdataset_leftright: 
         {
+            cacheChildrenTablesUsed(used, 2, max);
+
             IHqlExpression * leftDs = queryChild(0);
             IHqlExpression * rightDs = queryChild(1);
             IHqlExpression * selSeq = querySelSeq(this);
             OwnedHqlExpr left = createSelector(no_left, leftDs, selSeq);
             OwnedHqlExpr right = createSelector(no_right, rightDs, selSeq);
-            cacheChildrenTablesUsed(used, 2, max);
             used.removeActive(right);
             used.removeRows(this, left, right);
             if (op == no_normalize)
@@ -5146,10 +5219,9 @@ void CHqlExpressionWithTables::cacheTablesProcessChildScope(CUsedTablesBuilder &
                 if (!ignoreInputs)
                     cacheChildrenTablesUsed(used, 0, 2);
             }
-#ifdef GATHER_HIDDEN_SELECTORS
-            used.addHiddenTable(left, selSeq);
-            used.addHiddenTable(right, selSeq);
-#endif
+
+            used.addHiddenSelector(left);
+            used.addHiddenSelector(right);
             break;
         }
         break;
@@ -5731,7 +5803,11 @@ void CHqlField::onCreateField()
     case type_groupedtable:
         typeExpr = queryRecord();
 #ifdef _DEBUG
-        assertex(!recordRequiresLinkCount(typeExpr) || hasAttribute(_linkCounted_Atom));
+        if (!hasAttribute(_linkCounted_Atom))
+        {
+            OwnedHqlExpr unadornedRecord = getUnadornedRecordOrField(typeExpr->queryRecord());
+            assertex(!recordRequiresLinkCount(unadornedRecord));
+        }
 #endif
         break;
     }
@@ -5749,8 +5825,7 @@ bool CHqlField::equals(const IHqlExpression & r) const
 {
     if (CHqlExpression::equals(r))
     {
-        const CHqlField *fr = QUERYINTERFACE(&r, const CHqlField);
-        if (fr && fr->id==id)
+        if (id == r.queryId())
             return true;
     }
     return false;
@@ -9643,8 +9718,8 @@ bool CHqlVariable::equals(const IHqlExpression & r) const
 {
     if (CHqlExpression::equals(r))
     {
-        assertex(QUERYINTERFACE(&r, const CHqlVariable) == (const CHqlVariable *) &r);
-        const CHqlVariable *c = (const CHqlVariable *) &r;
+        dbgassertex(QUERYINTERFACE(&r, const CHqlVariable) == (const CHqlVariable *) &r);
+        const CHqlVariable *c = static_cast<const CHqlVariable *>(&r);
         return strcmp(name, c->name) == 0;
     }
     return false;
@@ -9691,7 +9766,7 @@ bool CHqlAttribute::equals(const IHqlExpression &r) const
 void CHqlAttribute::sethash()
 {
     CHqlExpression::sethash();
-    hashcode = hashc((unsigned char *) &name, sizeof(name), hashcode);
+    HASHFIELD(name);
 }
 
 IHqlExpression *CHqlAttribute::clone(HqlExprArray & args)
@@ -9763,8 +9838,8 @@ IInterface * CHqlUnknown::queryUnknownExtra()
 void CHqlUnknown::sethash()
 {
     CHqlExpression::sethash();
-    hashcode = hashc((unsigned char *) &name, sizeof(name), hashcode);
-    hashcode = hashc((unsigned char *) &extra, sizeof(extra), hashcode);
+    HASHFIELD(name);
+    HASHFIELD(extra);
 }
 
 IHqlExpression *CHqlUnknown::clone(HqlExprArray &newkids)
@@ -9807,8 +9882,8 @@ bool CHqlSequence::equals(const IHqlExpression &r) const
 void CHqlSequence::sethash()
 {
     CHqlExpression::sethash();
-    hashcode = hashc((unsigned char *) &name, sizeof(name), hashcode);
-    hashcode = hashc((unsigned char *) &seq, sizeof(seq), hashcode);
+    HASHFIELD(name);
+    HASHFIELD(seq);
 }
 
 IHqlExpression *CHqlSequence::clone(HqlExprArray &newkids)
@@ -10251,7 +10326,7 @@ extern IHqlExpression *createEnumType(ITypeInfo * _type, IHqlScope *_scope)
 void CHqlTemplateFunctionContext::sethash() 
 { 
     CHqlExpression::sethash(); 
-    hashcode = hashc((unsigned char *) context, sizeof(context), hashcode); 
+    HASHFIELD(context);
 }
 
 //==============================================================================================================
@@ -10352,9 +10427,9 @@ extern IHqlExpression *createValueF(node_operator op, ITypeInfo *type, ...)
         IHqlExpression *parm = va_arg(args, IHqlExpression *);
         if (!parm)
             break;
-#ifdef _DEBUG
-        assertex(QUERYINTERFACE(parm, IHqlExpression));
-#endif
+
+        dbgassertex(QUERYINTERFACE(parm, IHqlExpression));
+
         if (parm->getOperator() == no_comma)
         {
             parm->unwindList(children, no_comma);
@@ -10557,9 +10632,9 @@ extern IHqlExpression *createDatasetF(node_operator op, ...)
         IHqlExpression *parm = va_arg(args, IHqlExpression *);
         if (!parm)
             break;
-#ifdef _DEBUG
-        assertex(QUERYINTERFACE(parm, IHqlExpression));
-#endif
+
+        dbgassertex(QUERYINTERFACE(parm, IHqlExpression));
+
         if (parm->getOperator() == no_comma)
         {
             parm->unwindList(children, no_comma);
@@ -11591,9 +11666,9 @@ extern IHqlExpression *createRowF(node_operator op, ...)
         IHqlExpression *parm = va_arg(args, IHqlExpression *);
         if (!parm)
             break;
-#ifdef _DEBUG
-        assertex(QUERYINTERFACE(parm, IHqlExpression));
-#endif
+
+        dbgassertex(QUERYINTERFACE(parm, IHqlExpression));
+
         if (parm->getOperator() == no_comma)
         {
             parm->unwindList(children, no_comma);
@@ -13318,7 +13393,7 @@ static TransformTrackingInfo transformExtraState[NUM_PARALLEL_TRANSFORMS+1];
 static bool isActiveMask[NUM_PARALLEL_TRANSFORMS+1];
 
 #if NUM_PARALLEL_TRANSFORMS==1
-unsigned threadActiveExtraIndex=1;
+const unsigned threadActiveExtraIndex=1;
 #else
 #ifdef _WIN32
 __declspec(thread) unsigned threadActiveExtraIndex;
@@ -13504,6 +13579,7 @@ void TransformTrackingInfo::unlock()
     curTransformDepth--;
 }
 
+#if NUM_PARALLEL_TRANSFORMS!=1
 static void ensureThreadExtraIndex()
 {
     {
@@ -13535,6 +13611,7 @@ static void releaseExtraIndex()
     threadActiveExtraIndex = 0;
     transformSemaphore->signal();
 }
+#endif
 
 extern HQL_API void lockTransformMutex()
 {
@@ -15107,7 +15184,7 @@ IHqlExpression * queryExpression(ITypeInfo * type)
     ITypeInfo * indirect = queryModifier(type, typemod_indirect);
     if (indirect)
         return static_cast<IHqlExpression *>(indirect->queryModifierExtra());
-    return QUERYINTERFACE(queryUnqualifiedType(type), IHqlExpression);
+    return queryUnqualifiedType(type)->castToExpression();
 }
 
 IHqlExpression * queryExpression(IHqlDataset * ds)
@@ -15119,7 +15196,7 @@ IHqlExpression * queryExpression(IHqlDataset * ds)
 IHqlScope * queryScope(ITypeInfo * type)
 {
     if (!type) return NULL;
-    return QUERYINTERFACE(queryUnqualifiedType(type), IHqlScope);
+    return queryUnqualifiedType(type)->castToScope();
 }
 
 IHqlRemoteScope * queryRemoteScope(IHqlScope * scope)

ecl/hql/hqlexpr.ipp

@@ -96,24 +96,53 @@ private:
     unsigned numActiveTables;
 };
 
+class HQL_API UsedExpressionHashTable : public SuperHashTableOf<IHqlExpression, IHqlExpression>
+{
+public:
+    UsedExpressionHashTable() : SuperHashTableOf<IHqlExpression,IHqlExpression>(false) {}
+    ~UsedExpressionHashTable() { releaseAll(); }
+
+    inline void zap(IHqlExpression & expr) { remove(&expr); }
+
+protected:
+    virtual void onAdd(void *next) {}
+    virtual void onRemove(void *) {}
+    virtual bool matchesFindParam(const void * _element, const void * _key, unsigned fphash) const
+    {
+        const IHqlExpression * element = static_cast<const IHqlExpression *>(_element);
+        const IHqlExpression * key = static_cast<const IHqlExpression *>(_key);
+        return element==key;
+    }
+    virtual unsigned getHashFromElement(const void * et) const
+    {
+        return static_cast<const IHqlExpression *>(et)->getHash();
+    }
+    virtual unsigned getHashFromFindParam(const void * et) const
+    {
+        return static_cast<const IHqlExpression *>(et)->getHash();
+    }
+    inline const void * getFindParam(const void * et) const { return et; }
+};
+
+
 class HQL_API CUsedTablesBuilder
 {
 public:
     void addNewTable(IHqlExpression * expr);
-    void addHiddenTable(IHqlExpression * expr, IHqlExpression * selSeq);
+    void addHiddenSelector(IHqlExpression * expr);
     void addActiveTable(IHqlExpression * expr);
     void cleanupProduction();
-    inline void removeActive(IHqlExpression * expr) { inScopeTables.zap(*expr); }
+    inline void removeActive(IHqlExpression * expr) { inScopeTables.remove(expr); }
     void removeParent(IHqlExpression * expr);
     void removeActiveRecords();
     void removeRows(IHqlExpression * expr, IHqlExpression * left, IHqlExpression * right);
-    void set(CUsedTables & tables) { tables.set(inScopeTables, newScopeTables); }
+    void set(CUsedTables & tables);
 
     inline bool isIndependentOfScope() const { return inScopeTables.ordinality() == 0; }
 
 protected:
-    HqlExprCopyArray inScopeTables;     // may need to rename, since use has changed.
-    HqlExprCopyArray newScopeTables;
+    UsedExpressionHashTable inScopeTables;     // may need to rename, since use has changed.
+    UsedExpressionHashTable newScopeTables;
 };
 
 class HQL_API CHqlExpression : public CInterfaceOf<IHqlExpression>
@@ -193,8 +222,10 @@ protected:
     virtual IInterface * queryExistingProperty(ExprPropKind kind) const;
 
 public:
+#if (defined(GATHER_LINK_STATS) || defined(DEBUG_TRACK_INSTANCEID))
     virtual void Link(void) const;
     virtual bool Release(void) const;
+#endif
 
     virtual ~CHqlExpression();
 
@@ -694,8 +725,8 @@ protected:
 
 protected:
     Linked<ISourcePath> sourcePath;
-    int lineno;
-    int column;
+    unsigned lineno;
+    unsigned column;
 };
 
 class HQL_API CHqlAnnotationExtraBase: public CHqlAnnotation
@@ -1086,6 +1117,8 @@ public:
     virtual StringBuffer & appendStringFromMem(StringBuffer & out, const void * data) {assertex(!"tbd"); return out; }
     virtual unsigned getCrc();
     virtual bool assignableFrom(ITypeInfo * source);
+    virtual IHqlExpression * castToExpression() { return this; }
+    virtual IHqlScope * castToScope() { return this; }
 
     virtual void serialize(MemoryBuffer &) { UNIMPLEMENTED; }
     virtual void deserialize(MemoryBuffer &) { UNIMPLEMENTED; }
@@ -1608,6 +1641,8 @@ public:
     virtual unsigned getCrc();
 
     virtual const char *queryTypeName()  { return queryName()->str(); }
+    virtual IHqlExpression * castToExpression() { return this; }
+    virtual IHqlScope * castToScope() { return NULL; }
     
     virtual unsigned getCardinality()           { return 0; }
     virtual bool isInteger()                    { return false; }
@@ -1750,6 +1785,8 @@ public:
     virtual IInterface * queryModifierExtra()   { return NULL; }
     virtual StringBuffer & appendStringFromMem(StringBuffer & out, const void * data) {assertex(!"tbd"); return out; }
     virtual unsigned getCrc();
+    virtual IHqlExpression * castToExpression() { return this; }
+    virtual IHqlScope * castToScope() { return NULL; }
 
     virtual void serialize(MemoryBuffer &) { UNIMPLEMENTED; }
     virtual void deserialize(MemoryBuffer &) { UNIMPLEMENTED; }

ecl/hql/hqltrans.cpp

@@ -968,10 +968,17 @@ IHqlExpression * QuickHqlTransformer::createTransformedBody(IHqlExpression * exp
         }
     case no_select:
         {
+            IHqlExpression * ds = expr->queryChild(0);
             IHqlExpression * field = expr->queryChild(1);
-            OwnedHqlExpr newDs = transform(expr->queryChild(0));
+            OwnedHqlExpr newDs = transform(ds);
+            if (newDs == ds)
+                return LINK(expr);
+
             children.append(*LINK(newDs));
-            children.append(*lookupNewSelectedField(newDs, field));
+            if (ds->queryRecord() == newDs->queryRecord())
+                children.append(*LINK(field));
+            else
+                children.append(*lookupNewSelectedField(newDs, field));
             break;
         }
     }

ecl/hqlcpp/hqllib.cpp

@@ -121,7 +121,7 @@ unsigned HqlCppLibrary::getHash(const HqlExprArray & values, unsigned crc) const
         case type_groupedtable:
         case type_dictionary:
             {
-                OwnedHqlExpr normalizedRecord = normalizeRecord(translator, cur.queryRecord());
+                OwnedHqlExpr normalizedRecord = normalizeExpression(translator, cur.queryRecord());
                 OwnedHqlExpr serialized = getSerializedForm(normalizedRecord, diskAtom);
                 unsigned recordCrc = getExpressionCRC(serialized);
                 crc = hashc((const byte *)&tc, sizeof(tc), crc);
@@ -654,7 +654,8 @@ void HqlCppTranslator::buildLibraryGraph(BuildCtx & ctx, IHqlExpression * expr,
     for (unsigned i2 = outputLibrary->numStreamedInputs(); i2 < numParams; i2++)
     {
         IHqlExpression * parameter = outputLibrary->queryInputExpr(i2);
-        extractBuilder->evaluateExpression(evalctx, parameter, dummyTarget, NULL, false);
+        OwnedHqlExpr normalParameter = normalizeExpression(*this, parameter);
+        extractBuilder->evaluateExpression(evalctx, normalParameter, dummyTarget, NULL, false);
     }
 
     BuildCtx graphctx(initctx);

ecl/hqlcpp/hqlttcpp.cpp

@@ -12494,11 +12494,10 @@ IHqlExpression * HqlTreeNormalizer::createTransformedSelector(IHqlExpression * e
     throwUnexpected();
 }
 
-IHqlExpression * normalizeRecord(HqlCppTranslator & translator, IHqlExpression * record)
+IHqlExpression * normalizeExpression(HqlCppTranslator & translator, IHqlExpression * expr)
 {
     HqlTreeNormalizer normalizer(translator);
-    HqlExprArray transformed;
-    return normalizer.transformRoot(record);
+    return normalizer.transformRoot(expr);
 }
 
 void normalizeHqlTree(HqlCppTranslator & translator, HqlExprArray & exprs)

ecl/hqlcpp/hqlttcpp.ipp

@@ -1229,7 +1229,7 @@ protected:
 void normalizeHqlTree(HqlCppTranslator & translator, HqlExprArray & exprs);
 IHqlExpression * normalizeHqlTree(HqlCppTranslator & translator, IHqlExpression * expr);
 // more: This really shouldn't need a translator argument - but it is a bit of a god class, and error reporting needs splitting from it.
-IHqlExpression * normalizeRecord(HqlCppTranslator & translator, IHqlExpression * expr);     
+IHqlExpression * normalizeExpression(HqlCppTranslator & translator, IHqlExpression * expr);
 
 IHqlExpression * removeNamedSymbols(IHqlExpression * expr);
 void hoistNestedCompound(HqlCppTranslator & _translator, HqlExprArray & exprs);

system/jlib/jhash.cpp

@@ -70,11 +70,9 @@ MODULE_EXIT()
 }
 
 #define HASHONE(hash, c)        { hash *= 0x01000193; hash ^= c; }      // Fowler/Noll/Vo Hash... seems to work pretty well, and fast
-#define USE_FNV
 
 unsigned hashc( const unsigned char *k, unsigned length, unsigned initval)
 {
-#ifdef USE_FNV
     unsigned hash = initval;        // ^_rotl(length,2)??
     unsigned char c;
     while (length >= 8)
@@ -96,46 +94,36 @@ unsigned hashc( const unsigned char *k, unsigned length, unsigned initval)
     case 1: c = (*k++); HASHONE(hash, c);
     }
     return hash;
-#else
-   register unsigned a,b,c,len;
+}
 
-   /* Set up the internal state */
-   len = length;
-   a = b = 0x9e3779b9;  /* the golden ratio; an arbitrary value */
-   c = initval;         /* the previous hash value */
+template <typename T>
+inline unsigned doHashValue( T value, unsigned initval)
+{
+    //If values are to be consistent with hashn() assumes little-endian
+    unsigned hash = initval;
+    unsigned char c;
+    for (unsigned i=0; i < sizeof(T); i++)
+    {
+        c = (byte)value;
+        value >>= 8;
+        HASHONE(hash, c);
+    }
+    return hash;
+}
 
-   /*---------------------------------------- handle most of the key */
-   while (len >= 12)
-   {
-      a += GETWORD(k,0);
-      b += GETWORD(k,4);
-      c += GETWORD(k,8);
-      mix(a,b,c);
-      k += 12; len -= 12;
-   }
+unsigned hashvalue( unsigned value, unsigned initval)
+{
+    return doHashValue(value, initval);
+}
 
-   /*------------------------------------- handle the last 11 bytes */
-   c += length;
-   switch(len)              /* all the case statements fall through */
-   {
-   case 11: c+=GETBYTE3(7);
-   case 10: c+=GETBYTE2(7);
-   case 9 : c+=GETBYTE1(7);
-      /* the first byte of c is reserved for the length */
-   case 8 : b+=GETBYTE3(4);
-   case 7 : b+=GETBYTE2(4);
-   case 6 : b+=GETBYTE1(4);
-   case 5 : b+=GETBYTE0(4);
-   case 4 : a+=GETBYTE3(0);
-   case 3 : a+=GETBYTE2(0);
-   case 2 : a+=GETBYTE1(0);
-   case 1 : a+=GETBYTE0(0);
-     /* case 0: nothing left to add */
-   }
-   mix(a,b,c);
-   /*-------------------------------------------- report the result */
-   return c;
-#endif
+unsigned hashvalue( unsigned __int64 value, unsigned initval)
+{
+    return doHashValue(value, initval);
+}
+
+unsigned hashvalue( const void * value, unsigned initval)
+{
+    return doHashValue((memsize_t)value, initval);
 }
 
 #define GETWORDNC(k,n) ((GETBYTE0(n)+GETBYTE1(n)+GETBYTE2(n)+GETBYTE3(n))&0xdfdfdfdf)
@@ -143,7 +131,6 @@ unsigned hashc( const unsigned char *k, unsigned length, unsigned initval)
 
 unsigned hashnc( const unsigned char *k, unsigned length, unsigned initval)
 {
-#ifdef USE_FNV
     unsigned hash = initval;
     unsigned char c;
     while (length >= 8)
@@ -165,47 +152,6 @@ unsigned hashnc( const unsigned char *k, unsigned length, unsigned initval)
     case 1: c = (*k++)&0xdf; HASHONE(hash, c);
     }
     return hash;
-#else
-
-   register unsigned a,b,c,len;
-
-   /* Set up the internal state */
-   len = length;
-   a = b = 0x9e3779b9;  /* the golden ratio; an arbitrary value */
-   c = initval;         /* the previous hash value */
-
-   /*---------------------------------------- handle most of the key */
-   while (len >= 12)
-   {
-      a += GETWORDNC(k,0);
-      b += GETWORDNC(k,4);
-      c += GETWORDNC(k,8);
-      mix(a,b,c);
-      k += 12; len -= 12;
-   }
-
-   /*------------------------------------- handle the last 11 bytes */
-   c += length;
-   switch(len)              /* all the case statements fall through */
-   {
-   case 11: c+=GETBYTE3(7)&0xdf;
-   case 10: c+=GETBYTE2(7)&0xdf;
-   case 9 : c+=GETBYTE1(7)&0xdf;
-      /* the first byte of c is reserved for the length */
-   case 8 : b+=GETBYTE3(4)&0xdf;
-   case 7 : b+=GETBYTE2(4)&0xdf;
-   case 6 : b+=GETBYTE1(4)&0xdf;
-   case 5 : b+=GETBYTE0(4)&0xdf;
-   case 4 : a+=GETBYTE3(0)&0xdf;
-   case 3 : a+=GETBYTE2(0)&0xdf;
-   case 2 : a+=GETBYTE1(0)&0xdf;
-   case 1 : a+=GETBYTE0(0)&0xdf;
-     /* case 0: nothing left to add */
-   }
-   mix(a,b,c);
-   /*-------------------------------------------- report the result */
-   return c;
-#endif
 }
 
 
@@ -376,11 +322,9 @@ BlockHash:
       else
         h = hashc(bp,ksize,h);
    }
-#ifdef USE_FNV
       //Strings that only differ by a single character don't generate hashes very far apart without this
    h *= 0x01000193;
    //h = ((h << 7) ^ (h >> 25));
-#endif
    return h;
 }
 

system/jlib/jhash.hpp

@@ -427,6 +427,9 @@ extern jlib_decl IIdAtom * createIdAtom(const char *value, size32_t len);
 extern jlib_decl void releaseAtoms();
 extern jlib_decl unsigned hashc( const unsigned char *k, unsigned length, unsigned initval);
 extern jlib_decl unsigned hashnc( const unsigned char *k, unsigned length, unsigned initval);
+extern jlib_decl unsigned hashvalue( unsigned value, unsigned initval);
+extern jlib_decl unsigned hashvalue( unsigned __int64 value, unsigned initval);
+extern jlib_decl unsigned hashvalue( const void * value, unsigned initval);
 
 //================================================
 // Minimal Hash table template - slightly less overhead that HashTable/SuperHashTable

system/jlib/jsuperhash.cpp

@@ -48,6 +48,7 @@ SuperHashTable::SuperHashTable(void)
 #ifdef TRACE_HASH
     search_tot = 0;
     search_num = 0;
+    search_max = 0;
 #endif
 }
 
@@ -97,17 +98,14 @@ void SuperHashTable::reinit(unsigned initsize)
 SuperHashTable::~SuperHashTable()
 {
     doKill();
-#ifdef TRACE_HASH
-    dumpStats();
-#endif
 }
 
 #ifdef TRACE_HASH
 void SuperHashTable::dumpStats()
 {
     if (tablecount && search_tot && search_num)
-        printf("Hash table %d entries, %d size, average search length %d(%d/%d))\n", tablecount, tablesize, 
-               (int) (search_tot/search_num), search_tot, search_num);
+        printf("Hash table %d entries, %d size, average search length %d(%d/%d)) max %d\n", tablecount, tablesize,
+               (int) (search_tot/search_num), search_tot, search_num, search_max);
 }
 #endif
  
@@ -116,6 +114,8 @@ void SuperHashTable::note_searchlen(int len) const
 {
     search_tot += len;
     search_num++;
+    if (search_max < len)
+        search_max = len;
 }
 #endif
 
@@ -254,14 +254,18 @@ unsigned SuperHashTable::doFindExact(const void *et) const
         i = getHashFromElement(et) % tablesize;
 #endif
         unsigned is = i;
-        while (table[i]!=et)
+        loop
         {
+            const void * cur = table[i];
+            if (!cur || cur == et)
+                break;
             i++;
             if (i==tablesize)
                 i = 0;
-            if (!table[i] || i==is)
+            if (i==is)
                 break;
         }
+        setCache(i);
     }
     return i;
 }
@@ -522,6 +526,26 @@ void *SuperHashTable::addOrFind(void * donor)
     return et;
 }
 
+void *SuperHashTable::addOrFindExact(void * donor)
+{
+    unsigned vm = doFindExact(donor);
+    void *et = table[vm];
+    if(!et)
+    {
+        unsigned tablelim = getTableLimit(tablesize);
+        if (tablecount>=tablelim)
+        {
+            expand();
+            vm = doFindExact(donor);
+        }
+        tablecount++;
+        table[vm] = donor;
+        onAdd(donor);
+        return donor;
+    }
+    return et;
+}
+
 void *SuperHashTable::next(const void *et) const
 {
     unsigned i;

system/jlib/jsuperhash.hpp

@@ -59,6 +59,7 @@ protected:
     void             addNew(void * et); //use this when you are sure the key does not already exist in the table (saves some needless matching)
     void             addNew(void * donor, unsigned hash);
     void *           addOrFind(void *);
+    void *           addOrFindExact(void * donor);
     inline void *    find(const void * param) const { return table[doFind(param)]; }
     inline void *    find(unsigned hashcode, const void * param) const { return table[doFind(hashcode, param)]; }
     void *           findElement(unsigned hashcode, const void * searchE) const;
@@ -101,6 +102,7 @@ protected:
 #ifdef TRACE_HASH
     mutable int      search_tot;
     mutable int      search_num;
+    mutable int      search_max;
 #endif
 };
 
@@ -116,6 +118,8 @@ class SuperHashTableOf : public SuperHashTable
       { return SuperHashTable::replace(&et); }
     inline ET *      addOrFind(ET & et)
       { return static_cast<ET *>(SuperHashTable::addOrFind(&et)); }
+    inline ET *      addOrFindExact(ET & et)
+      { return static_cast<ET *>(SuperHashTable::addOrFindExact(&et)); }
     inline ET *      find(const FP * fp) const
       { return static_cast<ET *>(SuperHashTable::find(fp)); }
     inline ET *      next(const ET * et) const