HPCC-9701 Refactor meta calculation into separate properties

This is a fairly big refactoring of the way sort orders etc. are tracked in
the code generator.  Previously the type held some extra information which
was initialised whenever the dataset was created.  This patch changes that
so that the sort order/grouping/distribution is dynamically calculated when
it is required.

This means that when datasets are recreated in transforms (e.g., when binding
functions or when transforming expression graphs) it avoids unnecessary
calculations.  The code is also cleaner since the CHqlMetaInfo class implements
many functions as members which were previously global functions.

Signed-off-by: Gavin Halliday <gavin.halliday@lexisnexis.com>

common/deftype/deftype.cpp

@@ -1379,40 +1379,6 @@ bool CTableTypeInfo::assignableFrom(ITypeInfo *t2)
     return false;
 }
 
-IInterface *CTableTypeInfo::queryDistributeInfo()
-{
-    return distributeinfo;
-}
-
-IInterface *CTableTypeInfo::queryGlobalSortInfo()
-{
-    return globalsortinfo;
-}
-
-IInterface *CTableTypeInfo::queryLocalUngroupedSortInfo()
-{
-    return localsortinfo;
-}
-
-unsigned CTableTypeInfo::getHash() const
-{
-    unsigned hashcode = CBasedTypeInfo::getHash();
-    HASHFIELD(distributeinfo);
-    HASHFIELD(globalsortinfo);
-    HASHFIELD(localsortinfo);
-    return hashcode;
-}
-
-
-bool CTableTypeInfo::equals(const CTypeInfo & _other) const
-{
-    if (!CBasedTypeInfo::equals(_other))
-        return false;
-    const CTableTypeInfo & other = static_cast<const CTableTypeInfo &>(_other);
-    return (distributeinfo == other.distributeinfo) && (globalsortinfo == other.globalsortinfo) && (localsortinfo == other.localsortinfo);
-}
-
-
 StringBuffer & CTableTypeInfo::getECLType(StringBuffer & out)   
 { 
     ITypeInfo * recordType = ::queryRecordType(this);
@@ -1425,13 +1391,12 @@ StringBuffer & CTableTypeInfo::getECLType(StringBuffer & out)
     return out; 
 }
 
+
 void CTableTypeInfo::serialize(MemoryBuffer &tgt)
-{ 
-    assertex(!globalsortinfo && !localsortinfo && !distributeinfo);
+{
     CBasedTypeInfo::serializeSkipChild(tgt);
 }
 
-
 bool CGroupedTableTypeInfo::assignableFrom(ITypeInfo *t2)
 {
     if (getTypeCode()==t2->getTypeCode())
@@ -1446,55 +1411,6 @@ bool CGroupedTableTypeInfo::assignableFrom(ITypeInfo *t2)
     return false;
 }
 
-IInterface *CGroupedTableTypeInfo::queryDistributeInfo()
-{
-    return queryChildType()->queryDistributeInfo();
-}
-
-IInterface *CGroupedTableTypeInfo::queryGroupInfo()
-{
-    return groupinfo;
-}
-
-IInterface *CGroupedTableTypeInfo::queryGlobalSortInfo()
-{
-    return queryChildType()->queryGlobalSortInfo();
-}
-
-IInterface *CGroupedTableTypeInfo::queryLocalUngroupedSortInfo()
-{
-    return queryChildType()->queryLocalUngroupedSortInfo();
-}
-
-IInterface *CGroupedTableTypeInfo::queryGroupSortInfo()
-{
-    return groupsortinfo;
-}
-
-unsigned CGroupedTableTypeInfo::getHash() const
-{
-    unsigned hashcode = CBasedTypeInfo::getHash();
-    HASHFIELD(groupinfo);
-    HASHFIELD(groupsortinfo);
-    return hashcode;
-}
-
-
-bool CGroupedTableTypeInfo::equals(const CTypeInfo & _other) const
-{
-    if (!CBasedTypeInfo::equals(_other))
-        return false;
-    const CGroupedTableTypeInfo & other = static_cast<const CGroupedTableTypeInfo &>(_other);
-    return (groupsortinfo == other.groupsortinfo) && (groupinfo == other.groupinfo);
-}
-
-void CGroupedTableTypeInfo::serialize(MemoryBuffer &tgt)
-{ 
-    assertex(!groupsortinfo && !groupinfo);
-    CBasedTypeInfo::serialize(tgt);
-}
-
-
 bool CSetTypeInfo::assignableFrom(ITypeInfo *t2)
 {
     return getTypeCode()==t2->getTypeCode() && 
@@ -2014,10 +1930,10 @@ extern DEFTYPE_API ITypeInfo *makeRuleType(ITypeInfo *basetype)
     return commonUpType(new CRuleTypeInfo(basetype));
 }
 
-extern DEFTYPE_API ITypeInfo *makeTableType(ITypeInfo *basetype, IInterface * distributeinfo, IInterface *globalsortinfo, IInterface *localsortinfo)
+extern DEFTYPE_API ITypeInfo *makeTableType(ITypeInfo *basetype)
 {
     assertex(!basetype || basetype->getTypeCode() == type_row);
-    return commonUpType(new CTableTypeInfo(basetype, distributeinfo, globalsortinfo, localsortinfo));
+    return commonUpType(new CTableTypeInfo(basetype));
 }
 
 extern DEFTYPE_API ITypeInfo *makeDictionaryType(ITypeInfo *basetype)
@@ -2026,9 +1942,9 @@ extern DEFTYPE_API ITypeInfo *makeDictionaryType(ITypeInfo *basetype)
     return commonUpType(new CDictionaryTypeInfo(basetype));
 }
 
-extern DEFTYPE_API ITypeInfo *makeGroupedTableType(ITypeInfo *basetype, IInterface *groupinfo, IInterface *sortinfo)
+extern DEFTYPE_API ITypeInfo *makeGroupedTableType(ITypeInfo *basetype)
 {
-    return commonUpType(new CGroupedTableTypeInfo(basetype, groupinfo, sortinfo));
+    return commonUpType(new CGroupedTableTypeInfo(basetype));
 }
 
 extern DEFTYPE_API ITypeInfo *makeFunctionType(ITypeInfo *basetype, IInterface * parameters, IInterface * defaults)
@@ -3409,10 +3325,10 @@ ITypeInfo * replaceChildType(ITypeInfo * type, ITypeInfo * newChild)
         newType.setown(makeDictionaryType(LINK(newChild)));
         break;
     case type_table:
-        newType.setown(makeTableType(LINK(newChild), LINK(type->queryDistributeInfo()), LINK(type->queryGlobalSortInfo()), LINK(type->queryLocalUngroupedSortInfo())));
+        newType.setown(makeTableType(LINK(newChild)));
         break;
     case type_groupedtable:
-        newType.setown(makeGroupedTableType(LINK(newChild), LINK(type->queryGroupInfo()), LINK(type->queryGroupSortInfo())));
+        newType.setown(makeGroupedTableType(LINK(newChild)));
         break;
     case type_row:
         newType.setown(makeRowType(LINK(newChild)));
@@ -3631,12 +3547,12 @@ extern DEFTYPE_API ITypeInfo * deserializeType(MemoryBuffer &src)
     case type_table:
         {
             ITypeInfo *base = deserializeType(src);
-            return makeTableType(makeRowType(base), NULL, NULL, NULL);
+            return makeTableType(makeRowType(base));
         }
     case type_groupedtable:
         {
             ITypeInfo *base = deserializeType(src);
-            return makeGroupedTableType(base, NULL, NULL);
+            return makeGroupedTableType(base);
         }
 
         

common/deftype/deftype.hpp

@@ -194,11 +194,6 @@ public:
     virtual ICharsetInfo * queryCharset() = 0;
     virtual ICollationInfo * queryCollation() = 0;
     virtual IAtom * queryLocale() = 0;
-    virtual IInterface * queryDistributeInfo() = 0;
-    virtual IInterface * queryGroupInfo() = 0;
-    virtual IInterface * queryGlobalSortInfo() = 0;
-    virtual IInterface * queryLocalUngroupedSortInfo() = 0;
-    virtual IInterface * queryGroupSortInfo() = 0;
     virtual ITypeInfo * queryPromotedType() = 0;
     virtual ITypeInfo * queryTypeBase() = 0;
     virtual unsigned getCrc() = 0;      // must be run independant.
@@ -249,8 +244,8 @@ extern DEFTYPE_API ITypeInfo *makeType(type_t type, int size);
 extern DEFTYPE_API IEnumeratedTypeBuilder *makeEnumeratedTypeBuilder(ITypeInfo *base, aindex_t numvalues);
 extern DEFTYPE_API ITypeInfo *makeDecimalType(unsigned digits, unsigned prec, bool isSigned);
 extern DEFTYPE_API ITypeInfo *makeDictionaryType(ITypeInfo *basetype);
-extern DEFTYPE_API ITypeInfo *makeTableType(ITypeInfo *basetype, IInterface * distributeinfo, IInterface *gloalSortinfo, IInterface * localSortInfo);
-extern DEFTYPE_API ITypeInfo *makeGroupedTableType(ITypeInfo *basetype, IInterface *groupinfo, IInterface *sortinfo);
+extern DEFTYPE_API ITypeInfo *makeTableType(ITypeInfo *basetype);
+extern DEFTYPE_API ITypeInfo *makeGroupedTableType(ITypeInfo *basetype);
 extern DEFTYPE_API ITypeInfo *makeRowType(ITypeInfo *basetype);
 extern DEFTYPE_API ITypeInfo *makeSetType(ITypeInfo *basetype);
 extern DEFTYPE_API ITypeInfo *makeTransformType(ITypeInfo *basetype);

common/deftype/deftype.ipp

@@ -56,11 +56,6 @@ public:
     virtual bool isSigned()                     { return false; }
     virtual bool isSwappedEndian()              { return false; }
     virtual ITypeInfo * queryChildType()        { return NULL; }
-    virtual IInterface * queryDistributeInfo()  { return NULL; }
-    virtual IInterface * queryLocalUngroupedSortInfo()   { return NULL; }
-    virtual IInterface * queryGlobalSortInfo()  { return NULL; }
-    virtual IInterface * queryGroupInfo()       { return NULL; }
-    virtual IInterface * queryGroupSortInfo()   { return NULL; }
     virtual ITypeInfo * queryPromotedType()     { return this; }    // very common implementation
     virtual ICharsetInfo * queryCharset()       { return NULL; }
     virtual ICollationInfo * queryCollation()   { return NULL; }
@@ -649,21 +644,15 @@ public:
 class CTableTypeInfo : public CBasedTypeInfo
 {
 private:
-    Owned<IInterface> distributeinfo;
-    Owned<IInterface> globalsortinfo;
-    Owned<IInterface> localsortinfo;
 public:
-    CTableTypeInfo(ITypeInfo * _basetype, IInterface * _distributeinfo, IInterface *_globalsortinfo, IInterface *_localsortinfo) 
-        : CBasedTypeInfo(_basetype, UNKNOWN_LENGTH), distributeinfo(_distributeinfo), globalsortinfo(_globalsortinfo), localsortinfo(_localsortinfo) {}
+    CTableTypeInfo(ITypeInfo * _basetype)
+        : CBasedTypeInfo(_basetype, UNKNOWN_LENGTH) {}
 
     virtual type_t getTypeCode() const                          { return type_table; }
     virtual bool isScalar()                                 { return false; }
     virtual const char *queryTypeName()                     { return "table"; }
     virtual StringBuffer &getECLType(StringBuffer & out);
     virtual bool assignableFrom(ITypeInfo *t2);
-    virtual IInterface * queryDistributeInfo();
-    virtual IInterface * queryGlobalSortInfo();
-    virtual IInterface * queryLocalUngroupedSortInfo();
 
     virtual void serialize(MemoryBuffer &tgt);
     //MORE: Delete this when persist attributes change again
@@ -674,31 +663,21 @@ public:
                                                                 return crc;
                                                             }
 
-    virtual unsigned getHash() const;
-    virtual bool equals(const CTypeInfo & other) const;
 };
 
 class CGroupedTableTypeInfo : public CBasedTypeInfo
 {
 private:
-    Owned<IInterface> groupinfo;
-    Owned<IInterface> groupsortinfo;
 public:
-    CGroupedTableTypeInfo(ITypeInfo * _basetype, IInterface *_groupinfo, IInterface *_groupsortinfo) 
-    : CBasedTypeInfo(_basetype, UNKNOWN_LENGTH), groupinfo(_groupinfo), groupsortinfo(_groupsortinfo) {}
+    CGroupedTableTypeInfo(ITypeInfo * _basetype)
+    : CBasedTypeInfo(_basetype, UNKNOWN_LENGTH) {}
 
     virtual type_t getTypeCode() const                          { return type_groupedtable; }
     virtual StringBuffer &getECLType(StringBuffer & out)    { out.append(queryTypeName()).append(" of "); queryChildType()->getECLType(out); return out; };
     virtual const char *queryTypeName()                     { return "groupedtable"; }
     virtual bool isScalar()                                 { return false; }
     virtual bool assignableFrom(ITypeInfo *t2);
-    virtual IInterface * queryDistributeInfo();
-    virtual IInterface * queryGroupInfo();
-    virtual IInterface * queryGlobalSortInfo();
-    virtual IInterface * queryLocalUngroupedSortInfo();
-    virtual IInterface * queryGroupSortInfo();
 
-    virtual void serialize(MemoryBuffer &tgt);
     //MORE: Delete this when persist attributes change again
     virtual unsigned getCrc()                               {
                                                                 unsigned crc = getTypeCode();
@@ -707,8 +686,6 @@ public:
                                                                 return crc;
                                                             }
 
-    virtual unsigned getHash() const;
-    virtual bool equals(const CTypeInfo & other) const;
 };
 
 class CSetTypeInfo : public CBasedTypeInfo
@@ -850,11 +827,6 @@ public:
     virtual ICharsetInfo * queryCharset()                           { return baseType->queryCharset(); }
     virtual ICollationInfo * queryCollation()                       { return baseType->queryCollation(); }
     virtual IAtom * queryLocale()                                     { return baseType->queryLocale(); }
-    virtual IInterface * queryLocalUngroupedSortInfo()                      { return baseType->queryLocalUngroupedSortInfo(); }
-    virtual IInterface * queryGlobalSortInfo()                      { return baseType->queryGlobalSortInfo(); }
-    virtual IInterface * queryGroupInfo()                           { return baseType->queryGroupInfo(); }
-    virtual IInterface * queryGroupSortInfo()                       { return baseType->queryGroupSortInfo(); }
-    virtual IInterface * queryDistributeInfo()                      { return baseType->queryDistributeInfo(); }
     virtual ITypeInfo * queryPromotedType()                         { return baseType->queryPromotedType(); }
     virtual ITypeInfo * queryTypeBase()                             { return baseType; }
     virtual unsigned getCrc()                                       { return baseType->getCrc(); }

common/fileview2/fvsource.cpp

@@ -99,7 +99,7 @@ void DataSourceMetaItem::serialize(MemoryBuffer & out) const
 
 DataSourceDatasetItem::DataSourceDatasetItem(const char * _name, const char * _xpath, IHqlExpression * expr) : DataSourceMetaItem(FVFFdataset, NULL, NULL, NULL), record(expr->queryRecord(), 0, true, false, false)
 {
-    type.setown(makeTableType(NULL, NULL, NULL, NULL));
+    type.setown(makeTableType(NULL));
     name.set(_name);
     xpath.set(_xpath);
     splitXmlTagNamesFromXPath(_xpath, record.tagname, &tagname);
@@ -109,7 +109,7 @@ DataSourceDatasetItem::DataSourceDatasetItem(const char * _name, const char * _x
 
 DataSourceDatasetItem::DataSourceDatasetItem(unsigned flags, MemoryBuffer & in) : DataSourceMetaItem(FVFFdataset, NULL, NULL, NULL), record(in)
 {
-    type.setown(makeTableType(NULL, NULL, NULL, NULL));
+    type.setown(makeTableType(NULL));
     in.read(name);
 }
 

ecl/hql/hqlattr.cpp

@@ -37,16 +37,16 @@
 #include "hqlattr.hpp"
 #include "hqlmeta.hpp"
 
-static CriticalSection * propertyCS;
+static SpinLock * propertyLock;
 
 MODULE_INIT(INIT_PRIORITY_HQLINTERNAL)
 {
-    propertyCS = new CriticalSection;
+    propertyLock = new SpinLock;
     return true;
 }
 MODULE_EXIT()
 {
-    delete propertyCS;
+    delete propertyLock;
 }
 
 // This file should contain most of the derived property calculation for nodes in the expression tree,
@@ -831,13 +831,12 @@ static IHqlExpression * evaluateSerializedRecord(IHqlExpression * expr, IAtom *
 class CHqlExprMeta
 {
 public:
-    static inline IHqlExpression * addProperty(IHqlExpression * expr, ExprPropKind kind, IHqlExpression * value)
+    static inline void addProperty(IHqlExpression * expr, ExprPropKind kind, IInterface * value)
     {
         CHqlExpression * cexpr = static_cast<CHqlExpression *>(expr);
         cexpr->addProperty(kind, value);
-        return value;
     }
-    static inline IHqlExpression * queryExistingProperty(IHqlExpression * expr, ExprPropKind kind)
+    static inline IInterface * queryExistingProperty(IHqlExpression * expr, ExprPropKind kind)
     {
         CHqlExpression * cexpr = static_cast<CHqlExpression *>(expr);
         return cexpr->queryExistingProperty(kind);
@@ -856,7 +855,8 @@ static IHqlExpression * evaluatePropSerializedForm(IHqlExpression * expr, ExprPr
             //Tag serialized form so don't re-evaluated
             meta.addProperty(serialized, kind, serialized);
         }
-        return meta.addProperty(expr, kind, serialized);
+        meta.addProperty(expr, kind, serialized);
+        return serialized;
     }
     return NULL;
 }
@@ -893,7 +893,8 @@ static IHqlExpression * evaluateFieldAttrSize(IHqlExpression * expr)
                 else
                 {
                     IHqlExpression * ret = expr->queryRecord()->queryProperty(EPsize);
-                    return meta.addProperty(expr, EPsize, ret);
+                    meta.addProperty(expr, EPsize, ret);
+                    return ret;
                 }
                 break;
             }
@@ -1077,13 +1078,15 @@ static IHqlExpression * evaluateFieldAttrSize(IHqlExpression * expr)
     if ((thisSize == minSize) && (minSize == maxSize))
     {
         OwnedHqlExpr attr = getFixedSizeAttr(thisSize);
-        return meta.addProperty(expr, EPsize, attr);
+        meta.addProperty(expr, EPsize, attr);
+        return attr;
     }
 
     if (!thisMaxSizeExpr)
         thisMaxSizeExpr.setown((maxSize == UNKNOWN_LENGTH) ? createAttribute(unknownSizeFieldAtom) : getSizetConstant(maxSize));
     OwnedHqlExpr attr = createExprAttribute(_propSize_Atom, getSizetConstant(thisSize), getSizetConstant(minSize), thisMaxSizeExpr.getClear());
-    return meta.addProperty(expr, EPsize, attr);
+    meta.addProperty(expr, EPsize, attr);
+    return attr;
 }
 
 //no_ifblock
@@ -1092,7 +1095,8 @@ static IHqlExpression * evaluateIfBlockAttrSize(IHqlExpression * expr)
     IHqlExpression * size = expr->queryChild(1)->queryProperty(EPsize);
     unsigned averageSize = (unsigned)getIntValue(size->queryChild(0), 0)/2;
     OwnedHqlExpr attr = createExprAttribute(_propSize_Atom, getSizetConstant(averageSize), getSizetConstant(0), LINK(size->queryChild(2)));
-    return meta.addProperty(expr, EPsize, attr);
+    meta.addProperty(expr, EPsize, attr);
+    return attr;
 }
 
 //no_record
@@ -1212,7 +1216,8 @@ static IHqlExpression * evaluateRecordAttrSize(IHqlExpression * expr)
         args.append(*LINK(derivedSizeExpr));
 
     OwnedHqlExpr sizeAttr = createExprAttribute(_propSize_Atom, args);
-    return meta.addProperty(expr, EPsize, sizeAttr);
+    meta.addProperty(expr, EPsize, sizeAttr);
+    return sizeAttr;
 }
 
 
@@ -1232,7 +1237,8 @@ static IHqlExpression * evalautePropSize(IHqlExpression * expr)
             //MORE: This could calculate a better estimate for the size of the record by taking into account any constant values or datasets that are assigned.
             IHqlExpression * record = expr->queryRecord();
             IHqlExpression * recordSize = record->queryProperty(EPsize);
-            return meta.addProperty(expr, EPsize, recordSize);
+            meta.addProperty(expr, EPsize, recordSize);
+            return recordSize;
         }
     }
     IHqlExpression * record = expr->queryRecord();
@@ -1281,7 +1287,7 @@ ITypeInfo * getSerializedForm(ITypeInfo * type, IAtom * variation)
             OwnedITypeInfo newChild = getSerializedForm(childType, variation);
             if (variation == internalAtom)
                 return replaceChildType(noLinkCountType, newChild);
-            OwnedITypeInfo datasetType = makeTableType(LINK(newChild), NULL, NULL, NULL);
+            OwnedITypeInfo datasetType = makeTableType(LINK(newChild));
             return cloneModifiers(noLinkCountType, datasetType);
         }
     }
@@ -1311,9 +1317,9 @@ HqlCachedPropertyTransformer::HqlCachedPropertyTransformer(HqlTransformerInfo &
 
 IHqlExpression * HqlCachedPropertyTransformer::transform(IHqlExpression * expr)
 {
-    IHqlExpression * match = meta.queryExistingProperty(expr, propKind);
+    IInterface * match = meta.queryExistingProperty(expr, propKind);
     if (match)
-        return LINK(match);
+        return static_cast<IHqlExpression *>(LINK(match));
 
     OwnedHqlExpr transformed = QuickHqlTransformer::transform(expr);
 
@@ -1407,7 +1413,7 @@ static IHqlExpression * evalautePropUnadorned(IHqlExpression * expr)
     HqlUnadornedNormalizer normalizer;
     //NB: Also has the side-effect of adding any missing attributes
     OwnedHqlExpr dummy = normalizer.transform(expr);
-    return meta.queryExistingProperty(expr, EPunadorned);
+    return static_cast<IHqlExpression *>(meta.queryExistingProperty(expr, EPunadorned));
 }
 
 //---------------------------------------------------------------------------------
@@ -1577,11 +1583,15 @@ static IHqlExpression * evalautePropAligned(IHqlExpression * expr)
         same = false;
     OwnedHqlExpr newRecord = same ? LINK(expr) : expr->clone(result);
     if (expr == newRecord)
-        return meta.addProperty(expr, EPaligned, queryAlignedAttr());
+    {
+        meta.addProperty(expr, EPaligned, queryAlignedAttr());
+        return queryAlignedAttr();
+    }
     meta.addProperty(newRecord, EPaligned, queryAlignedAttr());
     
     OwnedHqlExpr alignAttr = createExprAttribute(_propAligned_Atom, newRecord.getClear());
-    return meta.addProperty(expr, EPaligned, alignAttr);
+    meta.addProperty(expr, EPaligned, alignAttr);
+    return alignAttr;
 }
 
 //---------------------------------------------------------------------------------
@@ -1705,8 +1715,8 @@ bool isLocalActivity(IHqlExpression * expr)
     default:
         {
             assertex(!localChangesActivity(expr));
-            ITypeInfo * exprType = expr->queryType();
-            if (exprType && (exprType->queryDistributeInfo() != NULL))
+            //MORE: What is this test here for??
+            if (!expr->isAction() && queryDistribution(expr))
                 return !isGroupedActivity(expr);
             return false;
         }
@@ -2969,7 +2979,8 @@ IHqlExpression * calcRowInformation(IHqlExpression * expr)
 static IHqlExpression * evalautePropRecordCount(IHqlExpression * expr)
 {
     OwnedHqlExpr info = calcRowInformation(expr);
-    return meta.addProperty(expr, EPrecordCount, info);
+    meta.addProperty(expr, EPrecordCount, info);
+    return info;
 }
 
 
@@ -3426,7 +3437,7 @@ IHqlExpression * HqlLocationIndependentNormalizer::doCreateTransformed(IHqlExpre
             if (expr->numChildren() != 0)
             {
                 IAtom * name = expr->queryName();
-                if (name != _countProject_Atom)
+                if ((name != _countProject_Atom) && (name != _metadata_Atom))
                     return createAttribute(expr->queryName());
             }
             return LINK(expr);
@@ -3475,9 +3486,9 @@ IHqlExpression * HqlLocationIndependentNormalizer::createTransformed(IHqlExpress
     if (isAlwaysLocationIndependent(expr))
         return LINK(expr);
 
-    IHqlExpression * match = meta.queryExistingProperty(expr, EPlocationIndependent);
+    IInterface * match = meta.queryExistingProperty(expr, EPlocationIndependent);
     if (match)
-        return LINK(match);
+        return static_cast<IHqlExpression *>(LINK(match));
 
     OwnedHqlExpr transformed = doCreateTransformed(expr);
 
@@ -3698,43 +3709,73 @@ ITypeInfo * setStreamedAttr(ITypeInfo * _type, bool setValue)
 
 //---------------------------------------------------------------------------------------------------------------------
 
-IHqlExpression * CHqlExpression::queryExistingProperty(ExprPropKind propKind) const
+IInterface * CHqlExpression::queryExistingProperty(ExprPropKind propKind) const
 {
-    CriticalBlock block(*propertyCS);
+    SpinBlock block(*propertyLock);
     CHqlDynamicProperty * cur = attributes;
     while (cur)
     {
         if (cur->kind == propKind)
         {
-            IHqlExpression * value = cur->value;
+            IInterface * value = cur->value;
             if (value)
                 return value;
-            return const_cast<CHqlExpression *>(this);
+            return static_cast<IHqlExpression *>(const_cast<CHqlExpression *>(this));
         }
         cur = cur->next;
     }
     return NULL;
 }
 
-void CHqlExpression::addProperty(ExprPropKind kind, IHqlExpression * value)
+void CHqlExpression::addProperty(ExprPropKind kind, IInterface * value)
 {
-    if (value == this)
+    if (value == static_cast<IHqlExpression *>(this))
         value = NULL;
+    CHqlDynamicProperty * attr = new CHqlDynamicProperty(kind, value);
 
-    CriticalBlock block(*propertyCS);
+    SpinBlock block(*propertyLock);
     //theoretically we should test if the attribute has already been added by another thread, but in practice there is no
     //problem if the attribute is present twice.
-    CHqlDynamicProperty * attr = new CHqlDynamicProperty(kind, value);
     attr->next = attributes;
     attributes = attr;
 }
 
 
+//A specialised version of addProperty/queryExistingProperty.  Uses sizeof(void*) all the time instead
+//of 3*sizeof(void *)+heap overhead when used.
+//Possibly saves a bit of time, but more useful as a proof of concept in case it was useful elsewhere..
+void CHqlDataset::addProperty(ExprPropKind kind, IInterface * value)
+{
+    if (kind == EPmeta)
+    {
+        SpinBlock block(*propertyLock);
+        //ensure once meta is set it is never modified
+        if (!metaProperty)
+            metaProperty.set(value);
+    }
+    else
+        CHqlExpression::addProperty(kind, value);
+
+}
+
+IInterface * CHqlDataset::queryExistingProperty(ExprPropKind kind) const
+{
+    if (kind == EPmeta)
+    {
+        SpinBlock block(*propertyLock);
+        return metaProperty;
+    }
+
+    return CHqlExpression::queryExistingProperty(kind);
+}
+
+//---------------------------------------------------------------------------------------------------------------------
+
 IHqlExpression * CHqlExpression::queryProperty(ExprPropKind kind)
 {
-    IHqlExpression * match = queryExistingProperty(kind);
+    IInterface * match = queryExistingProperty(kind);
     if (match)
-        return match;
+        return static_cast<IHqlExpression *>(match);
 
     switch (kind)
     {
@@ -3755,3 +3796,23 @@ IHqlExpression * CHqlExpression::queryProperty(ExprPropKind kind)
     }
     return NULL;
 }
+
+CHqlMetaProperty * queryMetaProperty(IHqlExpression * expr)
+{
+    IHqlExpression * body = expr->queryBody();
+    IInterface * match = CHqlExprMeta::queryExistingProperty(body, EPmeta);
+    if (match)
+        return static_cast<CHqlMetaProperty *>(match);
+
+    CHqlMetaProperty * simple = querySimpleDatasetMeta(body);
+    if (simple)
+    {
+        CHqlExprMeta::addProperty(body, EPmeta, simple);
+        return simple;
+    }
+
+    Owned<CHqlMetaProperty> info = new CHqlMetaProperty;
+    calculateDatasetMeta(info->meta, body);
+    CHqlExprMeta::addProperty(body, EPmeta, info);
+    return info;
+}

ecl/hql/hqlattr.hpp

@@ -81,4 +81,7 @@ extern HQL_API IHqlExpression * getRecordCountInfo(IHqlExpression * expr);
 extern HQL_API bool hasNoMoreRowsThan(IHqlExpression * expr, __int64 limit);
 extern HQL_API bool spillToWorkunitNotFile(IHqlExpression * expr, ClusterType platform);
 
+class CHqlMetaProperty;
+extern HQL_API CHqlMetaProperty * queryMetaProperty(IHqlExpression * expr);
+
 #endif

ecl/hql/hqlexpr.hpp

@@ -794,6 +794,7 @@ enum ExprPropKind
     EPaligned,
     EPunadorned,
     EPlocationIndependent,
+    EPmeta,
     EPmax
 };
 
@@ -1470,7 +1471,7 @@ extern HQL_API IHqlNamedAnnotation * queryNameAnnotation(IHqlExpression * expr);
 inline bool hasAnnotation(IHqlExpression * expr, annotate_kind search){ return queryAnnotation(expr, search) != NULL; }
 inline IHqlExpression * queryNamedSymbol(IHqlExpression * expr) { return queryAnnotation(expr, annotate_symbol); }
 inline bool hasNamedSymbol(IHqlExpression * expr) { return hasAnnotation(expr, annotate_symbol); }
-inline bool hasAttribute(IAtom * search, HqlExprArray & exprs) { return queryAttribute(search, exprs) != NULL; }
+inline bool hasAttribute(IAtom * search, const HqlExprArray & exprs) { return queryAttribute(search, exprs) != NULL; }
 
 extern HQL_API IHqlExpression * queryAnnotationProperty(IAtom * search, IHqlExpression * annotation);
 extern HQL_API IHqlExpression * queryMetaProperty(IAtom * search, IHqlExpression * expr);
@@ -1643,17 +1644,13 @@ inline bool isCast(IHqlExpression * expr)
 }
 extern HQL_API bool isKey(IHqlExpression * expr);
 
-inline IHqlExpression * queryGrouping(ITypeInfo * type)     { return static_cast<IHqlExpression *>(type->queryGroupInfo()); }
-inline IHqlExpression * queryDistribution(ITypeInfo * type) { return static_cast<IHqlExpression *>(type->queryDistributeInfo()); }
-inline IHqlExpression * queryGlobalSortOrder(ITypeInfo * type)  { return static_cast<IHqlExpression *>(type->queryGlobalSortInfo()); }
-inline IHqlExpression * queryLocalUngroupedSortOrder(ITypeInfo * type)  { return static_cast<IHqlExpression *>(type->queryLocalUngroupedSortInfo()); }
-inline IHqlExpression * queryGroupSortOrder(ITypeInfo * type)   { return static_cast<IHqlExpression *>(type->queryGroupSortInfo()); }
-inline IHqlExpression * queryGrouping(IHqlExpression * expr)        { return queryGrouping(expr->queryType()); }
-inline IHqlExpression * queryDistribution(IHqlExpression * expr)    { return queryDistribution(expr->queryType()); }
-inline IHqlExpression * queryGlobalSortOrder(IHqlExpression * expr) { return queryGlobalSortOrder(expr->queryType()); }
-inline IHqlExpression * queryLocalUngroupedSortOrder(IHqlExpression * expr) { return queryLocalUngroupedSortOrder(expr->queryType()); }
-inline IHqlExpression * queryGroupSortOrder(IHqlExpression * expr)  { return queryGroupSortOrder(expr->queryType()); }
-inline bool isGrouped(ITypeInfo * type)                     { return type && type->queryGroupInfo() != NULL; }
+IHqlExpression * queryGrouping(IHqlExpression * expr);
+IHqlExpression * queryDistribution(IHqlExpression * expr);
+IHqlExpression * queryGlobalSortOrder(IHqlExpression * expr);
+IHqlExpression * queryLocalUngroupedSortOrder(IHqlExpression * expr);
+IHqlExpression * queryGroupSortOrder(IHqlExpression * expr);
+
+inline bool isGrouped(ITypeInfo * type)                     { return type && (type->getTypeCode() == type_groupedtable); }
 inline bool isGrouped(IHqlExpression * expr)                { return isGrouped(expr->queryType()); }
 
 inline IFunctionTypeExtra * queryFunctionTypeExtra(ITypeInfo * type)    { return static_cast<IFunctionTypeExtra *>(queryUnqualifiedType(type)->queryModifierExtra()); }
@@ -1666,7 +1663,7 @@ inline IHqlExpression * querySelSeq(IHqlExpression * expr)
 {
     return expr->queryAttribute(_selectorSequence_Atom);
 }
-extern HQL_API IHqlExpression * createGroupedAttribute(ITypeInfo * type);
+extern HQL_API IHqlExpression * createGroupedAttribute(IHqlExpression * grouping);
 extern HQL_API bool isSameUnqualifiedType(ITypeInfo * l, ITypeInfo * r);
 extern HQL_API bool isSameFullyUnqualifiedType(ITypeInfo * l, ITypeInfo * r);
 extern HQL_API IHqlExpression * queryNewSelectAttrExpr();

ecl/hql/hqlexpr.ipp

@@ -47,7 +47,7 @@ class HQL_API CHqlDynamicProperty
 {
     friend class CHqlExpression;
 public:
-    inline CHqlDynamicProperty(ExprPropKind _kind, IHqlExpression *_value)
+    inline CHqlDynamicProperty(ExprPropKind _kind, IInterface *_value)
         : kind(_kind), value(_value)
     {
         next = NULL;
@@ -56,10 +56,11 @@ public:
 
 protected:
     CHqlDynamicProperty * next;
-    LinkedHqlExpr value;
     ExprPropKind kind;
+    Linked<IInterface> value;
 };
 
+
 class CUsedTablesBuilder;
 
 //Optimized representation of the number of used tables
@@ -121,7 +122,7 @@ public:
 
 protected:
     unsigned hashcode;          // CInterface is 4 byte aligned in 64bits, so use this to pad
-                                // Worth storing becuase it significantly speeds up equality checking
+                                // Worth storing because it significantly speeds up equality checking
     IInterface * transformExtra[NUM_PARALLEL_TRANSFORMS];
     unsigned cachedCRC;
     unsigned infoFlags;
@@ -168,7 +169,6 @@ protected:
         operands.append(child);
         onAppendOperand(child, which);
     }
-    IHqlExpression * queryExistingProperty(ExprPropKind kind) const;
 
     void initFlagsBeforeOperands();
     void updateFlagsAfterOperands();
@@ -178,7 +178,8 @@ protected:
     virtual unsigned getCachedEclCRC();
     void setInitialHash(unsigned typeHash);
 
-    void addProperty(ExprPropKind kind, IHqlExpression * value);
+    virtual void addProperty(ExprPropKind kind, IInterface * value);
+    virtual IInterface * queryExistingProperty(ExprPropKind kind) const;
 
 public:
     virtual void Link(void) const;
@@ -1065,11 +1066,6 @@ public:
     virtual IAtom * queryLocale()                 { return NULL; }
 //  virtual IAtom * queryName() const             { return id; }
     virtual ITypeInfo * queryChildType()        { return NULL; }
-    virtual IInterface * queryDistributeInfo()  { return NULL; }
-    virtual IInterface * queryGroupInfo()       { return NULL; }
-    virtual IInterface * queryGlobalSortInfo()  { return NULL; }
-    virtual IInterface * queryLocalUngroupedSortInfo()   { return NULL; }
-    virtual IInterface * queryGroupSortInfo()   { return NULL; }
     virtual ITypeInfo * queryPromotedType()     { return this; }
     virtual ITypeInfo * queryTypeBase()         { return this; }
     virtual typemod_t queryModifier()           { return typemod_none; }
@@ -1609,11 +1605,6 @@ public:
     virtual ICollationInfo * queryCollation()   { return NULL; }
     virtual IAtom * queryLocale()                 { return NULL; }
     virtual ITypeInfo * queryChildType()        { return NULL; }
-    virtual IInterface * queryDistributeInfo()  { return NULL; }
-    virtual IInterface * queryGroupInfo()       { return NULL; }
-    virtual IInterface * queryGlobalSortInfo()  { return NULL; }
-    virtual IInterface * queryLocalUngroupedSortInfo()   { return NULL; }
-    virtual IInterface * queryGroupSortInfo()   { return NULL; }
     virtual ITypeInfo * queryPromotedType()     { return this; }
     virtual ITypeInfo * queryTypeBase()         { return this; }
     virtual typemod_t queryModifier()           { return typemod_none; }
@@ -1661,10 +1652,14 @@ public:
     bool equals(const IHqlExpression & r) const;
     virtual IHqlExpression *clone(HqlExprArray &newkids);
 
+    virtual void addProperty(ExprPropKind kind, IInterface * value);
+    virtual IInterface * queryExistingProperty(ExprPropKind kind) const;
+
 protected:
     IHqlDataset *rootTable;
     IHqlExpression * container;
     OwnedHqlExpr normalized;
+    Owned<IInterface> metaProperty;
 
     void cacheParent();
 
@@ -1735,11 +1730,6 @@ public:
     virtual IAtom * queryName() const { return id->lower(); }
     virtual IIdAtom * queryId() const { return id; }
     virtual ITypeInfo * queryChildType() { return logical; }
-    virtual IInterface * queryDistributeInfo()  { return NULL; }
-    virtual IInterface * queryGroupInfo()       { return NULL; }
-    virtual IInterface * queryGlobalSortInfo()  { return NULL; }
-    virtual IInterface * queryLocalUngroupedSortInfo()   { return NULL; }
-    virtual IInterface * queryGroupSortInfo()   { return NULL; }
     virtual ITypeInfo * queryPromotedType()     { return logical->queryPromotedType(); }
     virtual ITypeInfo * queryTypeBase()         { return this; }
     virtual typemod_t queryModifier()           { return typemod_none; }

ecl/hql/hqlfold.cpp

@@ -31,6 +31,7 @@
 #include "hqlerrors.hpp"
 #include "hqlutil.hpp"
 #include "hqlpmap.hpp"
+#include "hqlmeta.hpp"
 
 #include "hqlfold.hpp"
 #include "hqlthql.hpp"
@@ -3379,7 +3380,7 @@ IHqlExpression * NullFolderMixin::foldNullDataset(IHqlExpression * expr)
             if (expr->hasAttribute(skewAtom))
             	break;
             //Careful - distribute also destroys grouping, so don't remove if input is grouped.
-            if ((expr->queryType()->queryDistributeInfo() == child->queryType()->queryDistributeInfo()) && !isGrouped(child))
+            if ((queryDistribution(expr) == queryDistribution(child)) && !isGrouped(child))
                 return removeParentNode(expr);
             break;
         }
@@ -3388,7 +3389,7 @@ IHqlExpression * NullFolderMixin::foldNullDataset(IHqlExpression * expr)
     case no_sorted:
         {
             //If action does not change the type information, then it can't have done anything...
-            if (expr->queryType() == child->queryType())
+            if (hasSameSortGroupDistribution(expr, child))
                 return removeParentNode(expr);
             if (isNull(child) || hasNoMoreRowsThan(child, 1))
                 return removeParentNode(expr);
@@ -3425,7 +3426,7 @@ IHqlExpression * NullFolderMixin::foldNullDataset(IHqlExpression * expr)
 //  case no_preservemeta:
         {
             //If action does not change the type information, then it can't have done anything...
-            if (expr->queryType() == child->queryType())
+            if (hasSameSortGroupDistribution(expr, child))
                 return removeParentNode(expr);
             if (isNull(child))
                 return replaceWithNull(expr);

ecl/hql/hqlgram.y

@@ -725,8 +725,7 @@ explicitDatasetType
     : explicitDatasetType1
     | GROUPED explicitDatasetType1
                         {
-                            IHqlExpression * grouping = getUnknownSortlist();
-                            $$.setType(makeGroupedTableType($2.getType(), grouping, NULL));
+                            $$.setType(makeGroupedTableType($2.getType()));
                             $$.setPosition($1);
                         }
     ;
@@ -734,7 +733,7 @@ explicitDatasetType
 explicitDatasetType1
     : DATASET
                         {
-                            $$.setType(makeTableType(makeRowType(queryNullRecord()->getType()), NULL, NULL, NULL));
+                            $$.setType(makeTableType(makeRowType(queryNullRecord()->getType())));
                             $$.setPosition($1);
                         }
     | DATASET '(' recordDef childDatasetOptions ')'
@@ -742,7 +741,7 @@ explicitDatasetType1
                             OwnedHqlExpr record = $3.getExpr();
                             OwnedHqlExpr options = $4.getExpr();
                             ITypeInfo * recordType = createRecordType(record);
-                            Owned<ITypeInfo> tableType = makeTableType(makeRowType(recordType), NULL, NULL, NULL);
+                            Owned<ITypeInfo> tableType = makeTableType(makeRowType(recordType));
                             if (options)
                                 tableType.setown(makeAttributeModifier(LINK(tableType), createAttribute(_childAttr_Atom, LINK(options))));
                             $$.setType(tableType.getClear());
@@ -852,7 +851,7 @@ paramType
     | abstractDataset   {
                             OwnedHqlExpr record = $1.getExpr();
                             OwnedHqlExpr abstractRecord = createAbstractRecord(record);
-                            OwnedITypeInfo type = makeTableType(makeRowType(abstractRecord->getType()), NULL, NULL, NULL);
+                            OwnedITypeInfo type = makeTableType(makeRowType(abstractRecord->getType()));
                             $$.setType(type.getClear(), $1);
                             parser->setTemplateAttribute();
                         }
@@ -3669,7 +3668,7 @@ paramDefinition
     | ANY DATASET knownOrUnknownId
                         {
                             $$.clear();
-                            parser->addParameter($1, $3.getId(), makeTableType(makeRowType(queryNullRecord()->getType()), NULL, NULL, NULL), NULL);
+                            parser->addParameter($1, $3.getId(), makeTableType(makeRowType(queryNullRecord()->getType())), NULL);
                         }
     | ANY knownOrUnknownId defvalue
                         {
@@ -4254,7 +4253,7 @@ fieldDef
                             }
 
                             IHqlExpression *value = $7.getExpr();
-                            Owned<ITypeInfo> datasetType = makeTableType(makeRowType(createRecordType(record)), NULL, NULL, NULL);
+                            Owned<ITypeInfo> datasetType = makeTableType(makeRowType(createRecordType(record)));
                             parser->addDatasetField($2, $2.getId(), datasetType, value, attrs.getClear());
                         }
     | setType knownOrUnknownId optFieldAttrs defaultValue
@@ -8200,19 +8199,6 @@ simpleDataSet
                             OwnedHqlExpr attrs;
                             IHqlExpression *groupOrder = parser->processSortList($6, no_group, input, sortItems, NULL, &attrs);
                             OwnedHqlExpr args = createComma(groupOrder, LINK(attrs));
-                            
-                            //Non-all Group only make sense if the rows are together.  This can occur if
-                            //the dataset is sorted, or if previous operation was a grouped aggregate
-                            bool isLocal = queryAttributeInList(localAtom, attrs) != NULL;
-                            if (groupOrder && !queryAttributeInList(allAtom, attrs) && !appearsToBeSorted(input->queryType(), isLocal, true))
-                            {
-                                bool ok = (input->getOperator() == no_usertable) &&
-                                          (datasetHasGroupBy(input));// || isGrouped(input->queryChild(0)));
-                                if (!ok && !isLocal && isPartitionedForGroup(input, groupOrder, false) && appearsToBeSorted(input->queryType(), true, true))
-                                    ok = true;
-                                if (!ok && (groupOrder && groupOrder->numChildren()))
-                                    parser->reportWarning(WRN_NOT_SORTED, $2.pos, "GROUP on a table that does not appear to be sorted, was this intended?");
-                            }
                             $$.setExpr(createDataset(no_group, input.getClear(), args.getClear()), $1);
                         }
     | GROUPED '(' startTopFilter startGROUP beginList sortList endGROUP
@@ -8282,7 +8268,7 @@ simpleDataSet
                             if (grouping && !queryAttributeInList(groupedAtom, attrs))
                             {
                                 parser->checkGrouping($7, dataset,record,grouping);
-                                if (dataset->getOperator() == no_group && dataset->queryType()->queryGroupInfo())
+                                if (dataset->getOperator() == no_group && isGrouped(dataset))
                                     parser->reportWarning(WRN_GROUPINGIGNORED, $3.pos, "Grouping of table input will have no effect, was this intended?");
                             }
 

ecl/hql/hqlgram2.cpp

@@ -2403,7 +2403,7 @@ void HqlGram::addDatasetField(const attribute &errpos, IIdAtom * name, ITypeInfo
         if (!dsType || queryRecord(dsType)->numChildren() == 0)
         {
             ITypeInfo * recordType = queryRecordType(valueType);
-            dsType.setown(makeTableType(makeRowType(LINK(recordType)), NULL, NULL, NULL));
+            dsType.setown(makeTableType(makeRowType(LINK(recordType))));
         }
         else if (!dsType->assignableFrom(valueType))
         {
@@ -7562,7 +7562,6 @@ static const char * getName(IHqlExpression * e)
 
 void HqlGram::checkDistribution(attribute &errpos, IHqlExpression *input, bool localSpecified, bool ignoreGrouping)
 {
-    IInterface *distribution = input->queryType()->queryDistributeInfo();
     bool inputIsGrouped = isGrouped(input);
     if (localSpecified)
     {
@@ -7571,7 +7570,7 @@ void HqlGram::checkDistribution(attribute &errpos, IHqlExpression *input, bool l
     }
     else
     {
-        if (distribution && isExplicitlyDistributed(input))
+        if (isExplicitlyDistributed(input))
         {
             if (!inputIsGrouped || ignoreGrouping)
             {
@@ -8432,9 +8431,9 @@ void HqlGram::checkRecordIsValid(const attribute &atr, IHqlExpression *record)
 void HqlGram::checkMergeInputSorted(attribute &atr, bool isLocal)
 {
     IHqlExpression * expr = atr.queryExpr();
-    if (appearsToBeSorted(expr->queryType(), isLocal, true))
+    if (appearsToBeSorted(expr, isLocal, true))
         return;
-    if (!isLocal && appearsToBeSorted(expr->queryType(), true, true))
+    if (!isLocal && appearsToBeSorted(expr, true, true))
     {
         reportWarning(WRN_MERGE_NOT_SORTED, atr.pos, "INPUT to MERGE appears to be sorted locally but not globally");
         return;
@@ -8453,7 +8452,7 @@ void HqlGram::checkMergeInputSorted(attribute &atr, bool isLocal)
         }
     }
     
-    if (isGrouped(expr) && appearsToBeSorted(expr->queryType(), false, false))
+    if (isGrouped(expr) && appearsToBeSorted(expr, false, false))
         reportWarning(WRN_MERGE_NOT_SORTED, atr.pos, "Input to MERGE is only sorted with the group");
     else
         reportWarning(WRN_MERGE_NOT_SORTED, atr.pos, "Input to MERGE doesn't appear to be sorted");
@@ -9105,7 +9104,9 @@ void HqlGram::defineSymbolProduction(attribute & nameattr, attribute & paramattr
                     {
                         //allow dataset with no record to match, as long as base type is the same
                         if (queryRecord(type) != queryNullRecord() || (type->getTypeCode() != matchType->getTypeCode()))
+                        {
                             reportError(ERR_SAME_TYPE_REQUIRED, nameattr, "Explicit type for %s doesn't match definition in base module", name->str());
+                        }
                         else
                         {
                             type.set(matchType);
@@ -11371,12 +11372,14 @@ void parseAttribute(IHqlScope * scope, IFileContents * contents, HqlLookupContex
 
 void testHqlInternals()
 {
-    printf("Sizes: const(%u) expr(%u) select(%u) dataset(%u) annotation(%u)\n",
+    printf("Sizes: const(%u) expr(%u) select(%u) dataset(%u) annotation(%u) prop(%u)\n",
             (unsigned)sizeof(CHqlConstant),
             (unsigned)sizeof(CHqlExpressionWithType),
             (unsigned)sizeof(CHqlSelectExpression),
             (unsigned)sizeof(CHqlDataset),
-            (unsigned)sizeof(CHqlAnnotation));
+            (unsigned)sizeof(CHqlAnnotation),
+            (unsigned)sizeof(CHqlDynamicProperty)
+            );
 
     //
     // test getOpString()

ecl/hql/hqlir.cpp

@@ -1875,6 +1875,7 @@ id_t ExpressionIRPlayer::doProcessType(ITypeInfo * type)
         case type_groupedtable:
         case type_dictionary:
         case type_function:
+        case type_pointer:
             {
                 CompoundTypeBuilderInfo info;
                 info.baseType = processType(type->queryChildType());
@@ -1886,7 +1887,6 @@ id_t ExpressionIRPlayer::doProcessType(ITypeInfo * type)
         case type_ifblock:
         case type_alias:
         case type_blob:
-        case type_pointer:
         case type_class:
         case type_array:
             throwUnexpected();

ecl/hql/hqllex.l

@@ -1333,7 +1333,7 @@ RECORDSET           {
                         if (!lookup)
                             return lookupIdentifierToken(returnToken, lexer, lookup, activeState, CUR_TOKEN_TEXT);
 
-                        returnToken.setType(makeTableType(NULL, NULL, NULL, NULL));
+                        returnToken.setType(makeTableType(NULL));
                         return(SIMPLE_TYPE);
                     }
 SWAPPED             { 

ecl/hql/hqlmeta.hpp

@@ -19,6 +19,59 @@
 
 class TableProjectMapper;
 
+class HQL_API CHqlMetaInfo
+{
+public:
+    inline bool isGrouped() const { return grouping != NULL; }
+
+    bool appearsToBeSorted(bool isLocal, bool ignoreGrouping);
+    void applyDistribute(IHqlExpression * newDistribution, IHqlExpression * optMergeOrder);
+    void applyGroupSort(IHqlExpression * sortOrder);
+    void applyGroupBy(IHqlExpression * groupBy, bool isLocal);
+    void applyLocalSort(IHqlExpression * sortOrder);
+    void applyGlobalSort(IHqlExpression * sortOrder);
+    void applyProject(TableProjectMapper & mapper);
+    void applySubSort(IHqlExpression * groupBy, IHqlExpression * sortOrder, bool isLocal);
+
+    void ensureAppearsSorted(bool isLocal, bool ignoreGrouping);
+    void getIntersection(const CHqlMetaInfo & other);
+    IHqlExpression * getLocalSortOrder() const;
+    bool hasUsefulInformation() const;
+    bool matches(const CHqlMetaInfo & other) const;
+
+    void preserveGrouping(IHqlExpression * dataset);
+
+    void removeActiveSort();
+    void removeAllKeepGrouping();
+    void removeAllSortOrders();
+    void removeAllAndUngroup(bool isLocal);
+    void removeDistribution();
+    void removeGroup();
+
+    void set(const CHqlMetaInfo & other);
+    void setMatchesAny();
+    void setUnknownDistribution();
+    void setUnknownGrouping();
+
+protected:
+    void clearGrouping();
+
+public:
+    LinkedHqlExpr distribution;
+    LinkedHqlExpr globalSortOrder;
+    LinkedHqlExpr localUngroupedSortOrder;
+    LinkedHqlExpr grouping;
+    LinkedHqlExpr groupSortOrder;
+};
+
+class CHqlMetaProperty : public CInterfaceOf<IInterface>
+{
+public:
+    CHqlMetaInfo meta;
+};
+
+void extractMeta(CHqlMetaInfo & meta, ITypeInfo * type);
+
 extern HQL_API IHqlExpression * queryUnknownAttribute();
 extern HQL_API IHqlExpression * queryUnknownSortlist();
 extern HQL_API IHqlExpression * queryMatchGroupOrderSortlist();
@@ -29,32 +82,14 @@ extern HQL_API IHqlExpression * queryAnyOrderSortlist();
 extern HQL_API IHqlExpression * queryAnyDistributionAttribute();
 
 extern HQL_API IHqlExpression * getExistingSortOrder(IHqlExpression * dataset, bool isLocal, bool ignoreGrouping);
+extern ITypeInfo * calculateDatasetType(node_operator op, const HqlExprArray & parms);
 
 //---------------------------------------------------------------------------------------------
 
 extern HQL_API bool isKnownDistribution(IHqlExpression * distribution);
-inline bool isKnownDistribution(ITypeInfo * type) { return isKnownDistribution(queryDistribution(type)); }
 extern HQL_API bool isPersistDistribution(IHqlExpression * distribution);
 extern HQL_API bool isSortedDistribution(IHqlExpression * distribution);
 
-extern HQL_API ITypeInfo * getTypeUngroup(ITypeInfo * prev);
-extern HQL_API ITypeInfo * getTypeGrouped(ITypeInfo * prev, IHqlExpression * grouping, bool isLocal);
-extern HQL_API ITypeInfo * getTypeGlobalSort(ITypeInfo * prev, IHqlExpression * sortOrder);
-extern HQL_API ITypeInfo * getTypeLocalSort(ITypeInfo * prev, IHqlExpression * sortOrder);
-extern HQL_API ITypeInfo * getTypeGroupSort(ITypeInfo * prev, IHqlExpression * sortOrder);
-extern HQL_API ITypeInfo * getTypeDistribute(ITypeInfo * prev, IHqlExpression * distribution, IHqlExpression * optMergeOrder);
-extern HQL_API ITypeInfo * getTypeFromMeta(IHqlExpression * record, IHqlExpression * meta, unsigned firstChild);
-extern HQL_API ITypeInfo * getTypeIntersection(ITypeInfo * leftType, ITypeInfo * rightType);
-extern HQL_API ITypeInfo * getTypeLoseDistributionKeepOrder(ITypeInfo * prev);
-extern HQL_API ITypeInfo * getTypeProject(ITypeInfo * prev, IHqlExpression * newRecord, TableProjectMapper & mapper);
-extern HQL_API ITypeInfo * getTypeSubSort(ITypeInfo * prev, IHqlExpression * grouping, IHqlExpression * sortOrder, bool isLocal);
-extern HQL_API ITypeInfo * getTypeCannotProject(ITypeInfo * prev, IHqlExpression * newRecord); // preserve grouping, but that's it.
-extern HQL_API ITypeInfo * getTypeUnknownDistribution(ITypeInfo * prev);
-extern HQL_API ITypeInfo * getTypeRemoveDistribution(ITypeInfo * prev);
-extern HQL_API ITypeInfo * getTypeRemoveActiveSort(ITypeInfo * prev);
-extern HQL_API ITypeInfo * getTypeRemoveAllSortOrders(ITypeInfo * prev);
-extern HQL_API bool hasUsefulMetaInformation(ITypeInfo * prev);
-
 //---------------------------------------------------------------------------------------------
 
 extern HQL_API IHqlExpression * mapJoinDistribution(TableProjectMapper & mapper, IHqlExpression * distribution, IHqlExpression * side);
@@ -71,7 +106,7 @@ extern HQL_API IHqlExpression * ensureSortedForGroup(IHqlExpression * table, IHq
 extern HQL_API bool matchDedupDistribution(IHqlExpression * distn, const HqlExprArray & equalities);
 extern HQL_API bool matchesAnyDistribution(IHqlExpression * distn);
 
-extern HQL_API bool appearsToBeSorted(ITypeInfo * type, bool isLocal, bool ignoreGrouping);
+extern HQL_API bool appearsToBeSorted(IHqlExpression * dataset, bool isLocal, bool ignoreGrouping);
 extern HQL_API bool isAlreadySorted(IHqlExpression * dataset, HqlExprArray & newSort, bool isLocal, bool ignoreGrouping);
 extern HQL_API bool isAlreadySorted(IHqlExpression * dataset, IHqlExpression * newSort, bool isLocal, bool ignoreGrouping);
 extern HQL_API IHqlExpression * ensureSorted(IHqlExpression * dataset, IHqlExpression * order, bool isLocal, bool ignoreGrouping, bool alwaysLocal, bool allowSubSort);
@@ -88,6 +123,10 @@ extern HQL_API IHqlExpression * preserveTableInfo(IHqlExpression * newTable, IHq
 extern HQL_API bool isDistributedCoLocally(IHqlExpression * dataset1, IHqlExpression * dataset2, const HqlExprArray & sort1, const HqlExprArray & sort2);
 extern HQL_API IHqlExpression * createMatchingDistribution(IHqlExpression * expr, const HqlExprArray & oldSort, const HqlExprArray & newSort);
 
+extern HQL_API void calculateDatasetMeta(CHqlMetaInfo & meta, IHqlExpression * expr);
+extern HQL_API CHqlMetaProperty * querySimpleDatasetMeta(IHqlExpression * expr);
+extern HQL_API bool hasSameSortGroupDistribution(IHqlExpression * expr, IHqlExpression * other);
+
 inline IHqlExpression * queryRemoveOmitted(IHqlExpression * expr)
 {
     if (expr &&  expr->isAttribute() && (expr->queryName() == _omitted_Atom))

ecl/hql/hqlthql.cpp

@@ -702,15 +702,14 @@ void HqltHql::toECL(IHqlExpression *expr, StringBuffer &s, bool paren, bool inTy
         {
             if (expr->isDataset())
             {
-                ITypeInfo * type = expr->queryType();
-                IHqlExpression * group = (IHqlExpression*)type->queryGroupInfo();
+                IHqlExpression * group = queryGroupInfo(expr);
                 if (group)
                     getExprECL(group, s.append("G(")).append(")");
-                IHqlExpression * distrib = queryDistribution(type);
+                IHqlExpression * distrib = queryDistribution(expr);
                 if (distrib) getExprECL(distrib, s.append("D(")).append(")");
-                appendSortOrder(s, "GO", queryGlobalSortOrder(type));
-                appendSortOrder(s, "LO", queryLocalUngroupedSortOrder(type));
-                appendSortOrder(s, "RO", queryGroupSortOrder(type));
+                appendSortOrder(s, "GO", queryGlobalSortOrder(expr));
+                appendSortOrder(s, "LO", queryLocalUngroupedSortOrder(expr));
+                appendSortOrder(s, "RO", queryGroupSortOrder(expr));
             }
         }
         s.append("]");

ecl/hql/hqlutil.cpp

@@ -3651,13 +3651,13 @@ IDefRecordElement * RecordMetaCreator::createField(IHqlExpression * cur, IHqlExp
         childDefRecord.setown(createRecord(fieldRecord, self));
         break;
     case type_table:
-        defType.setown(makeTableType(makeRowType(makeRecordType()), NULL, NULL, NULL));
+        defType.setown(makeTableType(makeRowType(makeRecordType())));
         childDefRecord.setown(::createMetaRecord(fieldRecord, callback));
         break;
     case type_groupedtable:
         {
-            ITypeInfo * tableType = makeTableType(makeRowType(makeRecordType()), NULL, NULL, NULL);
-            defType.setown(makeGroupedTableType(tableType, createAttribute(groupedAtom), NULL));
+            ITypeInfo * tableType = makeTableType(makeRowType(makeRecordType()));
+            defType.setown(makeGroupedTableType(tableType));
             childDefRecord.setown(::createMetaRecord(fieldRecord, callback));
             break;
         }

ecl/hqlcpp/hqlcppds.cpp

@@ -2080,7 +2080,7 @@ void HqlCppTranslator::doBuildDataset(BuildCtx & ctx, IHqlExpression * expr, CHq
             if (expr->isDictionary())
                 type.setown(makeDictionaryType(makeRowType(record->getType())));
             else
-                type.setown(makeTableType(makeRowType(record->getType()), NULL, NULL, NULL));
+                type.setown(makeTableType(makeRowType(record->getType())));
             if ((format == FormatLinkedDataset) || (format == FormatArrayDataset) || expr->isDictionary())
                 type.setown(setLinkCountedAttr(type, true));
             tgt.expr.setown(createValue(no_nullptr, makeReferenceModifier(type.getClear())));
@@ -2844,7 +2844,7 @@ bool HqlCppTranslator::doBuildDatasetInlineTable(BuildCtx & ctx, IHqlExpression
 
     unsigned maxRows = values->numChildren();
     Owned<ITypeInfo> declareType = makeConstantModifier(makeArrayType(LINK(rowType), maxRows));
-    OwnedITypeInfo rowsType = makeOutOfLineModifier(makeTableType(LINK(rowType), NULL, NULL, NULL));
+    OwnedITypeInfo rowsType = makeOutOfLineModifier(makeTableType(LINK(rowType)));
     if (options.canLinkConstantRows)
         rowsType.setown(setLinkCountedAttr(rowsType, true));
 
@@ -4573,7 +4573,7 @@ void HqlCppTranslator::convertBoundRowToDataset(BuildCtx & ctx, CHqlBoundExpr &
 {
     IHqlExpression * boundRow = row->queryBound();
     IHqlExpression * record = row->queryDataset()->queryRecord();
-    Owned<ITypeInfo> type = makeTableType(makeRowType(LINK(record->queryType())), NULL, NULL, NULL);
+    Owned<ITypeInfo> type = makeTableType(makeRowType(LINK(record->queryType())));
     Owned<ITypeInfo> refType = makeReferenceModifier(LINK(type));
     if (hasLinkCountedModifier(boundRow->queryType()) && (preferredFormat != FormatBlockedDataset))
     {

ecl/hqlcpp/hqlhtcpp.cpp

@@ -222,7 +222,7 @@ bool canIterateTableInline(IHqlExpression * expr)
     case no_newaggregate:
         {
             IHqlExpression * child = expr->queryChild(0);
-            if (!child->queryType()->queryGroupInfo())
+            if (!isGrouped(child))
                 return canIterateTableInline(child);
             return false;
         }
@@ -2146,7 +2146,7 @@ void ActivityInstance::createGraphNode(IPropertyTree * defaultSubGraph, bool alw
     addAttributeBool("local", isLocal);
 
 #ifdef _DEBUG
-    assertex(dataset->isAction() == isActivitySink(kind));
+//    assertex(dataset->isAction() == isActivitySink(kind));
 #endif
     if (dataset->isAction())
     {
@@ -2181,30 +2181,29 @@ void ActivityInstance::createGraphNode(IPropertyTree * defaultSubGraph, bool alw
     if (translator.queryOptions().showMetaInGraph)
     {
         StringBuffer s;
-        ITypeInfo * type = dataset->queryType();
         if (translator.targetThor())
         {
-            IHqlExpression * distribution = queryDistribution(type);
+            IHqlExpression * distribution = queryDistribution(dataset);
             if (distribution && distribution->queryName() != localAtom)
                 addAttribute("metaDistribution", getExprECL(distribution, s.clear(), true).str());
         }
 
-        IHqlExpression * grouping = (IHqlExpression *)type->queryGroupInfo();
+        IHqlExpression * grouping = queryGrouping(dataset);
         if (grouping)
             addAttribute("metaGrouping", getExprECL(grouping, s.clear(), true).str());
 
         if (translator.targetThor())
         {
-            IHqlExpression * globalSortOrder = (IHqlExpression *)type->queryGlobalSortInfo();
+            IHqlExpression * globalSortOrder = queryGlobalSortOrder(dataset);
             if (globalSortOrder)
                 addAttribute("metaGlobalSortOrder", getExprECL(globalSortOrder, s.clear(), true).str());
         }
 
-        IHqlExpression * localSortOrder = (IHqlExpression *)type->queryLocalUngroupedSortInfo();
+        IHqlExpression * localSortOrder = queryLocalUngroupedSortOrder(dataset);
         if (localSortOrder)
             addAttribute("metaLocalSortOrder", getExprECL(localSortOrder, s.clear(), true).str());
 
-        IHqlExpression * groupSortOrder = (IHqlExpression *)type->queryGroupSortInfo();
+        IHqlExpression * groupSortOrder = queryGroupSortOrder(dataset);
         if (groupSortOrder)
             addAttribute("metaGroupSortOrder", getExprECL(groupSortOrder, s.clear(), true).str());
     }
@@ -13535,7 +13534,7 @@ ABoundActivity * HqlCppTranslator::doBuildActivityDedup(BuildCtx & ctx, IHqlExpr
 
     IHqlExpression * dataset = expr->queryChild(0);
     IHqlExpression * selSeq = querySelSeq(expr);
-    bool isGrouped = (dataset->queryType()->queryGroupInfo() != NULL);
+    bool isGrouped = ::isGrouped(dataset);
     bool isLocal = isLocalActivity(expr);
     bool useHash = expr->hasAttribute(hashAtom);
     if (targetThor() && !isGrouped && !isLocal)
@@ -13571,7 +13570,7 @@ ABoundActivity * HqlCppTranslator::doBuildActivityDedup(BuildCtx & ctx, IHqlExpr
             ForEachItemIn(i1, info.equalities)
                 normalizedEqualities.append(*replaceSelector(info.equalities.item(i1).queryBody(), dataset, queryActiveTableSelector()));
 
-            IHqlExpression * grouping = static_cast<IHqlExpression *>(dataset->queryType()->queryGroupInfo());
+            IHqlExpression * grouping = queryGrouping(dataset);
             ForEachChild(i, grouping)
             {
                 IHqlExpression * curGroup = grouping->queryChild(i);
@@ -14912,12 +14911,12 @@ void HqlCppTranslator::bindRows(BuildCtx & ctx, node_operator side, IHqlExpressi
     OwnedHqlExpr selector = createSelector(side, dataset, selSeq);
     OwnedHqlExpr rowsExpr = createDataset(no_rows, LINK(selector), LINK(rowsid));
 
-    ITypeInfo * rowType = makeReferenceModifier(LINK(rowsExpr->queryType()->queryChildType()));
+    Owned<ITypeInfo> rowType = makeReferenceModifier(LINK(rowsExpr->queryType()->queryChildType()));
     if (rowsAreLinkCounted)
-        rowType = makeAttributeModifier(rowType, getLinkCountedAttr());
+        rowType.setown(setLinkCountedAttr(rowType, true));
 
     //Rows may be link counted, but rows() is not a linkable rowset
-    OwnedITypeInfo rowsType = makeReferenceModifier(makeTableType(rowType, NULL, NULL, NULL));
+    OwnedITypeInfo rowsType = makeReferenceModifier(makeTableType(rowType.getClear()));
     rowsType.setown(makeOutOfLineModifier(LINK(rowsType)));
 
     CHqlBoundExpr boundRows;
@@ -15527,7 +15526,7 @@ ABoundActivity * HqlCppTranslator::doBuildActivityGroup(BuildCtx & ctx, IHqlExpr
         child = child->queryChild(0);
 
     Owned<ABoundActivity> boundDataset = buildCachedActivity(ctx, child);
-    if (expr->queryType()->queryGroupInfo() == child->queryType()->queryGroupInfo())
+    if (queryGrouping(expr) == queryGrouping(child))
         return boundDataset.getClear();
 
     IHqlExpression * sortlist = queryRealChild(expr, 1);

ecl/hqlcpp/hqlresource.cpp

@@ -488,7 +488,7 @@ bool lightweightAndReducesDatasetSize(IHqlExpression * expr)
         return true;
     case no_group:
         //removing grouping will reduce size of the spill file.
-        if (expr->queryType()->queryGroupInfo() == NULL)
+        if (!isGrouped(expr))
             return true;
         break;
     }
@@ -1040,8 +1040,7 @@ void ResourcerInfo::clearProjected()
 
 void ResourcerInfo::addSpillFlags(HqlExprArray & args, bool isRead)
 {
-    IHqlExpression * grouping = (IHqlExpression *)original->queryType()->queryGroupInfo();
-    if (grouping)
+    if (isGrouped(original))
         args.append(*createAttribute(groupedAtom));
 
     if (outputToUseForSpill)
@@ -3123,7 +3122,7 @@ void EclResourcer::createInitialGraph(IHqlExpression * expr, IHqlExpression * ow
                 if (filename && (filename->getOperator() == no_constant) && !expr->hasAttribute(xmlAtom) && !expr->hasAttribute(csvAtom))
                 {
                     IHqlExpression * dataset = expr->queryChild(0);
-                    if (expr->hasAttribute(groupedAtom) == (dataset->queryType()->queryGroupInfo() != NULL))
+                    if (expr->hasAttribute(groupedAtom) == isGrouped(dataset))
                     {
                         StringBuffer filenameText;
                         filename->queryValue()->getStringValue(filenameText);

ecl/hqlcpp/hqlttcpp.cpp

@@ -3478,7 +3478,7 @@ IHqlExpression * ThorHqlTransformer::normalizeTableGrouping(IHqlExpression * exp
                     newsort.set(group);
 
                 LinkedHqlExpr ds = dataset;
-                if (ds->queryType()->queryGroupInfo())
+                if (isGrouped(ds))
                 {
                     ds.setown(createDataset(no_group, ds.getClear(), NULL));
                     ds.setown(cloneInheritedAnnotations(expr, ds));
@@ -4901,7 +4901,6 @@ void GlobalAttributeInfo::doSplitGlobalDefinition(ITypeInfo * type, IHqlExpressi
     ITypeInfo * valueType = value->queryType();
     if (value->isDataset() || value->isDictionary())
     {
-        IHqlExpression * groupOrder = (IHqlExpression *)valueType->queryGroupInfo();
         if (few)
         {
             splitSmallDataset(value, setOutput, getOutput);
@@ -4920,7 +4919,7 @@ void GlobalAttributeInfo::doSplitGlobalDefinition(ITypeInfo * type, IHqlExpressi
         if (valueType->getTypeCode() == type_groupedtable)
             args.append(*createAttribute(groupedAtom));
         else
-            assertex(groupOrder == NULL);
+            assertex(!isGrouped(valueType));
 
         bool compressFile = true;
         switch (persistOp)
@@ -4930,7 +4929,7 @@ void GlobalAttributeInfo::doSplitGlobalDefinition(ITypeInfo * type, IHqlExpressi
                 args.append(*createAttribute(_workflowPersist_Atom));
                 args.append(*createAttribute(sequenceAtom, getGlobalSequenceNumber()));
                 //add a flag to help get the resourcing right - may need to hash distribute on different size thor
-                IHqlExpression * distribution = queryDistribution(valueType);
+                IHqlExpression * distribution = queryDistribution(value);
                 if (distribution && !distribution->isAttribute())
                     args.append(*createAttribute(distributedAtom));
                 break;
@@ -4985,7 +4984,7 @@ void GlobalAttributeInfo::doSplitGlobalDefinition(ITypeInfo * type, IHqlExpressi
 
             if (persistOp == no_persist)
                 args.append(*createAttribute(_workflowPersist_Atom));
-            if (groupOrder)
+            if (isGrouped(value))
                 args.append(*createAttribute(groupedAtom));
             if (compressFile)
                 args.append(*createAttribute(__compressed__Atom));