Merge pull request #4711 from ghalliday/issue9701

HPCC-9701 Refactor meta calculation into separate properties

Reviewed-By: Richard Chapman <rchapman@hpccsystems.com>

common/deftype/deftype.cpp

@@ -1379,40 +1379,6 @@ bool CTableTypeInfo::assignableFrom(ITypeInfo *t2)
     return false;
 }
 
-IInterface *CTableTypeInfo::queryDistributeInfo()
-{
-    return distributeinfo;
-}
-
-IInterface *CTableTypeInfo::queryGlobalSortInfo()
-{
-    return globalsortinfo;
-}
-
-IInterface *CTableTypeInfo::queryLocalUngroupedSortInfo()
-{
-    return localsortinfo;
-}
-
-unsigned CTableTypeInfo::getHash() const
-{
-    unsigned hashcode = CBasedTypeInfo::getHash();
-    HASHFIELD(distributeinfo);
-    HASHFIELD(globalsortinfo);
-    HASHFIELD(localsortinfo);
-    return hashcode;
-}
-
-
-bool CTableTypeInfo::equals(const CTypeInfo & _other) const
-{
-    if (!CBasedTypeInfo::equals(_other))
-        return false;
-    const CTableTypeInfo & other = static_cast<const CTableTypeInfo &>(_other);
-    return (distributeinfo == other.distributeinfo) && (globalsortinfo == other.globalsortinfo) && (localsortinfo == other.localsortinfo);
-}
-
-
 StringBuffer & CTableTypeInfo::getECLType(StringBuffer & out)   
 { 
     ITypeInfo * recordType = ::queryRecordType(this);
@@ -1425,13 +1391,12 @@ StringBuffer & CTableTypeInfo::getECLType(StringBuffer & out)
     return out; 
 }
 
+
 void CTableTypeInfo::serialize(MemoryBuffer &tgt)
-{ 
-    assertex(!globalsortinfo && !localsortinfo && !distributeinfo);
+{
     CBasedTypeInfo::serializeSkipChild(tgt);
 }
 
-
 bool CGroupedTableTypeInfo::assignableFrom(ITypeInfo *t2)
 {
     if (getTypeCode()==t2->getTypeCode())
@@ -1446,55 +1411,6 @@ bool CGroupedTableTypeInfo::assignableFrom(ITypeInfo *t2)
     return false;
 }
 
-IInterface *CGroupedTableTypeInfo::queryDistributeInfo()
-{
-    return queryChildType()->queryDistributeInfo();
-}
-
-IInterface *CGroupedTableTypeInfo::queryGroupInfo()
-{
-    return groupinfo;
-}
-
-IInterface *CGroupedTableTypeInfo::queryGlobalSortInfo()
-{
-    return queryChildType()->queryGlobalSortInfo();
-}
-
-IInterface *CGroupedTableTypeInfo::queryLocalUngroupedSortInfo()
-{
-    return queryChildType()->queryLocalUngroupedSortInfo();
-}
-
-IInterface *CGroupedTableTypeInfo::queryGroupSortInfo()
-{
-    return groupsortinfo;
-}
-
-unsigned CGroupedTableTypeInfo::getHash() const
-{
-    unsigned hashcode = CBasedTypeInfo::getHash();
-    HASHFIELD(groupinfo);
-    HASHFIELD(groupsortinfo);
-    return hashcode;
-}
-
-
-bool CGroupedTableTypeInfo::equals(const CTypeInfo & _other) const
-{
-    if (!CBasedTypeInfo::equals(_other))
-        return false;
-    const CGroupedTableTypeInfo & other = static_cast<const CGroupedTableTypeInfo &>(_other);
-    return (groupsortinfo == other.groupsortinfo) && (groupinfo == other.groupinfo);
-}
-
-void CGroupedTableTypeInfo::serialize(MemoryBuffer &tgt)
-{ 
-    assertex(!groupsortinfo && !groupinfo);
-    CBasedTypeInfo::serialize(tgt);
-}
-
-
 bool CSetTypeInfo::assignableFrom(ITypeInfo *t2)
 {
     return getTypeCode()==t2->getTypeCode() && 
@@ -2014,10 +1930,10 @@ extern DEFTYPE_API ITypeInfo *makeRuleType(ITypeInfo *basetype)
     return commonUpType(new CRuleTypeInfo(basetype));
 }
 
-extern DEFTYPE_API ITypeInfo *makeTableType(ITypeInfo *basetype, IInterface * distributeinfo, IInterface *globalsortinfo, IInterface *localsortinfo)
+extern DEFTYPE_API ITypeInfo *makeTableType(ITypeInfo *basetype)
 {
     assertex(!basetype || basetype->getTypeCode() == type_row);
-    return commonUpType(new CTableTypeInfo(basetype, distributeinfo, globalsortinfo, localsortinfo));
+    return commonUpType(new CTableTypeInfo(basetype));
 }
 
 extern DEFTYPE_API ITypeInfo *makeDictionaryType(ITypeInfo *basetype)
@@ -2026,9 +1942,9 @@ extern DEFTYPE_API ITypeInfo *makeDictionaryType(ITypeInfo *basetype)
     return commonUpType(new CDictionaryTypeInfo(basetype));
 }
 
-extern DEFTYPE_API ITypeInfo *makeGroupedTableType(ITypeInfo *basetype, IInterface *groupinfo, IInterface *sortinfo)
+extern DEFTYPE_API ITypeInfo *makeGroupedTableType(ITypeInfo *basetype)
 {
-    return commonUpType(new CGroupedTableTypeInfo(basetype, groupinfo, sortinfo));
+    return commonUpType(new CGroupedTableTypeInfo(basetype));
 }
 
 extern DEFTYPE_API ITypeInfo *makeFunctionType(ITypeInfo *basetype, IInterface * parameters, IInterface * defaults)
@@ -3409,10 +3325,10 @@ ITypeInfo * replaceChildType(ITypeInfo * type, ITypeInfo * newChild)
         newType.setown(makeDictionaryType(LINK(newChild)));
         break;
     case type_table:
-        newType.setown(makeTableType(LINK(newChild), LINK(type->queryDistributeInfo()), LINK(type->queryGlobalSortInfo()), LINK(type->queryLocalUngroupedSortInfo())));
+        newType.setown(makeTableType(LINK(newChild)));
         break;
     case type_groupedtable:
-        newType.setown(makeGroupedTableType(LINK(newChild), LINK(type->queryGroupInfo()), LINK(type->queryGroupSortInfo())));
+        newType.setown(makeGroupedTableType(LINK(newChild)));
         break;
     case type_row:
         newType.setown(makeRowType(LINK(newChild)));
@@ -3631,12 +3547,12 @@ extern DEFTYPE_API ITypeInfo * deserializeType(MemoryBuffer &src)
     case type_table:
         {
             ITypeInfo *base = deserializeType(src);
-            return makeTableType(makeRowType(base), NULL, NULL, NULL);
+            return makeTableType(makeRowType(base));
         }
     case type_groupedtable:
         {
             ITypeInfo *base = deserializeType(src);
-            return makeGroupedTableType(base, NULL, NULL);
+            return makeGroupedTableType(base);
         }
 
         

common/deftype/deftype.hpp

@@ -194,11 +194,6 @@ public:
     virtual ICharsetInfo * queryCharset() = 0;
     virtual ICollationInfo * queryCollation() = 0;
     virtual IAtom * queryLocale() = 0;
-    virtual IInterface * queryDistributeInfo() = 0;
-    virtual IInterface * queryGroupInfo() = 0;
-    virtual IInterface * queryGlobalSortInfo() = 0;
-    virtual IInterface * queryLocalUngroupedSortInfo() = 0;
-    virtual IInterface * queryGroupSortInfo() = 0;
     virtual ITypeInfo * queryPromotedType() = 0;
     virtual ITypeInfo * queryTypeBase() = 0;
     virtual unsigned getCrc() = 0;      // must be run independant.
@@ -249,8 +244,8 @@ extern DEFTYPE_API ITypeInfo *makeType(type_t type, int size);
 extern DEFTYPE_API IEnumeratedTypeBuilder *makeEnumeratedTypeBuilder(ITypeInfo *base, aindex_t numvalues);
 extern DEFTYPE_API ITypeInfo *makeDecimalType(unsigned digits, unsigned prec, bool isSigned);
 extern DEFTYPE_API ITypeInfo *makeDictionaryType(ITypeInfo *basetype);
-extern DEFTYPE_API ITypeInfo *makeTableType(ITypeInfo *basetype, IInterface * distributeinfo, IInterface *gloalSortinfo, IInterface * localSortInfo);
-extern DEFTYPE_API ITypeInfo *makeGroupedTableType(ITypeInfo *basetype, IInterface *groupinfo, IInterface *sortinfo);
+extern DEFTYPE_API ITypeInfo *makeTableType(ITypeInfo *basetype);
+extern DEFTYPE_API ITypeInfo *makeGroupedTableType(ITypeInfo *basetype);
 extern DEFTYPE_API ITypeInfo *makeRowType(ITypeInfo *basetype);
 extern DEFTYPE_API ITypeInfo *makeSetType(ITypeInfo *basetype);
 extern DEFTYPE_API ITypeInfo *makeTransformType(ITypeInfo *basetype);

common/deftype/deftype.ipp

@@ -56,11 +56,6 @@ public:
     virtual bool isSigned()                     { return false; }
     virtual bool isSwappedEndian()              { return false; }
     virtual ITypeInfo * queryChildType()        { return NULL; }
-    virtual IInterface * queryDistributeInfo()  { return NULL; }
-    virtual IInterface * queryLocalUngroupedSortInfo()   { return NULL; }
-    virtual IInterface * queryGlobalSortInfo()  { return NULL; }
-    virtual IInterface * queryGroupInfo()       { return NULL; }
-    virtual IInterface * queryGroupSortInfo()   { return NULL; }
     virtual ITypeInfo * queryPromotedType()     { return this; }    // very common implementation
     virtual ICharsetInfo * queryCharset()       { return NULL; }
     virtual ICollationInfo * queryCollation()   { return NULL; }
@@ -649,21 +644,15 @@ public:
 class CTableTypeInfo : public CBasedTypeInfo
 {
 private:
-    Owned<IInterface> distributeinfo;
-    Owned<IInterface> globalsortinfo;
-    Owned<IInterface> localsortinfo;
 public:
-    CTableTypeInfo(ITypeInfo * _basetype, IInterface * _distributeinfo, IInterface *_globalsortinfo, IInterface *_localsortinfo) 
-        : CBasedTypeInfo(_basetype, UNKNOWN_LENGTH), distributeinfo(_distributeinfo), globalsortinfo(_globalsortinfo), localsortinfo(_localsortinfo) {}
+    CTableTypeInfo(ITypeInfo * _basetype)
+        : CBasedTypeInfo(_basetype, UNKNOWN_LENGTH) {}
 
     virtual type_t getTypeCode() const                          { return type_table; }
     virtual bool isScalar()                                 { return false; }
     virtual const char *queryTypeName()                     { return "table"; }
     virtual StringBuffer &getECLType(StringBuffer & out);
     virtual bool assignableFrom(ITypeInfo *t2);
-    virtual IInterface * queryDistributeInfo();
-    virtual IInterface * queryGlobalSortInfo();
-    virtual IInterface * queryLocalUngroupedSortInfo();
 
     virtual void serialize(MemoryBuffer &tgt);
     //MORE: Delete this when persist attributes change again
@@ -674,31 +663,21 @@ public:
                                                                 return crc;
                                                             }
 
-    virtual unsigned getHash() const;
-    virtual bool equals(const CTypeInfo & other) const;
 };
 
 class CGroupedTableTypeInfo : public CBasedTypeInfo
 {
 private:
-    Owned<IInterface> groupinfo;
-    Owned<IInterface> groupsortinfo;
 public:
-    CGroupedTableTypeInfo(ITypeInfo * _basetype, IInterface *_groupinfo, IInterface *_groupsortinfo) 
-    : CBasedTypeInfo(_basetype, UNKNOWN_LENGTH), groupinfo(_groupinfo), groupsortinfo(_groupsortinfo) {}
+    CGroupedTableTypeInfo(ITypeInfo * _basetype)
+    : CBasedTypeInfo(_basetype, UNKNOWN_LENGTH) {}
 
     virtual type_t getTypeCode() const                          { return type_groupedtable; }
     virtual StringBuffer &getECLType(StringBuffer & out)    { out.append(queryTypeName()).append(" of "); queryChildType()->getECLType(out); return out; };
     virtual const char *queryTypeName()                     { return "groupedtable"; }
     virtual bool isScalar()                                 { return false; }
     virtual bool assignableFrom(ITypeInfo *t2);
-    virtual IInterface * queryDistributeInfo();
-    virtual IInterface * queryGroupInfo();
-    virtual IInterface * queryGlobalSortInfo();
-    virtual IInterface * queryLocalUngroupedSortInfo();
-    virtual IInterface * queryGroupSortInfo();
 
-    virtual void serialize(MemoryBuffer &tgt);
     //MORE: Delete this when persist attributes change again
     virtual unsigned getCrc()                               {
                                                                 unsigned crc = getTypeCode();
@@ -707,8 +686,6 @@ public:
                                                                 return crc;
                                                             }
 
-    virtual unsigned getHash() const;
-    virtual bool equals(const CTypeInfo & other) const;
 };
 
 class CSetTypeInfo : public CBasedTypeInfo
@@ -850,11 +827,6 @@ public:
     virtual ICharsetInfo * queryCharset()                           { return baseType->queryCharset(); }
     virtual ICollationInfo * queryCollation()                       { return baseType->queryCollation(); }
     virtual IAtom * queryLocale()                                     { return baseType->queryLocale(); }
-    virtual IInterface * queryLocalUngroupedSortInfo()                      { return baseType->queryLocalUngroupedSortInfo(); }
-    virtual IInterface * queryGlobalSortInfo()                      { return baseType->queryGlobalSortInfo(); }
-    virtual IInterface * queryGroupInfo()                           { return baseType->queryGroupInfo(); }
-    virtual IInterface * queryGroupSortInfo()                       { return baseType->queryGroupSortInfo(); }
-    virtual IInterface * queryDistributeInfo()                      { return baseType->queryDistributeInfo(); }
     virtual ITypeInfo * queryPromotedType()                         { return baseType->queryPromotedType(); }
     virtual ITypeInfo * queryTypeBase()                             { return baseType; }
     virtual unsigned getCrc()                                       { return baseType->getCrc(); }

common/fileview2/fvsource.cpp

@@ -99,7 +99,7 @@ void DataSourceMetaItem::serialize(MemoryBuffer & out) const
 
 DataSourceDatasetItem::DataSourceDatasetItem(const char * _name, const char * _xpath, IHqlExpression * expr) : DataSourceMetaItem(FVFFdataset, NULL, NULL, NULL), record(expr->queryRecord(), 0, true, false, false)
 {
-    type.setown(makeTableType(NULL, NULL, NULL, NULL));
+    type.setown(makeTableType(NULL));
     name.set(_name);
     xpath.set(_xpath);
     splitXmlTagNamesFromXPath(_xpath, record.tagname, &tagname);
@@ -109,7 +109,7 @@ DataSourceDatasetItem::DataSourceDatasetItem(const char * _name, const char * _x
 
 DataSourceDatasetItem::DataSourceDatasetItem(unsigned flags, MemoryBuffer & in) : DataSourceMetaItem(FVFFdataset, NULL, NULL, NULL), record(in)
 {
-    type.setown(makeTableType(NULL, NULL, NULL, NULL));
+    type.setown(makeTableType(NULL));
     in.read(name);
 }
 

ecl/hql/hqlattr.cpp

@@ -37,16 +37,16 @@
 #include "hqlattr.hpp"
 #include "hqlmeta.hpp"
 
-static CriticalSection * propertyCS;
+static SpinLock * propertyLock;
 
 MODULE_INIT(INIT_PRIORITY_HQLINTERNAL)
 {
-    propertyCS = new CriticalSection;
+    propertyLock = new SpinLock;
     return true;
 }
 MODULE_EXIT()
 {
-    delete propertyCS;
+    delete propertyLock;
 }
 
 // This file should contain most of the derived property calculation for nodes in the expression tree,
@@ -831,13 +831,12 @@ static IHqlExpression * evaluateSerializedRecord(IHqlExpression * expr, IAtom *
 class CHqlExprMeta
 {
 public:
-    static inline IHqlExpression * addProperty(IHqlExpression * expr, ExprPropKind kind, IHqlExpression * value)
+    static inline void addProperty(IHqlExpression * expr, ExprPropKind kind, IInterface * value)
     {
         CHqlExpression * cexpr = static_cast<CHqlExpression *>(expr);
         cexpr->addProperty(kind, value);
-        return value;
     }
-    static inline IHqlExpression * queryExistingProperty(IHqlExpression * expr, ExprPropKind kind)
+    static inline IInterface * queryExistingProperty(IHqlExpression * expr, ExprPropKind kind)
     {
         CHqlExpression * cexpr = static_cast<CHqlExpression *>(expr);
         return cexpr->queryExistingProperty(kind);
@@ -856,7 +855,8 @@ static IHqlExpression * evaluatePropSerializedForm(IHqlExpression * expr, ExprPr
             //Tag serialized form so don't re-evaluated
             meta.addProperty(serialized, kind, serialized);
         }
-        return meta.addProperty(expr, kind, serialized);
+        meta.addProperty(expr, kind, serialized);
+        return serialized;
     }
     return NULL;
 }
@@ -893,7 +893,8 @@ static IHqlExpression * evaluateFieldAttrSize(IHqlExpression * expr)
                 else
                 {
                     IHqlExpression * ret = expr->queryRecord()->queryProperty(EPsize);
-                    return meta.addProperty(expr, EPsize, ret);
+                    meta.addProperty(expr, EPsize, ret);
+                    return ret;
                 }
                 break;
             }
@@ -1077,13 +1078,15 @@ static IHqlExpression * evaluateFieldAttrSize(IHqlExpression * expr)
     if ((thisSize == minSize) && (minSize == maxSize))
     {
         OwnedHqlExpr attr = getFixedSizeAttr(thisSize);
-        return meta.addProperty(expr, EPsize, attr);
+        meta.addProperty(expr, EPsize, attr);
+        return attr;
     }
 
     if (!thisMaxSizeExpr)
         thisMaxSizeExpr.setown((maxSize == UNKNOWN_LENGTH) ? createAttribute(unknownSizeFieldAtom) : getSizetConstant(maxSize));
     OwnedHqlExpr attr = createExprAttribute(_propSize_Atom, getSizetConstant(thisSize), getSizetConstant(minSize), thisMaxSizeExpr.getClear());
-    return meta.addProperty(expr, EPsize, attr);
+    meta.addProperty(expr, EPsize, attr);
+    return attr;
 }
 
 //no_ifblock
@@ -1092,7 +1095,8 @@ static IHqlExpression * evaluateIfBlockAttrSize(IHqlExpression * expr)
     IHqlExpression * size = expr->queryChild(1)->queryProperty(EPsize);
     unsigned averageSize = (unsigned)getIntValue(size->queryChild(0), 0)/2;
     OwnedHqlExpr attr = createExprAttribute(_propSize_Atom, getSizetConstant(averageSize), getSizetConstant(0), LINK(size->queryChild(2)));
-    return meta.addProperty(expr, EPsize, attr);
+    meta.addProperty(expr, EPsize, attr);
+    return attr;
 }
 
 //no_record
@@ -1212,7 +1216,8 @@ static IHqlExpression * evaluateRecordAttrSize(IHqlExpression * expr)
         args.append(*LINK(derivedSizeExpr));
 
     OwnedHqlExpr sizeAttr = createExprAttribute(_propSize_Atom, args);
-    return meta.addProperty(expr, EPsize, sizeAttr);
+    meta.addProperty(expr, EPsize, sizeAttr);
+    return sizeAttr;
 }
 
 
@@ -1232,7 +1237,8 @@ static IHqlExpression * evalautePropSize(IHqlExpression * expr)
             //MORE: This could calculate a better estimate for the size of the record by taking into account any constant values or datasets that are assigned.
             IHqlExpression * record = expr->queryRecord();
             IHqlExpression * recordSize = record->queryProperty(EPsize);
-            return meta.addProperty(expr, EPsize, recordSize);
+            meta.addProperty(expr, EPsize, recordSize);
+            return recordSize;
         }
     }
     IHqlExpression * record = expr->queryRecord();
@@ -1281,7 +1287,7 @@ ITypeInfo * getSerializedForm(ITypeInfo * type, IAtom * variation)
             OwnedITypeInfo newChild = getSerializedForm(childType, variation);
             if (variation == internalAtom)
                 return replaceChildType(noLinkCountType, newChild);
-            OwnedITypeInfo datasetType = makeTableType(LINK(newChild), NULL, NULL, NULL);
+            OwnedITypeInfo datasetType = makeTableType(LINK(newChild));
             return cloneModifiers(noLinkCountType, datasetType);
         }
     }
@@ -1311,9 +1317,9 @@ HqlCachedPropertyTransformer::HqlCachedPropertyTransformer(HqlTransformerInfo &
 
 IHqlExpression * HqlCachedPropertyTransformer::transform(IHqlExpression * expr)
 {
-    IHqlExpression * match = meta.queryExistingProperty(expr, propKind);
+    IInterface * match = meta.queryExistingProperty(expr, propKind);
     if (match)
-        return LINK(match);
+        return static_cast<IHqlExpression *>(LINK(match));
 
     OwnedHqlExpr transformed = QuickHqlTransformer::transform(expr);
 
@@ -1407,7 +1413,7 @@ static IHqlExpression * evalautePropUnadorned(IHqlExpression * expr)
     HqlUnadornedNormalizer normalizer;
     //NB: Also has the side-effect of adding any missing attributes
     OwnedHqlExpr dummy = normalizer.transform(expr);
-    return meta.queryExistingProperty(expr, EPunadorned);
+    return static_cast<IHqlExpression *>(meta.queryExistingProperty(expr, EPunadorned));
 }
 
 //---------------------------------------------------------------------------------
@@ -1577,11 +1583,15 @@ static IHqlExpression * evalautePropAligned(IHqlExpression * expr)
         same = false;
     OwnedHqlExpr newRecord = same ? LINK(expr) : expr->clone(result);
     if (expr == newRecord)
-        return meta.addProperty(expr, EPaligned, queryAlignedAttr());
+    {
+        meta.addProperty(expr, EPaligned, queryAlignedAttr());
+        return queryAlignedAttr();
+    }
     meta.addProperty(newRecord, EPaligned, queryAlignedAttr());
     
     OwnedHqlExpr alignAttr = createExprAttribute(_propAligned_Atom, newRecord.getClear());
-    return meta.addProperty(expr, EPaligned, alignAttr);
+    meta.addProperty(expr, EPaligned, alignAttr);
+    return alignAttr;
 }
 
 //---------------------------------------------------------------------------------
@@ -1705,8 +1715,8 @@ bool isLocalActivity(IHqlExpression * expr)
     default:
         {
             assertex(!localChangesActivity(expr));
-            ITypeInfo * exprType = expr->queryType();
-            if (exprType && (exprType->queryDistributeInfo() != NULL))
+            //MORE: What is this test here for??
+            if (!expr->isAction() && queryDistribution(expr))
                 return !isGroupedActivity(expr);
             return false;
         }
@@ -2969,7 +2979,8 @@ IHqlExpression * calcRowInformation(IHqlExpression * expr)
 static IHqlExpression * evalautePropRecordCount(IHqlExpression * expr)
 {
     OwnedHqlExpr info = calcRowInformation(expr);
-    return meta.addProperty(expr, EPrecordCount, info);
+    meta.addProperty(expr, EPrecordCount, info);
+    return info;
 }
 
 
@@ -3427,7 +3438,7 @@ IHqlExpression * HqlLocationIndependentNormalizer::doCreateTransformed(IHqlExpre
             if (expr->numChildren() != 0)
             {
                 IAtom * name = expr->queryName();
-                if (name != _countProject_Atom)
+                if ((name != _countProject_Atom) && (name != _metadata_Atom))
                     return createAttribute(expr->queryName());
             }
             return LINK(expr);
@@ -3476,9 +3487,9 @@ IHqlExpression * HqlLocationIndependentNormalizer::createTransformed(IHqlExpress
     if (isAlwaysLocationIndependent(expr))
         return LINK(expr);
 
-    IHqlExpression * match = meta.queryExistingProperty(expr, EPlocationIndependent);
+    IInterface * match = meta.queryExistingProperty(expr, EPlocationIndependent);
     if (match)
-        return LINK(match);
+        return static_cast<IHqlExpression *>(LINK(match));
 
     OwnedHqlExpr transformed = doCreateTransformed(expr);
 
@@ -3699,43 +3710,73 @@ ITypeInfo * setStreamedAttr(ITypeInfo * _type, bool setValue)
 
 //---------------------------------------------------------------------------------------------------------------------
 
-IHqlExpression * CHqlExpression::queryExistingProperty(ExprPropKind propKind) const
+IInterface * CHqlExpression::queryExistingProperty(ExprPropKind propKind) const
 {
-    CriticalBlock block(*propertyCS);
+    SpinBlock block(*propertyLock);
     CHqlDynamicProperty * cur = attributes;
     while (cur)
     {
         if (cur->kind == propKind)
         {
-            IHqlExpression * value = cur->value;
+            IInterface * value = cur->value;
             if (value)
                 return value;
-            return const_cast<CHqlExpression *>(this);
+            return static_cast<IHqlExpression *>(const_cast<CHqlExpression *>(this));
         }
         cur = cur->next;
     }
     return NULL;
 }
 
-void CHqlExpression::addProperty(ExprPropKind kind, IHqlExpression * value)
+void CHqlExpression::addProperty(ExprPropKind kind, IInterface * value)
 {
-    if (value == this)
+    if (value == static_cast<IHqlExpression *>(this))
         value = NULL;
+    CHqlDynamicProperty * attr = new CHqlDynamicProperty(kind, value);
 
-    CriticalBlock block(*propertyCS);
+    SpinBlock block(*propertyLock);
     //theoretically we should test if the attribute has already been added by another thread, but in practice there is no
     //problem if the attribute is present twice.
-    CHqlDynamicProperty * attr = new CHqlDynamicProperty(kind, value);
     attr->next = attributes;
     attributes = attr;
 }
 
 
+//A specialised version of addProperty/queryExistingProperty.  Uses sizeof(void*) all the time instead
+//of 3*sizeof(void *)+heap overhead when used.
+//Possibly saves a bit of time, but more useful as a proof of concept in case it was useful elsewhere..
+void CHqlDataset::addProperty(ExprPropKind kind, IInterface * value)
+{
+    if (kind == EPmeta)
+    {
+        SpinBlock block(*propertyLock);
+        //ensure once meta is set it is never modified
+        if (!metaProperty)
+            metaProperty.set(value);
+    }
+    else
+        CHqlExpression::addProperty(kind, value);
+
+}
+
+IInterface * CHqlDataset::queryExistingProperty(ExprPropKind kind) const
+{
+    if (kind == EPmeta)
+    {
+        SpinBlock block(*propertyLock);
+        return metaProperty;
+    }
+
+    return CHqlExpression::queryExistingProperty(kind);
+}
+
+//---------------------------------------------------------------------------------------------------------------------
+
 IHqlExpression * CHqlExpression::queryProperty(ExprPropKind kind)
 {
-    IHqlExpression * match = queryExistingProperty(kind);
+    IInterface * match = queryExistingProperty(kind);
     if (match)
-        return match;
+        return static_cast<IHqlExpression *>(match);
 
     switch (kind)
     {
@@ -3756,3 +3797,23 @@ IHqlExpression * CHqlExpression::queryProperty(ExprPropKind kind)
     }
     return NULL;
 }
+
+CHqlMetaProperty * queryMetaProperty(IHqlExpression * expr)
+{
+    IHqlExpression * body = expr->queryBody();
+    IInterface * match = CHqlExprMeta::queryExistingProperty(body, EPmeta);
+    if (match)
+        return static_cast<CHqlMetaProperty *>(match);
+
+    CHqlMetaProperty * simple = querySimpleDatasetMeta(body);
+    if (simple)
+    {
+        CHqlExprMeta::addProperty(body, EPmeta, simple);
+        return simple;
+    }
+
+    Owned<CHqlMetaProperty> info = new CHqlMetaProperty;
+    calculateDatasetMeta(info->meta, body);
+    CHqlExprMeta::addProperty(body, EPmeta, info);
+    return info;
+}

ecl/hql/hqlattr.hpp

@@ -81,4 +81,7 @@ extern HQL_API IHqlExpression * getRecordCountInfo(IHqlExpression * expr);
 extern HQL_API bool hasNoMoreRowsThan(IHqlExpression * expr, __int64 limit);
 extern HQL_API bool spillToWorkunitNotFile(IHqlExpression * expr, ClusterType platform);
 
+class CHqlMetaProperty;
+extern HQL_API CHqlMetaProperty * queryMetaProperty(IHqlExpression * expr);
+
 #endif

ecl/hql/hqlexpr.hpp

@@ -794,6 +794,7 @@ enum ExprPropKind
     EPaligned,
     EPunadorned,
     EPlocationIndependent,
+    EPmeta,
     EPmax
 };
 
@@ -1470,7 +1471,7 @@ extern HQL_API IHqlNamedAnnotation * queryNameAnnotation(IHqlExpression * expr);
 inline bool hasAnnotation(IHqlExpression * expr, annotate_kind search){ return queryAnnotation(expr, search) != NULL; }
 inline IHqlExpression * queryNamedSymbol(IHqlExpression * expr) { return queryAnnotation(expr, annotate_symbol); }
 inline bool hasNamedSymbol(IHqlExpression * expr) { return hasAnnotation(expr, annotate_symbol); }
-inline bool hasAttribute(IAtom * search, HqlExprArray & exprs) { return queryAttribute(search, exprs) != NULL; }
+inline bool hasAttribute(IAtom * search, const HqlExprArray & exprs) { return queryAttribute(search, exprs) != NULL; }
 
 extern HQL_API IHqlExpression * queryAnnotationProperty(IAtom * search, IHqlExpression * annotation);
 extern HQL_API IHqlExpression * queryMetaProperty(IAtom * search, IHqlExpression * expr);
@@ -1643,17 +1644,13 @@ inline bool isCast(IHqlExpression * expr)
 }
 extern HQL_API bool isKey(IHqlExpression * expr);
 
-inline IHqlExpression * queryGrouping(ITypeInfo * type)     { return static_cast<IHqlExpression *>(type->queryGroupInfo()); }
-inline IHqlExpression * queryDistribution(ITypeInfo * type) { return static_cast<IHqlExpression *>(type->queryDistributeInfo()); }
-inline IHqlExpression * queryGlobalSortOrder(ITypeInfo * type)  { return static_cast<IHqlExpression *>(type->queryGlobalSortInfo()); }
-inline IHqlExpression * queryLocalUngroupedSortOrder(ITypeInfo * type)  { return static_cast<IHqlExpression *>(type->queryLocalUngroupedSortInfo()); }
-inline IHqlExpression * queryGroupSortOrder(ITypeInfo * type)   { return static_cast<IHqlExpression *>(type->queryGroupSortInfo()); }
-inline IHqlExpression * queryGrouping(IHqlExpression * expr)        { return queryGrouping(expr->queryType()); }
-inline IHqlExpression * queryDistribution(IHqlExpression * expr)    { return queryDistribution(expr->queryType()); }
-inline IHqlExpression * queryGlobalSortOrder(IHqlExpression * expr) { return queryGlobalSortOrder(expr->queryType()); }
-inline IHqlExpression * queryLocalUngroupedSortOrder(IHqlExpression * expr) { return queryLocalUngroupedSortOrder(expr->queryType()); }
-inline IHqlExpression * queryGroupSortOrder(IHqlExpression * expr)  { return queryGroupSortOrder(expr->queryType()); }
-inline bool isGrouped(ITypeInfo * type)                     { return type && type->queryGroupInfo() != NULL; }
+IHqlExpression * queryGrouping(IHqlExpression * expr);
+IHqlExpression * queryDistribution(IHqlExpression * expr);
+IHqlExpression * queryGlobalSortOrder(IHqlExpression * expr);
+IHqlExpression * queryLocalUngroupedSortOrder(IHqlExpression * expr);
+IHqlExpression * queryGroupSortOrder(IHqlExpression * expr);
+
+inline bool isGrouped(ITypeInfo * type)                     { return type && (type->getTypeCode() == type_groupedtable); }
 inline bool isGrouped(IHqlExpression * expr)                { return isGrouped(expr->queryType()); }
 
 inline IFunctionTypeExtra * queryFunctionTypeExtra(ITypeInfo * type)    { return static_cast<IFunctionTypeExtra *>(queryUnqualifiedType(type)->queryModifierExtra()); }
@@ -1666,7 +1663,7 @@ inline IHqlExpression * querySelSeq(IHqlExpression * expr)
 {
     return expr->queryAttribute(_selectorSequence_Atom);
 }
-extern HQL_API IHqlExpression * createGroupedAttribute(ITypeInfo * type);
+extern HQL_API IHqlExpression * createGroupedAttribute(IHqlExpression * grouping);
 extern HQL_API bool isSameUnqualifiedType(ITypeInfo * l, ITypeInfo * r);
 extern HQL_API bool isSameFullyUnqualifiedType(ITypeInfo * l, ITypeInfo * r);
 extern HQL_API IHqlExpression * queryNewSelectAttrExpr();

ecl/hql/hqlexpr.ipp

@@ -47,7 +47,7 @@ class HQL_API CHqlDynamicProperty
 {
     friend class CHqlExpression;
 public:
-    inline CHqlDynamicProperty(ExprPropKind _kind, IHqlExpression *_value)
+    inline CHqlDynamicProperty(ExprPropKind _kind, IInterface *_value)
         : kind(_kind), value(_value)
     {
         next = NULL;
@@ -56,10 +56,11 @@ public:
 
 protected:
     CHqlDynamicProperty * next;
-    LinkedHqlExpr value;
     ExprPropKind kind;
+    Linked<IInterface> value;
 };
 
+
 class CUsedTablesBuilder;
 
 //Optimized representation of the number of used tables
@@ -121,7 +122,7 @@ public:
 
 protected:
     unsigned hashcode;          // CInterface is 4 byte aligned in 64bits, so use this to pad
-                                // Worth storing becuase it significantly speeds up equality checking
+                                // Worth storing because it significantly speeds up equality checking
     IInterface * transformExtra[NUM_PARALLEL_TRANSFORMS];
     unsigned cachedCRC;
     unsigned infoFlags;
@@ -168,7 +169,6 @@ protected:
         operands.append(child);
         onAppendOperand(child, which);
     }
-    IHqlExpression * queryExistingProperty(ExprPropKind kind) const;
 
     void initFlagsBeforeOperands();
     void updateFlagsAfterOperands();
@@ -178,7 +178,8 @@ protected:
     virtual unsigned getCachedEclCRC();
     void setInitialHash(unsigned typeHash);
 
-    void addProperty(ExprPropKind kind, IHqlExpression * value);
+    virtual void addProperty(ExprPropKind kind, IInterface * value);
+    virtual IInterface * queryExistingProperty(ExprPropKind kind) const;
 
 public:
     virtual void Link(void) const;
@@ -1065,11 +1066,6 @@ public:
     virtual IAtom * queryLocale()                 { return NULL; }
 //  virtual IAtom * queryName() const             { return id; }
     virtual ITypeInfo * queryChildType()        { return NULL; }
-    virtual IInterface * queryDistributeInfo()  { return NULL; }
-    virtual IInterface * queryGroupInfo()       { return NULL; }
-    virtual IInterface * queryGlobalSortInfo()  { return NULL; }
-    virtual IInterface * queryLocalUngroupedSortInfo()   { return NULL; }
-    virtual IInterface * queryGroupSortInfo()   { return NULL; }
     virtual ITypeInfo * queryPromotedType()     { return this; }
     virtual ITypeInfo * queryTypeBase()         { return this; }
     virtual typemod_t queryModifier()           { return typemod_none; }
@@ -1609,11 +1605,6 @@ public:
     virtual ICollationInfo * queryCollation()   { return NULL; }
     virtual IAtom * queryLocale()                 { return NULL; }
     virtual ITypeInfo * queryChildType()        { return NULL; }
-    virtual IInterface * queryDistributeInfo()  { return NULL; }
-    virtual IInterface * queryGroupInfo()       { return NULL; }
-    virtual IInterface * queryGlobalSortInfo()  { return NULL; }
-    virtual IInterface * queryLocalUngroupedSortInfo()   { return NULL; }
-    virtual IInterface * queryGroupSortInfo()   { return NULL; }
     virtual ITypeInfo * queryPromotedType()     { return this; }
     virtual ITypeInfo * queryTypeBase()         { return this; }
     virtual typemod_t queryModifier()           { return typemod_none; }
@@ -1661,10 +1652,14 @@ public:
     bool equals(const IHqlExpression & r) const;
     virtual IHqlExpression *clone(HqlExprArray &newkids);
 
+    virtual void addProperty(ExprPropKind kind, IInterface * value);
+    virtual IInterface * queryExistingProperty(ExprPropKind kind) const;
+
 protected:
     IHqlDataset *rootTable;
     IHqlExpression * container;
     OwnedHqlExpr normalized;
+    Owned<IInterface> metaProperty;
 
     void cacheParent();
 
@@ -1735,11 +1730,6 @@ public:
     virtual IAtom * queryName() const { return id->lower(); }
     virtual IIdAtom * queryId() const { return id; }
     virtual ITypeInfo * queryChildType() { return logical; }
-    virtual IInterface * queryDistributeInfo()  { return NULL; }
-    virtual IInterface * queryGroupInfo()       { return NULL; }
-    virtual IInterface * queryGlobalSortInfo()  { return NULL; }
-    virtual IInterface * queryLocalUngroupedSortInfo()   { return NULL; }
-    virtual IInterface * queryGroupSortInfo()   { return NULL; }
     virtual ITypeInfo * queryPromotedType()     { return logical->queryPromotedType(); }
     virtual ITypeInfo * queryTypeBase()         { return this; }
     virtual typemod_t queryModifier()           { return typemod_none; }

ecl/hql/hqlfold.cpp

@@ -31,6 +31,7 @@
 #include "hqlerrors.hpp"
 #include "hqlutil.hpp"
 #include "hqlpmap.hpp"
+#include "hqlmeta.hpp"
 
 #include "hqlfold.hpp"
 #include "hqlthql.hpp"
@@ -3379,7 +3380,7 @@ IHqlExpression * NullFolderMixin::foldNullDataset(IHqlExpression * expr)
             if (expr->hasAttribute(skewAtom))
             	break;
             //Careful - distribute also destroys grouping, so don't remove if input is grouped.
-            if ((expr->queryType()->queryDistributeInfo() == child->queryType()->queryDistributeInfo()) && !isGrouped(child))
+            if ((queryDistribution(expr) == queryDistribution(child)) && !isGrouped(child))
                 return removeParentNode(expr);
             break;
         }
@@ -3388,7 +3389,7 @@ IHqlExpression * NullFolderMixin::foldNullDataset(IHqlExpression * expr)
     case no_sorted:
         {
             //If action does not change the type information, then it can't have done anything...
-            if (expr->queryType() == child->queryType())
+            if (hasSameSortGroupDistribution(expr, child))
                 return removeParentNode(expr);
             if (isNull(child) || hasNoMoreRowsThan(child, 1))
                 return removeParentNode(expr);
@@ -3425,7 +3426,7 @@ IHqlExpression * NullFolderMixin::foldNullDataset(IHqlExpression * expr)
 //  case no_preservemeta:
         {
             //If action does not change the type information, then it can't have done anything...
-            if (expr->queryType() == child->queryType())
+            if (hasSameSortGroupDistribution(expr, child))
                 return removeParentNode(expr);
             if (isNull(child))
                 return replaceWithNull(expr);

ecl/hql/hqlgram.y

@@ -725,8 +725,7 @@ explicitDatasetType
     : explicitDatasetType1
     | GROUPED explicitDatasetType1
                         {
-                            IHqlExpression * grouping = getUnknownSortlist();
-                            $$.setType(makeGroupedTableType($2.getType(), grouping, NULL));
+                            $$.setType(makeGroupedTableType($2.getType()));
                             $$.setPosition($1);
                         }
     ;
@@ -734,7 +733,7 @@ explicitDatasetType
 explicitDatasetType1
     : DATASET
                         {
-                            $$.setType(makeTableType(makeRowType(queryNullRecord()->getType()), NULL, NULL, NULL));
+                            $$.setType(makeTableType(makeRowType(queryNullRecord()->getType())));
                             $$.setPosition($1);
                         }
     | DATASET '(' recordDef childDatasetOptions ')'
@@ -742,7 +741,7 @@ explicitDatasetType1
                             OwnedHqlExpr record = $3.getExpr();
                             OwnedHqlExpr options = $4.getExpr();
                             ITypeInfo * recordType = createRecordType(record);
-                            Owned<ITypeInfo> tableType = makeTableType(makeRowType(recordType), NULL, NULL, NULL);
+                            Owned<ITypeInfo> tableType = makeTableType(makeRowType(recordType));
                             if (options)
                                 tableType.setown(makeAttributeModifier(LINK(tableType), createAttribute(_childAttr_Atom, LINK(options))));
                             $$.setType(tableType.getClear());
@@ -852,7 +851,7 @@ paramType
     | abstractDataset   {
                             OwnedHqlExpr record = $1.getExpr();
                             OwnedHqlExpr abstractRecord = createAbstractRecord(record);
-                            OwnedITypeInfo type = makeTableType(makeRowType(abstractRecord->getType()), NULL, NULL, NULL);
+                            OwnedITypeInfo type = makeTableType(makeRowType(abstractRecord->getType()));
                             $$.setType(type.getClear(), $1);
                             parser->setTemplateAttribute();
                         }
@@ -3669,7 +3668,7 @@ paramDefinition
     | ANY DATASET knownOrUnknownId
                         {
                             $$.clear();
-                            parser->addParameter($1, $3.getId(), makeTableType(makeRowType(queryNullRecord()->getType()), NULL, NULL, NULL), NULL);
+                            parser->addParameter($1, $3.getId(), makeTableType(makeRowType(queryNullRecord()->getType())), NULL);
                         }
     | ANY knownOrUnknownId defvalue
                         {
@@ -4254,7 +4253,7 @@ fieldDef
                             }
 
                             IHqlExpression *value = $7.getExpr();
-                            Owned<ITypeInfo> datasetType = makeTableType(makeRowType(createRecordType(record)), NULL, NULL, NULL);
+                            Owned<ITypeInfo> datasetType = makeTableType(makeRowType(createRecordType(record)));
                             parser->addDatasetField($2, $2.getId(), datasetType, value, attrs.getClear());
                         }
     | setType knownOrUnknownId optFieldAttrs defaultValue
@@ -8200,19 +8199,6 @@ simpleDataSet
                             OwnedHqlExpr attrs;
                             IHqlExpression *groupOrder = parser->processSortList($6, no_group, input, sortItems, NULL, &attrs);
                             OwnedHqlExpr args = createComma(groupOrder, LINK(attrs));
-                            
-                            //Non-all Group only make sense if the rows are together.  This can occur if
-                            //the dataset is sorted, or if previous operation was a grouped aggregate
-                            bool isLocal = queryAttributeInList(localAtom, attrs) != NULL;
-                            if (groupOrder && !queryAttributeInList(allAtom, attrs) && !appearsToBeSorted(input->queryType(), isLocal, true))
-                            {
-                                bool ok = (input->getOperator() == no_usertable) &&
-                                          (datasetHasGroupBy(input));// || isGrouped(input->queryChild(0)));
-                                if (!ok && !isLocal && isPartitionedForGroup(input, groupOrder, false) && appearsToBeSorted(input->queryType(), true, true))
-                                    ok = true;
-                                if (!ok && (groupOrder && groupOrder->numChildren()))
-                                    parser->reportWarning(WRN_NOT_SORTED, $2.pos, "GROUP on a table that does not appear to be sorted, was this intended?");
-                            }
                             $$.setExpr(createDataset(no_group, input.getClear(), args.getClear()), $1);
                         }
     | GROUPED '(' startTopFilter startGROUP beginList sortList endGROUP
@@ -8282,7 +8268,7 @@ simpleDataSet
                             if (grouping && !queryAttributeInList(groupedAtom, attrs))
                             {
                                 parser->checkGrouping($7, dataset,record,grouping);
-                                if (dataset->getOperator() == no_group && dataset->queryType()->queryGroupInfo())
+                                if (dataset->getOperator() == no_group && isGrouped(dataset))
                                     parser->reportWarning(WRN_GROUPINGIGNORED, $3.pos, "Grouping of table input will have no effect, was this intended?");
                             }
 

ecl/hql/hqlgram2.cpp

@@ -2403,7 +2403,7 @@ void HqlGram::addDatasetField(const attribute &errpos, IIdAtom * name, ITypeInfo
         if (!dsType || queryRecord(dsType)->numChildren() == 0)
         {
             ITypeInfo * recordType = queryRecordType(valueType);
-            dsType.setown(makeTableType(makeRowType(LINK(recordType)), NULL, NULL, NULL));
+            dsType.setown(makeTableType(makeRowType(LINK(recordType))));
         }
         else if (!dsType->assignableFrom(valueType))
         {
@@ -7562,7 +7562,6 @@ static const char * getName(IHqlExpression * e)
 
 void HqlGram::checkDistribution(attribute &errpos, IHqlExpression *input, bool localSpecified, bool ignoreGrouping)
 {
-    IInterface *distribution = input->queryType()->queryDistributeInfo();
     bool inputIsGrouped = isGrouped(input);
     if (localSpecified)
     {
@@ -7571,7 +7570,7 @@ void HqlGram::checkDistribution(attribute &errpos, IHqlExpression *input, bool l
     }
     else
     {
-        if (distribution && isExplicitlyDistributed(input))
+        if (isExplicitlyDistributed(input))
         {
             if (!inputIsGrouped || ignoreGrouping)
             {
@@ -8432,9 +8431,9 @@ void HqlGram::checkRecordIsValid(const attribute &atr, IHqlExpression *record)
 void HqlGram::checkMergeInputSorted(attribute &atr, bool isLocal)
 {
     IHqlExpression * expr = atr.queryExpr();
-    if (appearsToBeSorted(expr->queryType(), isLocal, true))
+    if (appearsToBeSorted(expr, isLocal, true))
         return;
-    if (!isLocal && appearsToBeSorted(expr->queryType(), true, true))
+    if (!isLocal && appearsToBeSorted(expr, true, true))
     {
         reportWarning(WRN_MERGE_NOT_SORTED, atr.pos, "INPUT to MERGE appears to be sorted locally but not globally");
         return;
@@ -8453,7 +8452,7 @@ void HqlGram::checkMergeInputSorted(attribute &atr, bool isLocal)
         }
     }
     
-    if (isGrouped(expr) && appearsToBeSorted(expr->queryType(), false, false))
+    if (isGrouped(expr) && appearsToBeSorted(expr, false, false))
         reportWarning(WRN_MERGE_NOT_SORTED, atr.pos, "Input to MERGE is only sorted with the group");
     else
         reportWarning(WRN_MERGE_NOT_SORTED, atr.pos, "Input to MERGE doesn't appear to be sorted");
@@ -9105,7 +9104,9 @@ void HqlGram::defineSymbolProduction(attribute & nameattr, attribute & paramattr
                     {
                         //allow dataset with no record to match, as long as base type is the same
                         if (queryRecord(type) != queryNullRecord() || (type->getTypeCode() != matchType->getTypeCode()))
+                        {
                             reportError(ERR_SAME_TYPE_REQUIRED, nameattr, "Explicit type for %s doesn't match definition in base module", name->str());
+                        }
                         else
                         {
                             type.set(matchType);
@@ -11371,12 +11372,14 @@ void parseAttribute(IHqlScope * scope, IFileContents * contents, HqlLookupContex
 
 void testHqlInternals()
 {
-    printf("Sizes: const(%u) expr(%u) select(%u) dataset(%u) annotation(%u)\n",
+    printf("Sizes: const(%u) expr(%u) select(%u) dataset(%u) annotation(%u) prop(%u)\n",
             (unsigned)sizeof(CHqlConstant),
             (unsigned)sizeof(CHqlExpressionWithType),
             (unsigned)sizeof(CHqlSelectExpression),
             (unsigned)sizeof(CHqlDataset),
-            (unsigned)sizeof(CHqlAnnotation));
+            (unsigned)sizeof(CHqlAnnotation),
+            (unsigned)sizeof(CHqlDynamicProperty)
+            );
 
     //
     // test getOpString()

ecl/hql/hqlir.cpp

@@ -1875,6 +1875,7 @@ id_t ExpressionIRPlayer::doProcessType(ITypeInfo * type)
         case type_groupedtable:
         case type_dictionary:
         case type_function:
+        case type_pointer:
             {
                 CompoundTypeBuilderInfo info;
                 info.baseType = processType(type->queryChildType());
@@ -1886,7 +1887,6 @@ id_t ExpressionIRPlayer::doProcessType(ITypeInfo * type)
         case type_ifblock:
         case type_alias:
         case type_blob:
-        case type_pointer:
         case type_class:
         case type_array:
             throwUnexpected();

ecl/hql/hqllex.l

@@ -1333,7 +1333,7 @@ RECORDSET           {
                         if (!lookup)
                             return lookupIdentifierToken(returnToken, lexer, lookup, activeState, CUR_TOKEN_TEXT);
 
-                        returnToken.setType(makeTableType(NULL, NULL, NULL, NULL));
+                        returnToken.setType(makeTableType(NULL));
                         return(SIMPLE_TYPE);
                     }
 SWAPPED             { 

ecl/hql/hqlmeta.hpp

@@ -19,6 +19,59 @@
 
 class TableProjectMapper;
 
+class HQL_API CHqlMetaInfo
+{
+public:
+    inline bool isGrouped() const { return grouping != NULL; }
+
+    bool appearsToBeSorted(bool isLocal, bool ignoreGrouping);
+    void applyDistribute(IHqlExpression * newDistribution, IHqlExpression * optMergeOrder);
+    void applyGroupSort(IHqlExpression * sortOrder);
+    void applyGroupBy(IHqlExpression * groupBy, bool isLocal);
+    void applyLocalSort(IHqlExpression * sortOrder);
+    void applyGlobalSort(IHqlExpression * sortOrder);
+    void applyProject(TableProjectMapper & mapper);
+    void applySubSort(IHqlExpression * groupBy, IHqlExpression * sortOrder, bool isLocal);
+
+    void ensureAppearsSorted(bool isLocal, bool ignoreGrouping);
+    void getIntersection(const CHqlMetaInfo & other);
+    IHqlExpression * getLocalSortOrder() const;
+    bool hasUsefulInformation() const;
+    bool matches(const CHqlMetaInfo & other) const;
+
+    void preserveGrouping(IHqlExpression * dataset);
+
+    void removeActiveSort();
+    void removeAllKeepGrouping();
+    void removeAllSortOrders();
+    void removeAllAndUngroup(bool isLocal);
+    void removeDistribution();
+    void removeGroup();
+
+    void set(const CHqlMetaInfo & other);
+    void setMatchesAny();
+    void setUnknownDistribution();
+    void setUnknownGrouping();
+
+protected:
+    void clearGrouping();
+
+public:
+    LinkedHqlExpr distribution;
+    LinkedHqlExpr globalSortOrder;
+    LinkedHqlExpr localUngroupedSortOrder;
+    LinkedHqlExpr grouping;
+    LinkedHqlExpr groupSortOrder;
+};
+
+class CHqlMetaProperty : public CInterfaceOf<IInterface>
+{
+public:
+    CHqlMetaInfo meta;
+};
+
+void extractMeta(CHqlMetaInfo & meta, ITypeInfo * type);
+
 extern HQL_API IHqlExpression * queryUnknownAttribute();
 extern HQL_API IHqlExpression * queryUnknownSortlist();
 extern HQL_API IHqlExpression * queryMatchGroupOrderSortlist();
@@ -29,32 +82,14 @@ extern HQL_API IHqlExpression * queryAnyOrderSortlist();
 extern HQL_API IHqlExpression * queryAnyDistributionAttribute();
 
 extern HQL_API IHqlExpression * getExistingSortOrder(IHqlExpression * dataset, bool isLocal, bool ignoreGrouping);
+extern ITypeInfo * calculateDatasetType(node_operator op, const HqlExprArray & parms);
 
 //---------------------------------------------------------------------------------------------
 
 extern HQL_API bool isKnownDistribution(IHqlExpression * distribution);
-inline bool isKnownDistribution(ITypeInfo * type) { return isKnownDistribution(queryDistribution(type)); }
 extern HQL_API bool isPersistDistribution(IHqlExpression * distribution);
 extern HQL_API bool isSortedDistribution(IHqlExpression * distribution);
 
-extern HQL_API ITypeInfo * getTypeUngroup(ITypeInfo * prev);
-extern HQL_API ITypeInfo * getTypeGrouped(ITypeInfo * prev, IHqlExpression * grouping, bool isLocal);
-extern HQL_API ITypeInfo * getTypeGlobalSort(ITypeInfo * prev, IHqlExpression * sortOrder);
-extern HQL_API ITypeInfo * getTypeLocalSort(ITypeInfo * prev, IHqlExpression * sortOrder);
-extern HQL_API ITypeInfo * getTypeGroupSort(ITypeInfo * prev, IHqlExpression * sortOrder);
-extern HQL_API ITypeInfo * getTypeDistribute(ITypeInfo * prev, IHqlExpression * distribution, IHqlExpression * optMergeOrder);
-extern HQL_API ITypeInfo * getTypeFromMeta(IHqlExpression * record, IHqlExpression * meta, unsigned firstChild);
-extern HQL_API ITypeInfo * getTypeIntersection(ITypeInfo * leftType, ITypeInfo * rightType);
-extern HQL_API ITypeInfo * getTypeLoseDistributionKeepOrder(ITypeInfo * prev);
-extern HQL_API ITypeInfo * getTypeProject(ITypeInfo * prev, IHqlExpression * newRecord, TableProjectMapper & mapper);
-extern HQL_API ITypeInfo * getTypeSubSort(ITypeInfo * prev, IHqlExpression * grouping, IHqlExpression * sortOrder, bool isLocal);
-extern HQL_API ITypeInfo * getTypeCannotProject(ITypeInfo * prev, IHqlExpression * newRecord); // preserve grouping, but that's it.
-extern HQL_API ITypeInfo * getTypeUnknownDistribution(ITypeInfo * prev);
-extern HQL_API ITypeInfo * getTypeRemoveDistribution(ITypeInfo * prev);
-extern HQL_API ITypeInfo * getTypeRemoveActiveSort(ITypeInfo * prev);
-extern HQL_API ITypeInfo * getTypeRemoveAllSortOrders(ITypeInfo * prev);
-extern HQL_API bool hasUsefulMetaInformation(ITypeInfo * prev);
-
 //---------------------------------------------------------------------------------------------
 
 extern HQL_API IHqlExpression * mapJoinDistribution(TableProjectMapper & mapper, IHqlExpression * distribution, IHqlExpression * side);
@@ -71,7 +106,7 @@ extern HQL_API IHqlExpression * ensureSortedForGroup(IHqlExpression * table, IHq
 extern HQL_API bool matchDedupDistribution(IHqlExpression * distn, const HqlExprArray & equalities);
 extern HQL_API bool matchesAnyDistribution(IHqlExpression * distn);
 
-extern HQL_API bool appearsToBeSorted(ITypeInfo * type, bool isLocal, bool ignoreGrouping);
+extern HQL_API bool appearsToBeSorted(IHqlExpression * dataset, bool isLocal, bool ignoreGrouping);
 extern HQL_API bool isAlreadySorted(IHqlExpression * dataset, HqlExprArray & newSort, bool isLocal, bool ignoreGrouping);
 extern HQL_API bool isAlreadySorted(IHqlExpression * dataset, IHqlExpression * newSort, bool isLocal, bool ignoreGrouping);
 extern HQL_API IHqlExpression * ensureSorted(IHqlExpression * dataset, IHqlExpression * order, bool isLocal, bool ignoreGrouping, bool alwaysLocal, bool allowSubSort);
@@ -88,6 +123,10 @@ extern HQL_API IHqlExpression * preserveTableInfo(IHqlExpression * newTable, IHq
 extern HQL_API bool isDistributedCoLocally(IHqlExpression * dataset1, IHqlExpression * dataset2, const HqlExprArray & sort1, const HqlExprArray & sort2);
 extern HQL_API IHqlExpression * createMatchingDistribution(IHqlExpression * expr, const HqlExprArray & oldSort, const HqlExprArray & newSort);
 
+extern HQL_API void calculateDatasetMeta(CHqlMetaInfo & meta, IHqlExpression * expr);
+extern HQL_API CHqlMetaProperty * querySimpleDatasetMeta(IHqlExpression * expr);
+extern HQL_API bool hasSameSortGroupDistribution(IHqlExpression * expr, IHqlExpression * other);
+
 inline IHqlExpression * queryRemoveOmitted(IHqlExpression * expr)
 {
     if (expr &&  expr->isAttribute() && (expr->queryName() == _omitted_Atom))

ecl/hql/hqlthql.cpp

@@ -702,15 +702,14 @@ void HqltHql::toECL(IHqlExpression *expr, StringBuffer &s, bool paren, bool inTy
         {
             if (expr->isDataset())
             {
-                ITypeInfo * type = expr->queryType();
-                IHqlExpression * group = (IHqlExpression*)type->queryGroupInfo();
+                IHqlExpression * group = queryGroupInfo(expr);
                 if (group)
                     getExprECL(group, s.append("G(")).append(")");
-                IHqlExpression * distrib = queryDistribution(type);
+                IHqlExpression * distrib = queryDistribution(expr);
                 if (distrib) getExprECL(distrib, s.append("D(")).append(")");
-                appendSortOrder(s, "GO", queryGlobalSortOrder(type));
-                appendSortOrder(s, "LO", queryLocalUngroupedSortOrder(type));
-                appendSortOrder(s, "RO", queryGroupSortOrder(type));
+                appendSortOrder(s, "GO", queryGlobalSortOrder(expr));
+                appendSortOrder(s, "LO", queryLocalUngroupedSortOrder(expr));
+                appendSortOrder(s, "RO", queryGroupSortOrder(expr));
             }
         }
         s.append("]");

ecl/hql/hqlutil.cpp

@@ -3651,13 +3651,13 @@ IDefRecordElement * RecordMetaCreator::createField(IHqlExpression * cur, IHqlExp
         childDefRecord.setown(createRecord(fieldRecord, self));
         break;
     case type_table:
-        defType.setown(makeTableType(makeRowType(makeRecordType()), NULL, NULL, NULL));
+        defType.setown(makeTableType(makeRowType(makeRecordType())));
         childDefRecord.setown(::createMetaRecord(fieldRecord, callback));
         break;
     case type_groupedtable:
         {
-            ITypeInfo * tableType = makeTableType(makeRowType(makeRecordType()), NULL, NULL, NULL);
-            defType.setown(makeGroupedTableType(tableType, createAttribute(groupedAtom), NULL));
+            ITypeInfo * tableType = makeTableType(makeRowType(makeRecordType()));
+            defType.setown(makeGroupedTableType(tableType));
             childDefRecord.setown(::createMetaRecord(fieldRecord, callback));
             break;
         }

ecl/hqlcpp/hqlcppds.cpp

@@ -2080,7 +2080,7 @@ void HqlCppTranslator::doBuildDataset(BuildCtx & ctx, IHqlExpression * expr, CHq
             if (expr->isDictionary())
                 type.setown(makeDictionaryType(makeRowType(record->getType())));
             else
-                type.setown(makeTableType(makeRowType(record->getType()), NULL, NULL, NULL));
+                type.setown(makeTableType(makeRowType(record->getType())));
             if ((format == FormatLinkedDataset) || (format == FormatArrayDataset) || expr->isDictionary())
                 type.setown(setLinkCountedAttr(type, true));
             tgt.expr.setown(createValue(no_nullptr, makeReferenceModifier(type.getClear())));
@@ -2844,7 +2844,7 @@ bool HqlCppTranslator::doBuildDatasetInlineTable(BuildCtx & ctx, IHqlExpression
 
     unsigned maxRows = values->numChildren();
     Owned<ITypeInfo> declareType = makeConstantModifier(makeArrayType(LINK(rowType), maxRows));
-    OwnedITypeInfo rowsType = makeOutOfLineModifier(makeTableType(LINK(rowType), NULL, NULL, NULL));
+    OwnedITypeInfo rowsType = makeOutOfLineModifier(makeTableType(LINK(rowType)));
     if (options.canLinkConstantRows)
         rowsType.setown(setLinkCountedAttr(rowsType, true));
 
@@ -4573,7 +4573,7 @@ void HqlCppTranslator::convertBoundRowToDataset(BuildCtx & ctx, CHqlBoundExpr &
 {
     IHqlExpression * boundRow = row->queryBound();
     IHqlExpression * record = row->queryDataset()->queryRecord();
-    Owned<ITypeInfo> type = makeTableType(makeRowType(LINK(record->queryType())), NULL, NULL, NULL);
+    Owned<ITypeInfo> type = makeTableType(makeRowType(LINK(record->queryType())));
     Owned<ITypeInfo> refType = makeReferenceModifier(LINK(type));
     if (hasLinkCountedModifier(boundRow->queryType()) && (preferredFormat != FormatBlockedDataset))
     {

ecl/hqlcpp/hqlhtcpp.cpp

@@ -222,7 +222,7 @@ bool canIterateTableInline(IHqlExpression * expr)
     case no_newaggregate:
         {
             IHqlExpression * child = expr->queryChild(0);
-            if (!child->queryType()->queryGroupInfo())
+            if (!isGrouped(child))
                 return canIterateTableInline(child);
             return false;
         }
@@ -2146,7 +2146,7 @@ void ActivityInstance::createGraphNode(IPropertyTree * defaultSubGraph, bool alw
     addAttributeBool("local", isLocal);
 
 #ifdef _DEBUG
-    assertex(dataset->isAction() == isActivitySink(kind));
+//    assertex(dataset->isAction() == isActivitySink(kind));
 #endif
     if (dataset->isAction())
     {
@@ -2181,30 +2181,29 @@ void ActivityInstance::createGraphNode(IPropertyTree * defaultSubGraph, bool alw
     if (translator.queryOptions().showMetaInGraph)
     {
         StringBuffer s;
-        ITypeInfo * type = dataset->queryType();
         if (translator.targetThor())
         {
-            IHqlExpression * distribution = queryDistribution(type);
+            IHqlExpression * distribution = queryDistribution(dataset);
             if (distribution && distribution->queryName() != localAtom)
                 addAttribute("metaDistribution", getExprECL(distribution, s.clear(), true).str());
         }
 
-        IHqlExpression * grouping = (IHqlExpression *)type->queryGroupInfo();
+        IHqlExpression * grouping = queryGrouping(dataset);
         if (grouping)
             addAttribute("metaGrouping", getExprECL(grouping, s.clear(), true).str());
 
         if (translator.targetThor())
         {
-            IHqlExpression * globalSortOrder = (IHqlExpression *)type->queryGlobalSortInfo();
+            IHqlExpression * globalSortOrder = queryGlobalSortOrder(dataset);
             if (globalSortOrder)
                 addAttribute("metaGlobalSortOrder", getExprECL(globalSortOrder, s.clear(), true).str());
         }
 
-        IHqlExpression * localSortOrder = (IHqlExpression *)type->queryLocalUngroupedSortInfo();
+        IHqlExpression * localSortOrder = queryLocalUngroupedSortOrder(dataset);
         if (localSortOrder)
             addAttribute("metaLocalSortOrder", getExprECL(localSortOrder, s.clear(), true).str());
 
-        IHqlExpression * groupSortOrder = (IHqlExpression *)type->queryGroupSortInfo();
+        IHqlExpression * groupSortOrder = queryGroupSortOrder(dataset);
         if (groupSortOrder)
             addAttribute("metaGroupSortOrder", getExprECL(groupSortOrder, s.clear(), true).str());
     }
@@ -13535,7 +13534,7 @@ ABoundActivity * HqlCppTranslator::doBuildActivityDedup(BuildCtx & ctx, IHqlExpr
 
     IHqlExpression * dataset = expr->queryChild(0);
     IHqlExpression * selSeq = querySelSeq(expr);
-    bool isGrouped = (dataset->queryType()->queryGroupInfo() != NULL);
+    bool isGrouped = ::isGrouped(dataset);
     bool isLocal = isLocalActivity(expr);
     bool useHash = expr->hasAttribute(hashAtom);
     if (targetThor() && !isGrouped && !isLocal)
@@ -13571,7 +13570,7 @@ ABoundActivity * HqlCppTranslator::doBuildActivityDedup(BuildCtx & ctx, IHqlExpr
             ForEachItemIn(i1, info.equalities)
                 normalizedEqualities.append(*replaceSelector(info.equalities.item(i1).queryBody(), dataset, queryActiveTableSelector()));
 
-            IHqlExpression * grouping = static_cast<IHqlExpression *>(dataset->queryType()->queryGroupInfo());
+            IHqlExpression * grouping = queryGrouping(dataset);
             ForEachChild(i, grouping)
             {
                 IHqlExpression * curGroup = grouping->queryChild(i);
@@ -14912,12 +14911,12 @@ void HqlCppTranslator::bindRows(BuildCtx & ctx, node_operator side, IHqlExpressi
     OwnedHqlExpr selector = createSelector(side, dataset, selSeq);
     OwnedHqlExpr rowsExpr = createDataset(no_rows, LINK(selector), LINK(rowsid));
 
-    ITypeInfo * rowType = makeReferenceModifier(LINK(rowsExpr->queryType()->queryChildType()));
+    Owned<ITypeInfo> rowType = makeReferenceModifier(LINK(rowsExpr->queryType()->queryChildType()));
     if (rowsAreLinkCounted)
-        rowType = makeAttributeModifier(rowType, getLinkCountedAttr());
+        rowType.setown(setLinkCountedAttr(rowType, true));
 
     //Rows may be link counted, but rows() is not a linkable rowset
-    OwnedITypeInfo rowsType = makeReferenceModifier(makeTableType(rowType, NULL, NULL, NULL));
+    OwnedITypeInfo rowsType = makeReferenceModifier(makeTableType(rowType.getClear()));
     rowsType.setown(makeOutOfLineModifier(LINK(rowsType)));
 
     CHqlBoundExpr boundRows;
@@ -15527,7 +15526,7 @@ ABoundActivity * HqlCppTranslator::doBuildActivityGroup(BuildCtx & ctx, IHqlExpr
         child = child->queryChild(0);
 
     Owned<ABoundActivity> boundDataset = buildCachedActivity(ctx, child);
-    if (expr->queryType()->queryGroupInfo() == child->queryType()->queryGroupInfo())
+    if (queryGrouping(expr) == queryGrouping(child))
         return boundDataset.getClear();
 
     IHqlExpression * sortlist = queryRealChild(expr, 1);

ecl/hqlcpp/hqlresource.cpp

@@ -488,7 +488,7 @@ bool lightweightAndReducesDatasetSize(IHqlExpression * expr)
         return true;
     case no_group:
         //removing grouping will reduce size of the spill file.
-        if (expr->queryType()->queryGroupInfo() == NULL)
+        if (!isGrouped(expr))
             return true;
         break;
     }
@@ -1040,8 +1040,7 @@ void ResourcerInfo::clearProjected()
 
 void ResourcerInfo::addSpillFlags(HqlExprArray & args, bool isRead)
 {
-    IHqlExpression * grouping = (IHqlExpression *)original->queryType()->queryGroupInfo();
-    if (grouping)
+    if (isGrouped(original))
         args.append(*createAttribute(groupedAtom));
 
     if (outputToUseForSpill)
@@ -3123,7 +3122,7 @@ void EclResourcer::createInitialGraph(IHqlExpression * expr, IHqlExpression * ow
                 if (filename && (filename->getOperator() == no_constant) && !expr->hasAttribute(xmlAtom) && !expr->hasAttribute(csvAtom))
                 {
                     IHqlExpression * dataset = expr->queryChild(0);
-                    if (expr->hasAttribute(groupedAtom) == (dataset->queryType()->queryGroupInfo() != NULL))
+                    if (expr->hasAttribute(groupedAtom) == isGrouped(dataset))
                     {
                         StringBuffer filenameText;
                         filename->queryValue()->getStringValue(filenameText);

ecl/hqlcpp/hqlttcpp.cpp

@@ -3478,7 +3478,7 @@ IHqlExpression * ThorHqlTransformer::normalizeTableGrouping(IHqlExpression * exp
                     newsort.set(group);
 
                 LinkedHqlExpr ds = dataset;
-                if (ds->queryType()->queryGroupInfo())
+                if (isGrouped(ds))
                 {
                     ds.setown(createDataset(no_group, ds.getClear(), NULL));
                     ds.setown(cloneInheritedAnnotations(expr, ds));
@@ -4901,7 +4901,6 @@ void GlobalAttributeInfo::doSplitGlobalDefinition(ITypeInfo * type, IHqlExpressi
     ITypeInfo * valueType = value->queryType();
     if (value->isDataset() || value->isDictionary())
     {
-        IHqlExpression * groupOrder = (IHqlExpression *)valueType->queryGroupInfo();
         if (few)
         {
             splitSmallDataset(value, setOutput, getOutput);
@@ -4920,7 +4919,7 @@ void GlobalAttributeInfo::doSplitGlobalDefinition(ITypeInfo * type, IHqlExpressi
         if (valueType->getTypeCode() == type_groupedtable)
             args.append(*createAttribute(groupedAtom));
         else
-            assertex(groupOrder == NULL);
+            assertex(!isGrouped(valueType));
 
         bool compressFile = true;
         switch (persistOp)
@@ -4930,7 +4929,7 @@ void GlobalAttributeInfo::doSplitGlobalDefinition(ITypeInfo * type, IHqlExpressi
                 args.append(*createAttribute(_workflowPersist_Atom));
                 args.append(*createAttribute(sequenceAtom, getGlobalSequenceNumber()));
                 //add a flag to help get the resourcing right - may need to hash distribute on different size thor
-                IHqlExpression * distribution = queryDistribution(valueType);
+                IHqlExpression * distribution = queryDistribution(value);
                 if (distribution && !distribution->isAttribute())
                     args.append(*createAttribute(distributedAtom));
                 break;
@@ -4985,7 +4984,7 @@ void GlobalAttributeInfo::doSplitGlobalDefinition(ITypeInfo * type, IHqlExpressi
 
             if (persistOp == no_persist)
                 args.append(*createAttribute(_workflowPersist_Atom));
-            if (groupOrder)
+            if (isGrouped(value))
                 args.append(*createAttribute(groupedAtom));
             if (compressFile)
                 args.append(*createAttribute(__compressed__Atom));