Add dictionary support to ECL

Initial work on adding dictionary support to ECL. Grammar changes to
support syntax checks of simple dictionary constructs.

ecl regression suite passes (no changes to existing parse structures).

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

common/deftype/deftype.cpp

@@ -1332,6 +1332,25 @@ bool CRowTypeInfo::assignableFrom(ITypeInfo *t2)
 
 //===========================================================================
 
+StringBuffer & CDictionaryTypeInfo::getECLType(StringBuffer & out)
+{
+    ITypeInfo * recordType = ::queryRecordType(this);
+    out.append(queryTypeName());
+    if (recordType)
+    {
+        out.append(" of ");
+        recordType->getECLType(out);
+    }
+    return out;
+}
+
+void CDictionaryTypeInfo::serialize(MemoryBuffer &tgt)
+{
+    CBasedTypeInfo::serializeSkipChild(tgt);
+}
+
+//===========================================================================
+
 bool CTableTypeInfo::assignableFrom(ITypeInfo *t2)
 {
     if (getTypeCode()==t2->getTypeCode())
@@ -1985,6 +2004,12 @@ extern DEFTYPE_API ITypeInfo *makeTableType(ITypeInfo *basetype, IInterface * di
     return commonUpType(new CTableTypeInfo(basetype, distributeinfo, globalsortinfo, localsortinfo));
 }
 
+extern DEFTYPE_API ITypeInfo *makeDictionaryType(ITypeInfo *basetype)
+{
+    assertex(!basetype || basetype->getTypeCode() == type_row);
+    return commonUpType(new CDictionaryTypeInfo(basetype));
+}
+
 extern DEFTYPE_API ITypeInfo *makeGroupedTableType(ITypeInfo *basetype, IInterface *groupinfo, IInterface *sortinfo)
 {
     return commonUpType(new CGroupedTableTypeInfo(basetype, groupinfo, sortinfo));
@@ -3213,6 +3238,7 @@ inline ITypeInfo * queryChildType(ITypeInfo * t, type_t search)
         switch (code)
         {
         case type_set:
+        case type_dictionary:
         case type_groupedtable:
         case type_row:
         case type_table:
@@ -3341,6 +3367,9 @@ ITypeInfo * replaceChildType(ITypeInfo * type, ITypeInfo * newChild)
     OwnedITypeInfo newType;
     switch (type->getTypeCode())
     {
+    case type_dictionary:
+        newType.setown(makeDictionaryType(LINK(newChild)));
+        break;
     case type_table:
         newType.setown(makeTableType(LINK(newChild), LINK(type->queryDistributeInfo()), LINK(type->queryGlobalSortInfo()), LINK(type->queryLocalUngroupedSortInfo())));
         break;

common/deftype/deftype.hpp

@@ -104,6 +104,7 @@ type_unused5            = 29,
     type_ifblock        = 43,       // not a real type -but used for the rtlfield serialization
     type_function       = 44,
     type_sortlist       = 45,
+    type_dictionary     = 46,
 
     type_modifier       = 0xff,     // used by getKind()
     type_unsigned       = 0x100,  // combined with some of the above, when returning summary type information. Not returned by getTypeCode()
@@ -254,6 +255,7 @@ extern DEFTYPE_API ITypeInfo *makeAnyType();
 extern DEFTYPE_API ITypeInfo *makeType(type_t type, int size);
 extern DEFTYPE_API IEnumeratedTypeBuilder *makeEnumeratedTypeBuilder(ITypeInfo *base, aindex_t numvalues);
 extern DEFTYPE_API ITypeInfo *makeDecimalType(unsigned digits, unsigned prec, bool isSigned);
+extern DEFTYPE_API ITypeInfo *makeDictionaryType(ITypeInfo *basetype);
 extern DEFTYPE_API ITypeInfo *makeTableType(ITypeInfo *basetype, IInterface * distributeinfo, IInterface *gloalSortinfo, IInterface * localSortInfo);
 extern DEFTYPE_API ITypeInfo *makeGroupedTableType(ITypeInfo *basetype, IInterface *groupinfo, IInterface *sortinfo);
 extern DEFTYPE_API ITypeInfo *makeRowType(ITypeInfo *basetype);

common/deftype/deftype.ipp

@@ -631,6 +631,21 @@ public:
     virtual bool assignableFrom(ITypeInfo *t2);
 };
 
+class CDictionaryTypeInfo : public CBasedTypeInfo
+{
+public:
+    CDictionaryTypeInfo(ITypeInfo * _basetype)
+        : CBasedTypeInfo(_basetype, UNKNOWN_LENGTH)
+    {}
+
+    virtual type_t getTypeCode() const                      { return type_dictionary; }
+    virtual bool isScalar()                                 { return false; }
+    virtual const char *queryTypeName()                     { return "dictionary"; }
+    virtual StringBuffer &getECLType(StringBuffer & out);
+
+    virtual void serialize(MemoryBuffer &tgt);
+};
+
 class CTableTypeInfo : public CBasedTypeInfo
 {
 private:

common/fileview2/fvsource.cpp

@@ -355,6 +355,10 @@ void DataSourceMetaData::gatherFields(IHqlExpression * expr, bool isConditional)
                 gatherChildFields(expr->queryRecord(), isConditional);
                 fields.append(*new DataSourceMetaItem(FVFFendrecord, outname, xpath, voidType));
             }
+            else if ((tc == type_dictionary))
+            {
+                UNIMPLEMENTED;
+            }
             else if ((tc == type_table) || (tc == type_groupedtable))
             {
                 isStoredFixedWidth = false;

ecl/hql/hqlatoms.cpp

@@ -254,7 +254,6 @@ _ATOM _parameterScopeType_Atom;
 _ATOM partitionAtom;
 _ATOM partitionLeftAtom;
 _ATOM partitionRightAtom;
-_ATOM payloadAtom;
 _ATOM _payload_Atom;
 _ATOM persistAtom;
 _ATOM physicalFilenameAtom;
@@ -635,7 +634,6 @@ MODULE_INIT(INIT_PRIORITY_HQLATOM)
     MAKEATOM(partition);
     partitionLeftAtom = createLowerCaseAtom("partition left");
     partitionRightAtom = createLowerCaseAtom("partition right");
-    MAKEATOM(payload);
     MAKESYSATOM(payload);
     MAKEATOM(persist);
     MAKEATOM(physicalFilename);

ecl/hql/hqlattr.cpp

@@ -264,6 +264,7 @@ unsigned getOperatorMetaFlags(node_operator op)
     case no_newtransform:
 
 //Rows
+    case no_selectmap:
     case no_selectnth:
     case no_matchrow:
     case no_matchattr:      // and scalar
@@ -272,6 +273,9 @@ unsigned getOperatorMetaFlags(node_operator op)
     case no_newrow:
     case no_temprow:
 
+//Dictionaries
+    case no_inlinedictionary:
+
 //Datasets [see also selection operators]
     case no_rollup:
     case no_iterate:
@@ -607,13 +611,13 @@ unsigned getOperatorMetaFlags(node_operator op)
     case no_persist_check:
     case no_dataset_from_transform:
 
-    case no_unused3: case no_unused4: case no_unused5: case no_unused6:
+    case no_unused4: case no_unused5: case no_unused6:
     case no_unused13: case no_unused14: case no_unused15: case no_unused18: case no_unused19:
     case no_unused20: case no_unused21: case no_unused22: case no_unused23: case no_unused24: case no_unused25: case no_unused26: case no_unused27: case no_unused28: case no_unused29:
     case no_unused30: case no_unused31: case no_unused32: case no_unused33: case no_unused34: case no_unused35: case no_unused36: case no_unused37: case no_unused38:
     case no_unused40: case no_unused41: case no_unused42: case no_unused43: case no_unused44: case no_unused45: case no_unused46: case no_unused47: case no_unused48: case no_unused49:
     case no_unused50: case no_unused52:
-    case no_unused80: case no_unused82: case no_unused83:
+    case no_unused80: case no_unused83:
     case no_is_null:
     case no_position:
     case no_current_time:
@@ -868,6 +872,7 @@ static IHqlExpression * evaluateFieldAttrSize(IHqlExpression * expr)
                 }
                 break;
             }
+        case type_dictionary:
         case type_groupedtable:
         case type_table:
             {
@@ -3429,6 +3434,7 @@ bool isLinkedRowset(ITypeInfo * t)
     {
     case type_table:
     case type_groupedtable:
+    case type_dictionary:
         return hasLinkCountedModifier(t);
     }
     return false;
@@ -3441,6 +3447,7 @@ bool isArrayRowset(ITypeInfo * t)
     case type_table:
     case type_groupedtable:
     case type_array:
+    case type_dictionary:
         {
             if (hasLinkCountedModifier(t))
                 assertex(hasLinkCountedModifier(t->queryChildType()));
@@ -3463,6 +3470,7 @@ bool hasLinkedRow(ITypeInfo * t)
     {
     case type_table:
     case type_groupedtable:
+    case type_dictionary:
         return hasLinkedRow(t->queryChildType());
     case type_row:
         return hasLinkCountedModifier(t);
@@ -3479,6 +3487,7 @@ ITypeInfo * setLinkCountedAttr(ITypeInfo * _type, bool setValue)
     {
     case type_table:
     case type_groupedtable:
+    case type_dictionary:
         {
             ITypeInfo * rowType = type->queryChildType();
             Owned<ITypeInfo> newRowType = setLinkCountedAttr(rowType, setValue);

ecl/hql/hqlexpr.cpp

@@ -72,7 +72,7 @@
 //#define PARANOID
 //#define SEARCH_NAME1   "vL6R"
 //#define SEARCH_NAME2   "v19"
-//#define SEARCH_IEXPR 0x03289048
+//#define SEARCH_IEXPR 0x0681cb0
 //#define CHECK_SELSEQ_CONSISTENCY
 //#define GATHER_COMMON_STATS
 #define VERIFY_EXPR_INTEGRITY
@@ -1004,6 +1004,7 @@ const char *getOpString(node_operator op)
     case no_mergedscope: return "<scope>";
     case no_privatescope: return "<private_scope>";
     case no_list: return "<list>";
+    case no_selectmap: return "SELECT_MAP";
     case no_selectnth: return "SELECT_NTH";
     case no_filter: return "FILTER";
     case no_param: return "<parameter>";
@@ -1445,9 +1446,9 @@ const char *getOpString(node_operator op)
     case no_debug_option_value: return "__DEBUG__";
     case no_dataset_alias: return "TABLE";
     case no_childquery: return "no_childquery";
+    case no_inlinedictionary: return "DICTIONARY";
 
-
-    case no_unused3: case no_unused4: case no_unused5: case no_unused6:
+    case no_unused4: case no_unused5: case no_unused6:
     case no_unused13: case no_unused14: case no_unused15: case no_unused18: case no_unused19:
     case no_unused20: case no_unused21: case no_unused22: case no_unused23: case no_unused24: case no_unused25: case no_unused26: case no_unused27: case no_unused28: case no_unused29:
     case no_unused30: case no_unused31: case no_unused32: case no_unused33: case no_unused34: case no_unused35: case no_unused36: case no_unused37: case no_unused38:
@@ -1455,7 +1456,6 @@ const char *getOpString(node_operator op)
     case no_unused50: case no_unused52:
     case no_unused80:
     case no_unused81:
-    case no_unused82:
     case no_unused83:
         return "unused";
     /* if fail, use "hqltest -internal" to find out why. */
@@ -2517,7 +2517,7 @@ bool definesColumnList(IHqlExpression * dataset)
     case no_field:
         {
             type_t tc = dataset->queryType()->getTypeCode();
-            assertex(tc == type_table || tc == type_groupedtable);
+            assertex(tc == type_table || tc == type_groupedtable || tc == type_dictionary);
             return true;
         }
     case no_comma:
@@ -2883,6 +2883,7 @@ IHqlExpression * ensureExprType(IHqlExpression * expr, ITypeInfo * type, node_op
                 break;
             }
         }
+    case type_dictionary:
     case type_table:
     case type_groupedtable:
         if (recordTypesMatch(type, exprType))
@@ -3067,8 +3068,8 @@ void CHqlExpression::initFlagsBeforeOperands()
 {
 #ifdef _DEBUG
 #ifdef SEARCH_IEXPR
-    if ((unsigned)(IHqlExpression *)this == SEARCH_IEXPR)
-        op = op;
+    if ((memsize_t)(IHqlExpression *)this == SEARCH_IEXPR)
+        DBGLOG("initFlags");
 #endif
 #endif
 
@@ -3266,6 +3267,7 @@ void CHqlExpression::updateFlagsAfterOperands()
     case no_assert_ds:
         switch (queryType()->getTypeCode())
         {
+        case type_dictionary:
         case type_table:
         case type_groupedtable:
             infoFlags |= HEFthrowds;
@@ -3626,6 +3628,7 @@ switch (op)
                 }
                 break;
             }
+        case type_dictionary:
         case type_groupedtable:
         case type_table:
             {
@@ -4116,6 +4119,27 @@ bool CHqlExpression::isDataset()
     }
 }
 
+bool CHqlExpression::isDictionary()
+{
+    ITypeInfo * cur = queryType();
+    loop
+    {
+        if (!cur)
+            return false;
+
+        switch(cur->getTypeCode())
+        {
+        case type_dictionary:
+            return true;
+        case type_function:
+            cur = cur->queryChildType();
+            break;
+        default:
+            return false;
+        }
+    }
+}
+
 bool CHqlExpression::isDatarow() 
 {
     return matchesTypeCode(queryType(), type_row);
@@ -4249,8 +4273,8 @@ void CHqlExpression::Link(void) const
 #endif
 #ifdef _DEBUG
 #ifdef SEARCH_IEXPR
-    if ((unsigned)(IHqlExpression *)this == SEARCH_IEXPR)
-        op = op;
+    if ((memsize_t)(IHqlExpression *)this == SEARCH_IEXPR)
+        DBGLOG("Link");
 #endif
 #endif
     Parent::Link();
@@ -4265,8 +4289,8 @@ bool CHqlExpression::Release(void) const
 #endif
 #ifdef _DEBUG
 #ifdef SEARCH_IEXPR
-    if ((unsigned)(IHqlExpression *)this == SEARCH_IEXPR)
-        op = op;
+    if ((memsize_t)(IHqlExpression *)this == SEARCH_IEXPR)
+        DBGLOG("Release");
 #endif
 #endif
     return Parent::Release();
@@ -5089,6 +5113,7 @@ void CHqlExpressionWithTables::cacheTablesUsed()
                             switch (thisType->getTypeCode())
                             {
                             case type_void:
+                            case type_dictionary:
                             case type_table:
                             case type_groupedtable:
                             case type_row:
@@ -5428,6 +5453,7 @@ void CHqlField::onCreateField()
         break;
     case type_row:
         break;
+    case type_dictionary:
     case type_table:
     case type_groupedtable:
         typeExpr = queryRecord();
@@ -5641,6 +5667,29 @@ bool isInImplictScope(IHqlExpression * scope, IHqlExpression * dataset)
     return false;
 }
 
+//===========================================================================
+
+CHqlDictionary *CHqlDictionary::makeDictionary(node_operator _op, ITypeInfo *type, HqlExprArray &_ownedOperands)
+{
+    CHqlDictionary *e = new CHqlDictionary(_op, type, _ownedOperands);
+    return (CHqlDictionary *) e->closeExpr();
+}
+
+CHqlDictionary::CHqlDictionary(node_operator _op, ITypeInfo *_type, HqlExprArray &_ownedOperands)
+: CHqlExpressionWithType(_op, _type, _ownedOperands)
+{
+}
+
+CHqlDictionary::~CHqlDictionary()
+{
+}
+
+IHqlExpression *CHqlDictionary::clone(HqlExprArray &newkids)
+{
+    return createDictionary(op, newkids);
+}
+
+//===========================================================================
 
 CHqlDataset *CHqlDataset::makeDataset(node_operator _op, ITypeInfo *type, HqlExprArray &_ownedOperands)
 {
@@ -6082,6 +6131,7 @@ bool CHqlRecord::assignableFrom(ITypeInfo * source)
     switch(source->getTypeCode())
     {
     case type_groupedtable:
+    case type_dictionary:
     case type_table:
     case type_row:
     case type_transform:
@@ -6897,6 +6947,7 @@ extern HQL_API bool okToAddLocation(IHqlExpression * expr)
     ITypeInfo * type = expr->queryType();
     switch (type->getTypeCode())
     {
+    case type_dictionary:
     case type_table:
     case type_groupedtable:
     case type_void:
@@ -8754,6 +8805,9 @@ IHqlExpression * CHqlParameter::makeParameter(_ATOM _name, unsigned _idx, ITypeI
     type_t tc = _type->getTypeCode();
     switch (tc)
     {
+    case type_dictionary:
+        UNIMPLEMENTED;
+        break;
     case type_table:
     case type_groupedtable:
         e = new CHqlDatasetParameter(_name, _idx, _type);
@@ -9819,6 +9873,88 @@ extern IHqlExpression *createDatasetF(node_operator op, ...)
     return createDataset(op, children);
 }
 
+IHqlExpression *createDictionary(node_operator op, HqlExprArray & parms)
+{
+#ifdef GATHER_LINK_STATS
+    insideCreate++;
+#endif
+    Owned<ITypeInfo> type = NULL;
+
+    switch (op)
+    {
+    case no_null:
+        type.setown(makeDictionaryType(makeRowType(createRecordType(&parms.item(0)))));
+        break;
+    case no_inlinedictionary:
+        type.setown(makeDictionaryType(makeRowType(createRecordType(&parms.item(1)))));
+        break;
+    case no_select:
+        type.set(parms.item(1).queryType());
+        break;
+    case no_addfiles:
+        type.set(parms.item(0).queryType());  // It's an error if they don't all match, caught elsewhere (?)
+        break;
+    case no_if:
+        {
+            IHqlExpression & left = parms.item(1);
+            IHqlExpression & right = parms.item(2);
+            ITypeInfo * leftType = left.queryType();
+            ITypeInfo * rightType = right.queryType();
+            if (left.getOperator() == no_null)
+                type.set(rightType);
+            else if (right.getOperator() == no_null)
+                type.set(leftType);
+            else
+                type.setown(getTypeIntersection(leftType, rightType));
+            break;
+        }
+    case no_case:
+        //following is wrong, but they get removed pretty quickly so I don't really care
+        type.set(parms.item(1).queryType());
+        break;
+    case no_map:
+        //following is wrong, but they get removed pretty quickly so I don't really care
+        type.set(parms.item(0).queryType());
+        break;
+    case no_nofold:
+    case no_nohoist:
+    case no_thor:
+    case no_nothor:
+    case no_alias:
+    case no_translated:
+    case no_catch:
+        type.set(parms.item(0).queryType());
+        break;
+    default:
+        UNIMPLEMENTED_XY("Type calculation for dictionary operator", getOpString(op));
+        break;
+    }
+
+    IHqlExpression * ret = CHqlDictionary::makeDictionary(op, type.getClear(), parms);
+#ifdef GATHER_LINK_STATS
+    insideCreate--;
+#endif
+    return ret;
+}
+
+IHqlExpression *createDictionary(node_operator op, IHqlExpression *dictionary, IHqlExpression *list)
+{
+    HqlExprArray parms;
+    parms.append(*dictionary);
+    if (list)
+    {
+        list->unwindList(parms, no_comma);
+        list->Release();
+    }
+    return createDictionary(op, parms);
+}
+
+IHqlExpression *createDictionary(node_operator op, IHqlExpression *dictionary)
+{
+    HqlExprArray parms;
+    parms.append(*dictionary);
+    return createDictionary(op, parms);
+}
 
 IHqlExpression * createAliasOwn(IHqlExpression * expr, IHqlExpression * attr)
 {
@@ -9839,6 +9975,8 @@ IHqlExpression * createTypedValue(node_operator op, ITypeInfo * type, HqlExprArr
     case type_groupedtable:
     case type_table:
         return createDataset(op, args);
+    case type_dictionary:
+        return createDictionary(op, args);
     case type_row:
         return createRow(op, args);
     default:
@@ -11917,6 +12055,8 @@ extern IHqlExpression *createWrapper(node_operator op, IHqlExpression * e)
         case type_table:
         case type_groupedtable:
             return createDataset(op, e, NULL);
+        case type_dictionary:
+            return createDictionary(op, e, NULL);
         }
     }
     return createValue(op, LINK(type), e);
@@ -11931,6 +12071,8 @@ IHqlExpression *createWrapper(node_operator op, ITypeInfo * type, HqlExprArray &
         {
         case type_row:
             return createRow(op, args);
+        case type_dictionary:
+            return createDictionary(op, args);
         case type_table:
         case type_groupedtable:
             return createDataset(op, args);
@@ -12044,6 +12186,8 @@ extern IHqlExpression * createSelectExpr(IHqlExpression * _lhs, IHqlExpression *
     type_t t = rhs->queryType()->getTypeCode();
     if (t == type_table || t == type_groupedtable)
         return createDataset(no_select, LINK(normalLhs), createComma(rhs, LINK(newAttr)));
+    if (t == type_dictionary)
+        return createDictionary(no_select, LINK(normalLhs), createComma(rhs, LINK(newAttr)));
     if (t == type_row)
         return createRow(no_select, LINK(normalLhs), createComma(rhs, LINK(newAttr)));
 
@@ -12058,6 +12202,8 @@ static IHqlExpression * doCreateSelectExpr(HqlExprArray & args)
     type_t t = rhs->queryType()->getTypeCode();
     if (t == type_table || t == type_groupedtable)
         return createDataset(no_select, args);
+    if (t == type_dictionary)
+        return createDictionary(no_select, args);
     if (t == type_row)
         return createRow(no_select, args);
 
@@ -12548,6 +12694,13 @@ extern HQL_API IHqlExpression * createNullExpr(ITypeInfo * type)
             IHqlExpression * attr = queryProperty(type, _linkCounted_Atom);
             return createDataset(no_null, LINK(record), LINK(attr));
         }
+    case type_dictionary:
+        {
+            ITypeInfo * recordType = queryRecordType(type);
+            IHqlExpression * record = queryExpression(recordType);
+            IHqlExpression * attr = queryProperty(type, _linkCounted_Atom);
+            return createDictionary(no_null, LINK(record), LINK(attr));
+        }
     case type_row:
 #if 0
         {
@@ -12600,6 +12753,8 @@ extern HQL_API IHqlExpression * createPureVirtual(ITypeInfo * type)
             return createDataset(no_purevirtual, LINK(queryOriginalRecord(type)));
         case type_groupedtable:
             return createDataset(no_purevirtual, LINK(queryOriginalRecord(type)), createAttribute(groupedAtom));
+        case type_dictionary:
+            return createDictionary(no_purevirtual, LINK(queryOriginalRecord(type)), createAttribute(groupedAtom));
         case type_row:
         case type_record:
             return createRow(no_purevirtual, LINK(queryOriginalRecord(type)));
@@ -13733,7 +13888,7 @@ IHqlExpression * extractChildren(IHqlExpression * value)
 
 IHqlExpression * queryDatasetCursor(IHqlExpression * ds)
 {
-    while ((ds->getOperator() == no_select) && !ds->isDataset())
+    while ((ds->getOperator() == no_select) && !ds->isDataset() && !ds->isDictionary())
         ds = ds->queryChild(0);
     return ds;
 }
@@ -13743,7 +13898,7 @@ IHqlExpression * querySelectorDataset(IHqlExpression * expr, bool & isNew)
     assertex(expr->getOperator() == no_select);
     isNew = expr->hasProperty(newAtom);
     IHqlExpression * ds = expr->queryChild(0);
-    while ((ds->getOperator() == no_select) && !ds->isDataset())
+    while ((ds->getOperator() == no_select) && !ds->isDataset() && !ds->isDictionary())
     {
         if (ds->hasProperty(newAtom))
             isNew = true;
@@ -13759,7 +13914,7 @@ bool isNewSelector(IHqlExpression * expr)
     if (expr->hasProperty(newAtom))
         return true;
     IHqlExpression * ds = expr->queryChild(0);
-    while ((ds->getOperator() == no_select) && !ds->isDataset())
+    while ((ds->getOperator() == no_select) && !ds->isDataset() && !ds->isDictionary())
     {
         if (ds->hasProperty(newAtom))
             return true;
@@ -13781,7 +13936,7 @@ IHqlExpression * replaceSelectorDataset(IHqlExpression * expr, IHqlExpression *
 {
     assertex(expr->getOperator() == no_select);
     IHqlExpression * ds = expr->queryChild(0);
-    if ((ds->getOperator() == no_select) && !ds->isDataset())
+    if ((ds->getOperator() == no_select) && !ds->isDataset() && !ds->isDictionary())
     {
         OwnedHqlExpr newSelector = replaceSelectorDataset(ds, newDataset);
         return replaceChild(expr, 0, newSelector);
@@ -13841,6 +13996,7 @@ static ITypeInfo * doGetSimplifiedType(ITypeInfo * type, bool isConditional, boo
     case type_alien:
         return getSimplifiedType(promoted, isConditional, isSerialized);
     case type_row:
+    case type_dictionary:
     case type_table:
     case type_groupedtable:
     case type_transform:
@@ -14629,6 +14785,7 @@ extern HQL_API bool hasUnknownTransform(IHqlExpression * expr)
         if (expr->hasProperty(mergeTransformAtom))
             return true;
         break;
+    case no_inlinedictionary:
     case no_inlinetable:
         {
             IHqlExpression * transforms = expr->queryChild(0);
@@ -14740,6 +14897,11 @@ IHqlExpression * createNullDataset()
     return createDataset(no_null, LINK(queryNullRecord()));
 }
 
+IHqlExpression * createNullDictionary()
+{
+    return createDictionary(no_null, LINK(queryNullRecord()));
+}
+
 bool removeProperty(HqlExprArray & args, _ATOM name)
 {
     bool removed = false;
@@ -15617,6 +15779,9 @@ IHqlExpression * createTypeTransfer(IHqlExpression * expr, ITypeInfo * _newType)
     case type_groupedtable:
         return createDataset(no_typetransfer, LINK(queryOriginalRecord(newType)), expr);
         break;
+    case type_dictionary:
+        return createDictionary(no_typetransfer, LINK(queryOriginalRecord(newType)), expr);
+        break;
     default:
         return createValue(no_typetransfer, newType.getClear(), expr);
     }

ecl/hql/hqlexpr.hpp

@@ -227,7 +227,7 @@ enum _node_operator {
         no_comma,
         no_count,
         no_countgroup,
-    no_unused82,
+        no_selectmap,
         no_exists,
         no_within,
         no_notwithin,
@@ -358,7 +358,7 @@ enum _node_operator {
         no_dataset_from_transform,
         no_childquery,
         no_unknown,
-    no_unused3,
+        no_inlinedictionary,
     no_unused4,
     no_unused5,
         no_any,
@@ -1066,6 +1066,7 @@ interface IHqlExpression : public IInterface
     virtual bool isBoolean() = 0;
     virtual bool isDataset() = 0;
     virtual bool isDatarow() = 0;
+    virtual bool isDictionary() = 0;
     virtual bool isScope() = 0;
     virtual bool isType() = 0;
     virtual bool isAction() = 0;
@@ -1253,6 +1254,9 @@ extern HQL_API IHqlExpression *createDataset(node_operator op, IHqlExpression *d
 extern HQL_API IHqlExpression *createDataset(node_operator op, IHqlExpression *dataset, IHqlExpression *elist);
 extern HQL_API IHqlExpression *createDataset(node_operator op, HqlExprArray & parms);       // inScope should only be set internally.
 extern HQL_API IHqlExpression *createDatasetF(node_operator op, ...);
+extern HQL_API IHqlExpression *createDictionary(node_operator op, IHqlExpression *initializer, IHqlExpression *recordDef);
+extern HQL_API IHqlExpression *createDictionary(node_operator op, IHqlExpression *dictionary);
+extern HQL_API IHqlExpression *createDictionary(node_operator op, HqlExprArray & parms);
 extern HQL_API IHqlExpression *createNewDataset(IHqlExpression *name, IHqlExpression *recorddef, IHqlExpression *mode, IHqlExpression *parent, IHqlExpression *joinCondition, IHqlExpression * options = NULL);
 extern HQL_API IHqlExpression *createRow(node_operator op, IHqlExpression *Dataset, IHqlExpression *element = NULL);
 extern HQL_API IHqlExpression *createRow(node_operator op, HqlExprArray & args);
@@ -1542,6 +1546,7 @@ extern HQL_API bool isSimpleCountAggregate(IHqlExpression * aggregateExpr, bool
 extern HQL_API bool isSimpleCountExistsAggregate(IHqlExpression * aggregateExpr, bool canFilterArg, bool canCast);
 extern HQL_API bool isKeyedCountAggregate(IHqlExpression * aggregate);
 extern HQL_API IHqlExpression * createNullDataset();
+extern HQL_API IHqlExpression * createNullDictionary();
 extern HQL_API IHqlExpression * queryNullRecord();
 extern HQL_API IHqlExpression * queryNullRowRecord();
 extern HQL_API IHqlExpression * queryExpression(ITypeInfo * t);

ecl/hql/hqlexpr.ipp

@@ -200,6 +200,7 @@ public:
     virtual bool isBoolean();
     virtual bool isConstant();
     virtual bool isDataset();
+    virtual bool isDictionary();
     virtual bool isDatarow();
     virtual bool isScope();
     virtual bool isMacro();
@@ -1528,6 +1529,18 @@ public:
     void insertSymbols(IHqlExpression * expr);
 };
 
+class CHqlDictionary : public CHqlExpressionWithType // , implements IHqlDictionary
+{
+public:
+    IMPLEMENT_IINTERFACE_USING(CHqlExpression)
+    static CHqlDictionary *makeDictionary(node_operator op, ITypeInfo *type, HqlExprArray &operands);
+
+public:
+    CHqlDictionary(node_operator op, ITypeInfo *_type, HqlExprArray &_ownedOperands);
+    ~CHqlDictionary();
+
+    virtual IHqlExpression *clone(HqlExprArray &newkids);
+};
 
 
 class CHqlDataset : public CHqlExpressionWithType, implements IHqlDataset

ecl/hql/hqlfold.cpp

@@ -5571,6 +5571,10 @@ HqlConstantPercolator * CExprFolderTransformer::gatherConstants(IHqlExpression *
         //all bets are off.
         break;
 
+    case no_selectmap:
+        // MORE - maybe should be something here?
+        break;
+
     case no_selectnth:
         {
             //Careful - this can create a null row if it is out of range.

ecl/hql/hqlgram.hpp

@@ -166,6 +166,10 @@ public:
     {
         return queryExpr()->isDatarow();
     }
+    inline bool isDictionary() const
+    {
+        return queryExpr()->isDictionary();
+    }
 
     /* setters */
     inline void inherit(attribute & other)
@@ -416,6 +420,7 @@ public:
     void checkBooleanOrNumeric(attribute &atr);
     void checkDatarow(attribute &atr);
     void checkDataset(attribute &atr);
+    void checkDictionary(attribute &atr);
     void checkFieldnameValid(const attribute &errpos, _ATOM name);
     void checkList(attribute &atr);
     void checkScalar(attribute &atr);
@@ -508,6 +513,7 @@ public:
     ITypeInfo *promoteMapToSameType(HqlExprArray & exprs, attribute &eElse);
     ITypeInfo *promoteSetToSameType(HqlExprArray & exprs, attribute &errpos);
     ITypeInfo * queryElementType(const attribute & errpos, IHqlExpression * list);
+    IHqlExpression *createINDict(node_operator op, IHqlExpression *expr, IHqlExpression *dict, attribute &errpos);
     IHqlExpression *createINExpression(node_operator op, IHqlExpression *expr, IHqlExpression *set, attribute &errpos);
     IHqlExpression * createLoopCondition(IHqlExpression * left, IHqlExpression * arg1, IHqlExpression * arg2, IHqlExpression * seq, IHqlExpression * rowsid);
     void setTemplateAttribute();
@@ -628,12 +634,15 @@ public:
     IHqlExpression * attachMetaAttributes(IHqlExpression * ownedExpr, HqlExprArray & meta);
 
     void addDatasetField(const attribute &errpos, _ATOM name, IHqlExpression * record, IHqlExpression *value, IHqlExpression * attrs);
+    void addDictionaryField(const attribute &errpos, _ATOM name, IHqlExpression * record, IHqlExpression *value, IHqlExpression * attrs);
     void addField(const attribute &errpos, _ATOM name, ITypeInfo *type, IHqlExpression *value, IHqlExpression *attrs);
     void addFields(const attribute &errpos, IHqlExpression *record, IHqlExpression * dataset, bool clone);
     void addIfBlockToActive(const attribute &errpos, IHqlExpression * ifblock);
     void addToActiveRecord(IHqlExpression * newField);
     void beginIfBlock();
     IHqlExpression * endIfBlock();
+    void beginPayload();
+    IHqlExpression * endPayload();
 
     IHqlExpression * expandedSortListByReference(attribute * module, attribute & list);
     IHqlExpression *bindParameters(const attribute & errpos, IHqlExpression * function, HqlExprArray & ownedActuals);
@@ -650,11 +659,12 @@ public:
     int checkRecordTypes(IHqlExpression *left, IHqlExpression *right, attribute &atr, unsigned maxFields = (unsigned)-1);
     bool checkRecordCreateTransform(HqlExprArray & assigns, IHqlExpression *leftExpr, IHqlExpression *leftSelect, IHqlExpression *rightExpr, IHqlExpression *rightSelect, attribute &atr);
     IHqlExpression * checkEnsureRecordsMatch(IHqlExpression * left, IHqlExpression * right, attribute & errpos, bool rightIsRow);
-    void checkRecordIsValid(attribute &atr, IHqlExpression *record);
+    void checkRecordIsValid(const attribute &atr, IHqlExpression *record);
     void checkValidRecordMode(IHqlExpression * dataset, attribute & atr, attribute & modeatr);
     void checkValidCsvRecord(const attribute & errpos, IHqlExpression * record);
     void checkValidPipeRecord(const attribute & errpos, IHqlExpression * record, IHqlExpression * attrs, IHqlExpression * expr);
 
+    void createAppendDictionaries(attribute & targetAttr, attribute & leftAttr, attribute & rightAttr, _ATOM kind);
     void createAppendFiles(attribute & targetAttr, attribute & leftAttr, attribute & rightAttr, _ATOM kind);
     IHqlExpression * processIfProduction(attribute & condAttr, attribute & trueAttr, attribute * falseAttr);
 
@@ -919,6 +929,7 @@ protected:
 
     IHqlExpression * getActiveCounter(attribute & errpos);
     void pushRecord(IHqlExpression *);
+    IHqlExpression *endRecordDef();
     IHqlExpression *popRecord();
     IHqlExpression *queryTopScope();
     ITypeInfo * getPromotedECLType(HqlExprArray & args, ITypeInfo * otherType, bool allowVariableLength);
@@ -949,11 +960,11 @@ protected:
     IHqlExpression *queryCurrentTransformRecord();
     IHqlExpression* queryFieldMap(IHqlExpression* expr);
     IHqlExpression* bindFieldMap(IHqlExpression*, IHqlExpression*);
-    void applyPayloadAttribute(const attribute & errpos, IHqlExpression * record, SharedHqlExpr & extra);
     void extractRecordFromExtra(SharedHqlExpr & record, SharedHqlExpr & extra);
     void transferOptions(attribute & filenameAttr, attribute & optionsAttr);
     IHqlExpression * extractTransformFromExtra(SharedHqlExpr & extra);
     void expandPayload(HqlExprArray & fields, IHqlExpression * payload, IHqlSimpleScope * scope, ITypeInfo * & lastFieldType, const attribute & errpos);
+    void mergeDictionaryPayload(SharedHqlExpr & record, SharedHqlExpr & payload, const attribute & errpos);
     void modifyIndexPayloadRecord(SharedHqlExpr & record, SharedHqlExpr & payload, SharedHqlExpr & extra, const attribute & errpos);
 
     bool haveAssignedToChildren(IHqlExpression * select);

ecl/hql/hqlgram.y

@@ -166,6 +166,7 @@ static void eclsyntaxerror(HqlGram * parser, const char * s, short yystate, int
   DENORMALIZE
   DEPRECATED
   DESC
+  DICTIONARY
   DISTRIBUTE
   DISTRIBUTED
   DISTRIBUTION
@@ -332,7 +333,6 @@ static void eclsyntaxerror(HqlGram * parser, const char * s, short yystate, int
   PARTITION
   PARTITION_ATTR
   TOK_PATTERN
-  PAYLOAD
   PENALTY
   PERSIST
   PHYSICALFILENAME
@@ -479,6 +479,7 @@ static void eclsyntaxerror(HqlGram * parser, const char * s, short yystate, int
 
   DATAROW_ID
   DATASET_ID
+  DICTIONARY_ID
   SCOPE_ID
   VALUE_ID
   VALUE_ID_REF
@@ -499,6 +500,7 @@ static void eclsyntaxerror(HqlGram * parser, const char * s, short yystate, int
 
   DATAROW_FUNCTION
   DATASET_FUNCTION
+  DICTIONARY_FUNCTION
   VALUE_FUNCTION
   ACTION_FUNCTION
   PATTERN_FUNCTION
@@ -703,6 +705,7 @@ defineType
     : typeDef
     | setType
     | explicitDatasetType
+    | explicitDictionaryType
     | ROW               {
                             IHqlExpression* record = queryNullRecord();
                             $$.setType(makeRowType(record->getType()));
@@ -759,6 +762,22 @@ explicitDatasetType1
                         }
     ;
 
+explicitDictionaryType
+    : DICTIONARY
+                        {
+                            $$.setType(makeDictionaryType(makeRowType(queryNullRecord()->getType())));
+                            $$.setPosition($1);
+                        }
+    | DICTIONARY '(' recordDef ')'
+                        {
+                            OwnedHqlExpr record = $3.getExpr();
+                            ITypeInfo * recordType = createRecordType(record);
+                            $$.setType(makeDictionaryType(makeRowType(recordType)));
+                            $$.setPosition($1);
+                        }
+    ;
+
+
 transformType
     : TRANSFORM '(' recordDef ')'
                         {
@@ -862,6 +881,7 @@ object
 
 goodObject
     : dataSet
+    | dictionary
     | expression
                         {
                             //Remove later to allow sortlist attributes
@@ -1172,6 +1192,7 @@ knownOrUnknownId
 knownId
     : DATAROW_ID
     | DATASET_ID
+    | DICTIONARY_ID
     | VALUE_ID
     | ACTION_ID
     | RECORD_ID
@@ -1890,6 +1911,11 @@ transformation1
                             parser->addAssignment($1, $3);
                             $$.clear($1);
                         }
+    | transformDst ASSIGN dictionary
+                        {
+                            parser->addAssignment($1, $3);
+                            $$.clear($1);
+                        }
     | transformDst ASSIGN error ';'
                         {
                             $1.release();
@@ -1966,6 +1992,12 @@ transformDstSelect
                             parser->setDotScope(scope);
                             $$.setExpr(scope.getClear(), $1);
                         }
+    | DICTIONARY_ID
+                        {
+                            OwnedHqlExpr scope = $1.getExpr();
+                            parser->setDotScope(scope);
+                            $$.setExpr(scope.getClear(), $1);
+                        }
     ;
 
 transformDstField
@@ -2925,12 +2957,6 @@ indexFlag
                             $$.setExpr(createAttribute(fixedAtom));
                             $$.setPosition($1);
                         }
-    | PAYLOAD '(' expression ')'
-                        {
-                            parser->normalizeExpression($3);        //$3 indicates the first payload field
-                            $$.setExpr(createExprAttribute(payloadAtom, $3.getExpr()));
-                            $$.setPosition($1);
-                        }
     | COMPRESSED '(' compressMode ')'
                         {
                             $$.setExpr(createExprAttribute(compressedAtom, $3.getExpr()));
@@ -3721,6 +3747,15 @@ funcRetType
                         }
     ;
 
+payloadPart
+    :  		            {
+                            // NOTE - this reduction happens as soon as the GOESTO is seen,
+                            // so it ensures that the following fields go into the payload record def
+                            $$.setExpr(parser->endRecordDef());
+                            parser->beginRecord();
+                        }
+      GOESTO fieldDefs optSemiComma
+    ;
 
 recordDef
     : startrecord fieldDefs optSemiComma endrecord
@@ -3730,6 +3765,16 @@ recordDef
                             $$.setExpr(record.getClear());
                             $$.setPosition($1);
                         }
+
+    | startrecord fieldDefs payloadPart endrecord
+                        {
+                            OwnedHqlExpr record = $3.getExpr();
+                            OwnedHqlExpr payload = $4.getExpr();
+                            parser->mergeDictionaryPayload(record, payload, $1);
+                            $$.setExpr(record.getClear());
+                            $$.setPosition($1);
+                        }
+
     | startrecord recordOptions fieldDefs optSemiComma endrecord
                         {
                             OwnedHqlExpr record = $5.getExpr();
@@ -3737,6 +3782,16 @@ recordDef
                             $$.setExpr(record.getClear());
                             $$.setPosition($1);
                         }
+
+    | startrecord recordOptions fieldDefs payloadPart endrecord
+                        {
+                            OwnedHqlExpr record = $4.getExpr();
+                            OwnedHqlExpr payload = $5.getExpr();
+                            parser->mergeDictionaryPayload(record, payload, $1);
+                            $$.setExpr(record.getClear());
+                            $$.setPosition($1);
+                        }
+
     | startrecord recordBase optFieldDefs endrecord
                         {
                             OwnedHqlExpr record = $4.getExpr();
@@ -3744,6 +3799,16 @@ recordDef
                             $$.setExpr(record.getClear());
                             $$.setPosition($1);
                         }
+
+    | startrecord recordBase optFieldDefs payloadPart endrecord
+                        {
+                            OwnedHqlExpr record = $4.getExpr();
+                            OwnedHqlExpr payload = $5.getExpr();
+                            parser->mergeDictionaryPayload(record, payload, $1);
+                            $$.setExpr(record.getClear());
+                            $$.setPosition($1);
+                        }
+
     | startrecord recordBase recordOptions optFieldDefs endrecord
                         {
                             OwnedHqlExpr record = $5.getExpr();
@@ -3751,6 +3816,15 @@ recordDef
                             $$.setExpr(record.getClear());
                             $$.setPosition($1);
                         }
+
+    | startrecord recordBase recordOptions optFieldDefs payloadPart endrecord
+                        {
+                            OwnedHqlExpr record = $5.getExpr();
+                            OwnedHqlExpr payload = $6.getExpr();
+                            parser->mergeDictionaryPayload(record, payload, $1);
+                            $$.setExpr(record.getClear());
+                            $$.setPosition($1);
+                        }
     | simpleRecord
     | recordDef AND recordDef
                         {
@@ -3972,11 +4046,7 @@ recordOption
 endrecord
     : endOfRecordMarker
                         {
-                            IHqlExpression * record = parser->popRecord();
-                            record->Link();     // link should be in startrecord, but can only link after closeExpr()
-                            parser->popSelfScope();
-                            OwnedHqlExpr newRecord = record->closeExpr();
-                            $$.setExpr(newRecord.getClear());
+                            $$.setExpr(parser->endRecordDef());
                             parser->popLocale();
                             $$.setPosition($1);
                         }
@@ -4168,6 +4238,32 @@ fieldDef
                             parser->addDatasetField($1, $1.getName(), LINK(value->queryRecord()), value, $2.getExpr());
                             $$.clear();
                         }
+
+
+    | DICTIONARY '(' recordDef childDictionaryOptions ')' knownOrUnknownId optFieldAttrs
+                        {
+                            $$.clear($1);
+                            parser->addDictionaryField($6, $6.getName(), $3.getExpr(), NULL, createComma($4.getExpr(), $7.getExpr()));
+                        }
+    | DICTIONARY '(' recordDef childDictionaryOptions ')' knownOrUnknownId optFieldAttrs ASSIGN dictionary
+                        {
+                            $$.clear($1);
+                            parser->addDictionaryField($6, $6.getName(), $3.getExpr(), $9.getExpr(), createComma($4.getExpr(), $7.getExpr()));
+                        }
+    | DICTIONARY knownOrUnknownId optFieldAttrs ASSIGN dictionary
+                        {
+                            $$.clear($1);
+                            IHqlExpression * value = $5.getExpr();
+                            parser->addDictionaryField($2, $2.getName(), LINK(value->queryRecord()), value, $3.getExpr());
+                        }
+    | UNKNOWN_ID optFieldAttrs ASSIGN dictionary
+                        {
+                            IHqlExpression * value = $4.getExpr();
+                            parser->addDictionaryField($1, $1.getName(), LINK(value->queryRecord()), value, $2.getExpr());
+                            $$.clear();
+                        }
+
+
     | alienTypeInstance knownOrUnknownId optFieldAttrs defaultValue
                         {
                             $$.clear($1);
@@ -4593,6 +4689,24 @@ userTypedefPattern
     ;
 
 
+childDictionaryOptions
+    :
+                        { $$.setNullExpr(); }
+    | childDictionaryOptions ',' childDictionaryOption
+                        {
+                            $$.setExpr(createComma($1.getExpr(), $3.getExpr()));
+                        }
+    ;
+
+childDictionaryOption
+    : COUNT '(' expression ')'
+                        {
+                            parser->normalizeExpression($3);
+                            $$.setExpr(createLinkAttribute(countAtom, $3.getExpr()));
+                            $$.setPosition($1);
+                        }
+    ;
+
 childDatasetOptions
     :
                         { $$.setNullExpr(); }
@@ -4938,6 +5052,46 @@ compareExpr
                             $$.setExpr(parser->createINExpression(no_in, expr, set, $3));
                             $$.setPosition($2);
                         }
+    | expr NOT TOK_IN dictionary
+                        {
+                            parser->normalizeExpression($1);
+                            parser->normalizeExpression($4);
+                            parser->normalizeExpression($4, type_dictionary, false);
+                            IHqlExpression *dict = $4.getExpr();
+                            IHqlExpression *expr = $1.getExpr();
+                            $$.setExpr(parser->createINDict(no_notin, expr, dict, $4));
+                            $$.setPosition($3);
+                        }
+    | dataRow NOT TOK_IN dictionary
+                        {
+                            parser->normalizeExpression($1);
+                            parser->normalizeExpression($4);
+                            parser->normalizeExpression($4, type_dictionary, false);
+                            IHqlExpression *dict = $4.getExpr();
+                            IHqlExpression *expr = $1.getExpr();
+                            $$.setExpr(parser->createINDict(no_notin, expr, dict, $4));
+                            $$.setPosition($3);
+                        }
+    | expr TOK_IN dictionary
+                        {
+                            parser->normalizeExpression($1);
+                            parser->normalizeExpression($3);
+                            parser->normalizeExpression($3, type_dictionary, false);
+                            IHqlExpression *dict = $3.getExpr();
+                            IHqlExpression *expr = $1.getExpr();
+                            $$.setExpr(parser->createINDict(no_in, expr, dict, $3));
+                            $$.setPosition($2);
+                        }
+    | dataRow TOK_IN dictionary
+                        {
+                            parser->normalizeExpression($1);
+                            parser->normalizeExpression($3);
+                            parser->normalizeExpression($3, type_dictionary, false);
+                            IHqlExpression *dict = $3.getExpr();
+                            IHqlExpression *expr = $1.getExpr();
+                            $$.setExpr(parser->createINDict(no_in, expr, dict, $3));
+                            $$.setPosition($2);
+                        }
     | dataSet EQ dataSet    
                         {
                             parser->checkSameType($1, $3); $$.setExpr(createBoolExpr(no_eq, $1.getExpr(), $3.getExpr()));
@@ -6637,6 +6791,10 @@ dataRow
                             parser->normalizeExpression($3, type_int, false);
                             $$.setExpr(createRow(no_selectnth, $1.getExpr(), $3.getExpr()));    
                         }
+    | dictionary '[' expressionList ']'
+                        {
+                            $$.setExpr(createRow(no_selectmap, $1.getExpr(), $3.getExpr()));
+                        }
     | dataSet '[' NOBOUNDCHECK expression ']'
                         {   
                             parser->normalizeExpression($4, type_int, false);
@@ -6912,6 +7070,101 @@ simpleDataRow
                         }
     ;
 
+dictionary
+    : simpleDictionary
+    | dictionary '+' dictionary
+                        {   parser->createAppendDictionaries($$, $1, $3, NULL);    }
+    ;
+
+simpleDictionary
+    : scopedDictionaryId
+    | NOFOLD '(' dictionary ')'
+                        {
+                            $$.setExpr(createDictionary(no_nofold, $3.getExpr(), NULL));
+                            $$.setPosition($1);
+                        }
+    | NOHOIST '(' dictionary ')'
+                        {
+                            $$.setExpr(createDictionary(no_nohoist, $3.getExpr(), NULL));
+                            $$.setPosition($1);
+                        }
+/*
+    | DICTIONARY '(' dataSet ',' recordDef ')'
+                        {
+                            $$.setExpr($3.getExpr()); // MORE!
+                            $$.setPosition($1);
+                        }
+*/
+    | DICTIONARY '(' '[' beginList inlineDatasetValueList ']' ',' recordDef ')'
+                        {
+                            HqlExprArray values;
+                            parser->endList(values);
+                            OwnedHqlExpr table = createDataset(no_temptable, createValue(no_recordlist, NULL, values), $8.getExpr());
+                            $$.setExpr(convertTempTableToInlineDictionary(parser->errorHandler, $5.pos, table));
+                            $$.setPosition($1);
+                        }
+    | '(' dictionary  ')'  {
+                            $$.setExpr($2.getExpr());
+                            $$.setPosition($1);
+                        }
+    | IF '(' booleanExpr ',' dictionary ',' dictionary ')'
+                        {
+                            OwnedHqlExpr ds = parser->processIfProduction($3, $5, &$7);
+                            $$.setExpr(ds.getClear(), $1);
+                        }
+    | IF '(' booleanExpr ',' dictionary ')'
+                        {
+                            OwnedHqlExpr ds = parser->processIfProduction($3, $5, NULL);
+                            $$.setExpr(ds.getClear(), $1);
+                        }
+    | IFF '(' booleanExpr ',' dictionary ',' dictionary ')'
+                        {
+                            OwnedHqlExpr ds = parser->processIfProduction($3, $5, &$7);
+                            $$.setExpr(ds.getClear(), $1);
+                        }
+    | IFF '(' booleanExpr ',' dictionary ')'
+                        {
+                            OwnedHqlExpr ds = parser->processIfProduction($3, $5, NULL);
+                            $$.setExpr(ds.getClear(), $1);
+                        }
+// MORE - should do CASE and MAP
+    ;
+
+scopedDictionaryId
+    : globalScopedDictionaryId
+    | dotScope DICTIONARY_ID leaveScope
+                        {
+                            IHqlExpression *e1 = $1.getExpr();
+                            IHqlExpression *e2 = $2.getExpr();
+                            if (e1 && (e1->getOperator() != no_record) && (e2->getOperator() == no_field))
+                                $$.setExpr(parser->createSelect(e1, e2, $2));
+                            else
+                            {
+                                ::Release(e1);
+                                $$.setExpr(e2);
+                            }
+                        }
+/*
+    | dictionaryFunction '('
+                        {
+                            parser->beginFunctionCall($1);
+                        }
+    actualParameters ')'
+                        {
+                            $$.setExpr(parser->bindParameters($1, $4.getExpr()));
+                        }
+*/
+    ;
+
+globalScopedDictionaryId
+    : DICTIONARY_ID
+    | moduleScopeDot DICTIONARY_ID leaveScope
+                        {
+                            OwnedHqlExpr scope = $1.getExpr();
+                            $$.setExpr($2.getExpr());
+                        }
+    ;
+
 dataSet
     : simpleDataSet
     | startCompoundExpression beginInlineFunctionToken optDefinitions RETURN dataSet ';' endInlineFunctionToken
@@ -7859,7 +8112,6 @@ simpleDataSet
                             parser->extractRecordFromExtra(record, extra);
                             OwnedHqlExpr transform = parser->extractTransformFromExtra(extra);
 
-                            parser->applyPayloadAttribute($6, record, extra);
                             parser->inheritRecordMaxLength(dataset, record);
 
                             record.setown(parser->checkIndexRecord(record, $5));
@@ -7886,8 +8138,6 @@ simpleDataSet
                             OwnedHqlExpr record = $3.getExpr();
                             OwnedHqlExpr extra = $4.getExpr();
                             parser->extractRecordFromExtra(record, extra);
-                            parser->applyPayloadAttribute($6, record, extra);
-
                             $$.setExpr(parser->createIndexFromRecord(record, extra, $3));
                             parser->checkIndexRecordTypes($$.queryExpr(), $1);
                             $$.setPosition($1);
@@ -9443,8 +9693,13 @@ inlineFieldValues
     | inlineFieldValues ',' inlineFieldValue
     ;
 
+inlineFieldValuesWithGoesto
+    : inlineFieldValues optSemiComma
+    | inlineFieldValues GOESTO inlineFieldValues optSemiComma
+    ;
+
 inlineDatasetValue
-    : '{' beginList inlineFieldValues optSemiComma '}'
+    : '{' beginList inlineFieldValuesWithGoesto  '}'
                         {
                             HqlExprArray args;
                             parser->endList(args);

ecl/hql/hqlgram2.cpp

@@ -567,6 +567,14 @@ IHqlExpression* HqlGram::popRecord()
     return &activeRecords.pop();
 }                                       
 
+IHqlExpression* HqlGram::endRecordDef()
+{
+    IHqlExpression * record = popRecord();
+    record->Link();     // logically link should be in startrecord, but can only link after finished updating
+    popSelfScope();
+    OwnedHqlExpr newRecord = record->closeExpr();
+    return newRecord.getClear();
+}
 
 void HqlGram::beginFunctionCall(attribute & function)
 {
@@ -2329,6 +2337,22 @@ void HqlGram::addDatasetField(const attribute &errpos, _ATOM name, IHqlExpressio
     record->Release();
 }
 
+void HqlGram::addDictionaryField(const attribute &errpos, _ATOM name, IHqlExpression * record, IHqlExpression *value, IHqlExpression * attrs)
+{
+    if (!name)
+        name = createUnnamedFieldName();
+    checkFieldnameValid(errpos, name);
+    if (queryPropertyInList(virtualAtom, attrs))
+        reportError(ERR_BAD_FIELD_ATTR, errpos, "Virtual can only be specified on a scalar field");
+    if (!attrs)
+        attrs = extractAttrsFromExpr(value);
+
+    ITypeInfo * type = makeDictionaryType(makeRowType(createRecordType(record)));
+    IHqlExpression *newField = createField(name, type, value, attrs);
+    addToActiveRecord(newField);
+    record->Release();
+}
+
 void HqlGram::addIfBlockToActive(const attribute &errpos, IHqlExpression * ifblock)
 {
     activeRecords.tos().addOperand(LINK(ifblock));
@@ -3813,6 +3837,9 @@ void HqlGram::normalizeExpression(attribute & exprAttr, type_t expectedType, boo
     case type_set:
         checkList(exprAttr);
         break;
+    case type_dictionary:
+        checkDictionary(exprAttr);
+        break;
     case type_table:
         ensureDataset(exprAttr);
         break;
@@ -4907,6 +4934,15 @@ IHqlExpression *HqlGram::createINExpression(node_operator op, IHqlExpression *ex
     return createBoolExpr(op, expr, normalized.getClear());
 }
 
+IHqlExpression *HqlGram::createINDict(node_operator op, IHqlExpression *expr, IHqlExpression *dict, attribute &errpos)
+{
+    ITypeInfo * exprType = expr->queryType();
+    IHqlExpression * record = dict->queryRecord();
+    // Check that we have a value for every keyed field
+    // MORE
+    return createBoolExpr(op, expr, dict);
+}
+
 void HqlGram::checkCaseForDuplicates(HqlExprArray & exprs, attribute &err)
 {
     TransformMutexBlock lock;
@@ -7667,6 +7703,7 @@ void HqlGram::ensureDataset(attribute & attr)
     }
     checkDataset(attr);
 }
+
 void HqlGram::expandPayload(HqlExprArray & fields, IHqlExpression * payload, IHqlSimpleScope * scope, ITypeInfo * & lastFieldType, const attribute & errpos)
 {
     ForEachChild(i2, payload)
@@ -7708,10 +7745,45 @@ void HqlGram::expandPayload(HqlExprArray & fields, IHqlExpression * payload, IHq
     }
 }
 
+void HqlGram::mergeDictionaryPayload(SharedHqlExpr & record, SharedHqlExpr & payload, const attribute & errpos)
+{
+    checkRecordIsValid(errpos, record.get());
+    IHqlSimpleScope * scope = record->querySimpleScope();
+
+    // Move all the attributes to the front of the record
+    HqlExprArray fields;
+    ForEachChild(i3, record)
+    {
+        IHqlExpression * cur = record->queryChild(i3);
+        if (cur->isAttribute())
+            fields.append(*LINK(cur));
+    }
+    ForEachChild(i1, record)
+    {
+        IHqlExpression * cur = record->queryChild(i1);
+        if (!cur->isAttribute())
+            fields.append(*LINK(cur));
+    }
+
+    unsigned payloadCount = 0;
+    ITypeInfo * lastFieldType = NULL;
+    if (payload)
+    {
+        unsigned oldFields = fields.ordinality();
+        expandPayload(fields, payload,  scope, lastFieldType, errpos);
+        payloadCount = fields.ordinality() - oldFields;
+    }
+    fields.add(*createAttribute(_payload_Atom, createConstant((__int64) payloadCount)), 0);
+    record.setown(createRecord(fields));
+}
+
+
 void HqlGram::modifyIndexPayloadRecord(SharedHqlExpr & record, SharedHqlExpr & payload, SharedHqlExpr & extra, const attribute & errpos)
 {
+    checkRecordIsValid(errpos, record.get());
     IHqlSimpleScope * scope = record->querySimpleScope();
 
+    // Move all the attributes to the front of the record
     HqlExprArray fields;
     ForEachChild(i3, record)
     {
@@ -7758,7 +7830,7 @@ void HqlGram::modifyIndexPayloadRecord(SharedHqlExpr & record, SharedHqlExpr & p
         payloadCount++;
     }
 
-    extra.setown(createComma(extra.getClear(), createAttribute(_payload_Atom, createConstant((__int64)payloadCount))));
+    extra.setown(createComma(extra.getClear(), createAttribute(_payload_Atom, createConstant((__int64) payloadCount))));
     record.setown(createRecord(fields));
 }
 
@@ -7803,32 +7875,6 @@ IHqlExpression * HqlGram::extractTransformFromExtra(SharedHqlExpr & extra)
     }
     return ret;
 }
-            
-
-void HqlGram::applyPayloadAttribute(const attribute & errpos, IHqlExpression * record, SharedHqlExpr & extra)
-{
-    IHqlExpression * payload = queryPropertyInList(payloadAtom, extra);
-    if (payload)
-    {
-        HqlExprArray fields;
-        unwindChildren(fields, record);
-        IHqlExpression * search = payload->queryChild(0);
-        if (search->getOperator() == no_select)
-            search = search->queryChild(1);
-        unsigned match = fields.find(*search);
-        if (match != NotFound)
-        {
-            HqlExprArray args;
-            extra->unwindList(args, no_comma);
-            args.zap(*payload);
-            args.append(*createAttribute(_payload_Atom, createConstant((__int64)fields.ordinality()-match)));
-            extra.setown(createComma(args));
-        }
-        else
-            reportError(ERR_TYPEMISMATCH_RECORD, errpos, "The argument to the payload isn't found in the index record");
-    }
-}
-
 
 void HqlGram::checkBoolean(attribute &atr)
 {
@@ -7861,6 +7907,15 @@ void HqlGram::checkDataset(attribute &atr)
     }
 }
 
+void HqlGram::checkDictionary(attribute &atr)
+{
+    if (!atr.queryExpr()->isDictionary())
+    {
+        reportError(ERR_EXPECTED_DATASET, atr, "Expected dictionary expression");
+        atr.release().setExpr(createNullDictionary());
+    }
+}
+
 void HqlGram::checkDatarow(attribute &atr)
 {
     if (!atr.queryExpr()->isDatarow())
@@ -8214,7 +8269,7 @@ static bool isZeroSize(IHqlExpression * expr)
 }
 
 
-void HqlGram::checkRecordIsValid(attribute &atr, IHqlExpression *record)
+void HqlGram::checkRecordIsValid(const attribute &atr, IHqlExpression *record)
 {
     if (isZeroSize(record))
         reportError(ERR_ZEROSIZE_RECORD, atr, "Record must not be zero length");
@@ -8374,6 +8429,19 @@ void HqlGram::createAppendFiles(attribute & targetAttr, attribute & leftAttr, at
     targetAttr.setPosition(leftAttr);
 }
 
+void HqlGram::createAppendDictionaries(attribute & targetAttr, attribute & leftAttr, attribute & rightAttr, _ATOM kind)
+{
+    OwnedHqlExpr left = leftAttr.getExpr();
+    OwnedHqlExpr right = rightAttr.getExpr();
+//    if (left->isDatarow())
+//        left.setown(createDatasetFromRow(LINK(left)));
+    right.setown(checkEnsureRecordsMatch(left, right, rightAttr, right->isDatarow()));
+//    if (right->isDatarow())
+//        right.setown(createDatasetFromRow(LINK(right)));
+    IHqlExpression * attr = kind ? createAttribute(kind) : NULL;
+    targetAttr.setExpr(createDictionary(no_addfiles, LINK(left), createComma(LINK(right), attr)));
+    targetAttr.setPosition(leftAttr);
+}
 
 IHqlExpression * HqlGram::processIfProduction(attribute & condAttr, attribute & trueAttr, attribute * falseAttr)
 {
@@ -9841,6 +9909,7 @@ static void getTokenText(StringBuffer & msg, int token)
     case DENORMALIZE: msg.append("DENORMALIZE"); break;
     case DEPRECATED: msg.append("DEPRECATED"); break;
     case DESC: msg.append("DESC"); break;
+    case DICTIONARY: msg.append("DICTIONARY"); break;
     case DISTRIBUTE: msg.append("DISTRIBUTE"); break;
     case DISTRIBUTED: msg.append("DISTRIBUTED"); break;
     case DISTRIBUTION: msg.append("DISTRIBUTION"); break;
@@ -10011,7 +10080,6 @@ static void getTokenText(StringBuffer & msg, int token)
     case PARTITION: msg.append("PARTITION"); break;
     case PARTITION_ATTR: msg.append("PARTITION"); break;
     case TOK_PATTERN: msg.append("PATTERN"); break;
-    case PAYLOAD: msg.append("PAYLOAD"); break;
     case PENALTY: msg.append("PENALTY"); break;
     case PERSIST: msg.append("PERSIST"); break;
     case PHYSICALFILENAME: msg.append("PHYSICALFILENAME"); break;
@@ -10159,6 +10227,7 @@ static void getTokenText(StringBuffer & msg, int token)
 
     case DATASET_ID: msg.append("dataset"); break;
     case DATAROW_ID: msg.append("datarow"); break;
+    case DICTIONARY_ID: msg.append("dictionary"); break;
     case RECORD_ID: msg.append("record-name"); break;
     case RECORD_FUNCTION: msg.append("record-name"); break;
     case VALUE_ID: msg.append("identifier"); break;
@@ -10184,6 +10253,7 @@ static void getTokenText(StringBuffer & msg, int token)
     case SCOPE_FUNCTION: msg.append("module-name"); break;
     case TRANSFORM_FUNCTION: msg.append("transform-name"); break;
     case DATAROW_FUNCTION: msg.append("datarow"); break;
+    case DICTIONARY_FUNCTION: msg.append("dictionary"); break;
     case LIST_DATASET_FUNCTION: msg.append("identifier"); break;
 
     case VALUE_FUNCTION: 
@@ -10280,7 +10350,7 @@ void HqlGram::simplifyExpected(int *expected)
                        TOXML, '@', SECTION, EVENTEXTRA, EVENTNAME, __SEQUENCE__, IFF, OMITTED, GETENV, __DEBUG__, __STAND_ALONE__, 0);
     simplify(expected, DATA_CONST, REAL_CONST, STRING_CONST, INTEGER_CONST, UNICODE_CONST, 0);
     simplify(expected, VALUE_MACRO, DEFINITIONS_MACRO, 0);
-    simplify(expected, VALUE_ID, DATASET_ID, RECORD_ID, ACTION_ID, UNKNOWN_ID, SCOPE_ID, VALUE_FUNCTION, DATAROW_FUNCTION, DATASET_FUNCTION, LIST_DATASET_FUNCTION, LIST_DATASET_ID, ALIEN_ID, TYPE_ID, SET_TYPE_ID, TRANSFORM_ID, TRANSFORM_FUNCTION, RECORD_FUNCTION, FEATURE_ID, EVENT_ID, EVENT_FUNCTION, SCOPE_FUNCTION, ENUM_ID, PATTERN_TYPE_ID, 0); 
+    simplify(expected, VALUE_ID, DATASET_ID, DICTIONARY_ID, RECORD_ID, ACTION_ID, UNKNOWN_ID, SCOPE_ID, VALUE_FUNCTION, DATAROW_FUNCTION, DATASET_FUNCTION, DICTIONARY_FUNCTION, LIST_DATASET_FUNCTION, LIST_DATASET_ID, ALIEN_ID, TYPE_ID, SET_TYPE_ID, TRANSFORM_ID, TRANSFORM_FUNCTION, RECORD_FUNCTION, FEATURE_ID, EVENT_ID, EVENT_FUNCTION, SCOPE_FUNCTION, ENUM_ID, PATTERN_TYPE_ID, 0);
     simplify(expected, LIBRARY, LIBRARY, SCOPE_FUNCTION, STORED, PROJECT, INTERFACE, MODULE, 0);
     simplify(expected, MATCHROW, MATCHROW, LEFT, RIGHT, IF, IFF, ROW, HTTPCALL, SOAPCALL, PROJECT, GLOBAL, NOFOLD, NOHOIST, ALLNODES, THISNODE, SKIP, DATAROW_FUNCTION, TRANSFER, RIGHT_NN, FROMXML, 0);
     simplify(expected, TRANSFORM_ID, TRANSFORM_FUNCTION, TRANSFORM, '@', 0);
@@ -10328,6 +10398,7 @@ void HqlGram::syntaxError(const char *s, int token, int *expected)
     {
     case DATAROW_ID:
     case DATASET_ID:
+    case DICTIONARY_ID:
     case SCOPE_ID:
     case VALUE_ID:
     case VALUE_ID_REF:
@@ -10346,6 +10417,7 @@ void HqlGram::syntaxError(const char *s, int token, int *expected)
     case LIST_DATASET_ID:
     case DATAROW_FUNCTION:
     case DATASET_FUNCTION:
+    case DICTIONARY_FUNCTION:
     case VALUE_FUNCTION:
     case ACTION_FUNCTION:
     case PATTERN_FUNCTION:

ecl/hql/hqlir.cpp

@@ -798,8 +798,10 @@ static const char * getOperatorText(node_operator op)
     DUMP_CASE(no,debug_option_value);
     DUMP_CASE(no,dataset_alias);
     DUMP_CASE(no,childquery);
+    DUMP_CASE(no,selectmap);
+    DUMP_CASE(no,inlinedictionary);
 
-    case no_unused3: case no_unused4: case no_unused5: case no_unused6:
+    case no_unused4: case no_unused5: case no_unused6:
     case no_unused13: case no_unused14: case no_unused15: case no_unused18: case no_unused19:
     case no_unused20: case no_unused21: case no_unused22: case no_unused23: case no_unused24: case no_unused25: case no_unused26: case no_unused27: case no_unused28: case no_unused29:
     case no_unused30: case no_unused31: case no_unused32: case no_unused33: case no_unused34: case no_unused35: case no_unused36: case no_unused37: case no_unused38:

ecl/hql/hqllex.l

@@ -142,6 +142,9 @@ int HqlLex::lookupIdentifierToken(YYSTYPE & returnToken, HqlLex * lexer, bool lo
     case type_transform:
         token = isFunction ? TRANSFORM_FUNCTION : TRANSFORM_ID;
         break;
+    case type_dictionary:
+        token = isFunction ? DICTIONARY_FUNCTION : DICTIONARY_ID;
+        break;
     case type_groupedtable:
     case type_table:
         token = isFunction ? DATASET_FUNCTION : DATASET_ID;
@@ -659,6 +662,7 @@ DENORMALIZE         { RETURNSYM(DENORMALIZE); }
 DEPRECATED          { RETURNSYM(DEPRECATED); }
 DESC                { RETURNSYM(DESC); }
 DESCEND             { RETURNSYM(DESC); }
+DICTIONARY          { RETURNSYM(DICTIONARY); }
 DISTRIBUTE          { RETURNSYM(DISTRIBUTE); }
 DISTRIBUTED         { RETURNSYM(DISTRIBUTED); }
 DISTRIBUTION        { RETURNSYM(DISTRIBUTION); }
@@ -821,7 +825,6 @@ PARALLEL            { RETURNSYM(PARALLEL); }
 PARSE               { RETURNSYM(PARSE); }
 PARTITION           { RETURNSYM(PARTITION); }
 PATTERN             { RETURNSYM(TOK_PATTERN); }
-PAYLOAD             { RETURNSYM(PAYLOAD); }
 PENALTY             { RETURNSYM(PENALTY); }
 PERSIST             { RETURNSYM(PERSIST); }
 PHYSICALFILENAME    { RETURNSYM(PHYSICALFILENAME); }

ecl/hql/hqlthql.cpp

@@ -863,7 +863,7 @@ void HqltHql::toECL(IHqlExpression *expr, StringBuffer &s, bool paren, bool inTy
     }
     else
     {
-        if (expr->isDataset())
+        if (expr->isDataset() || expr->isDictionary())
         {
             if (!isNamedSymbol && expandProcessed && no != no_field && no != no_rows && !isTargetSelector(expr))
             {
@@ -874,7 +874,10 @@ void HqltHql::toECL(IHqlExpression *expr, StringBuffer &s, bool paren, bool inTy
                     
                     StringBuffer temp;
                     scope.append(NULL);
-                    temp.appendf("dataset%p ", expr);
+                    if (expr->isDataset())
+                        temp.appendf("dataset%p ", expr);
+                    else
+                        temp.appendf("dictionary%p ", expr);
 #ifdef SHOW_NORMALIZED
                     if (expandProcessed)
                         temp.appendf("[N%p] ", expr->queryNormalizedSelector());
@@ -889,7 +892,10 @@ void HqltHql::toECL(IHqlExpression *expr, StringBuffer &s, bool paren, bool inTy
                     insideNewTransform = wasInsideNewTransform;
                     expr->setTransformExtra(expr);
                 }   
-                s.appendf("dataset%p", expr);
+                if (expr->isDataset())
+                    s.appendf("dataset%p", expr);
+                else
+                    s.appendf("dictionary%p", expr);
                 if (paren)
                     s.append(')');
                 return;
@@ -1202,6 +1208,7 @@ void HqltHql::toECL(IHqlExpression *expr, StringBuffer &s, bool paren, bool inTy
         }
         case no_index:
         case no_selectnth:
+        case no_selectmap:
         case no_pat_index:
         {
             toECL(child0, s, child0->getPrecedence() < 0, inType);
@@ -2679,8 +2686,9 @@ StringBuffer &HqltHql::getFieldTypeString(IHqlExpression * e, StringBuffer &s)
         type = type->queryChildType();
         //fall through
     case type_table:
+    case type_dictionary:
         {
-            s.append("DATASET(");
+            s.append(type->getTypeCode()==type_dictionary ? "DICTIONARY(" : "DATASET(");
             getTypeString(type->queryChildType()->queryChildType(), s);
             ForEachChild(i, e)
             {
@@ -2872,6 +2880,7 @@ static bool addExplicitType(IHqlExpression * expr)
     {
     case type_transform:
         return true;
+    case type_dictionary:
     case type_row:
     case type_table:
     case type_groupedtable:

ecl/hql/hqlutil.cpp

@@ -1177,6 +1177,9 @@ IHqlExpression * createIf(IHqlExpression * cond, IHqlExpression * left, IHqlExpr
     if (left->isDatarow() || right->isDatarow())
         return createRow(no_if, cond, createComma(left, right));
 
+    if (left->isDictionary() || right->isDictionary())
+        return createDictionary(no_if, cond, createComma(left, right));
+
     ITypeInfo * type = ::getPromotedECLType(left->queryType(), right->queryType());
     return createValue(no_if, type, cond, left, right);
 }
@@ -1879,9 +1882,12 @@ unsigned isEmptyRecord(IHqlExpression * record)
 
 bool isTrivialSelectN(IHqlExpression * expr)
 {
-    IHqlExpression * index = expr->queryChild(1);
-    if (index->queryValue() && (index->queryValue()->getIntValue() == 1))
-        return hasSingleRow(expr->queryChild(0));
+    if (expr->getOperator() == no_index || expr->getOperator() == no_index)
+    {
+        IHqlExpression * index = expr->queryChild(1);
+        if (index->queryValue() && (index->queryValue()->getIntValue() == 1))
+            return hasSingleRow(expr->queryChild(0));
+    }
     return false;
 }
 
@@ -3239,6 +3245,8 @@ IHqlExpression * createGetResultFromSetResult(IHqlExpression * setResult, ITypeI
         return createDataset(no_getresult, LINK(queryOriginalRecord(valueType)), createComma(LINK(seqAttr), LINK(aliasAttr)));
     case type_groupedtable:
         return createDataset(no_getresult, LINK(queryOriginalRecord(valueType)), createComma(LINK(seqAttr), createAttribute(groupedAtom), LINK(aliasAttr)));
+    case type_dictionary:
+        return createDictionary(no_getresult, LINK(queryOriginalRecord(valueType)), createComma(LINK(seqAttr), createAttribute(groupedAtom), LINK(aliasAttr)));
     case type_row:
     case type_record:
          return createRow(no_getresult, LINK(queryOriginalRecord(valueType)), createComma(LINK(seqAttr), LINK(aliasAttr)));
@@ -5307,7 +5315,7 @@ IHqlExpression * convertTempRowToCreateRow(IErrorReceiver * errors, ECLlocation
     return expr->cloneAllAnnotations(ret);
 }
 
-IHqlExpression * convertTempTableToInlineTable(IErrorReceiver * errors, ECLlocation & location, IHqlExpression * expr)
+static IHqlExpression * convertTempTableToInline(IErrorReceiver * errors, ECLlocation & location, IHqlExpression * expr, bool isDictionary)
 {
     IHqlExpression * oldValues = expr->queryChild(0);
     IHqlExpression * record = expr->queryChild(1);
@@ -5344,10 +5352,19 @@ IHqlExpression * convertTempTableToInlineTable(IErrorReceiver * errors, ECLlocat
     HqlExprArray children;
     children.append(*createValue(no_transformlist, makeNullType(), transforms));
     children.append(*LINK(record));
-    OwnedHqlExpr ret = createDataset(no_inlinetable, children);
+    OwnedHqlExpr ret = isDictionary ? createDictionary(no_inlinedictionary, children) : createDataset(no_inlinetable, children);
     return expr->cloneAllAnnotations(ret);
 }
 
+IHqlExpression * convertTempTableToInlineTable(IErrorReceiver * errors, ECLlocation & location, IHqlExpression * expr)
+{
+    return convertTempTableToInline(errors, location, expr, false);
+}
+
+IHqlExpression * convertTempTableToInlineDictionary(IErrorReceiver * errors, ECLlocation & location, IHqlExpression * expr)
+{
+    return convertTempTableToInline(errors, location, expr, true);
+}
 
 bool areTypesComparable(ITypeInfo * leftType, ITypeInfo * rightType)
 {
@@ -5377,6 +5394,7 @@ bool areTypesComparable(ITypeInfo * leftType, ITypeInfo * rightType)
     case type_array:
         return areTypesComparable(leftType->queryChildType(), rightType->queryChildType());
     case type_row:
+    case type_dictionary:
     case type_table:
     case type_groupedtable:
         return recordTypesMatch(leftType, rightType);
@@ -6964,6 +6982,7 @@ void EclXmlSchemaBuilder::build(IHqlExpression * record) const
                         builder.addSetField(name, childName, *type);
                         break;
                     }
+                case type_dictionary:
                 case type_table:
                 case type_groupedtable:
                     {
@@ -7300,6 +7319,7 @@ bool ConstantRowCreator::processElement(IHqlExpression * expr, IHqlExpression *
                 if (rhsOp == no_createrow)
                     return createConstantRow(out, rhs->queryChild(0));
                 return false;
+            case type_dictionary:
             case type_table:
             case type_groupedtable:
                 if (!expr->hasProperty(countAtom) && !expr->hasProperty(sizeofAtom))

ecl/hql/hqlutil.hpp

@@ -449,6 +449,7 @@ extern HQL_API bool isNullList(IHqlExpression * expr);
 
 extern HQL_API IHqlExpression * convertTempRowToCreateRow(IErrorReceiver * errors, ECLlocation & location, IHqlExpression * expr);
 extern HQL_API IHqlExpression * convertTempTableToInlineTable(IErrorReceiver * errors, ECLlocation & location, IHqlExpression * expr);
+extern HQL_API IHqlExpression * convertTempTableToInlineDictionary(IErrorReceiver * errors, ECLlocation & location, IHqlExpression * expr);
 extern HQL_API bool areTypesComparable(ITypeInfo * leftType, ITypeInfo * rightType);
 extern HQL_API bool arraysMatch(const HqlExprArray & left, const HqlExprArray & right);
 extern HQL_API IHqlExpression * ensureTransformType(IHqlExpression * transform, node_operator op);

ecl/hqlcpp/hqlcpp.cpp

@@ -398,6 +398,7 @@ IHqlExpression * getPointer(IHqlExpression * source)
             return createValue(no_typetransfer, LINK(newType), LINK(cast));
         }
         break;
+    case type_dictionary:
     case type_table:
     case type_groupedtable:
     case type_row:
@@ -765,6 +766,8 @@ IHqlExpression * createTranslated(IHqlExpression * expr, IHqlExpression * length
     ITypeInfo * type = expr->queryType();
     switch (type->getTypeCode())
     {
+    case type_dictionary:
+        return createDictionary(no_translated, LINK(expr), LINK(length));
     case type_table:
     case type_groupedtable:
         return createDataset(no_translated, LINK(expr), LINK(length));
@@ -854,6 +857,7 @@ bool isNullAssign(const CHqlBoundTarget & target, IHqlExpression * expr)
         case type_string:
         case type_qstring:
             return exprType->getSize() == 0;
+        case type_dictionary:
         case type_table:
             return expr->getOperator() == no_null;
         }
@@ -895,6 +899,8 @@ IHqlExpression * CHqlBoundExpr::getTranslatedExpr() const
     ITypeInfo * type = expr->queryType();
     switch (type->getTypeCode())
     {
+    case type_dictionary:
+        return createDictionary(no_translated, args);
     case type_table:
     case type_groupedtable:
         return createDataset(no_translated, args);
@@ -4057,6 +4063,7 @@ IHqlExpression * HqlCppTranslator::createWrapperTemp(BuildCtx & ctx, ITypeInfo *
     case type_set:  //needed if we have sets with link counted elements
     case type_row:
     case type_array:
+    case type_dictionary:
     case type_table:
     case type_groupedtable:
         //Ensure row and dataset temporaries are active throughout a function, so pointers to the row
@@ -4081,6 +4088,7 @@ void HqlCppTranslator::createTempFor(BuildCtx & ctx, ITypeInfo * _exprType, CHql
     switch (tc)
     {
     case type_array:
+    case type_dictionary:
     case type_table:
     case type_groupedtable:
         {
@@ -4136,6 +4144,7 @@ void HqlCppTranslator::createTempFor(BuildCtx & ctx, ITypeInfo * _exprType, CHql
         case type_array:
         case type_table:
         case type_groupedtable:
+        case type_dictionary:
             break;
         default:
             {
@@ -4174,6 +4183,7 @@ void HqlCppTranslator::createTempFor(BuildCtx & ctx, ITypeInfo * _exprType, CHql
     case type_array:
     case type_table:
     case type_groupedtable:
+    case type_dictionary:
         {
             OwnedITypeInfo lenType = makeModifier(LINK(sizetType), modifier);
             target.expr.setown(createWrapperTemp(ctx, exprType, modifier));
@@ -4243,6 +4253,7 @@ void HqlCppTranslator::buildTempExpr(BuildCtx & ctx, BuildCtx & declareCtx, CHql
         break;
     case type_table:
     case type_groupedtable:
+    case type_dictionary:
         {
             createTempFor(declareCtx, type, tempTarget, modifier, format);
             IHqlStmt * stmt = subctx.addGroup();
@@ -4721,6 +4732,7 @@ void HqlCppTranslator::doBuildExprCompare(BuildCtx & ctx, IHqlExpression * expr,
                     return;
                 //fallthrough....
             }
+        case type_dictionary:
         case type_table:
         case type_groupedtable:
         case type_row:
@@ -5371,6 +5383,7 @@ IHqlExpression * getCastParameter(IHqlExpression * curParam, ITypeInfo * argType
                 ((ptc == type_varstring) && (argType->queryCharset() == paramType->queryCharset())))
                 return LINK(curParam);
             break;
+        case type_dictionary:
         case type_row:
         case type_table:
         case type_groupedtable:
@@ -5430,6 +5443,7 @@ IHqlExpression * getCastParameter(IHqlExpression * curParam, ITypeInfo * argType
             if (childType)
                 return ensureExprType(curParam, argType);
             break;
+        case type_dictionary:
         case type_table:
         case type_groupedtable:
         case type_row:
@@ -5657,6 +5671,7 @@ void HqlCppTranslator::doBuildCall(BuildCtx & ctx, const CHqlBoundTarget * tgt,
             returnByReference = true;
             break;
         }
+    case type_dictionary:
     case type_table:
     case type_groupedtable:
         {
@@ -5779,6 +5794,7 @@ void HqlCppTranslator::doBuildCall(BuildCtx & ctx, const CHqlBoundTarget * tgt,
         type_t atc = argType->getTypeCode();
         switch (atc)
         {
+        case type_dictionary:
         case type_table:
         case type_groupedtable:
             {
@@ -5858,6 +5874,7 @@ void HqlCppTranslator::doBuildCall(BuildCtx & ctx, const CHqlBoundTarget * tgt,
                 }
                 break;
             }
+        case type_dictionary:
         case type_table:
         case type_groupedtable:
             {
@@ -7991,6 +8008,7 @@ void HqlCppTranslator::doBuildAssignCompareElement(BuildCtx & ctx, EvaluateCompa
     bool useMemCmp = false;
     switch (tc)
     {
+    case type_dictionary:
     case type_table:
     case type_groupedtable:
         doBuildAssignCompareTable(ctx, info, left, right);
@@ -8736,6 +8754,7 @@ void HqlCppTranslator::doBuildAssignHashElement(BuildCtx & ctx, HashCodeCreator
         case type_row:
             throwUnexpected();
             break;
+        case type_dictionary:
         case type_groupedtable:
         case type_table:
             //MORE: Should be handle this differently, with an iterator for the link counted rows case?
@@ -10980,6 +10999,7 @@ void HqlCppTranslator::assignCastUnknownLength(BuildCtx & ctx, const CHqlBoundTa
             }
             break;
 
+        case type_dictionary:
         case type_table:
         case type_groupedtable:
             {
@@ -11205,6 +11225,7 @@ static IHqlExpression * replaceInlineParameters(IHqlExpression * funcdef, IHqlEx
         case type_data:
         case type_unicode:
         case type_utf8:
+        case type_dictionary:
         case type_table:
         case type_groupedtable:
             if (paramType->getSize() == UNKNOWN_LENGTH)
@@ -11393,6 +11414,7 @@ IHqlExpression * HqlCppTranslator::getBoundCount(const CHqlBoundExpr & bound)
                 return getSizetConstant(size / type->queryChildType()->getSize());
             UNIMPLEMENTED;
         }
+    case type_dictionary:
     case type_table:
     case type_groupedtable:
     case type_set:
@@ -11429,6 +11451,7 @@ IHqlExpression * HqlCppTranslator::getBoundLength(const CHqlBoundExpr & bound)
         }
     case type_set:
     case type_array:
+    case type_dictionary:
     case type_table:
     case type_groupedtable:
         assertex(!isArrayRowset(type));

ecl/hqlcpp/hqlcpp.ipp

@@ -838,6 +838,8 @@ public:
     IHqlExpression * ensureIteratedRowIsLive(BuildCtx & initctx, BuildCtx & searchctx, BuildCtx & iterctx, BoundRow * row, IHqlExpression * dataset, IHqlExpression * rowExpr);
     BoundRow * buildOptimizeSelectFirstRow(BuildCtx & ctx, IHqlExpression * expr);
 
+    IReferenceSelector * buildDatasetSelectMap(BuildCtx & ctx, IHqlExpression * expr);
+
 // Helper functions
 
     void ThrowStringException(int code,const char *format, ...) __attribute__((format(printf, 3, 4)));            // override the global function to try and add more context information
@@ -1089,6 +1091,7 @@ public:
     IHqlCppDatasetBuilder * createChoosenDatasetBuilder(IHqlExpression * record, IHqlExpression * maxCount);
     IHqlCppDatasetBuilder * createLimitedDatasetBuilder(IHqlExpression * record, IHqlExpression * maxCount);
     IHqlCppDatasetBuilder * createLinkedDatasetBuilder(IHqlExpression * record, IHqlExpression * choosenLimit = NULL);
+    IHqlCppDatasetBuilder * createLinkedDictionaryBuilder(IHqlExpression * record);
     IReferenceSelector * createSelfSelect(BuildCtx & ctx, IReferenceSelector * target, IHqlExpression * expr, IHqlExpression * rootSelector);
     IReferenceSelector * createReferenceSelector(BoundRow * cursor, IHqlExpression * path);
     IReferenceSelector * createReferenceSelector(BoundRow * cursor);
@@ -1561,6 +1564,7 @@ public:
     void buildConnectOrders(BuildCtx & ctx, ABoundActivity * slaveActivity, ABoundActivity * masterActivity);
     void buildDedupFilterFunction(BuildCtx & ctx, HqlExprArray & equalities, HqlExprArray & conds, IHqlExpression * dataset, IHqlExpression * selSeq);
     void buildDedupSerializeFunction(BuildCtx & ctx, const char * funcName, IHqlExpression * srcDataset, IHqlExpression * tgtDataset, HqlExprArray & srcValues, HqlExprArray & tgtValues, IHqlExpression * selSeq);
+    void buildDictionaryHashClass(BuildCtx &ctx, IHqlExpression *record, IHqlExpression *dictionary, StringBuffer &lookupHelperName);
     void buildHashClass(BuildCtx & ctx, const char * name, IHqlExpression * orderExpr, const DatasetReference & dataset);
     void buildHashOfExprsClass(BuildCtx & ctx, const char * name, IHqlExpression * cond, const DatasetReference & dataset, bool compareToSelf);
     void buildInstancePrefix(ActivityInstance * instance);

ecl/hqlcpp/hqlcppds.cpp

@@ -391,6 +391,8 @@ IReferenceSelector * HqlCppTranslator::buildNewRow(BuildCtx & ctx, IHqlExpressio
     case no_index:
     case no_selectnth:
         return buildDatasetIndex(ctx, expr);
+    case no_selectmap:
+        return buildDatasetSelectMap(ctx, expr);
     case no_left:
     case no_right:
     case no_self:
@@ -557,7 +559,7 @@ IReferenceSelector * HqlCppTranslator::buildActiveRow(BuildCtx & ctx, IHqlExpres
     case no_activerow:
         return buildActiveRow(ctx, expr->queryChild(0));
     default:
-        if (!expr->isDataset())
+        if (!expr->isDataset() && !expr->isDictionary())
             return buildNewRow(ctx, expr);
         break;
     }
@@ -2028,6 +2030,7 @@ void HqlCppTranslator::doBuildDataset(BuildCtx & ctx, IHqlExpression * expr, CHq
             break;
         }
     case no_inlinetable:
+    case no_inlinedictionary:
         if (doBuildDatasetInlineTable(ctx, expr, tgt, format))
             return;
         break;
@@ -2171,6 +2174,7 @@ void HqlCppTranslator::buildDatasetAssign(BuildCtx & ctx, const CHqlBoundTarget
             ctx.addAssign(target.expr, bound.expr);
             return;
         }
+    case no_inlinedictionary:
     case no_inlinetable:
         {
             //This will typically generate a loop.  If few items then it is more efficient to expand the assigns/clones out.
@@ -2401,14 +2405,20 @@ void HqlCppTranslator::buildDatasetAssign(BuildCtx & ctx, const CHqlBoundTarget
     Owned<IHqlCppDatasetBuilder> builder;
     if (targetOutOfLine)
     {
-        IHqlExpression * choosenLimit = NULL;
-        if ((op == no_choosen) && !isChooseNAllLimit(expr->queryChild(1)) && !queryRealChild(expr, 2))
+        if (target.queryType()->getTypeCode() == type_dictionary)
         {
-            choosenLimit = expr->queryChild(1);
-            expr = expr->queryChild(0);
+            builder.setown(createLinkedDictionaryBuilder(record));
+        }
+        else
+        {
+            IHqlExpression * choosenLimit = NULL;
+            if ((op == no_choosen) && !isChooseNAllLimit(expr->queryChild(1)) && !queryRealChild(expr, 2))
+            {
+                choosenLimit = expr->queryChild(1);
+                expr = expr->queryChild(0);
+            }
+            builder.setown(createLinkedDatasetBuilder(record, choosenLimit));
         }
-
-        builder.setown(createLinkedDatasetBuilder(record, choosenLimit));
     }
     else
         builder.setown(createBlockedDatasetBuilder(record));
@@ -2756,6 +2766,7 @@ void HqlCppTranslator::buildDatasetAssign(BuildCtx & ctx, IHqlCppDatasetBuilder
         buildDatasetAssignTempTable(subctx, target, expr);
         //MORE: Create rows and assign each one in turn.  Could possibly be done with a different dataset selector
         return;
+    case no_inlinedictionary:
     case no_inlinetable:
         buildDatasetAssignInlineTable(subctx, target, expr);
         return;
@@ -4180,6 +4191,7 @@ IReferenceSelector * HqlCppTranslator::buildDatasetIndex(BuildCtx & ctx, IHqlExp
             specialCase = !isMultiLevelDatasetSelector(expr, false);
             break;
         case no_if:
+        case no_inlinedictionary:
         case no_inlinetable:
         case no_join:
             //Always creates a temporary, so don't use an iterator
@@ -4223,6 +4235,42 @@ IReferenceSelector * HqlCppTranslator::buildDatasetIndex(BuildCtx & ctx, IHqlExp
     return createReferenceSelector(row);
 }
 
+IReferenceSelector * HqlCppTranslator::buildDatasetSelectMap(BuildCtx & ctx, IHqlExpression * expr)
+{
+    HqlExprAssociation * match = ctx.queryAssociation(expr, AssocRow, NULL);
+    if (match)
+        return createReferenceSelector(static_cast<BoundRow *>(match));
+
+    OwnedHqlExpr dataset = normalizeAnyDatasetAliases(expr->queryChild(0));
+    // Create a row to pass to the lookup function
+
+
+    BoundRow * row = NULL;
+    if (!canProcessInline(&ctx, expr))
+    {
+        CHqlBoundExpr bound;
+        OwnedHqlExpr dsExpr = expr->isDatarow() ? createDatasetFromRow(LINK(expr)) : LINK(expr);
+        buildDataset(ctx, dsExpr, bound, FormatNatural);
+        convertBoundDatasetToFirstRow(expr, bound);
+        row = bindRow(ctx, expr, bound.expr);
+    }
+
+    if (!row)
+    {
+        Owned<IHqlCppDatasetCursor> cursor = createDatasetSelector(ctx, dataset);
+        row = cursor->buildSelect(ctx, expr);
+
+        if (!row)
+        {
+            CHqlBoundExpr boundCleared;
+            buildDefaultRow(ctx, dataset, boundCleared);
+            OwnedHqlExpr defaultRowPtr = getPointer(boundCleared.expr);
+            row = bindRow(ctx, expr, defaultRowPtr);
+        }
+    }
+    return createReferenceSelector(row);
+}
+
 //---------------------------------------------------------------------------
 
 IHqlExpression * HqlCppTranslator::buildGetLocalResult(BuildCtx & ctx, IHqlExpression * expr, bool preferLinkedRows)
@@ -4327,6 +4375,7 @@ void HqlCppTranslator::doBuildExprGetGraphResult(BuildCtx & ctx, IHqlExpression
     {
     case type_row:
         throwUnexpected();
+    case type_dictionary:
     case type_table:
     case type_groupedtable:
         buildTempExpr(ctx, call, tgt);

ecl/hqlcpp/hqlcset.cpp

@@ -691,6 +691,33 @@ BoundRow * InlineLinkedDatasetCursor::buildSelect(BuildCtx & ctx, IHqlExpression
 }
 
 //---------------------------------------------------------------------------
+
+InlineLinkedDictionaryCursor::InlineLinkedDictionaryCursor(HqlCppTranslator & _translator, IHqlExpression * _ds, CHqlBoundExpr & _boundDs)
+  : InlineLinkedDatasetCursor(_translator, _ds, _boundDs)
+{
+}
+
+BoundRow * InlineLinkedDictionaryCursor::buildSelect(BuildCtx & ctx, IHqlExpression * indexExpr)
+{
+    OwnedHqlExpr index = foldHqlExpression(indexExpr->queryChild(1));
+
+    StringBuffer s, rowName;
+    OwnedHqlExpr row = createRow(ctx, "row", rowName, false); // MORE - what should this be?
+
+    CHqlBoundExpr boundIndex;
+    BuildCtx subctx(ctx);
+    translator.buildExpr(ctx, index, boundIndex);
+
+    OwnedHqlExpr address = getPointer(boundDs.expr);
+    OwnedHqlExpr indexedValue = createValue(no_selectmap, row->getType(), LINK(address), LINK(boundIndex.expr));
+    subctx.addAssign(row, indexedValue);
+
+    //MORE: Should mark as linked if it is.
+    BoundRow * cursor = translator.bindRow(ctx, indexExpr, row);
+    return cursor;
+}
+
+//---------------------------------------------------------------------------
 MultiLevelDatasetCursor::MultiLevelDatasetCursor(HqlCppTranslator & _translator, IHqlExpression * _ds)
 : BaseDatasetCursor(_translator, _ds, NULL)
 {
@@ -1299,7 +1326,10 @@ IHqlCppDatasetCursor * HqlCppTranslator::createDatasetSelector(BuildCtx & ctx, I
     buildDataset(ctx, expr, bound, FormatNatural);
     if (bound.expr->isDatarow() || !isArrayRowset(bound.expr->queryType()))
         return new InlineBlockDatasetCursor(*this, expr, bound);
-    return new InlineLinkedDatasetCursor(*this, expr, bound);
+    else if (bound.expr->isDictionary())
+        return new InlineLinkedDictionaryCursor(*this, expr, bound);
+    else
+        return new InlineLinkedDatasetCursor(*this, expr, bound);
 }
 
 
@@ -1709,6 +1739,162 @@ bool LinkedDatasetBuilder::buildAppendRows(BuildCtx & ctx, IHqlExpression * expr
 
 //---------------------------------------------------------------------------
 
+
+LinkedDictionaryBuilder::LinkedDictionaryBuilder(HqlCppTranslator & _translator, IHqlExpression * _record) : DatasetBuilderBase(_translator, _record, true)
+{
+}
+
+void LinkedDictionaryBuilder::buildDeclare(BuildCtx & ctx)
+{
+    StringBuffer decl, allocatorName;
+
+    OwnedHqlExpr curActivityId = translator.getCurrentActivityId(ctx);
+    translator.ensureRowAllocator(allocatorName, ctx, record, curActivityId);
+
+    StringBuffer lookupHelperName;
+    OwnedHqlExpr dict = createDictionary(no_null, record.getLink()); // MORE - is the actual dict not available?
+    translator.buildDictionaryHashClass(ctx, record, dict, lookupHelperName);
+
+    decl.append("RtlLinkedDictionaryBuilder ").append(instanceName).append("(");
+    decl.append(allocatorName).append(", &").append(lookupHelperName);
+    decl.append(");");
+
+    ctx.addQuoted(decl);
+}
+
+void LinkedDictionaryBuilder::finishRow(BuildCtx & ctx, BoundRow * selfCursor)
+{
+    OwnedHqlExpr size = translator.getRecordSize(selfCursor->querySelector());
+    CHqlBoundExpr boundSize;
+    translator.buildExpr(ctx, size, boundSize);
+
+    StringBuffer s;
+    s.append(instanceName).append(".finalizeRow(");
+    translator.generateExprCpp(s, boundSize.expr).append(");");
+    ctx.addQuoted(s);
+}
+
+void LinkedDictionaryBuilder::buildFinish(BuildCtx & ctx, const CHqlBoundTarget & target)
+{
+    //more: should I do this by really calling a function?
+    StringBuffer s;
+
+    s.append(instanceName).append(".getcount()");
+
+    if (hasWrapperModifier(target.queryType()))
+    {
+        translator.generateExprCpp(s.clear(), target.expr);
+        s.append(".setown(").append(instanceName).append(".getcount()");
+        s.append(",").append(instanceName).append(".linkrows());");
+        ctx.addQuoted(s);
+    }
+    else
+    {
+        OwnedHqlExpr countExpr = createQuoted(s.str(), LINK(unsignedType));
+        ctx.addAssign(target.count, countExpr);
+        s.clear().append(instanceName).append(".linkrows()");
+        OwnedHqlExpr rowsExpr = createQuoted(s.str(), dataset->getType());
+        ctx.addAssign(target.expr, rowsExpr);
+    }
+}
+
+
+void LinkedDictionaryBuilder::buildFinish(BuildCtx & ctx, CHqlBoundExpr & bound)
+{
+    StringBuffer s;
+    s.clear().append(instanceName).append(".getcount()");
+    bound.count.setown(createQuoted(s.str(), LINK(unsignedType)));
+    s.clear().append(instanceName).append(".queryrows()");
+    bound.expr.setown(createQuoted(s.str(), makeReferenceModifier(dataset->getType())));
+}
+
+bool LinkedDictionaryBuilder::buildLinkRow(BuildCtx & ctx, BoundRow * sourceRow)
+{
+    IHqlExpression * sourceRecord = sourceRow->queryRecord();
+    if (recordTypesMatch(sourceRecord, record) && sourceRow->isBinary())
+    {
+        OwnedHqlExpr source = getPointer(sourceRow->queryBound());
+        BuildCtx subctx(ctx);
+        if (sourceRow->isConditional())
+            subctx.addFilter(source);
+
+        if (sourceRow->isLinkCounted())
+        {
+            StringBuffer s;
+            s.append(instanceName).append(".append(");
+            translator.generateExprCpp(s, source);
+            s.append(");");
+            subctx.addQuoted(s);
+            return true;
+        }
+
+        IHqlExpression * sourceExpr = sourceRow->querySelector();
+        OwnedHqlExpr rowExpr = sourceExpr->isDataset() ? ensureActiveRow(sourceExpr) : LINK(sourceExpr);
+        OwnedHqlExpr size = createSizeof(rowExpr);
+        CHqlBoundExpr boundSize;
+        translator.buildExpr(ctx, size, boundSize);
+
+        StringBuffer s;
+        s.append(instanceName).append(".cloneRow(");
+        translator.generateExprCpp(s, boundSize.expr).append(",");
+        translator.generateExprCpp(s, source);
+        s.append(");");
+        subctx.addQuoted(s);
+        return true;
+    }
+    return false;
+}
+
+bool LinkedDictionaryBuilder::buildAppendRows(BuildCtx & ctx, IHqlExpression * expr)
+{
+    IHqlExpression * sourceRecord = expr->queryRecord();
+    if (recordTypesMatch(sourceRecord, record))
+    {
+        CHqlBoundExpr bound;
+        if (!ctx.getMatchExpr(expr, bound))
+        {
+            bool tryToOptimize = false;
+            switch (expr->getOperator())
+            {
+            case no_select:
+                if (isMultiLevelDatasetSelector(expr, false))
+                    break;
+                if (!hasLinkedRow(expr->queryType()))
+                    break;
+                tryToOptimize = true;
+                break;
+            default:
+                //Don't speculatively evaluate if the expression isn't pure
+                tryToOptimize = alwaysEvaluatesToBound(expr) && expr->isPure();
+                break;
+            }
+
+            if (tryToOptimize)
+                translator.buildDataset(ctx, expr, bound, FormatNatural);
+        }
+
+        if (bound.expr)
+        {
+            if (hasLinkedRow(bound.queryType()))
+            {
+                OwnedHqlExpr source = getPointer(bound.expr);
+                StringBuffer s;
+                s.append(instanceName).append(".appendRows(");
+                translator.generateExprCpp(s, bound.count);
+                s.append(",");
+                translator.generateExprCpp(s, source);
+                s.append(");");
+                ctx.addQuoted(s);
+                return true;
+            }
+        }
+    }
+    return false;
+}
+
+
+//---------------------------------------------------------------------------
+
 SetBuilder::SetBuilder(HqlCppTranslator & _translator, ITypeInfo * fieldType, IHqlExpression * _allVar) : translator(_translator)
 {
     HqlExprArray fields;
@@ -1795,6 +1981,11 @@ IHqlCppDatasetBuilder * HqlCppTranslator::createLinkedDatasetBuilder(IHqlExpress
     return new LinkedDatasetBuilder(*this, record, choosenLimit);
 }
 
+IHqlCppDatasetBuilder * HqlCppTranslator::createLinkedDictionaryBuilder(IHqlExpression * record)
+{
+    return new LinkedDictionaryBuilder(*this, record);
+}
+
 IHqlCppDatasetBuilder * HqlCppTranslator::createSingleRowTempDatasetBuilder(IHqlExpression * record, BoundRow * row)
 {
 //  if (translator.isFixedRecordSize(record))

ecl/hqlcpp/hqlcset.ipp

@@ -76,6 +76,16 @@ public:
     virtual void buildIterateClass(BuildCtx & ctx, StringBuffer & cursorName, BuildCtx * initctx);
 };
 
+class InlineLinkedDictionaryCursor : public InlineLinkedDatasetCursor
+{
+public:
+    InlineLinkedDictionaryCursor(HqlCppTranslator & _translator, IHqlExpression * _ds, CHqlBoundExpr & _boundDs);
+
+    virtual BoundRow * buildIterateLoop(BuildCtx & ctx, bool needToBreak) { throwUnexpected(); }
+    virtual BoundRow * buildSelect(BuildCtx & ctx, IHqlExpression * indexExpr);
+    virtual void buildIterateClass(BuildCtx & ctx, StringBuffer & cursorName, BuildCtx * initctx) { throwUnexpected(); }
+};
+
 class MultiLevelDatasetCursor : public BaseDatasetCursor
 {
 public:
@@ -275,6 +285,19 @@ protected:
     LinkedHqlExpr choosenLimit;
 };
 
+class LinkedDictionaryBuilder : public DatasetBuilderBase
+{
+public:
+    LinkedDictionaryBuilder(HqlCppTranslator & _translator, IHqlExpression * _record);
+
+    virtual void buildDeclare(BuildCtx & ctx);
+    virtual void buildFinish(BuildCtx & ctx, const CHqlBoundTarget & target);
+    virtual void buildFinish(BuildCtx & ctx, CHqlBoundExpr & bound);
+    virtual bool buildLinkRow(BuildCtx & ctx, BoundRow * sourceRow);
+    virtual bool buildAppendRows(BuildCtx & ctx, IHqlExpression * expr);
+    virtual void finishRow(BuildCtx & ctx, BoundRow * selfCursor);
+};
+
 //---------------------------------------------------------------------------
 
 class SetBuilder : public CInterface, implements IHqlCppSetBuilder

ecl/hqlcpp/hqlhtcpp.cpp

@@ -2946,6 +2946,7 @@ static bool anyXmlGeneratedForPass(IHqlExpression * expr, unsigned pass)
                 return anyXmlGeneratedForPass(queryRecord(type), pass);
             case type_set:
                 return (pass==1);
+            case type_dictionary:
             case type_table:
             case type_groupedtable:
                 return (pass == 1);
@@ -3052,6 +3053,7 @@ void HqlCppTranslator::doBuildFunctionReturn(BuildCtx & ctx, ITypeInfo * type, I
     case type_data:
     case type_unicode:
     case type_utf8:
+    case type_dictionary:
     case type_table:
     case type_groupedtable:
     case type_row:
@@ -3624,6 +3626,7 @@ unsigned HqlCppTranslator::buildRtlField(StringBuffer * instanceName, IHqlExpres
         case type_set:
             extractXmlName(xpathName, &xpathItem, NULL, field, "Item", false);
             break;
+        case type_dictionary:
         case type_table:
         case type_groupedtable:
             extractXmlName(xpathName, &xpathItem, NULL, field, "Row", false);
@@ -3918,6 +3921,7 @@ unsigned HqlCppTranslator::buildRtlType(StringBuffer & instanceName, ITypeInfo *
                 fieldType |= RFTMlinkcounted;
             break;
         }
+    case type_dictionary:
     case type_table:
     case type_groupedtable:
         {
@@ -4224,6 +4228,7 @@ public:
                             }
                             break;
                         }
+                    case type_dictionary:
                     case type_table:
                     case type_groupedtable:
                         {
@@ -4542,6 +4547,7 @@ IHqlExpression * HqlCppTranslator::convertBetweenCountAndSize(const CHqlBoundExp
     OwnedHqlExpr record;
     switch (type->getTypeCode())
     {
+    case type_dictionary:
     case type_table:
     case type_groupedtable:
         record.set(bound.expr->queryRecord());
@@ -4678,6 +4684,7 @@ void HqlCppTranslator::buildGetResultInfo(BuildCtx & ctx, IHqlExpression * expr,
     case type_swapint:  func = getResultIntAtom; break;
     case type_boolean:  func = getResultBoolAtom; break;
     case type_data:     func = getResultDataAtom; break;
+    case type_dictionary:
     case type_table:
     case type_groupedtable:
     case type_set:
@@ -4685,7 +4692,7 @@ void HqlCppTranslator::buildGetResultInfo(BuildCtx & ctx, IHqlExpression * expr,
         {
             OwnedHqlExpr record;
             bool ensureSerialized = true;
-            if (ttc == type_table || ttc == type_groupedtable)
+            if (ttc != type_set)
             {
                 overrideType.set(type);
                 record.set(::queryRecord(type));
@@ -4964,6 +4971,7 @@ void HqlCppTranslator::buildSetResultInfo(BuildCtx & ctx, IHqlExpression * origi
             schemaType.setown(makeSetType(elementType));
         }
         break;
+    case type_dictionary:
     case type_table:
     case type_groupedtable:
         {
@@ -5289,6 +5297,38 @@ void HqlCppTranslator::buildHashOfExprsClass(BuildCtx & ctx, const char * name,
     buildHashClass(ctx, name, hash, dataset);
 }
 
+
+void HqlCppTranslator::buildDictionaryHashClass(BuildCtx &ctx, IHqlExpression *record, IHqlExpression *dictionary, StringBuffer &lookupHelperName)
+{
+    BuildCtx declarectx(*code, declareAtom);
+    appendUniqueId(lookupHelperName.append("lu"), getConsistentUID(record));
+    OwnedHqlExpr attr = createAttribute(lookupAtom, LINK(record));
+    HqlExprAssociation * match = declarectx.queryMatchExpr(attr);
+    if (!match)
+    {
+        BuildCtx classctx(declarectx);
+        beginNestedClass(classctx, lookupHelperName, "IHThorHashLookupInfo");
+        HqlExprArray keyedFields;
+        IHqlExpression * payload = record ? record->queryProperty(_payload_Atom) : NULL;
+        unsigned payloadSize = payload ? getIntValue(payload->queryChild(0)) : 0;
+        unsigned max = record->numChildren() - payloadSize;
+        for (unsigned idx = 0; idx < max; idx++)
+        {
+            IHqlExpression *child = record->queryChild(idx);
+            if (!child->isAttribute())
+                keyedFields.append(*createSelectExpr(LINK(dictionary->queryNormalizedSelector()), LINK(child)));
+        }
+        OwnedHqlExpr keyedList = createValueSafe(no_sortlist, makeSortListType(NULL), keyedFields);
+        DatasetReference dictRef(dictionary, no_none, NULL);
+        buildHashOfExprsClass(classctx, "Hash", keyedList, dictRef, false);
+        buildCompareMember(classctx, "Compare", keyedList, dictRef);
+        endNestedClass();
+        OwnedHqlExpr temp = createVariable(lookupHelperName, makeVoidType());
+        declarectx.associateExpr(attr, temp);
+    }
+}
+
+
 //---------------------------------------------------------------------------
 
 IHqlExpression * queryImplementationInterface(IHqlExpression * moduleFunc)
@@ -6270,6 +6310,9 @@ ABoundActivity * HqlCppTranslator::buildActivity(BuildCtx & ctx, IHqlExpression
             case no_selectnth:
                 result = doBuildActivitySelectNth(ctx, expr);
                 break;
+            case no_selectmap:
+                UNIMPLEMENTED;
+                break;
             case no_join:
             case no_selfjoin:
                 result = doBuildActivityJoin(ctx, expr);
@@ -6997,6 +7040,7 @@ void HqlCppTranslator::ensureSerialized(const CHqlBoundTarget & variable, BuildC
             serializeName = serializeUtf8XAtom;
             deserializeName = deserializeUtf8XAtom;
             break;
+        case type_dictionary:
         case type_table:
         case type_groupedtable:
             {
@@ -7006,29 +7050,29 @@ void HqlCppTranslator::ensureSerialized(const CHqlBoundTarget & variable, BuildC
                     deserializeArgs.append(*createRowSerializer(deserializectx, record, deserializerAtom));
 
                     serializeArgs.append(*createRowSerializer(serializectx, record, serializerAtom));
-                    if (tc == type_table)
+                    if (tc == type_groupedtable)
                     {
-                        serializeName = serializeRowsetXAtom;
-                        deserializeName = deserializeRowsetXAtom;
+                        serializeName = serializeGroupedRowsetXAtom;
+                        deserializeName = deserializeGroupedRowsetXAtom;
                     }
                     else
                     {
-                        serializeName = serializeGroupedRowsetXAtom;
-                        deserializeName = deserializeGroupedRowsetXAtom;
+                        serializeName = serializeRowsetXAtom;
+                        deserializeName = deserializeRowsetXAtom;
                     }
                 }
                 else
                 {
                     assertex(!recordRequiresSerialization(record));
-                    if (tc == type_table)
+                    if (tc == type_groupedtable)
                     {
-                        serializeName = serializeDatasetXAtom;
-                        deserializeName = deserializeDatasetXAtom;
+                        serializeName = serializeGroupedDatasetXAtom;
+                        deserializeName = deserializeGroupedDatasetXAtom;
                     }
                     else
                     {
-                        serializeName = serializeGroupedDatasetXAtom;
-                        deserializeName = deserializeGroupedDatasetXAtom;
+                        serializeName = serializeDatasetXAtom;
+                        deserializeName = deserializeDatasetXAtom;
                     }
                 }
                 serializedType.set(type);
@@ -7269,6 +7313,7 @@ void HqlCppTranslator::doBuildStmtSetResult(BuildCtx & ctx, IHqlExpression * exp
             buildSetResultInfo(subctx, expr, normalized, setType, (persist != NULL), true);
             break;
         }
+    case type_dictionary:
     case type_table:
     case type_groupedtable:
         switch (value->getOperator())
@@ -7435,6 +7480,7 @@ void HqlCppTranslator::doBuildExprSizeof(BuildCtx & ctx, IHqlExpression * expr,
             unsigned size = UNKNOWN_LENGTH;
             switch (type->getTypeCode())
             {
+            case type_dictionary:
             case type_table:
             case type_groupedtable:
             case type_record:
@@ -7472,6 +7518,7 @@ void HqlCppTranslator::doBuildExprSizeof(BuildCtx & ctx, IHqlExpression * expr,
             unsigned size = UNKNOWN_LENGTH;
             switch (type->getTypeCode())
             {
+            case type_dictionary:
             case type_table:
             case type_groupedtable:
             case type_record:
@@ -7550,6 +7597,7 @@ void HqlCppTranslator::doBuildExprSizeof(BuildCtx & ctx, IHqlExpression * expr,
 
             switch (type->getTypeCode())
             {
+            case type_dictionary:
             case type_table:
             case type_groupedtable:
             case type_record:
@@ -7610,6 +7658,7 @@ void HqlCppTranslator::doBuildExprRowDiff(BuildCtx & ctx, const CHqlBoundTarget
                     return;
                 }
                 break;
+            case type_dictionary:
             case type_table:
             case type_groupedtable:
                 UNIMPLEMENTED;
@@ -10157,6 +10206,7 @@ void HqlCppTranslator::addSchemaField(IHqlExpression *field, MemoryBuffer &schem
     case type_bitfield:
         schemaType.set(schemaType->queryPromotedType());
         //fall through;
+    case type_dictionary:
     case type_table:
     case type_groupedtable:
     case type_row:
@@ -10528,6 +10578,7 @@ void HqlCppTranslator::buildXmlSerialize(BuildCtx & subctx, IHqlExpression * exp
                 case type_set:
                     buildXmlSerializeSet(subctx, expr, value);
                     break;
+                case type_dictionary:
                 case type_table:
                 case type_groupedtable:
                     buildXmlSerializeDataset(subctx, expr, value, assigns);

ecl/hqlcpp/hqlinline.cpp

@@ -151,6 +151,7 @@ static unsigned calcInlineFlags(BuildCtx * ctx, IHqlExpression * expr)
                 return RETiterate;
             return RETiterate|HEFspillinline;
         }
+    case no_selectmap:
     case no_selectnth:
         {
             IHqlExpression * ds = expr->queryChild(0);

ecl/hqlcpp/hqliproj.cpp

@@ -1048,7 +1048,7 @@ IHqlExpression * ComplexImplicitProjectInfo::createOutputProject(IHqlExpression
     OwnedHqlExpr left = createSelector(no_left, ds, seq);
     OwnedHqlExpr self = getSelf(queryOutputRecord());
     IHqlExpression * transform = createMappingTransform(self, left);
-    if (ds->isDataset())
+    if (ds->isDataset() || ds->isDictionary())
         return createDataset(no_hqlproject, LINK(ds), createComma(transform, LINK(seq)));
     return createRow(no_projectrow, LINK(ds), createComma(transform, LINK(seq)));
 }
@@ -1315,7 +1315,7 @@ void ImplicitProjectTransformer::analyseExpr(IHqlExpression * expr)
         case no_evaluate:
             throwUnexpected();
         case no_select:
-            if (expr->isDataset() || expr->isDatarow())
+            if (expr->isDataset() || expr->isDatarow() || expr->isDictionary())
             {
                 //MORE: These means that selects from a parent dataset don't project down the parent dataset.
                 //I'm not sure how big an issue that would be.
@@ -1354,7 +1354,7 @@ void ImplicitProjectTransformer::analyseExpr(IHqlExpression * expr)
             allowActivity = true;
             return;
         case no_thor:
-            if (expr->isDataset() || expr->isDatarow())
+            if (expr->isDataset() || expr->isDatarow() || expr->isDictionary())
             {
                 assertex(extra->activityKind() == SimpleActivity);
                 Parent::analyseExpr(expr);
@@ -1367,7 +1367,7 @@ void ImplicitProjectTransformer::analyseExpr(IHqlExpression * expr)
             }
             break;
         case no_compound:
-            if (expr->isDataset())
+            if (expr->isDataset() || expr->isDictionary())
             {
                 assertex(extra->activityKind() == SimpleActivity);
                 Parent::analyseExpr(expr);
@@ -1378,7 +1378,7 @@ void ImplicitProjectTransformer::analyseExpr(IHqlExpression * expr)
             Parent::analyseExpr(expr);
             break;
         case no_executewhen:
-            if (expr->isDataset())
+            if (expr->isDataset() || expr->isDictionary())
             {
                 assertex(extra->activityKind() == SimpleActivity);
                 Parent::analyseExpr(expr);
@@ -1402,7 +1402,7 @@ void ImplicitProjectTransformer::analyseExpr(IHqlExpression * expr)
                 ForEachChild(i, expr)
                 {
                     IHqlExpression * cur = expr->queryChild(i);
-                    if (cur->isDataset())
+                    if (cur->isDataset() || expr->isDictionary())
                     {
                         analyseExpr(cur);
                         queryBodyExtra(cur)->preventOptimization();
@@ -1425,7 +1425,7 @@ void ImplicitProjectTransformer::analyseExpr(IHqlExpression * expr)
         case no_ensureresult:
             {
                 IHqlExpression * value = expr->queryChild(0);
-                if (value->isDataset() || value->isDatarow())// || value->isList())
+                if (value->isDataset() || value->isDatarow() || expr->isDictionary())// || value->isList())
                 {
                     assertex(extra->activityKind() == FixedInputActivity);
                     analyseExpr(value);
@@ -1452,7 +1452,7 @@ void ImplicitProjectTransformer::analyseExpr(IHqlExpression * expr)
                 unsigned first = 0;
                 unsigned last = numArgs;
                 unsigned start = 0;
-                if (!expr->isAction() && !expr->isDataset() && !expr->isDatarow())
+                if (!expr->isAction() && !expr->isDataset() && !expr->isDatarow() && !expr->isDictionary())
                 {
                     switch (op)
                     {
@@ -1596,7 +1596,7 @@ void ImplicitProjectTransformer::analyseExpr(IHqlExpression * expr)
             setOriginal(complexExtra->rightFieldsRequired, expr->queryChild(1));
             break;
         case FixedInputActivity:
-            assertex(child && (child->isDataset() || child->isDatarow()));
+            assertex(child && (child->isDataset() || child->isDatarow() || child->isDictionary()));
             setOriginal(complexExtra->leftFieldsRequired, child);
             if (getNumChildTables(expr) >= 2)
                 setOriginal(complexExtra->rightFieldsRequired, expr->queryChild(1));
@@ -1672,7 +1672,7 @@ void ImplicitProjectTransformer::gatherFieldsUsed(IHqlExpression * expr, Implici
                         break;
                     }
                     node_operator dsOp = ds->getOperator();
-                    if (dsOp != no_select || ds->isDataset())
+                    if (dsOp != no_select || ds->isDataset() || expr->isDictionary())
                     {
                         if ((dsOp != no_self) && (dsOp != no_selfref))
                             extra->addActiveSelect(cur);
@@ -1913,7 +1913,7 @@ ProjectExprKind ImplicitProjectTransformer::getProjectExprKind(IHqlExpression *
     case no_evaluate:
         throwUnexpected();
     case no_select:
-        if (expr->isDataset() || expr->isDatarow())
+        if (expr->isDataset() || expr->isDatarow() || expr->isDictionary())
             return SourceActivity;
         if (isNewSelector(expr))
             return ScalarSelectActivity;
@@ -1925,21 +1925,21 @@ ProjectExprKind ImplicitProjectTransformer::getProjectExprKind(IHqlExpression *
     case no_attr_link:
         return NonActivity;
     case no_typetransfer:
-        if (expr->isDataset() || expr->isDatarow())
+        if (expr->isDataset() || expr->isDatarow() || expr->isDictionary())
             return SourceActivity;
         return NonActivity;
     case no_thor:
-        if (expr->isDataset() || expr->isDatarow())
+        if (expr->isDataset() || expr->isDatarow() || expr->isDictionary())
             return SimpleActivity;
         return NonActivity;
     case no_compound:
-        if (expr->isDataset())
+        if (expr->isDataset() || expr->isDictionary())
             return SimpleActivity;
         if (expr->isDatarow())
             return ComplexNonActivity;
         return NonActivity;
     case no_executewhen:
-        if (expr->isDataset() || expr->isDatarow())
+        if (expr->isDataset() || expr->isDatarow() || expr->isDictionary())
             return SimpleActivity;
         return NonActivity;
     case no_subgraph:
@@ -1954,7 +1954,7 @@ ProjectExprKind ImplicitProjectTransformer::getProjectExprKind(IHqlExpression *
     case no_ensureresult:
         {
             IHqlExpression * value = expr->queryChild(0);
-            if (value->isDataset() || value->isDatarow())
+            if (value->isDataset() || value->isDatarow() || expr->isDictionary())
                 return FixedInputActivity;
             return NonActivity;
         }
@@ -1971,6 +1971,7 @@ ProjectExprKind ImplicitProjectTransformer::getProjectExprKind(IHqlExpression *
         return CreateRecordActivity;
     case no_inlinetable:
     case no_dataset_from_transform:
+    case no_inlinedictionary:
         return CreateRecordSourceActivity;
     case no_extractresult:
     case no_apply:
@@ -1993,7 +1994,7 @@ ProjectExprKind ImplicitProjectTransformer::getProjectExprKind(IHqlExpression *
         return AnyTypeActivity;
     case no_skip:
     case no_fail:
-        if (expr->isDataset() || expr->isDatarow())
+        if (expr->isDataset() || expr->isDatarow() || expr->isDictionary())
             return AnyTypeActivity;
         return NonActivity;
     case no_table:
@@ -2055,13 +2056,13 @@ ProjectExprKind ImplicitProjectTransformer::getProjectExprKind(IHqlExpression *
     case no_newsoapcall:
     case no_libraryinput:
     case no_thisnode:
-        if (expr->isDataset() || expr->isDatarow())
+        if (expr->isDataset() || expr->isDatarow() || expr->isDictionary())
             return SourceActivity;
         return NonActivity;
     case no_pipe:
     case no_nofold:
     case no_nohoist:
-        if (expr->isDataset() || expr->isDatarow())
+        if (expr->isDataset() || expr->isDatarow() || expr->isDictionary())
             return FixedInputActivity;
         return NonActivity;
     case no_soapcall_ds:
@@ -2121,7 +2122,7 @@ ProjectExprKind ImplicitProjectTransformer::getProjectExprKind(IHqlExpression *
         return SinkActivity;
     case no_call:
     case no_externalcall:
-        if (expr->isDataset() || expr->isDatarow())
+        if (expr->isDataset() || expr->isDatarow() || expr->isDictionary())
             return SourceActivity;
         //MORE: What about parameters??
         return NonActivity;
@@ -2149,6 +2150,7 @@ ProjectExprKind ImplicitProjectTransformer::getProjectExprKind(IHqlExpression *
             return SinkActivity;
         return NonActivity;
     case type_row:
+    case type_dictionary:
     case type_table:
     case type_groupedtable:
         break;
@@ -2769,7 +2771,7 @@ IHqlExpression * ImplicitProjectTransformer::createTransformed(IHqlExpression *
         }
     case CreateRecordSourceActivity:
         {
-            assertex(expr->getOperator() == no_inlinetable || expr->getOperator() == no_dataset_from_transform);
+            assertex(expr->getOperator() == no_inlinetable || expr->getOperator() == no_dataset_from_transform || expr->getOperator() == no_inlinedictionary);
             //Always reduce things that create a new record so they only project the fields they need to
             if (complexExtra->outputChanged())
             {
@@ -2864,7 +2866,7 @@ IHqlExpression * ImplicitProjectTransformer::createTransformed(IHqlExpression *
             {
                 IHqlExpression * cur = expr->queryChild(i);
                 OwnedHqlExpr next = transform(cur);
-                if (cur->isDataset() || cur->isDatarow())
+                if (cur->isDataset() || cur->isDatarow() || expr->isDictionary())
                 {
                     //Ensure all inputs have same format..
                     if (cur->queryRecord() != complexExtra->queryOutputRecord())

ecl/hqlcpp/hqltcppc.cpp

@@ -2999,6 +2999,7 @@ CMemberInfo * ColumnToOffsetMap::createColumn(CContainerInfo * container, IHqlEx
         }
     case type_groupedtable:
     case type_table:
+    case type_dictionary:
         {
             IHqlExpression * count = NULL;
             IHqlExpression * size = NULL;
@@ -3195,6 +3196,7 @@ static bool cachedCanReadFromCsv(IHqlExpression * record)
                     if (!cachedCanReadFromCsv(cur->queryRecord()))
                         return false;
                     break;
+                case type_dictionary:
                 case type_table:
                 case type_groupedtable:
                 case type_set:

ecl/hqlcpp/hqltcppc2.cpp

@@ -763,4 +763,3 @@ AColumnInfo * CChildLinkedDatasetColumnInfo::lookupColumn(IHqlExpression * searc
     return NULL;
 }
 
-

ecl/hqlcpp/hqlttcpp.cpp

@@ -8486,6 +8486,7 @@ IHqlExpression * HqlLinkedChildRowTransformer::createTransformedBody(IHqlExpress
             ITypeInfo * type = expr->queryType();
             switch (type->getTypeCode())
             {
+            case type_dictionary:
             case type_table:
             case type_groupedtable:
                 if (expr->hasProperty(embeddedAtom))

ecl/hqlcpp/hqlwcpp.cpp

@@ -74,6 +74,7 @@ bool isTypePassedByAddress(ITypeInfo * type)
     case type_set:
     case type_row:
         return true;
+    case type_dictionary:
     case type_table:
     case type_groupedtable:
         return !isArrayRowset(type);
@@ -485,6 +486,7 @@ void HqlCppWriter::generateType(ITypeInfo * type, const char * name)
             //next = next->queryChildType();
             isPointer = false;
             break;
+        case type_dictionary:// MORE - is this right?
         case type_table:
             if (isArrayRowset(fullType))
             {
@@ -802,6 +804,7 @@ void HqlCppWriter::generateParamCpp(IHqlExpression * param)
     case type_data:
     case type_unicode:
     case type_utf8:
+    case type_dictionary:
     case type_table:
     case type_groupedtable:
         if (paramType->getSize() == UNKNOWN_LENGTH)
@@ -843,6 +846,7 @@ void HqlCppWriter::generateParamCpp(IHqlExpression * param)
     case type_record:
         out.append("IOutputMetaData * ");
         break;
+    case type_dictionary:
     case type_table:
     case type_groupedtable:
         if (isConst)
@@ -959,6 +963,7 @@ void HqlCppWriter::generateFunctionReturnType(StringBuffer & params, ITypeInfo *
         out.append("size32_t");
         params.append("ARowBuilder & __self");
         break;
+    case type_dictionary:
     case type_table:
     case type_groupedtable:
         if (hasStreamedModifier(retType))
@@ -1156,6 +1161,7 @@ StringBuffer & HqlCppWriter::generateExprCpp(IHqlExpression * expr)
                         out.append("getbytes()");
                         break;
                     case type_set:
+                    case type_dictionary:
                     case type_table:
                     case type_groupedtable:
                         if (hasLinkCountedModifier(type))
@@ -1198,6 +1204,12 @@ StringBuffer & HqlCppWriter::generateExprCpp(IHqlExpression * expr)
             generateChildExpr(expr, 1);
             out.append(']');
             break;
+        case no_selectmap:
+            generateChildExpr(expr, 0);
+            out.append("[<");
+            generateChildExpr(expr, 1);
+            out.append(">]");
+            break;
         case no_postinc:
         case no_postdec:
             generateChildExpr(expr, 0);
@@ -1226,6 +1238,7 @@ StringBuffer & HqlCppWriter::generateExprCpp(IHqlExpression * expr)
                         out.append("getbytes()");
                         break;
                     case type_set:
+                    case type_dictionary:
                     case type_table:
                     case type_groupedtable:
                         if (hasLinkCountedModifier(type))
@@ -1264,6 +1277,7 @@ StringBuffer & HqlCppWriter::generateExprCpp(IHqlExpression * expr)
                         out.append("getbytes()");       //????
                         break;
                     case type_set:
+                    case type_dictionary:
                     case type_table:
                     case type_groupedtable:
                         if (hasLinkCountedModifier(childType))
@@ -1709,6 +1723,7 @@ void HqlCppWriter::generateStmtAssign(IHqlStmt * assign)
             else
                 throwUnexpected();
             break;
+        case type_dictionary:
         case type_table:
         case type_groupedtable:
             if (hasWrapperModifier(type))
@@ -1778,6 +1793,7 @@ void HqlCppWriter::generateStmtAssignModify(IHqlStmt * assign)
     switch (type->getTypeCode())
     {
         case type_row:
+        case type_dictionary:
         case type_table:
         case type_groupedtable:
             //check it is a pointer increment

rtl/eclrtl/rtlds.cpp

@@ -332,7 +332,6 @@ RtlLinkedDatasetBuilder::~RtlLinkedDatasetBuilder()
 
 void RtlLinkedDatasetBuilder::append(const void * source)
 {
-    flush();
     if (count < choosenLimit)
     {
         ensure(count+1);
@@ -343,7 +342,6 @@ void RtlLinkedDatasetBuilder::append(const void * source)
 
 void RtlLinkedDatasetBuilder::appendRows(size32_t num, byte * * rows)
 {
-    flush();
     if (num && (count < choosenLimit))
     {
         unsigned numToAdd = (count + num < choosenLimit) ? num : choosenLimit - count;
@@ -356,7 +354,6 @@ void RtlLinkedDatasetBuilder::appendRows(size32_t num, byte * * rows)
 
 void RtlLinkedDatasetBuilder::appendOwn(const void * row)
 {
-    //flush() must have been called before this... otherwise the order will be messed up
     assertex(!builder.exists());
     if (count < choosenLimit)
     {
@@ -371,7 +368,6 @@ void RtlLinkedDatasetBuilder::appendOwn(const void * row)
 
 byte * RtlLinkedDatasetBuilder::createRow()
 {
-    flush();
     if (count >= choosenLimit)
         return NULL;
     return builder.getSelf();
@@ -395,7 +391,6 @@ public:
 
 void RtlLinkedDatasetBuilder::cloneRow(size32_t len, const void * row)
 {
-    flush();
     if (count >= choosenLimit)
         return;
 
@@ -415,7 +410,6 @@ void RtlLinkedDatasetBuilder::cloneRow(size32_t len, const void * row)
 
 void RtlLinkedDatasetBuilder::deserializeRow(IOutputRowDeserializer & deserializer, IRowDeserializerSource & in)
 {
-    flush();
     builder.ensureRow();
     size32_t rowSize = deserializer.deserialize(builder, in);
     finalizeRow(rowSize);
@@ -439,11 +433,15 @@ inline void doSerializeRowset(IRowSerializerTarget & out, IOutputRowSerializer *
 {
     for (unsigned i=0; i < count; i++)
     {
-        serializer->serialize(out, rows[i]);
-        if (isGrouped)
+        byte *row = rows[i];
+        if (row)   // When serializing a dictionary, there may be nulls in the rowset. These can be skipped (we rehash on deserialize)
         {
-            byte eogPending = (i+1 < count) && (rows[i+1] == NULL);
-            out.put(1, &eogPending);
+            serializer->serialize(out, rows[i]);
+            if (isGrouped)
+            {
+                byte eogPending = (i+1 < count) && (rows[i+1] == NULL);
+                out.put(1, &eogPending);
+            }
         }
     }
 }
@@ -451,8 +449,6 @@ inline void doSerializeRowset(IRowSerializerTarget & out, IOutputRowSerializer *
 
 void RtlLinkedDatasetBuilder::deserialize(IOutputRowDeserializer & deserializer, IRowDeserializerSource & in, bool isGrouped)
 {
-    flush();
-
     offset_t marker = in.beginNested();
     doDeserializeRowset(*this, deserializer, in, marker, isGrouped);
 }
@@ -460,17 +456,11 @@ void RtlLinkedDatasetBuilder::deserialize(IOutputRowDeserializer & deserializer,
 
 void RtlLinkedDatasetBuilder::finalizeRows()
 {
-    flush();
     if (count != max)
         resize(count);
 }
 
 
-void RtlLinkedDatasetBuilder::flush()
-{
-//  builder.clear();
-}
-
 void RtlLinkedDatasetBuilder::finalizeRow(size32_t rowSize)
 {
     assertex(builder.exists());
@@ -558,6 +548,119 @@ void rtlSerializeGroupedRowset(IRowSerializerTarget & out, IOutputRowSerializer
 
 //---------------------------------------------------------------------------
 
+RtlLinkedDictionaryBuilder::RtlLinkedDictionaryBuilder(IEngineRowAllocator * _rowAllocator, IHThorHashLookupInfo *_hashInfo, unsigned _initialSize)
+  : builder(_rowAllocator, false)
+{
+    init(_rowAllocator, _hashInfo, _initialSize);
+}
+
+RtlLinkedDictionaryBuilder::RtlLinkedDictionaryBuilder(IEngineRowAllocator * _rowAllocator, IHThorHashLookupInfo *_hashInfo)
+  : builder(_rowAllocator, false)
+{
+    init(_rowAllocator, _hashInfo, 8);
+}
+
+void RtlLinkedDictionaryBuilder::init(IEngineRowAllocator * _rowAllocator, IHThorHashLookupInfo *_hashInfo, unsigned _initialSize)
+{
+    hash  = _hashInfo->queryHash();
+    compare  = _hashInfo->queryCompare();
+    initialSize = _initialSize;
+    rowAllocator = LINK(_rowAllocator);
+    table = NULL;
+    usedCount = 0;
+    usedLimit = 0;
+    tableSize = 0;
+}
+
+RtlLinkedDictionaryBuilder::~RtlLinkedDictionaryBuilder()
+{
+//    builder.clear();
+    if (table)
+        rowAllocator->releaseRowset(tableSize, table);
+    ::Release(rowAllocator);
+}
+
+void RtlLinkedDictionaryBuilder::append(const void * source)
+{
+    if (source)
+    {
+        appendOwn(rowAllocator->linkRow(source));
+    }
+}
+
+void RtlLinkedDictionaryBuilder::appendOwn(const void * source)
+{
+    if (source)
+    {
+        checkSpace();
+        unsigned rowidx = hash->hash(source) % tableSize;
+        loop
+        {
+            const void *entry = table[rowidx];
+            if (entry && compare->docompare(source, entry)==0)
+            {
+                rowAllocator->releaseRow(entry);
+                usedCount--;
+                entry = NULL;
+            }
+            if (!entry)
+            {
+                table[rowidx] = (byte *) source;
+                usedCount++;
+                break;
+            }
+            rowidx++;
+            if (rowidx==tableSize)
+                rowidx = 0;
+        }
+    }
+}
+
+void RtlLinkedDictionaryBuilder::checkSpace()
+{
+    if (!table)
+    {
+        table = rowAllocator->createRowset(initialSize);
+        tableSize = initialSize;
+        memset(table, 0, tableSize*sizeof(byte *));
+        usedLimit = (tableSize * 3) / 4;
+        usedCount = 0;
+    }
+    else if (usedCount > usedLimit)
+    {
+        // Rehash
+        byte * * oldTable = table;
+        unsigned oldSize = tableSize;
+        tableSize = tableSize*2;
+        table = rowAllocator->createRowset(tableSize);
+        memset(table, 0, tableSize * sizeof(byte *));
+        usedLimit = (tableSize * 3) / 4;
+        usedCount = 0;
+        unsigned i;
+        for (i = 0; i < oldSize; i++)
+        {
+            append(oldTable[i]);  // we link the rows here...
+        }
+        rowAllocator->releaseRowset(oldSize, oldTable);   // ... because this will release them
+    }
+}
+
+void RtlLinkedDictionaryBuilder::appendRows(size32_t num, byte * * rows)
+{
+    // MORE - if we know that the source is already a hashtable, we can optimize the add to an empty table...
+    for (unsigned i=0; i < num; i++)
+        append(rows[i]);
+}
+
+void RtlLinkedDictionaryBuilder::finalizeRow(size32_t rowSize)
+{
+    assertex(builder.exists());
+    const void * next = builder.finalizeRowClear(rowSize);
+    appendOwn(next);
+}
+
+//---------------------------------------------------------------------------
+
 //These definitions should be shared with thorcommon, but to do that
 //they would need to be moved to an rtlds.ipp header, which thorcommon then included.
 

rtl/eclrtl/rtlds_imp.hpp

@@ -363,7 +363,6 @@ public:
     void appendRows(size32_t num, byte * * rows);
     inline void appendEOG() { appendOwn(NULL); }
     byte * createRow();
-    void cancelRow();
     void cloneRow(size32_t len, const void * ptr);
     void deserialize(IOutputRowDeserializer & deserializer, IRowDeserializerSource & in, bool isGrouped);
     void deserializeRow(IOutputRowDeserializer & deserializer, IRowDeserializerSource & in);
@@ -382,9 +381,6 @@ public:
     inline byte * row() const { return builder.row(); }
 
 protected:
-    void flush();
-
-protected:
     IEngineRowAllocator * rowAllocator;
     RtlDynamicRowBuilder builder;
     byte * * rowset;
@@ -393,6 +389,53 @@ protected:
     size32_t choosenLimit;
 };
 
+class ECLRTL_API RtlLinkedDictionaryBuilder
+{
+public:
+    RtlLinkedDictionaryBuilder(IEngineRowAllocator * _rowAllocator, IHThorHashLookupInfo *_hashInfo, unsigned _initialTableSize);
+    RtlLinkedDictionaryBuilder(IEngineRowAllocator * _rowAllocator, IHThorHashLookupInfo *_hashInfo);
+    ~RtlLinkedDictionaryBuilder();
+
+    void append(const void * source);
+    void appendOwn(const void * source);
+    void appendRows(size32_t num, byte * * rows);
+
+    inline size32_t getcount() { return tableSize; }
+    inline byte * * linkrows() { return rtlLinkRowset(table); }
+
+    byte * createRow()         { return builder.getSelf(); }
+    void finalizeRow(size32_t len);
+    inline RtlDynamicRowBuilder & rowBuilder() { return builder; }
+    /*
+        Not clear which if any of these we will want...
+    void cloneRow(size32_t len, const void * ptr);
+    void deserialize(IOutputRowDeserializer & deserializer, IRowDeserializerSource & in, bool isGrouped);
+    void deserializeRow(IOutputRowDeserializer & deserializer, IRowDeserializerSource & in);
+    void expand(size32_t required);
+    void resize(size32_t required);
+    void finalizeRows();
+
+    inline void ensure(size32_t required) { if (required > max) expand(required); }
+    inline byte * * queryrows() { finalizeRows(); return rowset; }
+
+    inline byte * row() const { return builder.row(); }
+*/
+protected:
+    void checkSpace();
+    void init(IEngineRowAllocator * _rowAllocator, IHThorHashLookupInfo *_hashInfo, unsigned _initialTableSize);
+
+protected:
+    IEngineRowAllocator *rowAllocator;
+    IHash *hash;
+    ICompare *compare;
+    RtlDynamicRowBuilder builder;
+    byte * * table;
+    size32_t usedCount;
+    size32_t usedLimit;
+    size32_t initialSize;
+    size32_t tableSize;
+};
+
 extern ECLRTL_API void appendRowsToRowset(size32_t & targetCount, byte * * & targetRowset, IEngineRowAllocator * rowAllocator, size32_t count, byte * * rows);
 
 

rtl/eclrtl/rtlfield.cpp

@@ -834,7 +834,8 @@ size32_t RtlDatasetTypeInfo::toXML(const byte * self, const byte * selfrow, cons
         for (unsigned i= 0; i < thisCount; i++)
         {
             const byte * row = rows[i];
-            child->toXML(row, row, field, target);
+            if (row)
+                child->toXML(row, row, field, target);
         }
         thisSize = sizeof(size32_t) + sizeof(void * *);
     }

rtl/include/eclhelper.hpp

@@ -2683,6 +2683,14 @@ struct IHThorExternalArg : public IHThorArg
     virtual IThorExternalRowProcessor * createProcessor() = 0;
 };
 
+//------------------------- Dictionary stuff -------------------------
+
+interface IHThorHashLookupInfo
+{
+    virtual IHash * queryHash() = 0;
+    virtual ICompare * queryCompare() = 0;
+};
+
 
 //------------------------- Other stuff -------------------------