Merge pull request #3100 from richardkchapman/hashmap4-310

DICTIONARY support

Reviewed-by: Gavin Halliday <ghalliday@hpccsystems.com>

common/deftype/deftype.cpp

@@ -1332,6 +1332,37 @@ bool CRowTypeInfo::assignableFrom(ITypeInfo *t2)
 
 //===========================================================================
 
+bool CDictionaryTypeInfo::assignableFrom(ITypeInfo *t2)
+{
+    if (getTypeCode()==t2->getTypeCode())
+    {
+        ITypeInfo *c1 = queryChildType();
+        ITypeInfo *c2 = t2->queryChildType();
+        if (c1==NULL || c2==NULL || c1->assignableFrom(c2))
+            return true;
+    }
+    return false;
+}
+
+StringBuffer & CDictionaryTypeInfo::getECLType(StringBuffer & out)
+{
+    ITypeInfo * recordType = ::queryRecordType(this);
+    out.append(queryTypeName());
+    if (recordType)
+    {
+        out.append(" of ");
+        recordType->getECLType(out);
+    }
+    return out;
+}
+
+void CDictionaryTypeInfo::serialize(MemoryBuffer &tgt)
+{
+    CBasedTypeInfo::serializeSkipChild(tgt);
+}
+
+//===========================================================================
+
 bool CTableTypeInfo::assignableFrom(ITypeInfo *t2)
 {
     if (getTypeCode()==t2->getTypeCode())
@@ -1985,6 +2016,12 @@ extern DEFTYPE_API ITypeInfo *makeTableType(ITypeInfo *basetype, IInterface * di
     return commonUpType(new CTableTypeInfo(basetype, distributeinfo, globalsortinfo, localsortinfo));
 }
 
+extern DEFTYPE_API ITypeInfo *makeDictionaryType(ITypeInfo *basetype)
+{
+    assertex(!basetype || basetype->getTypeCode() == type_row);
+    return commonUpType(new CDictionaryTypeInfo(basetype));
+}
+
 extern DEFTYPE_API ITypeInfo *makeGroupedTableType(ITypeInfo *basetype, IInterface *groupinfo, IInterface *sortinfo)
 {
     return commonUpType(new CGroupedTableTypeInfo(basetype, groupinfo, sortinfo));
@@ -3213,6 +3250,7 @@ inline ITypeInfo * queryChildType(ITypeInfo * t, type_t search)
         switch (code)
         {
         case type_set:
+        case type_dictionary:
         case type_groupedtable:
         case type_row:
         case type_table:
@@ -3341,6 +3379,9 @@ ITypeInfo * replaceChildType(ITypeInfo * type, ITypeInfo * newChild)
     OwnedITypeInfo newType;
     switch (type->getTypeCode())
     {
+    case type_dictionary:
+        newType.setown(makeDictionaryType(LINK(newChild)));
+        break;
     case type_table:
         newType.setown(makeTableType(LINK(newChild), LINK(type->queryDistributeInfo()), LINK(type->queryGlobalSortInfo()), LINK(type->queryLocalUngroupedSortInfo())));
         break;

common/deftype/deftype.hpp

@@ -104,6 +104,7 @@ type_unused5            = 29,
     type_ifblock        = 43,       // not a real type -but used for the rtlfield serialization
     type_function       = 44,
     type_sortlist       = 45,
+    type_dictionary     = 46,
 
     type_modifier       = 0xff,     // used by getKind()
     type_unsigned       = 0x100,  // combined with some of the above, when returning summary type information. Not returned by getTypeCode()
@@ -254,6 +255,7 @@ extern DEFTYPE_API ITypeInfo *makeAnyType();
 extern DEFTYPE_API ITypeInfo *makeType(type_t type, int size);
 extern DEFTYPE_API IEnumeratedTypeBuilder *makeEnumeratedTypeBuilder(ITypeInfo *base, aindex_t numvalues);
 extern DEFTYPE_API ITypeInfo *makeDecimalType(unsigned digits, unsigned prec, bool isSigned);
+extern DEFTYPE_API ITypeInfo *makeDictionaryType(ITypeInfo *basetype);
 extern DEFTYPE_API ITypeInfo *makeTableType(ITypeInfo *basetype, IInterface * distributeinfo, IInterface *gloalSortinfo, IInterface * localSortInfo);
 extern DEFTYPE_API ITypeInfo *makeGroupedTableType(ITypeInfo *basetype, IInterface *groupinfo, IInterface *sortinfo);
 extern DEFTYPE_API ITypeInfo *makeRowType(ITypeInfo *basetype);

common/deftype/deftype.ipp

@@ -631,6 +631,22 @@ public:
     virtual bool assignableFrom(ITypeInfo *t2);
 };
 
+class CDictionaryTypeInfo : public CBasedTypeInfo
+{
+public:
+    CDictionaryTypeInfo(ITypeInfo * _basetype)
+        : CBasedTypeInfo(_basetype, UNKNOWN_LENGTH)
+    {}
+
+    virtual type_t getTypeCode() const                      { return type_dictionary; }
+    virtual bool isScalar()                                 { return false; }
+    virtual const char *queryTypeName()                     { return "dictionary"; }
+    virtual StringBuffer &getECLType(StringBuffer & out);
+    virtual bool assignableFrom(ITypeInfo *t2);
+
+    virtual void serialize(MemoryBuffer &tgt);
+};
+
 class CTableTypeInfo : public CBasedTypeInfo
 {
 private:

common/fileview2/fvsource.cpp

@@ -355,7 +355,7 @@ void DataSourceMetaData::gatherFields(IHqlExpression * expr, bool isConditional)
                 gatherChildFields(expr->queryRecord(), isConditional);
                 fields.append(*new DataSourceMetaItem(FVFFendrecord, outname, xpath, voidType));
             }
-            else if ((tc == type_table) || (tc == type_groupedtable))
+            else if ((tc == type_dictionary) || (tc == type_table) || (tc == type_groupedtable))
             {
                 isStoredFixedWidth = false;
                 fields.append(*new DataSourceDatasetItem(outname, xpath, expr));

ecl/hql/hqlatoms.cpp

@@ -254,7 +254,6 @@ _ATOM _parameterScopeType_Atom;
 _ATOM partitionAtom;
 _ATOM partitionLeftAtom;
 _ATOM partitionRightAtom;
-_ATOM payloadAtom;
 _ATOM _payload_Atom;
 _ATOM persistAtom;
 _ATOM physicalFilenameAtom;
@@ -635,7 +634,6 @@ MODULE_INIT(INIT_PRIORITY_HQLATOM)
     MAKEATOM(partition);
     partitionLeftAtom = createLowerCaseAtom("partition left");
     partitionRightAtom = createLowerCaseAtom("partition right");
-    MAKEATOM(payload);
     MAKESYSATOM(payload);
     MAKEATOM(persist);
     MAKEATOM(physicalFilename);

ecl/hql/hqlattr.cpp

@@ -151,6 +151,7 @@ unsigned getOperatorMetaFlags(node_operator op)
     case no_hash32:
     case no_hash64:
     case no_wuid:
+    case no_countdict:
     case no_existslist:
     case no_countlist:
     case no_maxlist:
@@ -210,6 +211,7 @@ unsigned getOperatorMetaFlags(node_operator op)
     case no_notbetween:
     case no_between:
     case no_is_valid:
+    case no_indict:
 
 //Lists/Sets etc.
     case no_list:
@@ -264,6 +266,7 @@ unsigned getOperatorMetaFlags(node_operator op)
     case no_newtransform:
 
 //Rows
+    case no_selectmap:
     case no_selectnth:
     case no_matchrow:
     case no_matchattr:      // and scalar
@@ -272,6 +275,11 @@ unsigned getOperatorMetaFlags(node_operator op)
     case no_newrow:
     case no_temprow:
 
+//Dictionaries
+    case no_userdictionary:
+    case no_newuserdictionary:
+    case no_inlinedictionary:
+
 //Datasets [see also selection operators]
     case no_rollup:
     case no_iterate:
@@ -607,13 +615,13 @@ unsigned getOperatorMetaFlags(node_operator op)
     case no_persist_check:
     case no_dataset_from_transform:
 
-    case no_unused3: case no_unused4: case no_unused5: case no_unused6:
+    case no_unused6:
     case no_unused13: case no_unused14: case no_unused15: case no_unused18: case no_unused19:
-    case no_unused20: case no_unused21: case no_unused22: case no_unused23: case no_unused24: case no_unused25: case no_unused26: case no_unused27: case no_unused28: case no_unused29:
+    case no_unused20: case no_unused21: case no_unused22: case no_unused23: case no_unused24: case no_unused25: case no_unused28: case no_unused29:
     case no_unused30: case no_unused31: case no_unused32: case no_unused33: case no_unused34: case no_unused35: case no_unused36: case no_unused37: case no_unused38:
     case no_unused40: case no_unused41: case no_unused42: case no_unused43: case no_unused44: case no_unused45: case no_unused46: case no_unused47: case no_unused48: case no_unused49:
     case no_unused50: case no_unused52:
-    case no_unused80: case no_unused82: case no_unused83:
+    case no_unused80: case no_unused83:
     case no_is_null:
     case no_position:
     case no_current_time:
@@ -868,6 +876,7 @@ static IHqlExpression * evaluateFieldAttrSize(IHqlExpression * expr)
                 }
                 break;
             }
+        case type_dictionary:
         case type_groupedtable:
         case type_table:
             {
@@ -3429,6 +3438,7 @@ bool isLinkedRowset(ITypeInfo * t)
     {
     case type_table:
     case type_groupedtable:
+    case type_dictionary:
         return hasLinkCountedModifier(t);
     }
     return false;
@@ -3441,6 +3451,7 @@ bool isArrayRowset(ITypeInfo * t)
     case type_table:
     case type_groupedtable:
     case type_array:
+    case type_dictionary:
         {
             if (hasLinkCountedModifier(t))
                 assertex(hasLinkCountedModifier(t->queryChildType()));
@@ -3463,6 +3474,7 @@ bool hasLinkedRow(ITypeInfo * t)
     {
     case type_table:
     case type_groupedtable:
+    case type_dictionary:
         return hasLinkedRow(t->queryChildType());
     case type_row:
         return hasLinkCountedModifier(t);
@@ -3479,6 +3491,7 @@ ITypeInfo * setLinkCountedAttr(ITypeInfo * _type, bool setValue)
     {
     case type_table:
     case type_groupedtable:
+    case type_dictionary:
         {
             ITypeInfo * rowType = type->queryChildType();
             Owned<ITypeInfo> newRowType = setLinkCountedAttr(rowType, setValue);

ecl/hql/hqlerrors.hpp

@@ -458,6 +458,7 @@
 #define HQLWRN_CouldNotConstantFoldIf           3124
 #define HQLERR_UnexpectedOperator               3125
 #define HQLERR_UnexpectedType                   3126
+#define HQLERR_PayloadMismatch                  3127
 
 #define HQLERR_DedupFieldNotFound_Text          "Field removed from dedup could not be found"
 #define HQLERR_CycleWithModuleDefinition_Text   "Module definition contain an illegal cycle/recursive definition %s"
@@ -484,8 +485,9 @@
 #define HQLERR_IncompatiableInitailiser_Text    "Inline DATASET field '%s' cannot be initialized with a list of values"
 #define HQLERR_NoDefaultProvided_Text           "No value or default provided for field %s in inline table"
 #define HQLERR_TooManyInitializers_Text         "Too many initializers (value %s) for inline dataset definition"
-#define HQLERR_IncompatibleTypesForField_Text   "Initializer for field %s in inline dataset has the wrong type"
+#define HQLERR_IncompatibleTypesForField_Text   "Initializer for field %s has the wrong type"
 #define HQLWRN_CouldNotConstantFoldIf_Text      "Could not constant fold the condition on a IFBLOCK for a inline table"
+#define HQLERR_PayloadMismatch_Text             "Mismatched => in inline dictionary definition"
 
 /* parser error */
 #define ERR_PARSER_CANNOTRECOVER    3005  /* The parser can not recover from previous error(s) */

ecl/hql/hqlexpr.cpp

@@ -72,7 +72,7 @@
 //#define PARANOID
 //#define SEARCH_NAME1   "vL6R"
 //#define SEARCH_NAME2   "v19"
-//#define SEARCH_IEXPR 0x03289048
+//#define SEARCH_IEXPR 0x0681cb0
 //#define CHECK_SELSEQ_CONSISTENCY
 //#define GATHER_COMMON_STATS
 #define VERIFY_EXPR_INTEGRITY
@@ -973,11 +973,12 @@ const char *getOpString(node_operator op)
     case no_xor: return " XOR ";
     case no_notin: return " NOT IN ";
     case no_in: return " IN ";
+    case no_indict: return " IN ";
     case no_notbetween: return " NOT BETWEEN ";
     case no_between: return " BETWEEN ";
     case no_comma: return ",";
     case no_compound: return ",";
-    case no_count: case no_countlist: return "COUNT";
+    case no_count: case no_countlist: case no_countdict: return "COUNT";
     case no_counter: return "COUNTER";
     case no_countgroup: return "COUNT";
     case no_distribution: return "DISTRIBUTION";
@@ -1004,6 +1005,7 @@ const char *getOpString(node_operator op)
     case no_mergedscope: return "<scope>";
     case no_privatescope: return "<private_scope>";
     case no_list: return "<list>";
+    case no_selectmap: return "SELECT_MAP";
     case no_selectnth: return "SELECT_NTH";
     case no_filter: return "FILTER";
     case no_param: return "<parameter>";
@@ -1445,17 +1447,16 @@ const char *getOpString(node_operator op)
     case no_debug_option_value: return "__DEBUG__";
     case no_dataset_alias: return "TABLE";
     case no_childquery: return "no_childquery";
+    case no_inlinedictionary: case no_userdictionary: case no_newuserdictionary: return "DICTIONARY";
 
-
-    case no_unused3: case no_unused4: case no_unused5: case no_unused6:
+    case no_unused6:
     case no_unused13: case no_unused14: case no_unused15: case no_unused18: case no_unused19:
-    case no_unused20: case no_unused21: case no_unused22: case no_unused23: case no_unused24: case no_unused25: case no_unused26: case no_unused27: case no_unused28: case no_unused29:
+    case no_unused20: case no_unused21: case no_unused22: case no_unused23: case no_unused24: case no_unused25: case no_unused28: case no_unused29:
     case no_unused30: case no_unused31: case no_unused32: case no_unused33: case no_unused34: case no_unused35: case no_unused36: case no_unused37: case no_unused38:
     case no_unused40: case no_unused41: case no_unused42: case no_unused43: case no_unused44: case no_unused45: case no_unused46: case no_unused47: case no_unused48: case no_unused49:
     case no_unused50: case no_unused52:
     case no_unused80:
     case no_unused81:
-    case no_unused82:
     case no_unused83:
         return "unused";
     /* if fail, use "hqltest -internal" to find out why. */
@@ -1653,6 +1654,7 @@ int getPrecedence(node_operator op)
     case no_notbetween:
     case no_in:
     case no_notin:
+    case no_indict:
     case no_comma:
     case no_compound:
     case no_eq:
@@ -1814,6 +1816,8 @@ childDatasetType getChildDatasetType(IHqlExpression * expr)
     case no_transformascii:
     case no_selectfields:
     case no_newaggregate:
+    case no_userdictionary:
+    case no_newuserdictionary:
     case no_newusertable:
     case no_usertable:
     case no_alias_project:
@@ -2100,6 +2104,7 @@ inline unsigned doGetNumChildTables(IHqlExpression * dataset)
     case no_compound_inline:
     case no_transformascii:
     case no_transformebcdic:
+    case no_newuserdictionary:
     case no_newusertable:
     case no_aggregate:
     case no_usertable:
@@ -2517,7 +2522,7 @@ bool definesColumnList(IHqlExpression * dataset)
     case no_field:
         {
             type_t tc = dataset->queryType()->getTypeCode();
-            assertex(tc == type_table || tc == type_groupedtable);
+            assertex(tc == type_table || tc == type_groupedtable || tc == type_dictionary);
             return true;
         }
     case no_comma:
@@ -2604,6 +2609,7 @@ IHqlExpression * queryNewColumnProvider(IHqlExpression * expr)
     case no_createrow:
     case no_typetransfer:
         return expr->queryChild(0);
+    case no_userdictionary:
     case no_usertable:
     case no_selectfields:
     case no_transformebcdic:
@@ -2621,6 +2627,7 @@ IHqlExpression * queryNewColumnProvider(IHqlExpression * expr)
     case no_newkeyindex:
     case no_aggregate:
     case no_newaggregate:
+    case no_newuserdictionary:
     case no_newusertable:
     case no_normalize:
     case no_xmlparse:
@@ -2883,6 +2890,7 @@ IHqlExpression * ensureExprType(IHqlExpression * expr, ITypeInfo * type, node_op
                 break;
             }
         }
+    case type_dictionary:
     case type_table:
     case type_groupedtable:
         if (recordTypesMatch(type, exprType))
@@ -3067,8 +3075,8 @@ void CHqlExpression::initFlagsBeforeOperands()
 {
 #ifdef _DEBUG
 #ifdef SEARCH_IEXPR
-    if ((unsigned)(IHqlExpression *)this == SEARCH_IEXPR)
-        op = op;
+    if ((memsize_t)(IHqlExpression *)this == SEARCH_IEXPR)
+        DBGLOG("initFlags");
 #endif
 #endif
 
@@ -3266,6 +3274,7 @@ void CHqlExpression::updateFlagsAfterOperands()
     case no_assert_ds:
         switch (queryType()->getTypeCode())
         {
+        case type_dictionary:
         case type_table:
         case type_groupedtable:
             infoFlags |= HEFthrowds;
@@ -3626,6 +3635,7 @@ switch (op)
                 }
                 break;
             }
+        case type_dictionary:
         case type_groupedtable:
         case type_table:
             {
@@ -4116,6 +4126,11 @@ bool CHqlExpression::isDataset()
     }
 }
 
+bool CHqlExpression::isDictionary()
+{
+    return matchesTypeCode(queryType(), type_dictionary);
+}
+
 bool CHqlExpression::isDatarow() 
 {
     return matchesTypeCode(queryType(), type_row);
@@ -4249,8 +4264,8 @@ void CHqlExpression::Link(void) const
 #endif
 #ifdef _DEBUG
 #ifdef SEARCH_IEXPR
-    if ((unsigned)(IHqlExpression *)this == SEARCH_IEXPR)
-        op = op;
+    if ((memsize_t)(IHqlExpression *)this == SEARCH_IEXPR)
+        DBGLOG("Link");
 #endif
 #endif
     Parent::Link();
@@ -4265,8 +4280,8 @@ bool CHqlExpression::Release(void) const
 #endif
 #ifdef _DEBUG
 #ifdef SEARCH_IEXPR
-    if ((unsigned)(IHqlExpression *)this == SEARCH_IEXPR)
-        op = op;
+    if ((memsize_t)(IHqlExpression *)this == SEARCH_IEXPR)
+        DBGLOG("Release");
 #endif
 #endif
     return Parent::Release();
@@ -5089,6 +5104,7 @@ void CHqlExpressionWithTables::cacheTablesUsed()
                             switch (thisType->getTypeCode())
                             {
                             case type_void:
+                            case type_dictionary:
                             case type_table:
                             case type_groupedtable:
                             case type_row:
@@ -5428,6 +5444,7 @@ void CHqlField::onCreateField()
         break;
     case type_row:
         break;
+    case type_dictionary:
     case type_table:
     case type_groupedtable:
         typeExpr = queryRecord();
@@ -5641,6 +5658,29 @@ bool isInImplictScope(IHqlExpression * scope, IHqlExpression * dataset)
     return false;
 }
 
+//===========================================================================
+
+CHqlDictionary *CHqlDictionary::makeDictionary(node_operator _op, ITypeInfo *type, HqlExprArray &_ownedOperands)
+{
+    CHqlDictionary *e = new CHqlDictionary(_op, type, _ownedOperands);
+    return (CHqlDictionary *) e->closeExpr();
+}
+
+CHqlDictionary::CHqlDictionary(node_operator _op, ITypeInfo *_type, HqlExprArray &_ownedOperands)
+: CHqlExpressionWithType(_op, _type, _ownedOperands)
+{
+}
+
+CHqlDictionary::~CHqlDictionary()
+{
+}
+
+IHqlExpression *CHqlDictionary::clone(HqlExprArray &newkids)
+{
+    return createDictionary(op, newkids);
+}
+
+//===========================================================================
 
 CHqlDataset *CHqlDataset::makeDataset(node_operator _op, ITypeInfo *type, HqlExprArray &_ownedOperands)
 {
@@ -6082,6 +6122,7 @@ bool CHqlRecord::assignableFrom(ITypeInfo * source)
     switch(source->getTypeCode())
     {
     case type_groupedtable:
+    case type_dictionary:
     case type_table:
     case type_row:
     case type_transform:
@@ -6089,7 +6130,7 @@ bool CHqlRecord::assignableFrom(ITypeInfo * source)
 
     case type_record:
         {
-            if (recordTypesMatch(source, this))
+            if  (numChildren() == 0 || recordTypesMatch(source, this))
                 return true;
 
             //Record inheritance.  If the first entry in the source record is also a record, then check if compatible.
@@ -6897,6 +6938,7 @@ extern HQL_API bool okToAddLocation(IHqlExpression * expr)
     ITypeInfo * type = expr->queryType();
     switch (type->getTypeCode())
     {
+    case type_dictionary:
     case type_table:
     case type_groupedtable:
     case type_void:
@@ -8754,6 +8796,9 @@ IHqlExpression * CHqlParameter::makeParameter(_ATOM _name, unsigned _idx, ITypeI
     type_t tc = _type->getTypeCode();
     switch (tc)
     {
+    case type_dictionary:
+        e = new CHqlDictionaryParameter(_name, _idx, _type);
+        break;
     case type_table:
     case type_groupedtable:
         e = new CHqlDatasetParameter(_name, _idx, _type);
@@ -9819,6 +9864,97 @@ extern IHqlExpression *createDatasetF(node_operator op, ...)
     return createDataset(op, children);
 }
 
+IHqlExpression *createDictionary(node_operator op, HqlExprArray & parms)
+{
+#ifdef GATHER_LINK_STATS
+    insideCreate++;
+#endif
+    Owned<ITypeInfo> type = NULL;
+
+    switch (op)
+    {
+    case no_newuserdictionary:
+    case no_userdictionary:
+    case no_inlinedictionary:
+        type.setown(makeDictionaryType(makeRowType(createRecordType(&parms.item(1)))));
+        break;
+    case no_select:
+        type.set(parms.item(1).queryType());
+        break;
+    case no_addfiles:
+        type.set(parms.item(0).queryType());  // It's an error if they don't all match, caught elsewhere (?)
+        break;
+    case no_if:
+        type.set(parms.item(1).queryType());  // It's an error if they don't match, caught elsewhere
+        break;
+    case no_case:
+        //following is wrong, but they get removed pretty quickly so I don't really care
+        type.set(parms.item(1).queryType());
+        break;
+    case no_map:
+        //following is wrong, but they get removed pretty quickly so I don't really care
+        type.set(parms.item(0).queryType());
+        break;
+    case no_null:
+    case no_fail:
+    case no_anon:
+    {
+        IHqlExpression * record = &parms.item(0);
+        IHqlExpression * metadata = queryProperty(_metadata_Atom, parms);
+        bool linkCounted = (queryProperty(_linkCounted_Atom, parms) || recordRequiresSerialization(record));
+        if (!metadata)
+        {
+            ITypeInfo * recordType = createRecordType(record);
+            assertex(recordType->getTypeCode() == type_record);
+            ITypeInfo * rowType = makeRowType(recordType);
+            type.setown(makeDictionaryType(rowType));
+        }
+        else
+            UNIMPLEMENTED_XY("Type calculation for dictionary operator", getOpString(op));
+
+        if (linkCounted)
+            type.setown(setLinkCountedAttr(type, true));
+        break;
+    }
+    case no_nofold:
+    case no_nohoist:
+    case no_thor:
+    case no_nothor:
+    case no_alias:
+    case no_translated:
+    case no_catch:
+        type.set(parms.item(0).queryType());
+        break;
+    default:
+        UNIMPLEMENTED_XY("Type calculation for dictionary operator", getOpString(op));
+        break;
+    }
+
+    IHqlExpression * ret = CHqlDictionary::makeDictionary(op, type.getClear(), parms);
+#ifdef GATHER_LINK_STATS
+    insideCreate--;
+#endif
+    return ret;
+}
+
+IHqlExpression *createDictionary(node_operator op, IHqlExpression *dictionary, IHqlExpression *list)
+{
+    HqlExprArray parms;
+    parms.append(*dictionary);
+    if (list)
+    {
+        list->unwindList(parms, no_comma);
+        list->Release();
+    }
+    return createDictionary(op, parms);
+}
+
+IHqlExpression *createDictionary(node_operator op, IHqlExpression *dictionary)
+{
+    HqlExprArray parms;
+    parms.append(*dictionary);
+    return createDictionary(op, parms);
+}
 
 IHqlExpression * createAliasOwn(IHqlExpression * expr, IHqlExpression * attr)
 {
@@ -9839,6 +9975,8 @@ IHqlExpression * createTypedValue(node_operator op, ITypeInfo * type, HqlExprArr
     case type_groupedtable:
     case type_table:
         return createDataset(op, args);
+    case type_dictionary:
+        return createDictionary(op, args);
     case type_row:
         return createRow(op, args);
     default:
@@ -10353,6 +10491,10 @@ static void normalizeCallParameters(HqlExprArray & resolvedActuals, IHqlExpressi
                     }
                 }
                 break;
+            case type_dictionary:
+                // MORE - needs some code
+                // For now, never cast
+                break;
             case type_row:
             case type_transform:
             case type_function:
@@ -11917,6 +12059,8 @@ extern IHqlExpression *createWrapper(node_operator op, IHqlExpression * e)
         case type_table:
         case type_groupedtable:
             return createDataset(op, e, NULL);
+        case type_dictionary:
+            return createDictionary(op, e, NULL);
         }
     }
     return createValue(op, LINK(type), e);
@@ -11931,6 +12075,8 @@ IHqlExpression *createWrapper(node_operator op, ITypeInfo * type, HqlExprArray &
         {
         case type_row:
             return createRow(op, args);
+        case type_dictionary:
+            return createDictionary(op, args);
         case type_table:
         case type_groupedtable:
             return createDataset(op, args);
@@ -12044,6 +12190,8 @@ extern IHqlExpression * createSelectExpr(IHqlExpression * _lhs, IHqlExpression *
     type_t t = rhs->queryType()->getTypeCode();
     if (t == type_table || t == type_groupedtable)
         return createDataset(no_select, LINK(normalLhs), createComma(rhs, LINK(newAttr)));
+    if (t == type_dictionary)
+        return createDictionary(no_select, LINK(normalLhs), createComma(rhs, LINK(newAttr)));
     if (t == type_row)
         return createRow(no_select, LINK(normalLhs), createComma(rhs, LINK(newAttr)));
 
@@ -12058,6 +12206,8 @@ static IHqlExpression * doCreateSelectExpr(HqlExprArray & args)
     type_t t = rhs->queryType()->getTypeCode();
     if (t == type_table || t == type_groupedtable)
         return createDataset(no_select, args);
+    if (t == type_dictionary)
+        return createDictionary(no_select, args);
     if (t == type_row)
         return createRow(no_select, args);
 
@@ -12548,6 +12698,13 @@ extern HQL_API IHqlExpression * createNullExpr(ITypeInfo * type)
             IHqlExpression * attr = queryProperty(type, _linkCounted_Atom);
             return createDataset(no_null, LINK(record), LINK(attr));
         }
+    case type_dictionary:
+        {
+            ITypeInfo * recordType = queryRecordType(type);
+            IHqlExpression * record = queryExpression(recordType);
+            IHqlExpression * attr = queryProperty(type, _linkCounted_Atom);
+            return createDictionary(no_null, LINK(record), LINK(attr));
+        }
     case type_row:
 #if 0
         {
@@ -12600,6 +12757,8 @@ extern HQL_API IHqlExpression * createPureVirtual(ITypeInfo * type)
             return createDataset(no_purevirtual, LINK(queryOriginalRecord(type)));
         case type_groupedtable:
             return createDataset(no_purevirtual, LINK(queryOriginalRecord(type)), createAttribute(groupedAtom));
+        case type_dictionary:
+            return createDictionary(no_purevirtual, LINK(queryOriginalRecord(type)), createAttribute(groupedAtom));
         case type_row:
         case type_record:
             return createRow(no_purevirtual, LINK(queryOriginalRecord(type)));
@@ -13733,7 +13892,7 @@ IHqlExpression * extractChildren(IHqlExpression * value)
 
 IHqlExpression * queryDatasetCursor(IHqlExpression * ds)
 {
-    while ((ds->getOperator() == no_select) && !ds->isDataset())
+    while ((ds->getOperator() == no_select) && !ds->isDataset() && !ds->isDictionary())
         ds = ds->queryChild(0);
     return ds;
 }
@@ -13743,7 +13902,7 @@ IHqlExpression * querySelectorDataset(IHqlExpression * expr, bool & isNew)
     assertex(expr->getOperator() == no_select);
     isNew = expr->hasProperty(newAtom);
     IHqlExpression * ds = expr->queryChild(0);
-    while ((ds->getOperator() == no_select) && !ds->isDataset())
+    while ((ds->getOperator() == no_select) && !ds->isDataset() && !ds->isDictionary())
     {
         if (ds->hasProperty(newAtom))
             isNew = true;
@@ -13759,7 +13918,7 @@ bool isNewSelector(IHqlExpression * expr)
     if (expr->hasProperty(newAtom))
         return true;
     IHqlExpression * ds = expr->queryChild(0);
-    while ((ds->getOperator() == no_select) && !ds->isDataset())
+    while ((ds->getOperator() == no_select) && !ds->isDataset() && !ds->isDictionary())
     {
         if (ds->hasProperty(newAtom))
             return true;
@@ -13781,7 +13940,7 @@ IHqlExpression * replaceSelectorDataset(IHqlExpression * expr, IHqlExpression *
 {
     assertex(expr->getOperator() == no_select);
     IHqlExpression * ds = expr->queryChild(0);
-    if ((ds->getOperator() == no_select) && !ds->isDataset())
+    if ((ds->getOperator() == no_select) && !ds->isDataset() && !ds->isDictionary())
     {
         OwnedHqlExpr newSelector = replaceSelectorDataset(ds, newDataset);
         return replaceChild(expr, 0, newSelector);
@@ -13841,6 +14000,7 @@ static ITypeInfo * doGetSimplifiedType(ITypeInfo * type, bool isConditional, boo
     case type_alien:
         return getSimplifiedType(promoted, isConditional, isSerialized);
     case type_row:
+    case type_dictionary:
     case type_table:
     case type_groupedtable:
     case type_transform:
@@ -14629,6 +14789,7 @@ extern HQL_API bool hasUnknownTransform(IHqlExpression * expr)
         if (expr->hasProperty(mergeTransformAtom))
             return true;
         break;
+    case no_inlinedictionary:
     case no_inlinetable:
         {
             IHqlExpression * transforms = expr->queryChild(0);
@@ -14693,7 +14854,7 @@ bool transformHasSkipAttr(IHqlExpression * transform)
 
 bool isPureInlineDataset(IHqlExpression * expr)
 {
-    assertex(expr->getOperator() == no_inlinetable);
+    assertex(expr->getOperator() == no_inlinetable || expr->getOperator() == no_inlinedictionary);
     IHqlExpression * values = expr->queryChild(0);
     ForEachChild(i, values)
     {
@@ -14740,6 +14901,11 @@ IHqlExpression * createNullDataset()
     return createDataset(no_null, LINK(queryNullRecord()));
 }
 
+IHqlExpression * createNullDictionary()
+{
+    return createDictionary(no_null, LINK(queryNullRecord()));
+}
+
 bool removeProperty(HqlExprArray & args, _ATOM name)
 {
     bool removed = false;
@@ -15055,6 +15221,14 @@ bool recordTypesMatch(IHqlExpression * left, IHqlExpression * right)
 }
 
 
+bool recordTypesMatchIgnorePayload(IHqlExpression *left, IHqlExpression *right)
+{
+    OwnedHqlExpr simpleLeft = removeProperty(left->queryRecord(), _payload_Atom);
+    OwnedHqlExpr simpleRight = removeProperty(right->queryRecord(), _payload_Atom);
+    return recordTypesMatch(simpleLeft->queryType(), simpleRight->queryType());
+}
+
+
 IHqlExpression * queryTransformSingleAssign(IHqlExpression * transform)
 {
     if (transform->numChildren() != 1)
@@ -15616,7 +15790,8 @@ IHqlExpression * createTypeTransfer(IHqlExpression * expr, ITypeInfo * _newType)
     case type_table:
     case type_groupedtable:
         return createDataset(no_typetransfer, LINK(queryOriginalRecord(newType)), expr);
-        break;
+    case type_dictionary:
+        return createDictionary(no_typetransfer, LINK(queryOriginalRecord(newType)), expr);
     default:
         return createValue(no_typetransfer, newType.getClear(), expr);
     }

ecl/hql/hqlexpr.hpp

@@ -227,7 +227,7 @@ enum _node_operator {
         no_comma,
         no_count,
         no_countgroup,
-    no_unused82,
+        no_selectmap,
         no_exists,
         no_within,
         no_notwithin,
@@ -358,12 +358,12 @@ enum _node_operator {
         no_dataset_from_transform,
         no_childquery,
         no_unknown,
-    no_unused3,
-    no_unused4,
-    no_unused5,
+        no_inlinedictionary,
+        no_indict,
+        no_countdict,
         no_any,
-    no_unused27,
-    no_unused26,
+        no_userdictionary,
+        no_newuserdictionary,
     no_unused25,
     no_unused28,  
     no_unused29,
@@ -1066,6 +1066,7 @@ interface IHqlExpression : public IInterface
     virtual bool isBoolean() = 0;
     virtual bool isDataset() = 0;
     virtual bool isDatarow() = 0;
+    virtual bool isDictionary() = 0;
     virtual bool isScope() = 0;
     virtual bool isType() = 0;
     virtual bool isAction() = 0;
@@ -1253,6 +1254,9 @@ extern HQL_API IHqlExpression *createDataset(node_operator op, IHqlExpression *d
 extern HQL_API IHqlExpression *createDataset(node_operator op, IHqlExpression *dataset, IHqlExpression *elist);
 extern HQL_API IHqlExpression *createDataset(node_operator op, HqlExprArray & parms);       // inScope should only be set internally.
 extern HQL_API IHqlExpression *createDatasetF(node_operator op, ...);
+extern HQL_API IHqlExpression *createDictionary(node_operator op, IHqlExpression *initializer, IHqlExpression *recordDef);
+extern HQL_API IHqlExpression *createDictionary(node_operator op, IHqlExpression *dictionary);
+extern HQL_API IHqlExpression *createDictionary(node_operator op, HqlExprArray & parms);
 extern HQL_API IHqlExpression *createNewDataset(IHqlExpression *name, IHqlExpression *recorddef, IHqlExpression *mode, IHqlExpression *parent, IHqlExpression *joinCondition, IHqlExpression * options = NULL);
 extern HQL_API IHqlExpression *createRow(node_operator op, IHqlExpression *Dataset, IHqlExpression *element = NULL);
 extern HQL_API IHqlExpression *createRow(node_operator op, HqlExprArray & args);
@@ -1542,6 +1546,7 @@ extern HQL_API bool isSimpleCountAggregate(IHqlExpression * aggregateExpr, bool
 extern HQL_API bool isSimpleCountExistsAggregate(IHqlExpression * aggregateExpr, bool canFilterArg, bool canCast);
 extern HQL_API bool isKeyedCountAggregate(IHqlExpression * aggregate);
 extern HQL_API IHqlExpression * createNullDataset();
+extern HQL_API IHqlExpression * createNullDictionary();
 extern HQL_API IHqlExpression * queryNullRecord();
 extern HQL_API IHqlExpression * queryNullRowRecord();
 extern HQL_API IHqlExpression * queryExpression(ITypeInfo * t);
@@ -1559,6 +1564,7 @@ extern HQL_API IHqlExpression * extractChildren(IHqlExpression * value);
 extern HQL_API IHqlExpression * queryOnlyField(IHqlExpression * record);
 extern HQL_API bool recordTypesMatch(ITypeInfo * left, ITypeInfo * right);
 extern HQL_API bool recordTypesMatch(IHqlExpression * left, IHqlExpression * right);
+extern HQL_API bool recordTypesMatchIgnorePayload(IHqlExpression *left, IHqlExpression *right);
 extern HQL_API IHqlExpression * queryOriginalRecord(IHqlExpression * expr);
 extern HQL_API IHqlExpression * queryOriginalRecord(ITypeInfo * type);
 extern HQL_API IHqlExpression * queryOriginalTypeExpression(ITypeInfo * type);

ecl/hql/hqlexpr.ipp

@@ -200,6 +200,7 @@ public:
     virtual bool isBoolean();
     virtual bool isConstant();
     virtual bool isDataset();
+    virtual bool isDictionary();
     virtual bool isDatarow();
     virtual bool isScope();
     virtual bool isMacro();
@@ -1325,6 +1326,33 @@ public:
     virtual IHqlSimpleScope* querySimpleScope() { return CHqlParameter::querySimpleScope(); }
 };
 
+class CHqlDictionaryParameter : public CHqlParameter, implements IHqlDataset
+{
+public:
+    IMPLEMENT_IINTERFACE_USING(CHqlParameter)
+
+    CHqlDictionaryParameter(_ATOM name, unsigned idx, ITypeInfo *type)
+     : CHqlParameter(name, idx, type) { }
+
+//IHqlExpression
+    virtual bool assignableFrom(ITypeInfo * source) { type_t tc = source->getTypeCode(); return tc==type_dictionary; }
+    virtual IHqlDataset *queryDataset() { return this; }
+
+//CHqlParameter
+
+    //virtual IHqlSimpleScope *querySimpleScope();
+
+//IHqlDataset
+    virtual IHqlDataset* queryTable() { return this; }
+    virtual IHqlDataset * queryRootTable() { return this; }
+    virtual IHqlExpression * queryContainer() { return NULL; }
+
+    virtual bool isAggregate() { return false; }
+
+//Overlapped methods
+    virtual IHqlSimpleScope* querySimpleScope() { return CHqlParameter::querySimpleScope(); }
+};
+
 class CHqlScopeParameter : public CHqlScope
 {
 protected:
@@ -1528,6 +1556,18 @@ public:
     void insertSymbols(IHqlExpression * expr);
 };
 
+class CHqlDictionary : public CHqlExpressionWithType // , implements IHqlDictionary
+{
+public:
+    IMPLEMENT_IINTERFACE_USING(CHqlExpression)
+    static CHqlDictionary *makeDictionary(node_operator op, ITypeInfo *type, HqlExprArray &operands);
+
+public:
+    CHqlDictionary(node_operator op, ITypeInfo *_type, HqlExprArray &_ownedOperands);
+    ~CHqlDictionary();
+
+    virtual IHqlExpression *clone(HqlExprArray &newkids);
+};
 
 
 class CHqlDataset : public CHqlExpressionWithType, implements IHqlDataset

ecl/hql/hqlfold.cpp

@@ -2996,6 +2996,19 @@ IHqlExpression * foldConstantOperator(IHqlExpression * expr, unsigned foldOption
                 return createConstant(value->castTo(expr->queryType()));
         }
         break;
+    case no_countdict:
+        {
+            IHqlExpression * child = expr->queryChild(0);
+            node_operator childOp = child->getOperator();
+            switch (childOp)
+            {
+            case no_inlinedictionary:
+                if (isPureInlineDataset(child))
+                    return createConstant(expr->queryType()->castFrom(false, (__int64)child->queryChild(0)->numChildren()));
+                break;
+            }
+            break;
+        }
     case no_countlist:
         {
             IHqlExpression * child = expr->queryChild(0);
@@ -5571,6 +5584,13 @@ HqlConstantPercolator * CExprFolderTransformer::gatherConstants(IHqlExpression *
         //all bets are off.
         break;
 
+    case no_newuserdictionary:
+    case no_userdictionary:
+    case no_inlinedictionary:
+    case no_selectmap:
+        // MORE - maybe should be something here?
+        break;
+
     case no_selectnth:
         {
             //Careful - this can create a null row if it is out of range.

ecl/hql/hqlgram.hpp

@@ -166,6 +166,10 @@ public:
     {
         return queryExpr()->isDatarow();
     }
+    inline bool isDictionary() const
+    {
+        return queryExpr()->isDictionary();
+    }
 
     /* setters */
     inline void inherit(attribute & other)
@@ -416,6 +420,7 @@ public:
     void checkBooleanOrNumeric(attribute &atr);
     void checkDatarow(attribute &atr);
     void checkDataset(attribute &atr);
+    void checkDictionary(attribute &atr);
     void checkFieldnameValid(const attribute &errpos, _ATOM name);
     void checkList(attribute &atr);
     void checkScalar(attribute &atr);
@@ -628,12 +633,15 @@ public:
     IHqlExpression * attachMetaAttributes(IHqlExpression * ownedExpr, HqlExprArray & meta);
 
     void addDatasetField(const attribute &errpos, _ATOM name, IHqlExpression * record, IHqlExpression *value, IHqlExpression * attrs);
+    void addDictionaryField(const attribute &errpos, _ATOM name, IHqlExpression * record, IHqlExpression *value, IHqlExpression * attrs);
     void addField(const attribute &errpos, _ATOM name, ITypeInfo *type, IHqlExpression *value, IHqlExpression *attrs);
     void addFields(const attribute &errpos, IHqlExpression *record, IHqlExpression * dataset, bool clone);
     void addIfBlockToActive(const attribute &errpos, IHqlExpression * ifblock);
     void addToActiveRecord(IHqlExpression * newField);
     void beginIfBlock();
     IHqlExpression * endIfBlock();
+    void beginPayload();
+    IHqlExpression * endPayload();
 
     IHqlExpression * expandedSortListByReference(attribute * module, attribute & list);
     IHqlExpression *bindParameters(const attribute & errpos, IHqlExpression * function, HqlExprArray & ownedActuals);
@@ -650,11 +658,12 @@ public:
     int checkRecordTypes(IHqlExpression *left, IHqlExpression *right, attribute &atr, unsigned maxFields = (unsigned)-1);
     bool checkRecordCreateTransform(HqlExprArray & assigns, IHqlExpression *leftExpr, IHqlExpression *leftSelect, IHqlExpression *rightExpr, IHqlExpression *rightSelect, attribute &atr);
     IHqlExpression * checkEnsureRecordsMatch(IHqlExpression * left, IHqlExpression * right, attribute & errpos, bool rightIsRow);
-    void checkRecordIsValid(attribute &atr, IHqlExpression *record);
+    void checkRecordIsValid(const attribute &atr, IHqlExpression *record);
     void checkValidRecordMode(IHqlExpression * dataset, attribute & atr, attribute & modeatr);
     void checkValidCsvRecord(const attribute & errpos, IHqlExpression * record);
     void checkValidPipeRecord(const attribute & errpos, IHqlExpression * record, IHqlExpression * attrs, IHqlExpression * expr);
 
+    void createAppendDictionaries(attribute & targetAttr, attribute & leftAttr, attribute & rightAttr, _ATOM kind);
     void createAppendFiles(attribute & targetAttr, attribute & leftAttr, attribute & rightAttr, _ATOM kind);
     IHqlExpression * processIfProduction(attribute & condAttr, attribute & trueAttr, attribute * falseAttr);
 
@@ -919,6 +928,7 @@ protected:
 
     IHqlExpression * getActiveCounter(attribute & errpos);
     void pushRecord(IHqlExpression *);
+    IHqlExpression *endRecordDef();
     IHqlExpression *popRecord();
     IHqlExpression *queryTopScope();
     ITypeInfo * getPromotedECLType(HqlExprArray & args, ITypeInfo * otherType, bool allowVariableLength);
@@ -949,11 +959,11 @@ protected:
     IHqlExpression *queryCurrentTransformRecord();
     IHqlExpression* queryFieldMap(IHqlExpression* expr);
     IHqlExpression* bindFieldMap(IHqlExpression*, IHqlExpression*);
-    void applyPayloadAttribute(const attribute & errpos, IHqlExpression * record, SharedHqlExpr & extra);
     void extractRecordFromExtra(SharedHqlExpr & record, SharedHqlExpr & extra);
     void transferOptions(attribute & filenameAttr, attribute & optionsAttr);
     IHqlExpression * extractTransformFromExtra(SharedHqlExpr & extra);
     void expandPayload(HqlExprArray & fields, IHqlExpression * payload, IHqlSimpleScope * scope, ITypeInfo * & lastFieldType, const attribute & errpos);
+    void mergeDictionaryPayload(SharedHqlExpr & record, SharedHqlExpr & payload, const attribute & errpos);
     void modifyIndexPayloadRecord(SharedHqlExpr & record, SharedHqlExpr & payload, SharedHqlExpr & extra, const attribute & errpos);
 
     bool haveAssignedToChildren(IHqlExpression * select);

ecl/hql/hqlgram.y

@@ -166,6 +166,7 @@ static void eclsyntaxerror(HqlGram * parser, const char * s, short yystate, int
   DENORMALIZE
   DEPRECATED
   DESC
+  DICTIONARY
   DISTRIBUTE
   DISTRIBUTED
   DISTRIBUTION
@@ -332,7 +333,6 @@ static void eclsyntaxerror(HqlGram * parser, const char * s, short yystate, int
   PARTITION
   PARTITION_ATTR
   TOK_PATTERN
-  PAYLOAD
   PENALTY
   PERSIST
   PHYSICALFILENAME
@@ -479,6 +479,7 @@ static void eclsyntaxerror(HqlGram * parser, const char * s, short yystate, int
 
   DATAROW_ID
   DATASET_ID
+  DICTIONARY_ID
   SCOPE_ID
   VALUE_ID
   VALUE_ID_REF
@@ -499,6 +500,7 @@ static void eclsyntaxerror(HqlGram * parser, const char * s, short yystate, int
 
   DATAROW_FUNCTION
   DATASET_FUNCTION
+  DICTIONARY_FUNCTION
   VALUE_FUNCTION
   ACTION_FUNCTION
   PATTERN_FUNCTION
@@ -703,6 +705,7 @@ defineType
     : typeDef
     | setType
     | explicitDatasetType
+    | explicitDictionaryType
     | ROW               {
                             IHqlExpression* record = queryNullRecord();
                             $$.setType(makeRowType(record->getType()));
@@ -759,6 +762,28 @@ explicitDatasetType1
                         }
     ;
 
+explicitDictionaryType
+    : DICTIONARY
+                        {
+                            $$.setType(makeDictionaryType(makeRowType(queryNullRecord()->getType())));
+                            $$.setPosition($1);
+                        }
+    | DICTIONARY '(' recordDef ')'
+                        {
+                            OwnedHqlExpr record = $3.getExpr();
+                            ITypeInfo * recordType = createRecordType(record);
+                            $$.setType(makeDictionaryType(makeRowType(recordType)));
+                            $$.setPosition($1);
+                        }
+    | _LINKCOUNTED_ explicitDictionaryType
+                        {
+                            Owned<ITypeInfo> dsType = $2.getType();
+                            $$.setType(setLinkCountedAttr(dsType, true));
+                            $$.setPosition($1);
+                        }
+    ;
+
+
 transformType
     : TRANSFORM '(' recordDef ')'
                         {
@@ -819,11 +844,17 @@ paramType
                             parser->setTemplateAttribute();
                         }
     | explicitDatasetType
+    | explicitDictionaryType
     | ROW               {
                             IHqlExpression* record = queryNullRecord();
                             $$.setType(makeRowType(record->getType()));
                             $$.setPosition($1);
                         }
+    | _LINKCOUNTED_ ROW {
+                            IHqlExpression* record = queryNullRecord();
+                            $$.setType(setLinkCountedAttr(makeRowType(record->getType()), true));
+                            $$.setPosition($1);
+                        }
     | abstractModule
                         {
                             OwnedHqlExpr scope = $1.getExpr();
@@ -862,6 +893,7 @@ object
 
 goodObject
     : dataSet
+    | dictionary
     | expression
                         {
                             //Remove later to allow sortlist attributes
@@ -1172,6 +1204,7 @@ knownOrUnknownId
 knownId
     : DATAROW_ID
     | DATASET_ID
+    | DICTIONARY_ID
     | VALUE_ID
     | ACTION_ID
     | RECORD_ID
@@ -1187,6 +1220,7 @@ knownId
 knownFunction1
     : DATAROW_FUNCTION
     | DATASET_FUNCTION
+    | DICTIONARY_FUNCTION
     | VALUE_FUNCTION
     | ACTION_FUNCTION
     | PATTERN_FUNCTION
@@ -1890,6 +1924,11 @@ transformation1
                             parser->addAssignment($1, $3);
                             $$.clear($1);
                         }
+    | transformDst ASSIGN dictionary
+                        {
+                            parser->addAssignment($1, $3);
+                            $$.clear($1);
+                        }
     | transformDst ASSIGN error ';'
                         {
                             $1.release();
@@ -1966,6 +2005,12 @@ transformDstSelect
                             parser->setDotScope(scope);
                             $$.setExpr(scope.getClear(), $1);
                         }
+    | DICTIONARY_ID
+                        {
+                            OwnedHqlExpr scope = $1.getExpr();
+                            parser->setDotScope(scope);
+                            $$.setExpr(scope.getClear(), $1);
+                        }
     ;
 
 transformDstField
@@ -2925,12 +2970,6 @@ indexFlag
                             $$.setExpr(createAttribute(fixedAtom));
                             $$.setPosition($1);
                         }
-    | PAYLOAD '(' expression ')'
-                        {
-                            parser->normalizeExpression($3);        //$3 indicates the first payload field
-                            $$.setExpr(createExprAttribute(payloadAtom, $3.getExpr()));
-                            $$.setPosition($1);
-                        }
     | COMPRESSED '(' compressMode ')'
                         {
                             $$.setExpr(createExprAttribute(compressedAtom, $3.getExpr()));
@@ -3698,6 +3737,7 @@ funcRetType
     | propType
     | setType
     | explicitDatasetType
+    | explicitDictionaryType
     | transformType
  // A plain record would be better, but that then causes a s/r error in knownOrUnknownId because scope
     | ROW '(' recordDef ')'     
@@ -3721,35 +3761,75 @@ funcRetType
                         }
     ;
 
+payloadPart
+    :  		            {
+                            // NOTE - this reduction happens as soon as the GOESTO is seen,
+                            // so it ensures that the following fields go into the payload record def
+                            $$.setExpr(parser->endRecordDef());
+                            parser->beginRecord();
+                        }
+      GOESTO fieldDefs optSemiComma
+    ;
 
 recordDef
     : startrecord fieldDefs optSemiComma endrecord
                         {
                             OwnedHqlExpr record = $4.getExpr();
                             parser->checkRecordIsValid($1, record);
-                            $$.setExpr(record.getClear());
-                            $$.setPosition($1);
+                            $$.setExpr(record.getClear(), $1);
                         }
+
+    | startrecord fieldDefs payloadPart endrecord
+                        {
+                            OwnedHqlExpr record = $3.getExpr();
+                            OwnedHqlExpr payload = $4.getExpr();
+                            parser->mergeDictionaryPayload(record, payload, $1);
+                            $$.setExpr(record.getClear(), $1);
+                        }
+
     | startrecord recordOptions fieldDefs optSemiComma endrecord
                         {
                             OwnedHqlExpr record = $5.getExpr();
                             parser->checkRecordIsValid($1, record);
-                            $$.setExpr(record.getClear());
-                            $$.setPosition($1);
+                            $$.setExpr(record.getClear(), $1);
                         }
+
+    | startrecord recordOptions fieldDefs payloadPart endrecord
+                        {
+                            OwnedHqlExpr record = $4.getExpr();
+                            OwnedHqlExpr payload = $5.getExpr();
+                            parser->mergeDictionaryPayload(record, payload, $1);
+                            $$.setExpr(record.getClear(), $1);
+                        }
+
     | startrecord recordBase optFieldDefs endrecord
                         {
                             OwnedHqlExpr record = $4.getExpr();
                             parser->checkRecordIsValid($1, record);
-                            $$.setExpr(record.getClear());
-                            $$.setPosition($1);
+                            $$.setExpr(record.getClear(), $1);
                         }
+
+    | startrecord recordBase optFieldDefs payloadPart endrecord
+                        {
+                            OwnedHqlExpr record = $4.getExpr();
+                            OwnedHqlExpr payload = $5.getExpr();
+                            parser->mergeDictionaryPayload(record, payload, $1);
+                            $$.setExpr(record.getClear(), $1);
+                        }
+
     | startrecord recordBase recordOptions optFieldDefs endrecord
                         {
                             OwnedHqlExpr record = $5.getExpr();
                             parser->checkRecordIsValid($1, record);
-                            $$.setExpr(record.getClear());
-                            $$.setPosition($1);
+                            $$.setExpr(record.getClear(), $1);
+                        }
+
+    | startrecord recordBase recordOptions optFieldDefs payloadPart endrecord
+                        {
+                            OwnedHqlExpr record = $5.getExpr();
+                            OwnedHqlExpr payload = $6.getExpr();
+                            parser->mergeDictionaryPayload(record, payload, $1);
+                            $$.setExpr(record.getClear(), $1);
                         }
     | simpleRecord
     | recordDef AND recordDef
@@ -3972,11 +4052,7 @@ recordOption
 endrecord
     : endOfRecordMarker
                         {
-                            IHqlExpression * record = parser->popRecord();
-                            record->Link();     // link should be in startrecord, but can only link after closeExpr()
-                            parser->popSelfScope();
-                            OwnedHqlExpr newRecord = record->closeExpr();
-                            $$.setExpr(newRecord.getClear());
+                            $$.setExpr(parser->endRecordDef());
                             parser->popLocale();
                             $$.setPosition($1);
                         }
@@ -4168,6 +4244,32 @@ fieldDef
                             parser->addDatasetField($1, $1.getName(), LINK(value->queryRecord()), value, $2.getExpr());
                             $$.clear();
                         }
+
+
+    | DICTIONARY '(' recordDef ')' knownOrUnknownId optFieldAttrs
+                        {
+                            $$.clear($1);
+                            parser->addDictionaryField($5, $5.getName(), $3.getExpr(), NULL, $6.getExpr());
+                        }
+    | DICTIONARY '(' recordDef ')' knownOrUnknownId optFieldAttrs ASSIGN dictionary
+                        {
+                            $$.clear($1);
+                            parser->addDictionaryField($5, $5.getName(), $3.getExpr(), $8.getExpr(), $6.getExpr());
+                        }
+    | DICTIONARY knownOrUnknownId optFieldAttrs ASSIGN dictionary
+                        {
+                            $$.clear($1);
+                            IHqlExpression * value = $5.getExpr();
+                            parser->addDictionaryField($2, $2.getName(), LINK(value->queryRecord()), value, $3.getExpr());
+                        }
+    | UNKNOWN_ID optFieldAttrs ASSIGN dictionary
+                        {
+                            IHqlExpression * value = $4.getExpr();
+                            parser->addDictionaryField($1, $1.getName(), LINK(value->queryRecord()), value, $2.getExpr());
+                            $$.clear();
+                        }
+
+
     | alienTypeInstance knownOrUnknownId optFieldAttrs defaultValue
                         {
                             $$.clear($1);
@@ -4938,6 +5040,48 @@ compareExpr
                             $$.setExpr(parser->createINExpression(no_in, expr, set, $3));
                             $$.setPosition($2);
                         }
+    | expr NOT TOK_IN dictionary
+                        {
+                            parser->normalizeExpression($1);
+                            parser->normalizeExpression($4);
+                            parser->normalizeExpression($4, type_dictionary, false);
+                            IHqlExpression *dict = $4.getExpr();
+                            OwnedHqlExpr row = createValue(no_rowvalue, makeNullType(), $1.getExpr());
+                            OwnedHqlExpr indict = createINDictExpr(parser->errorHandler, $4.pos, row, dict);
+                            $$.setExpr(getInverse(indict));
+                            $$.setPosition($3);
+                        }
+    | dataRow NOT TOK_IN dictionary
+                        {
+                            parser->normalizeExpression($1);
+                            parser->normalizeExpression($4);
+                            parser->normalizeExpression($4, type_dictionary, false);
+                            IHqlExpression *dict = $4.getExpr();
+                            IHqlExpression *row = $1.getExpr();
+                            OwnedHqlExpr indict = createINDictRow(parser->errorHandler, $4.pos, row, dict);
+                            $$.setExpr(getInverse(indict));
+                            $$.setPosition($3);
+                        }
+    | expr TOK_IN dictionary
+                        {
+                            parser->normalizeExpression($1);
+                            parser->normalizeExpression($3);
+                            parser->normalizeExpression($3, type_dictionary, false);
+                            IHqlExpression *dict = $3.getExpr();
+                            OwnedHqlExpr row = createValue(no_rowvalue, makeNullType(), $1.getExpr());
+                            $$.setExpr(createINDictExpr(parser->errorHandler, $3.pos, row, dict));
+                            $$.setPosition($2);
+                        }
+    | dataRow TOK_IN dictionary
+                        {
+                            parser->normalizeExpression($1);
+                            parser->normalizeExpression($3);
+                            parser->normalizeExpression($3, type_dictionary, false);
+                            IHqlExpression *dict = $3.getExpr();
+                            IHqlExpression *row = $1.getExpr();
+                            $$.setExpr(createINDictRow(parser->errorHandler, $3.pos, row, dict));
+                            $$.setPosition($2);
+                        }
     | dataSet EQ dataSet    
                         {
                             parser->checkSameType($1, $3); $$.setExpr(createBoolExpr(no_eq, $1.getExpr(), $3.getExpr()));
@@ -5263,6 +5407,10 @@ primexpr1
                             //or a no_param, in which case it doesn't matter what we return
                             $$.setExpr(getSizetConstant(list->numChildren()), $1);
                         }
+    | COUNT '(' dictionary ')'
+                        {
+                            $$.setExpr(createValue(no_countdict, LINK(parser->defaultIntegralType), $3.getExpr()));
+                        }
     | CHOOSE '(' expression ',' chooseList ')'
                         {
                             parser->normalizeExpression($3, type_int, false);
@@ -6637,6 +6785,13 @@ dataRow
                             parser->normalizeExpression($3, type_int, false);
                             $$.setExpr(createRow(no_selectnth, $1.getExpr(), $3.getExpr()));    
                         }
+    | dictionary '[' expressionList ']'
+                        {
+                            HqlExprArray args;
+                            $3.unwindCommaList(args);
+                            OwnedHqlExpr row = createValue(no_rowvalue, makeNullType(), args);
+                            $$.setExpr(createSelectMapRow(parser->errorHandler, $3.pos, $1.getExpr(), row.getClear()));
+                        }
     | dataSet '[' NOBOUNDCHECK expression ']'
                         {   
                             parser->normalizeExpression($4, type_int, false);
@@ -6912,6 +7067,115 @@ simpleDataRow
                         }
     ;
 
+dictionary
+    : simpleDictionary
+    | dictionary '+' dictionary
+                        {   parser->createAppendDictionaries($$, $1, $3, NULL);    }
+    ;
+
+simpleDictionary
+    : scopedDictionaryId
+    | NOFOLD '(' dictionary ')'
+                        {
+                            $$.setExpr(createDictionary(no_nofold, $3.getExpr(), NULL));
+                            $$.setPosition($1);
+                        }
+    | NOHOIST '(' dictionary ')'
+                        {
+                            $$.setExpr(createDictionary(no_nohoist, $3.getExpr(), NULL));
+                            $$.setPosition($1);
+                        }
+
+    | DICTIONARY '(' startTopFilter ',' recordDef ')' endTopFilter
+                        {
+                            OwnedHqlExpr dataset = $3.getExpr();
+                            parser->checkOutputRecord($5, false);
+                            OwnedHqlExpr record = $5.getExpr();
+                            HqlExprArray args;
+                            args.append(*LINK(dataset));
+                            args.append(*LINK(record));
+                            $$.setExpr(createDictionary(no_userdictionary, args));
+                            parser->checkProjectedFields($$.queryExpr(), $5);
+                            $$.setPosition($1);
+                        }
+
+    | DICTIONARY '(' '[' ']' ',' recordDef ')'
+                        {
+                            HqlExprArray values;  // Empty list
+                            OwnedHqlExpr table = createDataset(no_temptable, createValue(no_recordlist, NULL, values), $6.getExpr());
+                            $$.setExpr(convertTempTableToInlineDictionary(parser->errorHandler, $4.pos, table));
+                            $$.setPosition($1);
+                        }
+    | DICTIONARY '(' '[' beginList inlineDatasetValueList ']' ',' recordDef ')'
+                        {
+                            HqlExprArray values;
+                            parser->endList(values);
+                            OwnedHqlExpr table = createDataset(no_temptable, createValue(no_recordlist, NULL, values), $8.getExpr());
+                            $$.setExpr(convertTempTableToInlineDictionary(parser->errorHandler, $5.pos, table));
+                            $$.setPosition($1);
+                        }
+    | '(' dictionary  ')'  {
+                            $$.setExpr($2.getExpr());
+                            $$.setPosition($1);
+                        }
+    | IF '(' booleanExpr ',' dictionary ',' dictionary ')'
+                        {
+                            OwnedHqlExpr ds = parser->processIfProduction($3, $5, &$7);
+                            $$.setExpr(ds.getClear(), $1);
+                        }
+    | IF '(' booleanExpr ',' dictionary ')'
+                        {
+                            OwnedHqlExpr ds = parser->processIfProduction($3, $5, NULL);
+                            $$.setExpr(ds.getClear(), $1);
+                        }
+    | IFF '(' booleanExpr ',' dictionary ',' dictionary ')'
+                        {
+                            OwnedHqlExpr ds = parser->processIfProduction($3, $5, &$7);
+                            $$.setExpr(ds.getClear(), $1);
+                        }
+    | IFF '(' booleanExpr ',' dictionary ')'
+                        {
+                            OwnedHqlExpr ds = parser->processIfProduction($3, $5, NULL);
+                            $$.setExpr(ds.getClear(), $1);
+                        }
+// MORE - should do CASE and MAP
+    ;
+
+scopedDictionaryId
+    : globalScopedDictionaryId
+    | dotScope DICTIONARY_ID leaveScope
+                        {
+                            IHqlExpression *e1 = $1.getExpr();
+                            IHqlExpression *e2 = $2.getExpr();
+                            if (e1 && (e1->getOperator() != no_record) && (e2->getOperator() == no_field))
+                                $$.setExpr(parser->createSelect(e1, e2, $2));
+                            else
+                            {
+                                ::Release(e1);
+                                $$.setExpr(e2);
+                            }
+                        }
+/*
+    | dictionaryFunction '('
+                        {
+                            parser->beginFunctionCall($1);
+                        }
+    actualParameters ')'
+                        {
+                            $$.setExpr(parser->bindParameters($1, $4.getExpr()));
+                        }
+*/
+    ;
+
+globalScopedDictionaryId
+    : DICTIONARY_ID
+    | moduleScopeDot DICTIONARY_ID leaveScope
+                        {
+                            OwnedHqlExpr scope = $1.getExpr();
+                            $$.setExpr($2.getExpr());
+                        }
+    ;
+
 dataSet
     : simpleDataSet
     | startCompoundExpression beginInlineFunctionToken optDefinitions RETURN dataSet ';' endInlineFunctionToken
@@ -7859,7 +8123,6 @@ simpleDataSet
                             parser->extractRecordFromExtra(record, extra);
                             OwnedHqlExpr transform = parser->extractTransformFromExtra(extra);
 
-                            parser->applyPayloadAttribute($6, record, extra);
                             parser->inheritRecordMaxLength(dataset, record);
 
                             record.setown(parser->checkIndexRecord(record, $5));
@@ -7886,8 +8149,6 @@ simpleDataSet
                             OwnedHqlExpr record = $3.getExpr();
                             OwnedHqlExpr extra = $4.getExpr();
                             parser->extractRecordFromExtra(record, extra);
-                            parser->applyPayloadAttribute($6, record, extra);
-
                             $$.setExpr(parser->createIndexFromRecord(record, extra, $3));
                             parser->checkIndexRecordTypes($$.queryExpr(), $1);
                             $$.setPosition($1);
@@ -9437,17 +9698,31 @@ inlineFieldValue
                         }
     ;
 
+inlineFieldValueGoesTo
+        : GOESTO        {
+                            parser->addListElement(createAttribute(_payload_Atom));
+                            $$.clear();
+                            $$.setPosition($1);
+                        }
+        ;
+
 inlineFieldValues
     : inlineFieldValue
     | inlineFieldValues ';' inlineFieldValue
     | inlineFieldValues ',' inlineFieldValue
     ;
 
+inlineFieldValuesWithGoesto
+    : inlineFieldValues optSemiComma
+    | inlineFieldValues inlineFieldValueGoesTo  inlineFieldValues optSemiComma
+    ;
+
 inlineDatasetValue
-    : '{' beginList inlineFieldValues optSemiComma '}'
+    : '{' beginList inlineFieldValuesWithGoesto  '}'
                         {
                             HqlExprArray args;
                             parser->endList(args);
+                            setPayloadAttribute(args);
 //                          args.append(*createLocationAttr($1));           // improves the error reporting, but slows it down, and changes the expression crcs
                             $$.setExpr(createValue(no_rowvalue, makeNullType(), args));
                         }
@@ -10231,6 +10506,7 @@ actualValue
                             $$.setExpr(parser->bindFieldMap(expr,map));
                         }
     | dataRow
+    | dictionary
     | TOK_PATTERN pattern
                         {   $$.setExpr($2.getExpr()); }
     | TOKEN pattern     {   $$.setExpr($2.getExpr()); }

ecl/hql/hqlgram2.cpp

@@ -567,6 +567,13 @@ IHqlExpression* HqlGram::popRecord()
     return &activeRecords.pop();
 }                                       
 
+IHqlExpression* HqlGram::endRecordDef()
+{
+    IHqlExpression * record = popRecord();
+    record->Link();     // logically link should be in startrecord, but can only link after finished updating
+    popSelfScope();
+    return record->closeExpr();
+}
 
 void HqlGram::beginFunctionCall(attribute & function)
 {
@@ -2329,6 +2336,22 @@ void HqlGram::addDatasetField(const attribute &errpos, _ATOM name, IHqlExpressio
     record->Release();
 }
 
+void HqlGram::addDictionaryField(const attribute &errpos, _ATOM name, IHqlExpression * record, IHqlExpression *value, IHqlExpression * attrs)
+{
+    if (!name)
+        name = createUnnamedFieldName();
+    checkFieldnameValid(errpos, name);
+    if (queryPropertyInList(virtualAtom, attrs))
+        reportError(ERR_BAD_FIELD_ATTR, errpos, "Virtual can only be specified on a scalar field");
+    if (!attrs)
+        attrs = extractAttrsFromExpr(value);
+
+    ITypeInfo * type = makeDictionaryType(makeRowType(createRecordType(record)));
+    IHqlExpression *newField = createField(name, type, value, attrs);
+    addToActiveRecord(newField);
+    record->Release();
+}
+
 void HqlGram::addIfBlockToActive(const attribute &errpos, IHqlExpression * ifblock)
 {
     activeRecords.tos().addOperand(LINK(ifblock));
@@ -3813,6 +3836,9 @@ void HqlGram::normalizeExpression(attribute & exprAttr, type_t expectedType, boo
     case type_set:
         checkList(exprAttr);
         break;
+    case type_dictionary:
+        checkDictionary(exprAttr);
+        break;
     case type_table:
         ensureDataset(exprAttr);
         break;
@@ -7667,6 +7693,7 @@ void HqlGram::ensureDataset(attribute & attr)
     }
     checkDataset(attr);
 }
+
 void HqlGram::expandPayload(HqlExprArray & fields, IHqlExpression * payload, IHqlSimpleScope * scope, ITypeInfo * & lastFieldType, const attribute & errpos)
 {
     ForEachChild(i2, payload)
@@ -7708,10 +7735,45 @@ void HqlGram::expandPayload(HqlExprArray & fields, IHqlExpression * payload, IHq
     }
 }
 
+void HqlGram::mergeDictionaryPayload(SharedHqlExpr & record, SharedHqlExpr & payload, const attribute & errpos)
+{
+    checkRecordIsValid(errpos, record.get());
+    IHqlSimpleScope * scope = record->querySimpleScope();
+
+    // Move all the attributes to the front of the record
+    HqlExprArray fields;
+    ForEachChild(i3, record)
+    {
+        IHqlExpression * cur = record->queryChild(i3);
+        if (cur->isAttribute())
+            fields.append(*LINK(cur));
+    }
+    ForEachChild(i1, record)
+    {
+        IHqlExpression * cur = record->queryChild(i1);
+        if (!cur->isAttribute())
+            fields.append(*LINK(cur));
+    }
+
+    unsigned payloadCount = 0;
+    ITypeInfo * lastFieldType = NULL;
+    if (payload)
+    {
+        unsigned oldFields = fields.ordinality();
+        expandPayload(fields, payload,  scope, lastFieldType, errpos);
+        payloadCount = fields.ordinality() - oldFields;
+    }
+    fields.add(*createAttribute(_payload_Atom, createConstant((__int64) payloadCount)), 0);
+    record.setown(createRecord(fields));
+}
+
+
 void HqlGram::modifyIndexPayloadRecord(SharedHqlExpr & record, SharedHqlExpr & payload, SharedHqlExpr & extra, const attribute & errpos)
 {
+    checkRecordIsValid(errpos, record.get());
     IHqlSimpleScope * scope = record->querySimpleScope();
 
+    // Move all the attributes to the front of the record
     HqlExprArray fields;
     ForEachChild(i3, record)
     {
@@ -7758,7 +7820,7 @@ void HqlGram::modifyIndexPayloadRecord(SharedHqlExpr & record, SharedHqlExpr & p
         payloadCount++;
     }
 
-    extra.setown(createComma(extra.getClear(), createAttribute(_payload_Atom, createConstant((__int64)payloadCount))));
+    extra.setown(createComma(extra.getClear(), createAttribute(_payload_Atom, createConstant((__int64) payloadCount))));
     record.setown(createRecord(fields));
 }
 
@@ -7803,32 +7865,6 @@ IHqlExpression * HqlGram::extractTransformFromExtra(SharedHqlExpr & extra)
     }
     return ret;
 }
-            
-
-void HqlGram::applyPayloadAttribute(const attribute & errpos, IHqlExpression * record, SharedHqlExpr & extra)
-{
-    IHqlExpression * payload = queryPropertyInList(payloadAtom, extra);
-    if (payload)
-    {
-        HqlExprArray fields;
-        unwindChildren(fields, record);
-        IHqlExpression * search = payload->queryChild(0);
-        if (search->getOperator() == no_select)
-            search = search->queryChild(1);
-        unsigned match = fields.find(*search);
-        if (match != NotFound)
-        {
-            HqlExprArray args;
-            extra->unwindList(args, no_comma);
-            args.zap(*payload);
-            args.append(*createAttribute(_payload_Atom, createConstant((__int64)fields.ordinality()-match)));
-            extra.setown(createComma(args));
-        }
-        else
-            reportError(ERR_TYPEMISMATCH_RECORD, errpos, "The argument to the payload isn't found in the index record");
-    }
-}
-
 
 void HqlGram::checkBoolean(attribute &atr)
 {
@@ -7861,6 +7897,15 @@ void HqlGram::checkDataset(attribute &atr)
     }
 }
 
+void HqlGram::checkDictionary(attribute &atr)
+{
+    if (!atr.queryExpr()->isDictionary())
+    {
+        reportError(ERR_EXPECTED_DATASET, atr, "Expected dictionary expression");
+        atr.release().setExpr(createNullDictionary());
+    }
+}
+
 void HqlGram::checkDatarow(attribute &atr)
 {
     if (!atr.queryExpr()->isDatarow())
@@ -8214,7 +8259,7 @@ static bool isZeroSize(IHqlExpression * expr)
 }
 
 
-void HqlGram::checkRecordIsValid(attribute &atr, IHqlExpression *record)
+void HqlGram::checkRecordIsValid(const attribute &atr, IHqlExpression *record)
 {
     if (isZeroSize(record))
         reportError(ERR_ZEROSIZE_RECORD, atr, "Record must not be zero length");
@@ -8374,6 +8419,21 @@ void HqlGram::createAppendFiles(attribute & targetAttr, attribute & leftAttr, at
     targetAttr.setPosition(leftAttr);
 }
 
+void HqlGram::createAppendDictionaries(attribute & targetAttr, attribute & leftAttr, attribute & rightAttr, _ATOM kind)
+{
+    OwnedHqlExpr left = leftAttr.getExpr();
+    OwnedHqlExpr right = rightAttr.getExpr();
+    assertex(left->isDictionary());
+    if (!right->isDictionary())
+        reportError(WRN_UNSUPPORTED_FEATURE, rightAttr, "Only dictionary may be appended to dictionary");
+    right.setown(checkEnsureRecordsMatch(left, right, rightAttr, right->isDatarow()));
+    // TODO: support for dict + row, dict + dataset
+//    if (right->isDatarow())
+//        right.setown(createDatasetFromRow(LINK(right)));
+    IHqlExpression * attr = kind ? createAttribute(kind) : NULL;
+    targetAttr.setExpr(createDictionary(no_addfiles, LINK(left), createComma(LINK(right), attr)));
+    targetAttr.setPosition(leftAttr);
+}
 
 IHqlExpression * HqlGram::processIfProduction(attribute & condAttr, attribute & trueAttr, attribute * falseAttr)
 {
@@ -9841,6 +9901,7 @@ static void getTokenText(StringBuffer & msg, int token)
     case DENORMALIZE: msg.append("DENORMALIZE"); break;
     case DEPRECATED: msg.append("DEPRECATED"); break;
     case DESC: msg.append("DESC"); break;
+    case DICTIONARY: msg.append("DICTIONARY"); break;
     case DISTRIBUTE: msg.append("DISTRIBUTE"); break;
     case DISTRIBUTED: msg.append("DISTRIBUTED"); break;
     case DISTRIBUTION: msg.append("DISTRIBUTION"); break;
@@ -10011,7 +10072,6 @@ static void getTokenText(StringBuffer & msg, int token)
     case PARTITION: msg.append("PARTITION"); break;
     case PARTITION_ATTR: msg.append("PARTITION"); break;
     case TOK_PATTERN: msg.append("PATTERN"); break;
-    case PAYLOAD: msg.append("PAYLOAD"); break;
     case PENALTY: msg.append("PENALTY"); break;
     case PERSIST: msg.append("PERSIST"); break;
     case PHYSICALFILENAME: msg.append("PHYSICALFILENAME"); break;
@@ -10159,6 +10219,7 @@ static void getTokenText(StringBuffer & msg, int token)
 
     case DATASET_ID: msg.append("dataset"); break;
     case DATAROW_ID: msg.append("datarow"); break;
+    case DICTIONARY_ID: msg.append("dictionary"); break;
     case RECORD_ID: msg.append("record-name"); break;
     case RECORD_FUNCTION: msg.append("record-name"); break;
     case VALUE_ID: msg.append("identifier"); break;
@@ -10184,6 +10245,7 @@ static void getTokenText(StringBuffer & msg, int token)
     case SCOPE_FUNCTION: msg.append("module-name"); break;
     case TRANSFORM_FUNCTION: msg.append("transform-name"); break;
     case DATAROW_FUNCTION: msg.append("datarow"); break;
+    case DICTIONARY_FUNCTION: msg.append("dictionary"); break;
     case LIST_DATASET_FUNCTION: msg.append("identifier"); break;
 
     case VALUE_FUNCTION: 
@@ -10280,7 +10342,7 @@ void HqlGram::simplifyExpected(int *expected)
                        TOXML, '@', SECTION, EVENTEXTRA, EVENTNAME, __SEQUENCE__, IFF, OMITTED, GETENV, __DEBUG__, __STAND_ALONE__, 0);
     simplify(expected, DATA_CONST, REAL_CONST, STRING_CONST, INTEGER_CONST, UNICODE_CONST, 0);
     simplify(expected, VALUE_MACRO, DEFINITIONS_MACRO, 0);
-    simplify(expected, VALUE_ID, DATASET_ID, RECORD_ID, ACTION_ID, UNKNOWN_ID, SCOPE_ID, VALUE_FUNCTION, DATAROW_FUNCTION, DATASET_FUNCTION, LIST_DATASET_FUNCTION, LIST_DATASET_ID, ALIEN_ID, TYPE_ID, SET_TYPE_ID, TRANSFORM_ID, TRANSFORM_FUNCTION, RECORD_FUNCTION, FEATURE_ID, EVENT_ID, EVENT_FUNCTION, SCOPE_FUNCTION, ENUM_ID, PATTERN_TYPE_ID, 0); 
+    simplify(expected, VALUE_ID, DATASET_ID, DICTIONARY_ID, RECORD_ID, ACTION_ID, UNKNOWN_ID, SCOPE_ID, VALUE_FUNCTION, DATAROW_FUNCTION, DATASET_FUNCTION, DICTIONARY_FUNCTION, LIST_DATASET_FUNCTION, LIST_DATASET_ID, ALIEN_ID, TYPE_ID, SET_TYPE_ID, TRANSFORM_ID, TRANSFORM_FUNCTION, RECORD_FUNCTION, FEATURE_ID, EVENT_ID, EVENT_FUNCTION, SCOPE_FUNCTION, ENUM_ID, PATTERN_TYPE_ID, 0);
     simplify(expected, LIBRARY, LIBRARY, SCOPE_FUNCTION, STORED, PROJECT, INTERFACE, MODULE, 0);
     simplify(expected, MATCHROW, MATCHROW, LEFT, RIGHT, IF, IFF, ROW, HTTPCALL, SOAPCALL, PROJECT, GLOBAL, NOFOLD, NOHOIST, ALLNODES, THISNODE, SKIP, DATAROW_FUNCTION, TRANSFER, RIGHT_NN, FROMXML, 0);
     simplify(expected, TRANSFORM_ID, TRANSFORM_FUNCTION, TRANSFORM, '@', 0);
@@ -10328,6 +10390,7 @@ void HqlGram::syntaxError(const char *s, int token, int *expected)
     {
     case DATAROW_ID:
     case DATASET_ID:
+    case DICTIONARY_ID:
     case SCOPE_ID:
     case VALUE_ID:
     case VALUE_ID_REF:
@@ -10346,6 +10409,7 @@ void HqlGram::syntaxError(const char *s, int token, int *expected)
     case LIST_DATASET_ID:
     case DATAROW_FUNCTION:
     case DATASET_FUNCTION:
+    case DICTIONARY_FUNCTION:
     case VALUE_FUNCTION:
     case ACTION_FUNCTION:
     case PATTERN_FUNCTION:

ecl/hql/hqlir.cpp

@@ -798,10 +798,16 @@ static const char * getOperatorText(node_operator op)
     DUMP_CASE(no,debug_option_value);
     DUMP_CASE(no,dataset_alias);
     DUMP_CASE(no,childquery);
-
-    case no_unused3: case no_unused4: case no_unused5: case no_unused6:
+    DUMP_CASE(no,selectmap);
+    DUMP_CASE(no,inlinedictionary);
+    DUMP_CASE(no,indict);
+    DUMP_CASE(no,countdict);
+    DUMP_CASE(no,userdictionary);
+    DUMP_CASE(no,newuserdictionary);
+
+    case no_unused6:
     case no_unused13: case no_unused14: case no_unused15: case no_unused18: case no_unused19:
-    case no_unused20: case no_unused21: case no_unused22: case no_unused23: case no_unused24: case no_unused25: case no_unused26: case no_unused27: case no_unused28: case no_unused29:
+    case no_unused20: case no_unused21: case no_unused22: case no_unused23: case no_unused24: case no_unused25: case no_unused28: case no_unused29:
     case no_unused30: case no_unused31: case no_unused32: case no_unused33: case no_unused34: case no_unused35: case no_unused36: case no_unused37: case no_unused38:
     case no_unused40: case no_unused41: case no_unused42: case no_unused43: case no_unused44: case no_unused45: case no_unused46: case no_unused47: case no_unused48: case no_unused49:
     case no_unused50: case no_unused52:
@@ -855,6 +861,7 @@ static const char * getTypeText(type_t type)
     DUMP_CASE(type,ifblock);
     DUMP_CASE(type,function);
     DUMP_CASE(type,sortlist);
+    DUMP_CASE(type,dictionary);
 
     case type_unused1:
     case type_unused2:

ecl/hql/hqllex.l

@@ -142,6 +142,9 @@ int HqlLex::lookupIdentifierToken(YYSTYPE & returnToken, HqlLex * lexer, bool lo
     case type_transform:
         token = isFunction ? TRANSFORM_FUNCTION : TRANSFORM_ID;
         break;
+    case type_dictionary:
+        token = isFunction ? DICTIONARY_FUNCTION : DICTIONARY_ID;
+        break;
     case type_groupedtable:
     case type_table:
         token = isFunction ? DATASET_FUNCTION : DATASET_ID;
@@ -659,6 +662,7 @@ DENORMALIZE         { RETURNSYM(DENORMALIZE); }
 DEPRECATED          { RETURNSYM(DEPRECATED); }
 DESC                { RETURNSYM(DESC); }
 DESCEND             { RETURNSYM(DESC); }
+DICTIONARY          { RETURNSYM(DICTIONARY); }
 DISTRIBUTE          { RETURNSYM(DISTRIBUTE); }
 DISTRIBUTED         { RETURNSYM(DISTRIBUTED); }
 DISTRIBUTION        { RETURNSYM(DISTRIBUTION); }
@@ -821,7 +825,6 @@ PARALLEL            { RETURNSYM(PARALLEL); }
 PARSE               { RETURNSYM(PARSE); }
 PARTITION           { RETURNSYM(PARTITION); }
 PATTERN             { RETURNSYM(TOK_PATTERN); }
-PAYLOAD             { RETURNSYM(PAYLOAD); }
 PENALTY             { RETURNSYM(PENALTY); }
 PERSIST             { RETURNSYM(PERSIST); }
 PHYSICALFILENAME    { RETURNSYM(PHYSICALFILENAME); }

ecl/hql/hqlpmap.cpp

@@ -973,11 +973,10 @@ static bool isTrivialTransform(IHqlExpression * expr, IHqlExpression * selector)
     return true;
 }
 
-
 bool isNullProject(IHqlExpression * expr, bool canLoseFieldsFromEnd)
 {
     IHqlExpression * ds = expr->queryChild(0);
-    if (!recordTypesMatch(expr, ds))
+    if (!recordTypesMatchIgnorePayload(expr, ds))
     {
         if (canLoseFieldsFromEnd)
         {
@@ -1001,6 +1000,7 @@ bool isSimpleProject(IHqlExpression * expr)
     case no_projectrow:
         selector.setown(createSelector(no_left, ds, querySelSeq(expr)));
         break;
+    case no_newuserdictionary:
     case no_newusertable:
          if (isAggregateDataset(expr))
              return false;

ecl/hql/hqlthql.cpp

@@ -863,7 +863,7 @@ void HqltHql::toECL(IHqlExpression *expr, StringBuffer &s, bool paren, bool inTy
     }
     else
     {
-        if (expr->isDataset())
+        if (expr->isDataset() || expr->isDictionary())
         {
             if (!isNamedSymbol && expandProcessed && no != no_field && no != no_rows && !isTargetSelector(expr))
             {
@@ -874,7 +874,10 @@ void HqltHql::toECL(IHqlExpression *expr, StringBuffer &s, bool paren, bool inTy
                     
                     StringBuffer temp;
                     scope.append(NULL);
-                    temp.appendf("dataset%p ", expr);
+                    if (expr->isDataset())
+                        temp.appendf("dataset%p ", expr);
+                    else
+                        temp.appendf("dictionary%p ", expr);
 #ifdef SHOW_NORMALIZED
                     if (expandProcessed)
                         temp.appendf("[N%p] ", expr->queryNormalizedSelector());
@@ -889,7 +892,10 @@ void HqltHql::toECL(IHqlExpression *expr, StringBuffer &s, bool paren, bool inTy
                     insideNewTransform = wasInsideNewTransform;
                     expr->setTransformExtra(expr);
                 }   
-                s.appendf("dataset%p", expr);
+                if (expr->isDataset())
+                    s.appendf("dataset%p", expr);
+                else
+                    s.appendf("dictionary%p", expr);
                 if (paren)
                     s.append(')');
                 return;
@@ -1202,6 +1208,7 @@ void HqltHql::toECL(IHqlExpression *expr, StringBuffer &s, bool paren, bool inTy
         }
         case no_index:
         case no_selectnth:
+        case no_selectmap:
         case no_pat_index:
         {
             toECL(child0, s, child0->getPrecedence() < 0, inType);
@@ -1236,6 +1243,7 @@ void HqltHql::toECL(IHqlExpression *expr, StringBuffer &s, bool paren, bool inTy
         case no_order:
         case no_notin:
         case no_in:
+        case no_indict:
         case no_colon:
         case no_pat_select:
         case no_lshift:
@@ -2679,8 +2687,9 @@ StringBuffer &HqltHql::getFieldTypeString(IHqlExpression * e, StringBuffer &s)
         type = type->queryChildType();
         //fall through
     case type_table:
+    case type_dictionary:
         {
-            s.append("DATASET(");
+            s.append(type->getTypeCode()==type_dictionary ? "DICTIONARY(" : "DATASET(");
             getTypeString(type->queryChildType()->queryChildType(), s);
             ForEachChild(i, e)
             {
@@ -2872,6 +2881,7 @@ static bool addExplicitType(IHqlExpression * expr)
     {
     case type_transform:
         return true;
+    case type_dictionary:
     case type_row:
     case type_table:
     case type_groupedtable:

ecl/hql/hqltrans.cpp

@@ -3375,6 +3375,7 @@ void ScopedTransformer::analyseChildren(IHqlExpression * expr)
     case no_setgraphresult:
     case no_setgraphloopresult:
     case no_extractresult:
+    case no_newuserdictionary:
         {
             IHqlExpression * dataset = expr->queryChild(0);
             pushScope();
@@ -3752,6 +3753,7 @@ IHqlExpression * ScopedTransformer::createTransformed(IHqlExpression * expr)
     case no_setgraphresult:
     case no_setgraphloopresult:
     case no_extractresult:
+    case no_newuserdictionary:
         {
             IHqlExpression * dataset = expr->queryChild(0);
             pushScope();

ecl/hql/hqlutil.cpp

@@ -1174,6 +1174,9 @@ IHqlExpression * createIf(IHqlExpression * cond, IHqlExpression * left, IHqlExpr
     if (left->isDataset() || right->isDataset())
         return createDataset(no_if, cond, createComma(left, right));
 
+    if (left->isDictionary() || right->isDictionary())
+        return createDictionary(no_if, cond, createComma(left, right));
+
     if (left->isDatarow() || right->isDatarow())
         return createRow(no_if, cond, createComma(left, right));
 
@@ -1879,9 +1882,12 @@ unsigned isEmptyRecord(IHqlExpression * record)
 
 bool isTrivialSelectN(IHqlExpression * expr)
 {
-    IHqlExpression * index = expr->queryChild(1);
-    if (index->queryValue() && (index->queryValue()->getIntValue() == 1))
-        return hasSingleRow(expr->queryChild(0));
+    if (expr->getOperator() == no_index || expr->getOperator() == no_selectnth)
+    {
+        IHqlExpression * index = expr->queryChild(1);
+        if (matchesConstantValue(index, 1))
+            return hasSingleRow(expr->queryChild(0));
+    }
     return false;
 }
 
@@ -3239,6 +3245,8 @@ IHqlExpression * createGetResultFromSetResult(IHqlExpression * setResult, ITypeI
         return createDataset(no_getresult, LINK(queryOriginalRecord(valueType)), createComma(LINK(seqAttr), LINK(aliasAttr)));
     case type_groupedtable:
         return createDataset(no_getresult, LINK(queryOriginalRecord(valueType)), createComma(LINK(seqAttr), createAttribute(groupedAtom), LINK(aliasAttr)));
+    case type_dictionary:
+        return createDictionary(no_getresult, LINK(queryOriginalRecord(valueType)), createComma(LINK(seqAttr), createAttribute(groupedAtom), LINK(aliasAttr)));
     case type_row:
     case type_record:
          return createRow(no_getresult, LINK(queryOriginalRecord(valueType)), createComma(LINK(seqAttr), LINK(aliasAttr)));
@@ -5028,7 +5036,9 @@ bool isNullList(IHqlExpression * expr)
 class TempTableTransformer
 {
 public:
-    TempTableTransformer(IErrorReceiver * _errors, ECLlocation & _location) : errors(_errors), defaultLocation(_location) {}
+    TempTableTransformer(IErrorReceiver * _errors, ECLlocation & _location, bool _strictTypeChecking = false)
+      : errors(_errors), defaultLocation(_location), strictTypeChecking(_strictTypeChecking)
+    {}
 
     IHqlExpression * createTempTableTransform(IHqlExpression * curRow, IHqlExpression * record);
 
@@ -5042,6 +5052,7 @@ protected:
 protected:
     IErrorReceiver * errors;
     ECLlocation & defaultLocation;
+    bool strictTypeChecking;
 };
 
 
@@ -5058,6 +5069,24 @@ IHqlExpression * TempTableTransformer::createTempTableTransform(IHqlExpression *
     OwnedHqlExpr self = getSelf(record);
     HqlMapTransformer mapping;
     unsigned col = 0;
+    IHqlExpression * rowPayloadAttr = curRow->queryProperty(_payload_Atom);
+    IHqlExpression * recordPayloadAttr = record->queryProperty(_payload_Atom);
+    if (rowPayloadAttr)
+    {
+        unsigned rowPayload =  (unsigned) getIntValue(rowPayloadAttr->queryChild(0));
+        col++;
+        if (recordPayloadAttr)
+        {
+            unsigned recordPayload =  (unsigned) getIntValue(recordPayloadAttr->queryChild(0));
+            if (rowPayload != recordPayload)
+                ERRORAT(curRow->queryProperty(_location_Atom), HQLERR_PayloadMismatch);
+        }
+        else
+            ERRORAT(curRow->queryProperty(_location_Atom), HQLERR_PayloadMismatch);
+    }
+    else if (recordPayloadAttr)
+        ERRORAT(curRow->queryProperty(_location_Atom), HQLERR_PayloadMismatch);
+
     OwnedHqlExpr ret = createTempTableTransform(self, curRow, record, col, self, mapping, true);
     if (queryRealChild(curRow, col))
     {
@@ -5207,11 +5236,13 @@ void TempTableTransformer::createTempTableAssign(HqlExprArray & assigns, IHqlExp
                     }
                     else if (type->isScalar() != src->queryType()->isScalar())
                     {
-//                  if (!type->assignableFrom(src->queryType()))            // stricter would be better, but might cause problems.
                         ERRORAT1(curRow->queryProperty(_location_Atom), HQLERR_IncompatibleTypesForField, expr->queryName()->str());
                         return;
                     }
-
+                    else if (strictTypeChecking && !type->assignableFrom(src->queryType()))
+                    {
+                        ERRORAT1(curRow->queryProperty(_location_Atom), HQLERR_IncompatibleTypesForField, expr->queryName()->str());
+                    }
                     castValue.setown(ensureExprType(src, type));
                 }
                 else
@@ -5293,11 +5324,51 @@ void TempTableTransformer::reportWarning(IHqlExpression * location, int code,con
     errors->reportWarning(code, errorMsg.str(), where->sourcePath->str(), where->lineno, where->column, where->position);
 }
 
+IHqlExpression *getDictionaryKeyRecord(IHqlExpression *record)
+{
+    // MORE - should probably use an attr to cache this?
+    IHqlExpression * payload = record->queryProperty(_payload_Atom);
+    unsigned payloadSize = payload ? getIntValue(payload->queryChild(0)) : 0;
+    unsigned max = record->numChildren() - payloadSize;
+    IHqlExpression *newrec = createRecord();
+    for (unsigned idx = 0; idx < max; idx++)
+    {
+        IHqlExpression *child = record->queryChild(idx);
+        if (!child->isAttribute() || child->queryName()!=_payload_Atom)  // Strip off the payload attribute
+            newrec->addOperand(LINK(child));
+    }
+    return newrec->closeExpr();
+}
+
+IHqlExpression * createSelectMapRow(IErrorReceiver * errors, ECLlocation & location, IHqlExpression * dict, IHqlExpression *values)
+{
+    OwnedHqlExpr record = getDictionaryKeyRecord(dict->queryRecord());
+    TempTableTransformer transformer(errors, location, true);
+    OwnedHqlExpr newTransform = transformer.createTempTableTransform(values, record);
+    return createRow(no_selectmap, dict, createRow(no_createrow, newTransform.getClear()));
+}
+
+IHqlExpression *createINDictExpr(IErrorReceiver * errors, ECLlocation & location, IHqlExpression *expr, IHqlExpression *dict)
+{
+    OwnedHqlExpr record = getDictionaryKeyRecord(dict->queryRecord());
+    TempTableTransformer transformer(errors, location, true);
+    OwnedHqlExpr newTransform = transformer.createTempTableTransform(expr, record);
+    return createBoolExpr(no_indict, createRow(no_createrow, newTransform.getClear()), dict);
+}
+
+IHqlExpression *createINDictRow(IErrorReceiver * errors, ECLlocation & location, IHqlExpression *row, IHqlExpression *dict)
+{
+    OwnedHqlExpr record = getDictionaryKeyRecord(dict->queryRecord());
+    if (!record->queryType()->assignableFrom(row->queryType()))
+        errors->reportError(ERR_TYPE_DIFFER, "Type mismatch", location.sourcePath->str(), location.lineno, location.column, location.position);
+    return createBoolExpr(no_indict, row, dict);
+}
+
 IHqlExpression * convertTempRowToCreateRow(IErrorReceiver * errors, ECLlocation & location, IHqlExpression * expr)
 {
     IHqlExpression * oldValues = expr->queryChild(0);
     IHqlExpression * record = expr->queryChild(1);
-    OwnedHqlExpr values = normalizeListCasts(oldValues);
+    OwnedHqlExpr values = normalizeListCasts(oldValues); // ??? not used
 
     TempTableTransformer transformer(errors, location);
     OwnedHqlExpr newTransform = transformer.createTempTableTransform(oldValues, record);
@@ -5307,7 +5378,7 @@ IHqlExpression * convertTempRowToCreateRow(IErrorReceiver * errors, ECLlocation
     return expr->cloneAllAnnotations(ret);
 }
 
-IHqlExpression * convertTempTableToInlineTable(IErrorReceiver * errors, ECLlocation & location, IHqlExpression * expr)
+static IHqlExpression * convertTempTableToInline(IErrorReceiver * errors, ECLlocation & location, IHqlExpression * expr, bool isDictionary)
 {
     IHqlExpression * oldValues = expr->queryChild(0);
     IHqlExpression * record = expr->queryChild(1);
@@ -5344,10 +5415,40 @@ IHqlExpression * convertTempTableToInlineTable(IErrorReceiver * errors, ECLlocat
     HqlExprArray children;
     children.append(*createValue(no_transformlist, makeNullType(), transforms));
     children.append(*LINK(record));
-    OwnedHqlExpr ret = createDataset(no_inlinetable, children);
+    OwnedHqlExpr ret = isDictionary ? createDictionary(no_inlinedictionary, children) : createDataset(no_inlinetable, children);
     return expr->cloneAllAnnotations(ret);
 }
 
+IHqlExpression * convertTempTableToInlineTable(IErrorReceiver * errors, ECLlocation & location, IHqlExpression * expr)
+{
+    return convertTempTableToInline(errors, location, expr, false);
+}
+
+IHqlExpression * convertTempTableToInlineDictionary(IErrorReceiver * errors, ECLlocation & location, IHqlExpression * expr)
+{
+    return convertTempTableToInline(errors, location, expr, true);
+}
+
+void setPayloadAttribute(HqlExprArray &args)
+{
+    // Locate a payload attribute in  an initializer value list. If found, move it to front and give it a position
+    int payloadPos = -1;
+    ForEachItemIn(idx, args)
+    {
+        IHqlExpression *cur = &args.item(idx);
+        if (cur->isAttribute())
+        {
+            assertex(payloadPos==-1);
+            assertex(cur->queryName()==_payload_Atom);
+            payloadPos = idx;
+        }
+    }
+    if (payloadPos != -1)
+    {
+        args.remove(payloadPos);
+        args.add(*createAttribute(_payload_Atom, createConstant((__int64) args.length()-payloadPos)), 0);
+    }
+}
 
 bool areTypesComparable(ITypeInfo * leftType, ITypeInfo * rightType)
 {
@@ -5377,6 +5478,7 @@ bool areTypesComparable(ITypeInfo * leftType, ITypeInfo * rightType)
     case type_array:
         return areTypesComparable(leftType->queryChildType(), rightType->queryChildType());
     case type_row:
+    case type_dictionary:
     case type_table:
     case type_groupedtable:
         return recordTypesMatch(leftType, rightType);
@@ -6964,6 +7066,7 @@ void EclXmlSchemaBuilder::build(IHqlExpression * record) const
                         builder.addSetField(name, childName, *type);
                         break;
                     }
+                case type_dictionary:
                 case type_table:
                 case type_groupedtable:
                     {
@@ -7300,6 +7403,7 @@ bool ConstantRowCreator::processElement(IHqlExpression * expr, IHqlExpression *
                 if (rhsOp == no_createrow)
                     return createConstantRow(out, rhs->queryChild(0));
                 return false;
+            case type_dictionary:
             case type_table:
             case type_groupedtable:
                 if (!expr->hasProperty(countAtom) && !expr->hasProperty(sizeofAtom))

ecl/hql/hqlutil.hpp

@@ -447,8 +447,14 @@ extern HQL_API IHqlExpression * extractCppBodyAttrs(unsigned len, const char * v
 extern HQL_API unsigned cleanupEmbeddedCpp(unsigned len, char * buffer);
 extern HQL_API bool isNullList(IHqlExpression * expr);
 
+extern HQL_API IHqlExpression * createSelectMapRow(IErrorReceiver * errors, ECLlocation & location, IHqlExpression * dict, IHqlExpression *values);
+extern HQL_API IHqlExpression * createINDictExpr(IErrorReceiver * errors, ECLlocation & location, IHqlExpression *expr, IHqlExpression *dict);
+extern HQL_API IHqlExpression *createINDictRow(IErrorReceiver * errors, ECLlocation & location, IHqlExpression *row, IHqlExpression *dict);
 extern HQL_API IHqlExpression * convertTempRowToCreateRow(IErrorReceiver * errors, ECLlocation & location, IHqlExpression * expr);
 extern HQL_API IHqlExpression * convertTempTableToInlineTable(IErrorReceiver * errors, ECLlocation & location, IHqlExpression * expr);
+extern HQL_API IHqlExpression * convertTempTableToInlineDictionary(IErrorReceiver * errors, ECLlocation & location, IHqlExpression * expr);
+extern HQL_API void setPayloadAttribute(HqlExprArray &args);
+
 extern HQL_API bool areTypesComparable(ITypeInfo * leftType, ITypeInfo * rightType);
 extern HQL_API bool arraysMatch(const HqlExprArray & left, const HqlExprArray & right);
 extern HQL_API IHqlExpression * ensureTransformType(IHqlExpression * transform, node_operator op);

ecl/hqlcpp/hqlcatom.cpp

@@ -239,6 +239,9 @@ _ATOM deserializerSkipVUniAtom;
 _ATOM destroyRegexAtom;
 _ATOM destroyWRegexAtom;
 _ATOM destructMetaMemberAtom;
+_ATOM dictionaryCountAtom;
+_ATOM dictionaryLookupAtom;
+_ATOM dictionaryLookupExistsAtom;
 _ATOM doNotifyAtom;
 _ATOM doNotifyTargetAtom;
 _ATOM ebcdic2asciiAtom;
@@ -944,6 +947,9 @@ MODULE_INIT(INIT_PRIORITY_HQLATOM-1)
     MAKEATOM(destroyRegex);
     MAKEATOM(destroyWRegex);
     MAKEATOM(destructMetaMember);
+    MAKEATOM(dictionaryCount);
+    MAKEATOM(dictionaryLookup);
+    MAKEATOM(dictionaryLookupExists);
     MAKEATOM(doNotify);
     MAKEATOM(doNotifyTarget);
     MAKEATOM(ebcdic2ascii);

ecl/hqlcpp/hqlcatom.hpp

@@ -239,6 +239,9 @@ extern _ATOM deserializerSkipVUniAtom;
 extern _ATOM destroyRegexAtom;
 extern _ATOM destroyWRegexAtom;
 extern _ATOM destructMetaMemberAtom;
+extern _ATOM dictionaryCountAtom;
+extern _ATOM dictionaryLookupAtom;
+extern _ATOM dictionaryLookupExistsAtom;
 extern _ATOM doNotifyAtom;
 extern _ATOM doNotifyTargetAtom;
 extern _ATOM ebcdic2asciiAtom;

ecl/hqlcpp/hqlcpp.cpp

@@ -398,6 +398,7 @@ IHqlExpression * getPointer(IHqlExpression * source)
             return createValue(no_typetransfer, LINK(newType), LINK(cast));
         }
         break;
+    case type_dictionary:
     case type_table:
     case type_groupedtable:
     case type_row:
@@ -765,6 +766,8 @@ IHqlExpression * createTranslated(IHqlExpression * expr, IHqlExpression * length
     ITypeInfo * type = expr->queryType();
     switch (type->getTypeCode())
     {
+    case type_dictionary:
+        return createDictionary(no_translated, LINK(expr), LINK(length));
     case type_table:
     case type_groupedtable:
         return createDataset(no_translated, LINK(expr), LINK(length));
@@ -854,6 +857,7 @@ bool isNullAssign(const CHqlBoundTarget & target, IHqlExpression * expr)
         case type_string:
         case type_qstring:
             return exprType->getSize() == 0;
+        case type_dictionary:
         case type_table:
             return expr->getOperator() == no_null;
         }
@@ -895,6 +899,8 @@ IHqlExpression * CHqlBoundExpr::getTranslatedExpr() const
     ITypeInfo * type = expr->queryType();
     switch (type->getTypeCode())
     {
+    case type_dictionary:
+        return createDictionary(no_translated, args);
     case type_table:
     case type_groupedtable:
         return createDataset(no_translated, args);
@@ -2813,6 +2819,10 @@ void HqlCppTranslator::buildExpr(BuildCtx & ctx, IHqlExpression * expr, CHqlBoun
         if (!(expr->isPure() && ctx.getMatchExpr(expr, tgt)))
             doBuildExprExists(ctx, expr, tgt);
         return;
+    case no_countdict:
+        if (!(expr->isPure() && ctx.getMatchExpr(expr, tgt)))
+            doBuildExprCountDict(ctx, expr, tgt);
+        return;
     case no_existslist:
         doBuildAggregateList(ctx, NULL, expr, &tgt);
         return;
@@ -3050,6 +3060,9 @@ void HqlCppTranslator::buildExpr(BuildCtx & ctx, IHqlExpression * expr, CHqlBoun
             }
             return;
         }
+    case no_indict:
+        doBuildExprInDict(ctx, expr, tgt);
+        return;
     case no_case:
     case no_choose:
     case no_concat:
@@ -4057,6 +4070,7 @@ IHqlExpression * HqlCppTranslator::createWrapperTemp(BuildCtx & ctx, ITypeInfo *
     case type_set:  //needed if we have sets with link counted elements
     case type_row:
     case type_array:
+    case type_dictionary:
     case type_table:
     case type_groupedtable:
         //Ensure row and dataset temporaries are active throughout a function, so pointers to the row
@@ -4081,6 +4095,7 @@ void HqlCppTranslator::createTempFor(BuildCtx & ctx, ITypeInfo * _exprType, CHql
     switch (tc)
     {
     case type_array:
+    case type_dictionary:
     case type_table:
     case type_groupedtable:
         {
@@ -4136,6 +4151,7 @@ void HqlCppTranslator::createTempFor(BuildCtx & ctx, ITypeInfo * _exprType, CHql
         case type_array:
         case type_table:
         case type_groupedtable:
+        case type_dictionary:
             break;
         default:
             {
@@ -4174,6 +4190,7 @@ void HqlCppTranslator::createTempFor(BuildCtx & ctx, ITypeInfo * _exprType, CHql
     case type_array:
     case type_table:
     case type_groupedtable:
+    case type_dictionary:
         {
             OwnedITypeInfo lenType = makeModifier(LINK(sizetType), modifier);
             target.expr.setown(createWrapperTemp(ctx, exprType, modifier));
@@ -4243,6 +4260,7 @@ void HqlCppTranslator::buildTempExpr(BuildCtx & ctx, BuildCtx & declareCtx, CHql
         break;
     case type_table:
     case type_groupedtable:
+    case type_dictionary:
         {
             createTempFor(declareCtx, type, tempTarget, modifier, format);
             IHqlStmt * stmt = subctx.addGroup();
@@ -4721,6 +4739,7 @@ void HqlCppTranslator::doBuildExprCompare(BuildCtx & ctx, IHqlExpression * expr,
                     return;
                 //fallthrough....
             }
+        case type_dictionary:
         case type_table:
         case type_groupedtable:
         case type_row:
@@ -5113,6 +5132,23 @@ void HqlCppTranslator::doBuildAssignIn(BuildCtx & ctx, const CHqlBoundTarget & t
 
 //---------------------------------------------------------------------------
 
+void HqlCppTranslator::doBuildExprInDict(BuildCtx & ctx, IHqlExpression * expr, CHqlBoundExpr & tgt)
+{
+    IHqlExpression *dict = expr->queryChild(1);
+    Owned<IHqlCppDatasetCursor> cursor = createDatasetSelector(ctx, dict);
+    cursor->buildInDataset(ctx, expr, tgt);
+}
+
+void HqlCppTranslator::doBuildExprCountDict(BuildCtx & ctx, IHqlExpression * expr, CHqlBoundExpr & tgt)
+{
+    IHqlExpression *dict = expr->queryChild(0);
+    Owned<IHqlCppDatasetCursor> cursor = createDatasetSelector(ctx, dict);
+    cursor->buildCountDict(ctx, tgt); // not the same as buildCount - that is the size of the table, we want the number of populated entries
+}
+
+
+//---------------------------------------------------------------------------
+
 void HqlCppTranslator::doBuildExprArith(BuildCtx & ctx, IHqlExpression * expr, CHqlBoundExpr & tgt)
 {
     ITypeInfo * type = expr->queryType();
@@ -5371,6 +5407,7 @@ IHqlExpression * getCastParameter(IHqlExpression * curParam, ITypeInfo * argType
                 ((ptc == type_varstring) && (argType->queryCharset() == paramType->queryCharset())))
                 return LINK(curParam);
             break;
+        case type_dictionary:
         case type_row:
         case type_table:
         case type_groupedtable:
@@ -5430,6 +5467,7 @@ IHqlExpression * getCastParameter(IHqlExpression * curParam, ITypeInfo * argType
             if (childType)
                 return ensureExprType(curParam, argType);
             break;
+        case type_dictionary:
         case type_table:
         case type_groupedtable:
         case type_row:
@@ -5657,6 +5695,7 @@ void HqlCppTranslator::doBuildCall(BuildCtx & ctx, const CHqlBoundTarget * tgt,
             returnByReference = true;
             break;
         }
+    case type_dictionary:
     case type_table:
     case type_groupedtable:
         {
@@ -5779,6 +5818,7 @@ void HqlCppTranslator::doBuildCall(BuildCtx & ctx, const CHqlBoundTarget * tgt,
         type_t atc = argType->getTypeCode();
         switch (atc)
         {
+        case type_dictionary:
         case type_table:
         case type_groupedtable:
             {
@@ -5788,8 +5828,16 @@ void HqlCppTranslator::doBuildCall(BuildCtx & ctx, const CHqlBoundTarget * tgt,
             }
         case type_row:
             {
-                Owned<IReferenceSelector> selector = buildNewRow(ctx, castParam);
-                selector->buildAddress(ctx, bound);
+                if (hasLinkCountedModifier(argType))
+                {
+                    doBuildAliasValue(ctx, castParam, bound);
+//                    buildTempExpr(ctx, castParam, bound, FormatLinkedDataset);
+                }
+                else
+                {
+                    Owned<IReferenceSelector> selector = buildNewRow(ctx, castParam);
+                    selector->buildAddress(ctx, bound);
+                }
     //          buildExpr(ctx, castParam, bound);       // more this needs more work I think
                 break;
             }
@@ -5858,6 +5906,7 @@ void HqlCppTranslator::doBuildCall(BuildCtx & ctx, const CHqlBoundTarget * tgt,
                 }
                 break;
             }
+        case type_dictionary:
         case type_table:
         case type_groupedtable:
             {
@@ -7991,6 +8040,7 @@ void HqlCppTranslator::doBuildAssignCompareElement(BuildCtx & ctx, EvaluateCompa
     bool useMemCmp = false;
     switch (tc)
     {
+    case type_dictionary:
     case type_table:
     case type_groupedtable:
         doBuildAssignCompareTable(ctx, info, left, right);
@@ -8736,6 +8786,7 @@ void HqlCppTranslator::doBuildAssignHashElement(BuildCtx & ctx, HashCodeCreator
         case type_row:
             throwUnexpected();
             break;
+        case type_dictionary:
         case type_groupedtable:
         case type_table:
             //MORE: Should be handle this differently, with an iterator for the link counted rows case?
@@ -10980,6 +11031,7 @@ void HqlCppTranslator::assignCastUnknownLength(BuildCtx & ctx, const CHqlBoundTa
             }
             break;
 
+        case type_dictionary:
         case type_table:
         case type_groupedtable:
             {
@@ -11205,6 +11257,7 @@ static IHqlExpression * replaceInlineParameters(IHqlExpression * funcdef, IHqlEx
         case type_data:
         case type_unicode:
         case type_utf8:
+        case type_dictionary:
         case type_table:
         case type_groupedtable:
             if (paramType->getSize() == UNKNOWN_LENGTH)
@@ -11393,6 +11446,7 @@ IHqlExpression * HqlCppTranslator::getBoundCount(const CHqlBoundExpr & bound)
                 return getSizetConstant(size / type->queryChildType()->getSize());
             UNIMPLEMENTED;
         }
+    case type_dictionary:
     case type_table:
     case type_groupedtable:
     case type_set:
@@ -11429,6 +11483,7 @@ IHqlExpression * HqlCppTranslator::getBoundLength(const CHqlBoundExpr & bound)
         }
     case type_set:
     case type_array:
+    case type_dictionary:
     case type_table:
     case type_groupedtable:
         assertex(!isArrayRowset(type));

ecl/hqlcpp/hqlcpp.ipp

@@ -173,8 +173,11 @@ interface IHqlCppDatasetCursor : public IInterface
     virtual void buildExists(BuildCtx & ctx, CHqlBoundExpr & tgt) = 0;
     virtual BoundRow * buildIterateLoop(BuildCtx & ctx, bool needToBreak) = 0;
     virtual void buildIterateClass(BuildCtx & ctx, SharedHqlExpr & iter, SharedHqlExpr & row) = 0;
-    virtual BoundRow * buildSelect(BuildCtx & ctx, IHqlExpression * indexExpr) = 0;
+    virtual BoundRow * buildSelectNth(BuildCtx & ctx, IHqlExpression * indexExpr) = 0;
+    virtual BoundRow * buildSelectMap(BuildCtx & ctx, IHqlExpression * indexExpr) = 0;
+    virtual void buildInDataset(BuildCtx & ctx, IHqlExpression * inExpr, CHqlBoundExpr & tgt) = 0;
     virtual void buildIterateMembers(BuildCtx & declarectx, BuildCtx & initctx) = 0;
+    virtual void buildCountDict(BuildCtx & ctx, CHqlBoundExpr & tgt) = 0;
 };
 
 interface IHqlCppSetCursor : public IInterface
@@ -838,6 +841,8 @@ public:
     IHqlExpression * ensureIteratedRowIsLive(BuildCtx & initctx, BuildCtx & searchctx, BuildCtx & iterctx, BoundRow * row, IHqlExpression * dataset, IHqlExpression * rowExpr);
     BoundRow * buildOptimizeSelectFirstRow(BuildCtx & ctx, IHqlExpression * expr);
 
+    IReferenceSelector * buildDatasetSelectMap(BuildCtx & ctx, IHqlExpression * expr);
+
 // Helper functions
 
     void ThrowStringException(int code,const char *format, ...) __attribute__((format(printf, 3, 4)));            // override the global function to try and add more context information
@@ -1089,6 +1094,7 @@ public:
     IHqlCppDatasetBuilder * createChoosenDatasetBuilder(IHqlExpression * record, IHqlExpression * maxCount);
     IHqlCppDatasetBuilder * createLimitedDatasetBuilder(IHqlExpression * record, IHqlExpression * maxCount);
     IHqlCppDatasetBuilder * createLinkedDatasetBuilder(IHqlExpression * record, IHqlExpression * choosenLimit = NULL);
+    IHqlCppDatasetBuilder * createLinkedDictionaryBuilder(IHqlExpression * record);
     IReferenceSelector * createSelfSelect(BuildCtx & ctx, IReferenceSelector * target, IHqlExpression * expr, IHqlExpression * rootSelector);
     IReferenceSelector * createReferenceSelector(BoundRow * cursor, IHqlExpression * path);
     IReferenceSelector * createReferenceSelector(BoundRow * cursor);
@@ -1248,6 +1254,7 @@ public:
     void doBuildExprCast(BuildCtx & ctx, ITypeInfo * type, CHqlBoundExpr & pure, CHqlBoundExpr & tgt);
     void doBuildExprCompare(BuildCtx & ctx, IHqlExpression * expr, CHqlBoundExpr & tgt);
     void doBuildExprCompareElement(BuildCtx & ctx, node_operator comp_op, IHqlExpression * lhs, IHqlExpression * rhs, CHqlBoundExpr & tgt);
+    void doBuildExprCountDict(BuildCtx & ctx, IHqlExpression * expr, CHqlBoundExpr & tgt);
     void doBuildExprCount(BuildCtx & ctx, IHqlExpression * expr, CHqlBoundExpr & tgt);
     void doBuildExprCounter(BuildCtx & ctx, IHqlExpression * expr, CHqlBoundExpr & tgt);
     void doBuildExprCppBody(BuildCtx & ctx, IHqlExpression * expr, CHqlBoundExpr * tgt);
@@ -1264,6 +1271,7 @@ public:
     void doBuildExprIdToBlob(BuildCtx & ctx, IHqlExpression * expr, CHqlBoundExpr & tgt);
     void doBuildExprIf(BuildCtx & ctx, IHqlExpression * expr, CHqlBoundExpr & tgt);
     void doBuildExprIndex(BuildCtx & ctx, IHqlExpression * expr, CHqlBoundExpr & tgt);
+    void doBuildExprInDict(BuildCtx & ctx, IHqlExpression * expr, CHqlBoundExpr & tgt);
     void doBuildExprIsValid(BuildCtx & ctx, IHqlExpression * expr, CHqlBoundExpr & tgt);
     void doBuildExprList(BuildCtx & ctx, IHqlExpression * expr, CHqlBoundExpr & tgt);
     void doBuildExprConstList(BuildCtx & ctx, IHqlExpression * expr, CHqlBoundExpr & tgt);
@@ -1561,6 +1569,7 @@ public:
     void buildConnectOrders(BuildCtx & ctx, ABoundActivity * slaveActivity, ABoundActivity * masterActivity);
     void buildDedupFilterFunction(BuildCtx & ctx, HqlExprArray & equalities, HqlExprArray & conds, IHqlExpression * dataset, IHqlExpression * selSeq);
     void buildDedupSerializeFunction(BuildCtx & ctx, const char * funcName, IHqlExpression * srcDataset, IHqlExpression * tgtDataset, HqlExprArray & srcValues, HqlExprArray & tgtValues, IHqlExpression * selSeq);
+    void buildDictionaryHashClass(BuildCtx &ctx, IHqlExpression *record, IHqlExpression *dictionary, StringBuffer &lookupHelperName);
     void buildHashClass(BuildCtx & ctx, const char * name, IHqlExpression * orderExpr, const DatasetReference & dataset);
     void buildHashOfExprsClass(BuildCtx & ctx, const char * name, IHqlExpression * cond, const DatasetReference & dataset, bool compareToSelf);
     void buildInstancePrefix(ActivityInstance * instance);

ecl/hqlcpp/hqlcppds.cpp

@@ -391,6 +391,8 @@ IReferenceSelector * HqlCppTranslator::buildNewRow(BuildCtx & ctx, IHqlExpressio
     case no_index:
     case no_selectnth:
         return buildDatasetIndex(ctx, expr);
+    case no_selectmap:
+        return buildDatasetSelectMap(ctx, expr);
     case no_left:
     case no_right:
     case no_self:
@@ -557,7 +559,7 @@ IReferenceSelector * HqlCppTranslator::buildActiveRow(BuildCtx & ctx, IHqlExpres
     case no_activerow:
         return buildActiveRow(ctx, expr->queryChild(0));
     default:
-        if (!expr->isDataset())
+        if (!expr->isDataset() && !expr->isDictionary())
             return buildNewRow(ctx, expr);
         break;
     }
@@ -2100,15 +2102,22 @@ void HqlCppTranslator::doBuildDataset(BuildCtx & ctx, IHqlExpression * expr, CHq
 
             if (format == FormatLinkedDataset || format == FormatArrayDataset)
             {
-                IHqlExpression * choosenLimit = NULL;
-                if ((op == no_choosen) && !isChooseNAllLimit(expr->queryChild(1)) && !queryRealChild(expr, 2))
+                if (expr->isDictionary())
                 {
-                    choosenLimit = expr->queryChild(1);
-                    expr = expr->queryChild(0);
+                    builder.setown(createLinkedDictionaryBuilder(record));
                 }
+                else
+                {
+                    IHqlExpression * choosenLimit = NULL;
+                    if ((op == no_choosen) && !isChooseNAllLimit(expr->queryChild(1)) && !queryRealChild(expr, 2))
+                    {
+                        choosenLimit = expr->queryChild(1);
+                        expr = expr->queryChild(0);
+                    }
 
-                //MORE: Extract limit and choosen and pass as parameters
-                builder.setown(createLinkedDatasetBuilder(record, choosenLimit));
+                    //MORE: Extract limit and choosen and pass as parameters
+                    builder.setown(createLinkedDatasetBuilder(record, choosenLimit));
+                }
             }
             else if ((op == no_choosen) && !isChooseNAllLimit(expr->queryChild(1)) && !queryRealChild(expr, 2))
             {
@@ -2171,6 +2180,7 @@ void HqlCppTranslator::buildDatasetAssign(BuildCtx & ctx, const CHqlBoundTarget
             ctx.addAssign(target.expr, bound.expr);
             return;
         }
+    case no_inlinedictionary:
     case no_inlinetable:
         {
             //This will typically generate a loop.  If few items then it is more efficient to expand the assigns/clones out.
@@ -2401,14 +2411,20 @@ void HqlCppTranslator::buildDatasetAssign(BuildCtx & ctx, const CHqlBoundTarget
     Owned<IHqlCppDatasetBuilder> builder;
     if (targetOutOfLine)
     {
-        IHqlExpression * choosenLimit = NULL;
-        if ((op == no_choosen) && !isChooseNAllLimit(expr->queryChild(1)) && !queryRealChild(expr, 2))
+        if (target.queryType()->getTypeCode() == type_dictionary)
         {
-            choosenLimit = expr->queryChild(1);
-            expr = expr->queryChild(0);
+            builder.setown(createLinkedDictionaryBuilder(record));
+        }
+        else
+        {
+            IHqlExpression * choosenLimit = NULL;
+            if ((op == no_choosen) && !isChooseNAllLimit(expr->queryChild(1)) && !queryRealChild(expr, 2))
+            {
+                choosenLimit = expr->queryChild(1);
+                expr = expr->queryChild(0);
+            }
+            builder.setown(createLinkedDatasetBuilder(record, choosenLimit));
         }
-
-        builder.setown(createLinkedDatasetBuilder(record, choosenLimit));
     }
     else
         builder.setown(createBlockedDatasetBuilder(record));
@@ -2649,6 +2665,11 @@ void HqlCppTranslator::buildDatasetAssignProject(BuildCtx & ctx, IHqlCppDatasetB
 
     if (sourceCursor)
     {
+        if (isNullProject(expr, false))
+        {
+            if (target->buildLinkRow(iterctx, sourceCursor))
+                return;
+        }
         BoundRow * targetRow = target->buildCreateRow(iterctx);
         HqlExprAssociation * skipAssociation = NULL;
         if (containsSkip)
@@ -2663,6 +2684,7 @@ void HqlCppTranslator::buildDatasetAssignProject(BuildCtx & ctx, IHqlCppDatasetB
         case no_hqlproject:
             doBuildRowAssignProject(iterctx, targetRef, expr);
             break;
+        case no_newuserdictionary:
         case no_newusertable:
             doBuildRowAssignUserTable(iterctx, targetRef, expr);
             break;
@@ -2756,6 +2778,7 @@ void HqlCppTranslator::buildDatasetAssign(BuildCtx & ctx, IHqlCppDatasetBuilder
         buildDatasetAssignTempTable(subctx, target, expr);
         //MORE: Create rows and assign each one in turn.  Could possibly be done with a different dataset selector
         return;
+    case no_inlinedictionary:
     case no_inlinetable:
         buildDatasetAssignInlineTable(subctx, target, expr);
         return;
@@ -2797,6 +2820,7 @@ void HqlCppTranslator::buildDatasetAssign(BuildCtx & ctx, IHqlCppDatasetBuilder
         return;
     case no_hqlproject:
     case no_newusertable:
+    case no_newuserdictionary:
         buildDatasetAssignProject(subctx, target, expr);
         return;
     case no_compound_childread:
@@ -4180,6 +4204,7 @@ IReferenceSelector * HqlCppTranslator::buildDatasetIndex(BuildCtx & ctx, IHqlExp
             specialCase = !isMultiLevelDatasetSelector(expr, false);
             break;
         case no_if:
+        case no_inlinedictionary:
         case no_inlinetable:
         case no_join:
             //Always creates a temporary, so don't use an iterator
@@ -4210,7 +4235,33 @@ IReferenceSelector * HqlCppTranslator::buildDatasetIndex(BuildCtx & ctx, IHqlExp
     if (!row)
     {
         Owned<IHqlCppDatasetCursor> cursor = createDatasetSelector(ctx, dataset);
-        row = cursor->buildSelect(ctx, expr);
+        row = cursor->buildSelectNth(ctx, expr);
+
+        if (!row)
+        {
+            CHqlBoundExpr boundCleared;
+            buildDefaultRow(ctx, dataset, boundCleared);
+            OwnedHqlExpr defaultRowPtr = getPointer(boundCleared.expr);
+            row = bindRow(ctx, expr, defaultRowPtr);
+        }
+    }
+    return createReferenceSelector(row);
+}
+
+IReferenceSelector * HqlCppTranslator::buildDatasetSelectMap(BuildCtx & ctx, IHqlExpression * expr)
+{
+    HqlExprAssociation * match = ctx.queryAssociation(expr, AssocRow, NULL);
+    if (match)
+        return createReferenceSelector(static_cast<BoundRow *>(match));
+
+    OwnedHqlExpr dataset = normalizeAnyDatasetAliases(expr->queryChild(0));
+    BoundRow * row = NULL;
+    assertex(canProcessInline(&ctx, expr));
+
+    if (!row)
+    {
+        Owned<IHqlCppDatasetCursor> cursor = createDatasetSelector(ctx, dataset);
+        row = cursor->buildSelectMap(ctx, expr);
 
         if (!row)
         {
@@ -4327,6 +4378,7 @@ void HqlCppTranslator::doBuildExprGetGraphResult(BuildCtx & ctx, IHqlExpression
     {
     case type_row:
         throwUnexpected();
+    case type_dictionary:
     case type_table:
     case type_groupedtable:
         buildTempExpr(ctx, call, tgt);

ecl/hqlcpp/hqlcppsys.ecl

@@ -796,6 +796,11 @@ const char * cppSystemText[]  = {
     "   unsigned4 serializerBeginNested() : omethod,entrypoint='beginNested';",
     "   serializerEndNested(unsigned4 pos) : omethod,entrypoint='endNested';",
 
+    // Dictionary support
+    "    integer8 dictionaryCount(_linkcounted_ dictionary dict) : eclrtl,include,pure,entrypoint='rtlDictionaryCount';",
+    "   _linkcounted_ row(dummyRecord) dictionaryLookup(boolean meta, _linkcounted_ dictionary dict, row key, _linkcounted_ row defaultrow) : eclrtl,include,pure,entrypoint='rtlDictionaryLookup';",
+    "    boolean dictionaryLookupExists(boolean meta, _linkcounted_ dictionary dict, row key) : eclrtl,include,pure,entrypoint='rtlDictionaryLookupExists';",
+
     "   END;",
     NULL };
 

ecl/hqlcpp/hqlcset.cpp

@@ -128,7 +128,25 @@ void BaseDatasetCursor::buildIterateMembers(BuildCtx & declarectx, BuildCtx & in
     translator.bindTableCursor(declarectx, ds, row);
 }
 
-BoundRow * BaseDatasetCursor::buildSelect(BuildCtx & ctx, IHqlExpression * indexExpr)
+BoundRow * BaseDatasetCursor::buildSelectMap(BuildCtx & ctx, IHqlExpression * indexExpr)
+{
+    // Should only be seen for dictionaries, for now
+    throwUnexpected();
+}
+
+void BaseDatasetCursor::buildCountDict(BuildCtx & ctx, CHqlBoundExpr & tgt)
+{
+    // Should only be seen for dictionaries
+    throwUnexpected();
+}
+
+void BaseDatasetCursor::buildInDataset(BuildCtx & ctx, IHqlExpression * inExpr, CHqlBoundExpr & tgt)
+{
+    // Should only be seen for dictionaries, for now
+    throwUnexpected();
+}
+
+BoundRow * BaseDatasetCursor::buildSelectNth(BuildCtx & ctx, IHqlExpression * indexExpr)
 {
     //MORE: Check if the cursor already exists....
 
@@ -409,7 +427,7 @@ BoundRow * InlineBlockDatasetCursor::buildSelectFirst(BuildCtx & ctx, IHqlExpres
 }
 
 
-BoundRow * InlineBlockDatasetCursor::buildSelect(BuildCtx & ctx, IHqlExpression * indexExpr)
+BoundRow * InlineBlockDatasetCursor::buildSelectNth(BuildCtx & ctx, IHqlExpression * indexExpr)
 {
     assertex(!isArrayRowset(boundDs.expr->queryType()));        // I don't think this can ever be called at the moment
 
@@ -418,7 +436,7 @@ BoundRow * InlineBlockDatasetCursor::buildSelect(BuildCtx & ctx, IHqlExpression
     {
         if (matchesConstantValue(index, 1))
             return buildSelectFirst(ctx, indexExpr, CREATE_DEAULT_ROW_IF_NULL_VALUE);
-        return BlockDatasetCursor::buildSelect(ctx, indexExpr);
+        return BlockDatasetCursor::buildSelectNth(ctx, indexExpr);
     }
     if (matchesConstantValue(index, 1))
         return buildSelectFirst(ctx, indexExpr, CREATE_DEAULT_ROW_IF_NULL_VALUE);
@@ -560,7 +578,7 @@ void InlineLinkedDatasetCursor::buildIterateClass(BuildCtx & ctx, StringBuffer &
     ctx.addQuoted(decl);
 }
 
-BoundRow * InlineLinkedDatasetCursor::buildIterateLoop(BuildCtx & ctx, bool needToBreak)
+BoundRow * InlineLinkedDatasetCursor::doBuildIterateLoop(BuildCtx & ctx, bool needToBreak, bool checkForNull)
 {
     StringBuffer rowName;
     OwnedHqlExpr row = createRow(ctx, "row", rowName, false);
@@ -602,12 +620,14 @@ BoundRow * InlineLinkedDatasetCursor::buildIterateLoop(BuildCtx & ctx, bool need
 
     ctx.addLoop(test, NULL, false);
     ctx.addQuoted(s.clear().append(rowName).append(" = *").append(cursorName).append("++;"));
+    if (checkForNull)
+        ctx.addQuoted(s.clear().append("if (!").append(rowName).append(") continue;"));
     BoundRow * cursor = translator.bindTableCursor(ctx, ds, row);
 
     return cursor;
 }
 
-BoundRow * InlineLinkedDatasetCursor::buildSelect(BuildCtx & ctx, IHqlExpression * indexExpr)
+BoundRow * InlineLinkedDatasetCursor::buildSelectNth(BuildCtx & ctx, IHqlExpression * indexExpr)
 {
     OwnedHqlExpr index = foldHqlExpression(indexExpr->queryChild(1));
 
@@ -691,6 +711,60 @@ BoundRow * InlineLinkedDatasetCursor::buildSelect(BuildCtx & ctx, IHqlExpression
 }
 
 //---------------------------------------------------------------------------
+
+InlineLinkedDictionaryCursor::InlineLinkedDictionaryCursor(HqlCppTranslator & _translator, IHqlExpression * _ds, CHqlBoundExpr & _boundDs)
+  : InlineLinkedDatasetCursor(_translator, _ds, _boundDs)
+{
+}
+
+BoundRow * InlineLinkedDictionaryCursor::buildSelectMap(BuildCtx & ctx, IHqlExpression * mapExpr)
+{
+    Owned<BoundRow> tempRow = translator.declareLinkedRow(ctx, mapExpr, false);
+    IHqlExpression *record = ds->queryRecord();
+
+    StringBuffer lookupHelperName;
+    OwnedHqlExpr dict = createDictionary(no_null, LINK(record));
+    translator.buildDictionaryHashClass(ctx, record, dict, lookupHelperName);
+    CHqlBoundTarget target;
+    target.expr.set(tempRow->queryBound());
+
+    HqlExprArray args;
+    args.append(*createQuoted(lookupHelperName, makeBoolType()));
+    args.append(*LINK(mapExpr->queryChild(0)));
+    args.append(*LINK(mapExpr->queryChild(1)));
+    args.append(*::createRow(no_null, LINK(record)));
+    Owned<ITypeInfo> resultType = makeReferenceModifier(makeAttributeModifier(makeRowType(record->getType()), getLinkCountedAttr()));
+    OwnedHqlExpr call = translator.bindFunctionCall(dictionaryLookupAtom, args, resultType);
+    translator.buildExprAssign(ctx, target, call);
+    ctx.associate(*tempRow);
+    return tempRow.getClear();
+}
+
+void InlineLinkedDictionaryCursor::buildInDataset(BuildCtx & ctx, IHqlExpression * inExpr, CHqlBoundExpr & tgt)
+{
+    IHqlExpression *record = ds->queryRecord();
+
+    StringBuffer lookupHelperName;
+    OwnedHqlExpr dict = createDictionary(no_null, LINK(record));
+    translator.buildDictionaryHashClass(ctx, record, dict, lookupHelperName);
+
+    HqlExprArray args;
+    args.append(*createQuoted(lookupHelperName, makeBoolType()));
+    args.append(*LINK(inExpr->queryChild(1)));
+    args.append(*LINK(inExpr->queryChild(0)));
+    OwnedHqlExpr call = translator.bindFunctionCall(dictionaryLookupExistsAtom, args, makeBoolType());
+    translator.buildExpr(ctx, call, tgt);
+}
+
+void InlineLinkedDictionaryCursor::buildCountDict(BuildCtx & ctx, CHqlBoundExpr & tgt)
+{
+    HqlExprArray args;
+    args.append(*LINK(ds));
+    OwnedHqlExpr call = translator.bindFunctionCall(dictionaryCountAtom, args, makeBoolType());
+    translator.buildExpr(ctx, call, tgt);
+}
+
+//---------------------------------------------------------------------------
 MultiLevelDatasetCursor::MultiLevelDatasetCursor(HqlCppTranslator & _translator, IHqlExpression * _ds)
 : BaseDatasetCursor(_translator, _ds, NULL)
 {
@@ -721,7 +795,7 @@ BoundRow * MultiLevelDatasetCursor::buildIterateLoop(BuildCtx & ctx, bool needTo
     return doBuildIterateLoop(ctx, ds, breakVar, true);
 }
 
-BoundRow * MultiLevelDatasetCursor::buildSelect(BuildCtx & ctx, IHqlExpression * indexExpr)
+BoundRow * MultiLevelDatasetCursor::buildSelectNth(BuildCtx & ctx, IHqlExpression * indexExpr)
 {
     //Declare row for final level, iterate the appropriate number of times, and then assign and break.
     BuildCtx initctx(ctx);
@@ -1091,7 +1165,7 @@ void GeneralSetCursor::buildExprSelect(BuildCtx & ctx, IHqlExpression * indexExp
             checkNotAll(ctx);
 
         OwnedHqlExpr dsIndexExpr = createDatasetSelect(indexExpr);
-        BoundRow * cursor = dsCursor->buildSelect(ctx, dsIndexExpr);
+        BoundRow * cursor = dsCursor->buildSelectNth(ctx, dsIndexExpr);
         OwnedHqlExpr select = createSelectExpr(LINK(dsIndexExpr), LINK(element));
         translator.buildExpr(ctx, select, tgt);
     }
@@ -1109,7 +1183,7 @@ void GeneralSetCursor::buildAssignSelect(BuildCtx & ctx, const CHqlBoundTarget &
         checkNotAll(ctx);
 
     OwnedHqlExpr dsIndexExpr = createDatasetSelect(indexExpr);
-    BoundRow * cursor = dsCursor->buildSelect(ctx, dsIndexExpr);
+    BoundRow * cursor = dsCursor->buildSelectNth(ctx, dsIndexExpr);
 
     if (cursor)
     {
@@ -1299,7 +1373,10 @@ IHqlCppDatasetCursor * HqlCppTranslator::createDatasetSelector(BuildCtx & ctx, I
     buildDataset(ctx, expr, bound, FormatNatural);
     if (bound.expr->isDatarow() || !isArrayRowset(bound.expr->queryType()))
         return new InlineBlockDatasetCursor(*this, expr, bound);
-    return new InlineLinkedDatasetCursor(*this, expr, bound);
+    else if (bound.expr->isDictionary())
+        return new InlineLinkedDictionaryCursor(*this, expr, bound);
+    else
+        return new InlineLinkedDatasetCursor(*this, expr, bound);
 }
 
 
@@ -1359,6 +1436,12 @@ BoundRow * DatasetBuilderBase::buildDeserializeRow(BuildCtx & ctx, IHqlExpressio
 void DatasetBuilderBase::finishRow(BuildCtx & ctx, BoundRow * selfCursor)
 {
     OwnedHqlExpr size = createSizeof(selfCursor->querySelector());
+    doFinishRow(ctx, selfCursor, size);
+}
+
+
+void DatasetBuilderBase::doFinishRow(BuildCtx & ctx, BoundRow * selfCursor, IHqlExpression *size)
+{
     CHqlBoundExpr boundSize;
     translator.buildExpr(ctx, size, boundSize);
 
@@ -1369,7 +1452,6 @@ void DatasetBuilderBase::finishRow(BuildCtx & ctx, BoundRow * selfCursor)
     ctx.removeAssociation(selfCursor);
 }
 
-
 //---------------------------------------------------------------------------
 
 
@@ -1550,45 +1632,17 @@ void InlineDatasetBuilder::finishRow(BuildCtx & ctx, BoundRow * selfCursor)
 
 //---------------------------------------------------------------------------
 
-
-LinkedDatasetBuilder::LinkedDatasetBuilder(HqlCppTranslator & _translator, IHqlExpression * _record, IHqlExpression * _choosenLimit) : DatasetBuilderBase(_translator, _record, true)
+LinkedDatasetBuilderBase::LinkedDatasetBuilderBase(HqlCppTranslator & _translator, IHqlExpression * _record) : DatasetBuilderBase(_translator, _record, true)
 {
-    choosenLimit.set(_choosenLimit);
 }
 
-void LinkedDatasetBuilder::buildDeclare(BuildCtx & ctx)
-{
-    StringBuffer decl, allocatorName;
-
-    OwnedHqlExpr curActivityId = translator.getCurrentActivityId(ctx);
-    translator.ensureRowAllocator(allocatorName, ctx, record, curActivityId);
-
-    decl.append("RtlLinkedDatasetBuilder ").append(instanceName).append("(");
-    decl.append(allocatorName);
-    if (choosenLimit)
-    {
-        CHqlBoundExpr boundLimit;
-        translator.buildExpr(ctx, choosenLimit, boundLimit);
-        translator.generateExprCpp(decl.append(", "), boundLimit.expr);
-    }
-    decl.append(");");
-
-    ctx.addQuoted(decl);
-}
-
-void LinkedDatasetBuilder::finishRow(BuildCtx & ctx, BoundRow * selfCursor)
+void LinkedDatasetBuilderBase::finishRow(BuildCtx & ctx, BoundRow * selfCursor)
 {
     OwnedHqlExpr size = translator.getRecordSize(selfCursor->querySelector());
-    CHqlBoundExpr boundSize;
-    translator.buildExpr(ctx, size, boundSize);
-
-    StringBuffer s;
-    s.append(instanceName).append(".finalizeRow(");
-    translator.generateExprCpp(s, boundSize.expr).append(");");
-    ctx.addQuoted(s);
+    doFinishRow(ctx, selfCursor, size);
 }
 
-void LinkedDatasetBuilder::buildFinish(BuildCtx & ctx, const CHqlBoundTarget & target)
+void LinkedDatasetBuilderBase::buildFinish(BuildCtx & ctx, const CHqlBoundTarget & target)
 {
     //more: should I do this by really calling a function?
     StringBuffer s;
@@ -1613,7 +1667,7 @@ void LinkedDatasetBuilder::buildFinish(BuildCtx & ctx, const CHqlBoundTarget & t
 }
 
 
-void LinkedDatasetBuilder::buildFinish(BuildCtx & ctx, CHqlBoundExpr & bound)
+void LinkedDatasetBuilderBase::buildFinish(BuildCtx & ctx, CHqlBoundExpr & bound)
 {
     StringBuffer s;
     s.clear().append(instanceName).append(".getcount()");
@@ -1622,10 +1676,10 @@ void LinkedDatasetBuilder::buildFinish(BuildCtx & ctx, CHqlBoundExpr & bound)
     bound.expr.setown(createQuoted(s.str(), makeReferenceModifier(dataset->getType())));
 }
 
-bool LinkedDatasetBuilder::buildLinkRow(BuildCtx & ctx, BoundRow * sourceRow)
+bool LinkedDatasetBuilderBase::buildLinkRow(BuildCtx & ctx, BoundRow * sourceRow)
 {
     IHqlExpression * sourceRecord = sourceRow->queryRecord();
-    if (recordTypesMatch(sourceRecord, record) && sourceRow->isBinary())
+    if (recordTypesMatchIgnorePayload(sourceRecord, record) && sourceRow->isBinary())
     {
         OwnedHqlExpr source = getPointer(sourceRow->queryBound());
         BuildCtx subctx(ctx);
@@ -1659,7 +1713,7 @@ bool LinkedDatasetBuilder::buildLinkRow(BuildCtx & ctx, BoundRow * sourceRow)
     return false;
 }
 
-bool LinkedDatasetBuilder::buildAppendRows(BuildCtx & ctx, IHqlExpression * expr) 
+bool LinkedDatasetBuilderBase::buildAppendRows(BuildCtx & ctx, IHqlExpression * expr)
 {
     IHqlExpression * sourceRecord = expr->queryRecord();
     if (recordTypesMatch(sourceRecord, record))
@@ -1706,6 +1760,55 @@ bool LinkedDatasetBuilder::buildAppendRows(BuildCtx & ctx, IHqlExpression * expr
     return false;
 }
 
+LinkedDatasetBuilder::LinkedDatasetBuilder(HqlCppTranslator & _translator, IHqlExpression * _record, IHqlExpression * _choosenLimit) : LinkedDatasetBuilderBase(_translator, _record)
+{
+    choosenLimit.set(_choosenLimit);
+}
+
+void LinkedDatasetBuilder::buildDeclare(BuildCtx & ctx)
+{
+    StringBuffer decl, allocatorName;
+
+    OwnedHqlExpr curActivityId = translator.getCurrentActivityId(ctx);
+    translator.ensureRowAllocator(allocatorName, ctx, record, curActivityId);
+
+    decl.append("RtlLinkedDatasetBuilder ").append(instanceName).append("(");
+    decl.append(allocatorName);
+    if (choosenLimit)
+    {
+        CHqlBoundExpr boundLimit;
+        translator.buildExpr(ctx, choosenLimit, boundLimit);
+        translator.generateExprCpp(decl.append(", "), boundLimit.expr);
+    }
+    decl.append(");");
+
+    ctx.addQuoted(decl);
+}
+
+
+LinkedDictionaryBuilder::LinkedDictionaryBuilder(HqlCppTranslator & _translator, IHqlExpression * _record) : LinkedDatasetBuilderBase(_translator, _record)
+{
+    dataset.setown(createDictionary(no_anon, LINK(record), createComma(getSelfAttr(), getLinkCountedAttr())));
+}
+
+void LinkedDictionaryBuilder::buildDeclare(BuildCtx & ctx)
+{
+    StringBuffer decl, allocatorName;
+
+    OwnedHqlExpr curActivityId = translator.getCurrentActivityId(ctx);
+    translator.ensureRowAllocator(allocatorName, ctx, record, curActivityId);
+
+    StringBuffer lookupHelperName;
+    OwnedHqlExpr dict = createDictionary(no_null, record.getLink()); // MORE - is the actual dict not available?
+    translator.buildDictionaryHashClass(ctx, record, dict, lookupHelperName);
+
+    decl.append("RtlLinkedDictionaryBuilder ").append(instanceName).append("(");
+    decl.append(allocatorName).append(", &").append(lookupHelperName);
+    decl.append(");");
+
+    ctx.addQuoted(decl);
+}
+
 
 //---------------------------------------------------------------------------
 
@@ -1795,6 +1898,11 @@ IHqlCppDatasetBuilder * HqlCppTranslator::createLinkedDatasetBuilder(IHqlExpress
     return new LinkedDatasetBuilder(*this, record, choosenLimit);
 }
 
+IHqlCppDatasetBuilder * HqlCppTranslator::createLinkedDictionaryBuilder(IHqlExpression * record)
+{
+    return new LinkedDictionaryBuilder(*this, record);
+}
+
 IHqlCppDatasetBuilder * HqlCppTranslator::createSingleRowTempDatasetBuilder(IHqlExpression * record, BoundRow * row)
 {
 //  if (translator.isFixedRecordSize(record))

ecl/hqlcpp/hqlcset.ipp

@@ -26,8 +26,11 @@ public:
 
     virtual BoundRow * buildIterateLoop(BuildCtx & ctx, bool needToBreak);
     virtual void buildIterateClass(BuildCtx & ctx, SharedHqlExpr & iter, SharedHqlExpr & row);
-    virtual BoundRow * buildSelect(BuildCtx & ctx, IHqlExpression * indexExpr);
+    virtual BoundRow * buildSelectNth(BuildCtx & ctx, IHqlExpression * indexExpr);
+    virtual BoundRow * buildSelectMap(BuildCtx & ctx, IHqlExpression * indexExpr);
+    virtual void buildInDataset(BuildCtx & ctx, IHqlExpression * inExpr, CHqlBoundExpr & tgt);
     virtual void buildIterateMembers(BuildCtx & declarectx, BuildCtx & initctx);
+    virtual void buildCountDict(BuildCtx & ctx, CHqlBoundExpr & tgt);
 
 protected:
     virtual void buildIterateClass(BuildCtx & ctx, StringBuffer & cursorName, BuildCtx * initctx) = 0;
@@ -58,7 +61,7 @@ public:
     InlineBlockDatasetCursor(HqlCppTranslator & _translator, IHqlExpression * _ds, CHqlBoundExpr & _boundDs);
 
     virtual BoundRow * buildIterateLoop(BuildCtx & ctx, bool needToBreak);
-    virtual BoundRow * buildSelect(BuildCtx & ctx, IHqlExpression * indexExpr);
+    virtual BoundRow * buildSelectNth(BuildCtx & ctx, IHqlExpression * indexExpr);
 
 protected:
     BoundRow * buildSelectFirst(BuildCtx & ctx, IHqlExpression * indexExpr, bool createDefaultRowIfNull);
@@ -71,9 +74,24 @@ public:
 
     virtual void buildCount(BuildCtx & ctx, CHqlBoundExpr & tgt);
     virtual void buildExists(BuildCtx & ctx, CHqlBoundExpr & tgt);
-    virtual BoundRow * buildIterateLoop(BuildCtx & ctx, bool needToBreak);
-    virtual BoundRow * buildSelect(BuildCtx & ctx, IHqlExpression * indexExpr);
+    virtual BoundRow * buildIterateLoop(BuildCtx & ctx, bool needToBreak) { return doBuildIterateLoop(ctx, needToBreak, false); }
+    virtual BoundRow * buildSelectNth(BuildCtx & ctx, IHqlExpression * indexExpr);
     virtual void buildIterateClass(BuildCtx & ctx, StringBuffer & cursorName, BuildCtx * initctx);
+
+protected:
+    BoundRow * doBuildIterateLoop(BuildCtx & ctx, bool needToBreak, bool checkForNull);
+};
+
+class InlineLinkedDictionaryCursor : public InlineLinkedDatasetCursor
+{
+public:
+    InlineLinkedDictionaryCursor(HqlCppTranslator & _translator, IHqlExpression * _ds, CHqlBoundExpr & _boundDs);
+
+    virtual BoundRow * buildIterateLoop(BuildCtx & ctx, bool needToBreak) { return doBuildIterateLoop(ctx, needToBreak, true); }
+    virtual BoundRow * buildSelectMap(BuildCtx & ctx, IHqlExpression * indexExpr);
+    virtual void buildInDataset(BuildCtx & ctx, IHqlExpression * inExpr, CHqlBoundExpr & tgt);
+    virtual void buildIterateClass(BuildCtx & ctx, StringBuffer & cursorName, BuildCtx * initctx) { throwUnexpected(); }
+    virtual void buildCountDict(BuildCtx & ctx, CHqlBoundExpr & tgt);
 };
 
 class MultiLevelDatasetCursor : public BaseDatasetCursor
@@ -84,7 +102,7 @@ public:
     virtual void buildCount(BuildCtx & ctx, CHqlBoundExpr & tgt);
     virtual void buildExists(BuildCtx & ctx, CHqlBoundExpr & tgt);
     virtual BoundRow * buildIterateLoop(BuildCtx & ctx, bool needToBreak);
-    virtual BoundRow * buildSelect(BuildCtx & ctx, IHqlExpression * indexExpr);
+    virtual BoundRow * buildSelectNth(BuildCtx & ctx, IHqlExpression * indexExpr);
     virtual void buildIterateClass(BuildCtx & ctx, StringBuffer & cursorName, BuildCtx * initctx) { UNIMPLEMENTED; }
 
 protected:
@@ -216,6 +234,8 @@ public:
     virtual void finishRow(BuildCtx & ctx, BoundRow * selfCursor);
 
 protected:
+    void doFinishRow(BuildCtx & ctx, BoundRow * selfCursor, IHqlExpression *size);
+
     StringBuffer instanceName;
     StringBuffer builderName;
     OwnedHqlExpr dataset;
@@ -258,23 +278,38 @@ protected:
     Owned<BoundRow> cursor;
 };
 
-class LinkedDatasetBuilder : public DatasetBuilderBase
+class LinkedDatasetBuilderBase : public DatasetBuilderBase
 {
 public:
-    LinkedDatasetBuilder(HqlCppTranslator & _translator, IHqlExpression * _record, IHqlExpression * _choosenLimit);
+    LinkedDatasetBuilderBase(HqlCppTranslator & _translator, IHqlExpression * _record);
 
-    virtual void buildDeclare(BuildCtx & ctx);
     virtual void buildFinish(BuildCtx & ctx, const CHqlBoundTarget & target);
     virtual void buildFinish(BuildCtx & ctx, CHqlBoundExpr & bound);
     virtual bool buildLinkRow(BuildCtx & ctx, BoundRow * sourceRow);
     virtual bool buildAppendRows(BuildCtx & ctx, IHqlExpression * expr);
     virtual void finishRow(BuildCtx & ctx, BoundRow * selfCursor);
-    virtual bool isRestricted()                             { return choosenLimit != NULL; }
+};
+
+class LinkedDatasetBuilder : public LinkedDatasetBuilderBase
+{
+public:
+    LinkedDatasetBuilder(HqlCppTranslator & _translator, IHqlExpression * _record, IHqlExpression * _choosenLimit);
+
+    virtual void buildDeclare(BuildCtx & ctx);
+    virtual bool isRestricted() { return choosenLimit != NULL; }
 
 protected:
     LinkedHqlExpr choosenLimit;
 };
 
+class LinkedDictionaryBuilder : public LinkedDatasetBuilderBase
+{
+public:
+    LinkedDictionaryBuilder(HqlCppTranslator & _translator, IHqlExpression * _record);
+
+    virtual void buildDeclare(BuildCtx & ctx);
+};
+
 //---------------------------------------------------------------------------
 
 class SetBuilder : public CInterface, implements IHqlCppSetBuilder

ecl/hqlcpp/hqlhtcpp.cpp

@@ -2946,6 +2946,7 @@ static bool anyXmlGeneratedForPass(IHqlExpression * expr, unsigned pass)
                 return anyXmlGeneratedForPass(queryRecord(type), pass);
             case type_set:
                 return (pass==1);
+            case type_dictionary:
             case type_table:
             case type_groupedtable:
                 return (pass == 1);
@@ -3052,6 +3053,7 @@ void HqlCppTranslator::doBuildFunctionReturn(BuildCtx & ctx, ITypeInfo * type, I
     case type_data:
     case type_unicode:
     case type_utf8:
+    case type_dictionary:
     case type_table:
     case type_groupedtable:
     case type_row:
@@ -3624,6 +3626,7 @@ unsigned HqlCppTranslator::buildRtlField(StringBuffer * instanceName, IHqlExpres
         case type_set:
             extractXmlName(xpathName, &xpathItem, NULL, field, "Item", false);
             break;
+        case type_dictionary:
         case type_table:
         case type_groupedtable:
             extractXmlName(xpathName, &xpathItem, NULL, field, "Row", false);
@@ -3918,6 +3921,7 @@ unsigned HqlCppTranslator::buildRtlType(StringBuffer & instanceName, ITypeInfo *
                 fieldType |= RFTMlinkcounted;
             break;
         }
+    case type_dictionary:
     case type_table:
     case type_groupedtable:
         {
@@ -4224,6 +4228,7 @@ public:
                             }
                             break;
                         }
+                    case type_dictionary:
                     case type_table:
                     case type_groupedtable:
                         {
@@ -4542,6 +4547,7 @@ IHqlExpression * HqlCppTranslator::convertBetweenCountAndSize(const CHqlBoundExp
     OwnedHqlExpr record;
     switch (type->getTypeCode())
     {
+    case type_dictionary:
     case type_table:
     case type_groupedtable:
         record.set(bound.expr->queryRecord());
@@ -4678,6 +4684,7 @@ void HqlCppTranslator::buildGetResultInfo(BuildCtx & ctx, IHqlExpression * expr,
     case type_swapint:  func = getResultIntAtom; break;
     case type_boolean:  func = getResultBoolAtom; break;
     case type_data:     func = getResultDataAtom; break;
+    case type_dictionary:
     case type_table:
     case type_groupedtable:
     case type_set:
@@ -4685,7 +4692,7 @@ void HqlCppTranslator::buildGetResultInfo(BuildCtx & ctx, IHqlExpression * expr,
         {
             OwnedHqlExpr record;
             bool ensureSerialized = true;
-            if (ttc == type_table || ttc == type_groupedtable)
+            if (ttc != type_set)
             {
                 overrideType.set(type);
                 record.set(::queryRecord(type));
@@ -4964,6 +4971,7 @@ void HqlCppTranslator::buildSetResultInfo(BuildCtx & ctx, IHqlExpression * origi
             schemaType.setown(makeSetType(elementType));
         }
         break;
+    case type_dictionary:
     case type_table:
     case type_groupedtable:
         {
@@ -5289,6 +5297,38 @@ void HqlCppTranslator::buildHashOfExprsClass(BuildCtx & ctx, const char * name,
     buildHashClass(ctx, name, hash, dataset);
 }
 
+
+void HqlCppTranslator::buildDictionaryHashClass(BuildCtx &ctx, IHqlExpression *record, IHqlExpression *dictionary, StringBuffer &lookupHelperName)
+{
+    BuildCtx declarectx(*code, declareAtom);
+    appendUniqueId(lookupHelperName.append("lu"), getConsistentUID(record));
+    OwnedHqlExpr attr = createAttribute(lookupAtom, LINK(record));
+    HqlExprAssociation * match = declarectx.queryMatchExpr(attr);
+    if (!match)
+    {
+        BuildCtx classctx(declarectx);
+        beginNestedClass(classctx, lookupHelperName, "IHThorHashLookupInfo");
+        HqlExprArray keyedFields;
+        IHqlExpression * payload = record->queryProperty(_payload_Atom);
+        unsigned payloadSize = payload ? getIntValue(payload->queryChild(0)) : 0;
+        unsigned max = record->numChildren() - payloadSize;
+        for (unsigned idx = 0; idx < max; idx++)
+        {
+            IHqlExpression *child = record->queryChild(idx);
+            if (!child->isAttribute())
+                keyedFields.append(*createSelectExpr(LINK(dictionary->queryNormalizedSelector()), LINK(child)));
+        }
+        OwnedHqlExpr keyedList = createValueSafe(no_sortlist, makeSortListType(NULL), keyedFields);
+        DatasetReference dictRef(dictionary, no_none, NULL);
+        buildHashOfExprsClass(classctx, "Hash", keyedList, dictRef, false);
+        buildCompareMember(classctx, "Compare", keyedList, dictRef);
+        endNestedClass();
+        OwnedHqlExpr temp = createVariable(lookupHelperName, makeVoidType());
+        declarectx.associateExpr(attr, temp);
+    }
+}
+
+
 //---------------------------------------------------------------------------
 
 IHqlExpression * queryImplementationInterface(IHqlExpression * moduleFunc)
@@ -6270,6 +6310,9 @@ ABoundActivity * HqlCppTranslator::buildActivity(BuildCtx & ctx, IHqlExpression
             case no_selectnth:
                 result = doBuildActivitySelectNth(ctx, expr);
                 break;
+            case no_selectmap:
+                UNIMPLEMENTED;
+                break;
             case no_join:
             case no_selfjoin:
                 result = doBuildActivityJoin(ctx, expr);
@@ -6997,6 +7040,7 @@ void HqlCppTranslator::ensureSerialized(const CHqlBoundTarget & variable, BuildC
             serializeName = serializeUtf8XAtom;
             deserializeName = deserializeUtf8XAtom;
             break;
+        case type_dictionary:
         case type_table:
         case type_groupedtable:
             {
@@ -7006,29 +7050,29 @@ void HqlCppTranslator::ensureSerialized(const CHqlBoundTarget & variable, BuildC
                     deserializeArgs.append(*createRowSerializer(deserializectx, record, deserializerAtom));
 
                     serializeArgs.append(*createRowSerializer(serializectx, record, serializerAtom));
-                    if (tc == type_table)
+                    if (tc == type_groupedtable)
                     {
-                        serializeName = serializeRowsetXAtom;
-                        deserializeName = deserializeRowsetXAtom;
+                        serializeName = serializeGroupedRowsetXAtom;
+                        deserializeName = deserializeGroupedRowsetXAtom;
                     }
                     else
                     {
-                        serializeName = serializeGroupedRowsetXAtom;
-                        deserializeName = deserializeGroupedRowsetXAtom;
+                        serializeName = serializeRowsetXAtom;
+                        deserializeName = deserializeRowsetXAtom;
                     }
                 }
                 else
                 {
                     assertex(!recordRequiresSerialization(record));
-                    if (tc == type_table)
+                    if (tc == type_groupedtable)
                     {
-                        serializeName = serializeDatasetXAtom;
-                        deserializeName = deserializeDatasetXAtom;
+                        serializeName = serializeGroupedDatasetXAtom;
+                        deserializeName = deserializeGroupedDatasetXAtom;
                     }
                     else
                     {
-                        serializeName = serializeGroupedDatasetXAtom;
-                        deserializeName = deserializeGroupedDatasetXAtom;
+                        serializeName = serializeDatasetXAtom;
+                        deserializeName = deserializeDatasetXAtom;
                     }
                 }
                 serializedType.set(type);
@@ -7269,6 +7313,7 @@ void HqlCppTranslator::doBuildStmtSetResult(BuildCtx & ctx, IHqlExpression * exp
             buildSetResultInfo(subctx, expr, normalized, setType, (persist != NULL), true);
             break;
         }
+    case type_dictionary:
     case type_table:
     case type_groupedtable:
         switch (value->getOperator())
@@ -7435,6 +7480,7 @@ void HqlCppTranslator::doBuildExprSizeof(BuildCtx & ctx, IHqlExpression * expr,
             unsigned size = UNKNOWN_LENGTH;
             switch (type->getTypeCode())
             {
+            case type_dictionary:
             case type_table:
             case type_groupedtable:
             case type_record:
@@ -7472,6 +7518,7 @@ void HqlCppTranslator::doBuildExprSizeof(BuildCtx & ctx, IHqlExpression * expr,
             unsigned size = UNKNOWN_LENGTH;
             switch (type->getTypeCode())
             {
+            case type_dictionary:
             case type_table:
             case type_groupedtable:
             case type_record:
@@ -7550,6 +7597,7 @@ void HqlCppTranslator::doBuildExprSizeof(BuildCtx & ctx, IHqlExpression * expr,
 
             switch (type->getTypeCode())
             {
+            case type_dictionary:
             case type_table:
             case type_groupedtable:
             case type_record:
@@ -7610,6 +7658,7 @@ void HqlCppTranslator::doBuildExprRowDiff(BuildCtx & ctx, const CHqlBoundTarget
                     return;
                 }
                 break;
+            case type_dictionary:
             case type_table:
             case type_groupedtable:
                 UNIMPLEMENTED;
@@ -10157,6 +10206,7 @@ void HqlCppTranslator::addSchemaField(IHqlExpression *field, MemoryBuffer &schem
     case type_bitfield:
         schemaType.set(schemaType->queryPromotedType());
         //fall through;
+    case type_dictionary:
     case type_table:
     case type_groupedtable:
     case type_row:
@@ -10528,6 +10578,7 @@ void HqlCppTranslator::buildXmlSerialize(BuildCtx & subctx, IHqlExpression * exp
                 case type_set:
                     buildXmlSerializeSet(subctx, expr, value);
                     break;
+                case type_dictionary:
                 case type_table:
                 case type_groupedtable:
                     buildXmlSerializeDataset(subctx, expr, value, assigns);

ecl/hqlcpp/hqlinline.cpp

@@ -75,7 +75,7 @@ static unsigned calcInlineFlags(BuildCtx * ctx, IHqlExpression * expr)
     //But it would be really good if the code could be made context independent - then it could go in hqlattr and be cached.
     if (ctx)
     {
-        if (expr->isDataset())
+        if (expr->isDataset() || expr->isDictionary())
         {
             if (ctx->queryMatchExpr(expr))
                 return RETevaluate;
@@ -151,6 +151,7 @@ static unsigned calcInlineFlags(BuildCtx * ctx, IHqlExpression * expr)
                 return RETiterate;
             return RETiterate|HEFspillinline;
         }
+    case no_selectmap:
     case no_selectnth:
         {
             IHqlExpression * ds = expr->queryChild(0);
@@ -215,6 +216,11 @@ static unsigned calcInlineFlags(BuildCtx * ctx, IHqlExpression * expr)
         return 0;       // for the moment always do this out of line 
     case no_table:
         return 0;
+    case no_newuserdictionary:
+    case no_userdictionary:
+        return RETassign;
+    case no_inlinedictionary:
+        return RETassign;
     case no_owned_ds:
         {
             unsigned childFlags = getInlineFlags(ctx, expr->queryChild(0));

ecl/hqlcpp/hqliproj.cpp

@@ -1050,6 +1050,8 @@ IHqlExpression * ComplexImplicitProjectInfo::createOutputProject(IHqlExpression
     IHqlExpression * transform = createMappingTransform(self, left);
     if (ds->isDataset())
         return createDataset(no_hqlproject, LINK(ds), createComma(transform, LINK(seq)));
+    else
+        assertex(!ds->isDictionary());
     return createRow(no_projectrow, LINK(ds), createComma(transform, LINK(seq)));
 }
 
@@ -1596,7 +1598,7 @@ void ImplicitProjectTransformer::analyseExpr(IHqlExpression * expr)
             setOriginal(complexExtra->rightFieldsRequired, expr->queryChild(1));
             break;
         case FixedInputActivity:
-            assertex(child && (child->isDataset() || child->isDatarow()));
+            assertex(child && (child->isDataset() || child->isDatarow() || child->isDictionary()));
             setOriginal(complexExtra->leftFieldsRequired, child);
             if (getNumChildTables(expr) >= 2)
                 setOriginal(complexExtra->rightFieldsRequired, expr->queryChild(1));
@@ -1620,7 +1622,7 @@ void ImplicitProjectTransformer::analyseExpr(IHqlExpression * expr)
     }
 
     IHqlExpression * record = expr->queryRecord();
-    if (record && !isPatternType(type) && !expr->isTransform())
+    if (record && !isPatternType(type) && !expr->isTransform() && !expr->isDictionary())
     {
         assertex(complexExtra);
         complexExtra->setOriginalRecord(queryBodyComplexExtra(record));
@@ -1972,6 +1974,11 @@ ProjectExprKind ImplicitProjectTransformer::getProjectExprKind(IHqlExpression *
     case no_inlinetable:
     case no_dataset_from_transform:
         return CreateRecordSourceActivity;
+    case no_inlinedictionary:
+        return NonActivity;
+    case no_indict:
+    case no_selectmap:
+        return FixedInputActivity;
     case no_extractresult:
     case no_apply:
         return SinkActivity;
@@ -2152,6 +2159,7 @@ ProjectExprKind ImplicitProjectTransformer::getProjectExprKind(IHqlExpression *
     case type_table:
     case type_groupedtable:
         break;
+    case type_dictionary:
     case type_transform:
         return NonActivity;
     default:
@@ -2239,7 +2247,7 @@ void ImplicitProjectTransformer::calculateFieldsUsed(IHqlExpression * expr)
 
     if (!extra->okToOptimize())
     {
-        if (expr->queryRecord())
+        if (expr->queryRecord() && !expr->isDictionary())
             extra->addAllOutputs();
     }
     else

ecl/hqlcpp/hqltcppc.cpp

@@ -2999,6 +2999,7 @@ CMemberInfo * ColumnToOffsetMap::createColumn(CContainerInfo * container, IHqlEx
         }
     case type_groupedtable:
     case type_table:
+    case type_dictionary:
         {
             IHqlExpression * count = NULL;
             IHqlExpression * size = NULL;
@@ -3195,6 +3196,7 @@ static bool cachedCanReadFromCsv(IHqlExpression * record)
                     if (!cachedCanReadFromCsv(cur->queryRecord()))
                         return false;
                     break;
+                case type_dictionary:
                 case type_table:
                 case type_groupedtable:
                 case type_set:

ecl/hqlcpp/hqltcppc2.cpp

@@ -763,4 +763,3 @@ AColumnInfo * CChildLinkedDatasetColumnInfo::lookupColumn(IHqlExpression * searc
     return NULL;
 }
 
-

ecl/hqlcpp/hqlttcpp.cpp

@@ -8486,6 +8486,7 @@ IHqlExpression * HqlLinkedChildRowTransformer::createTransformedBody(IHqlExpress
             ITypeInfo * type = expr->queryType();
             switch (type->getTypeCode())
             {
+            case type_dictionary:
             case type_table:
             case type_groupedtable:
                 if (expr->hasProperty(embeddedAtom))
@@ -8693,7 +8694,7 @@ IHqlExpression * HqlScopeTagger::transformSelect(IHqlExpression * expr)
     IHqlExpression * cursor = queryDatasetCursor(ds);
     if (cursor->isDataset())
     {
-        if (expr->isDataset())
+        if (expr->isDataset() || expr->isDictionary())
         {
             if (!isValidNormalizeSelector(cursor))
             {
@@ -10200,7 +10201,7 @@ IHqlExpression * HqlTreeNormalizer::convertSelectToProject(IHqlExpression * newR
     unsigned numChildren = expr->numChildren();
     for (unsigned idx = 2; idx < numChildren; idx++)
         args.append(*transform(expr->queryChild(idx)));
-    OwnedHqlExpr project = createDataset(no_newusertable, args);
+    OwnedHqlExpr project = expr->isDictionary() ? createDictionary(no_newuserdictionary, args) : createDataset(no_newusertable, args);
     return expr->cloneAllAnnotations(project);
 }
 
@@ -11268,6 +11269,7 @@ IHqlExpression * HqlTreeNormalizer::createTransformedBody(IHqlExpression * expr)
             }
             return Parent::createTransformed(cleaned);
         }
+    case no_userdictionary:
     case no_usertable:
     case no_selectfields:
         {

ecl/hqlcpp/hqlwcpp.cpp

@@ -74,6 +74,7 @@ bool isTypePassedByAddress(ITypeInfo * type)
     case type_set:
     case type_row:
         return true;
+    case type_dictionary:
     case type_table:
     case type_groupedtable:
         return !isArrayRowset(type);
@@ -485,6 +486,7 @@ void HqlCppWriter::generateType(ITypeInfo * type, const char * name)
             //next = next->queryChildType();
             isPointer = false;
             break;
+        case type_dictionary:// MORE - is this right?
         case type_table:
             if (isArrayRowset(fullType))
             {
@@ -802,6 +804,7 @@ void HqlCppWriter::generateParamCpp(IHqlExpression * param)
     case type_data:
     case type_unicode:
     case type_utf8:
+    case type_dictionary:
     case type_table:
     case type_groupedtable:
         if (paramType->getSize() == UNKNOWN_LENGTH)
@@ -843,6 +846,7 @@ void HqlCppWriter::generateParamCpp(IHqlExpression * param)
     case type_record:
         out.append("IOutputMetaData * ");
         break;
+    case type_dictionary:
     case type_table:
     case type_groupedtable:
         if (isConst)
@@ -959,6 +963,7 @@ void HqlCppWriter::generateFunctionReturnType(StringBuffer & params, ITypeInfo *
         out.append("size32_t");
         params.append("ARowBuilder & __self");
         break;
+    case type_dictionary:
     case type_table:
     case type_groupedtable:
         if (hasStreamedModifier(retType))
@@ -1156,6 +1161,7 @@ StringBuffer & HqlCppWriter::generateExprCpp(IHqlExpression * expr)
                         out.append("getbytes()");
                         break;
                     case type_set:
+                    case type_dictionary:
                     case type_table:
                     case type_groupedtable:
                         if (hasLinkCountedModifier(type))
@@ -1226,6 +1232,7 @@ StringBuffer & HqlCppWriter::generateExprCpp(IHqlExpression * expr)
                         out.append("getbytes()");
                         break;
                     case type_set:
+                    case type_dictionary:
                     case type_table:
                     case type_groupedtable:
                         if (hasLinkCountedModifier(type))
@@ -1264,6 +1271,7 @@ StringBuffer & HqlCppWriter::generateExprCpp(IHqlExpression * expr)
                         out.append("getbytes()");       //????
                         break;
                     case type_set:
+                    case type_dictionary:
                     case type_table:
                     case type_groupedtable:
                         if (hasLinkCountedModifier(childType))
@@ -1709,6 +1717,7 @@ void HqlCppWriter::generateStmtAssign(IHqlStmt * assign)
             else
                 throwUnexpected();
             break;
+        case type_dictionary:
         case type_table:
         case type_groupedtable:
             if (hasWrapperModifier(type))
@@ -1778,6 +1787,7 @@ void HqlCppWriter::generateStmtAssignModify(IHqlStmt * assign)
     switch (type->getTypeCode())
     {
         case type_row:
+        case type_dictionary:
         case type_table:
         case type_groupedtable:
             //check it is a pointer increment

ecl/regress/dict3e.ecl

@@ -0,0 +1,8 @@
+// Some invalid dictionary operations...
+
+d1a := dictionary([{1 => 'Richard'}], { integer id, string name });  // mismatch payload position
+d1b := dictionary([{1 => 2, 'Richard'}], { integer id1, integer id2 => string name }); // mismatch payload position
+d1c := dictionary([{1, 2, 'Richard'}], { integer id1, integer id2 => string name }); // mismatch payload position
+
+d1 := dictionary([{5 => 'Richard'}], { integer id => string name });
+'5' in d1;  // wrong type

ecl/regress/payload.ecl

@@ -1,40 +0,0 @@
-/*##############################################################################
-
-    Copyright (C) 2011 HPCC Systems.
-
-    All rights reserved. This program is free software: you can redistribute it and/or modify
-    it under the terms of the GNU Affero General Public License as
-    published by the Free Software Foundation, either version 3 of the
-    License, or (at your option) any later version.
-
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU Affero General Public License for more details.
-
-    You should have received a copy of the GNU Affero General Public License
-    along with this program.  If not, see <http://www.gnu.org/licenses/>.
-############################################################################## */
-
-
-namesRecord :=
-            RECORD
-string20        surname;
-string10        forename;
-unsigned8       age := 25;
-            END;
-
-namesTable := dataset('x',namesRecord,FLAT);
-
-indexRecord := { namesTable };
-
-myIndex := index(namesRecord, 'i', payload(forename));
-
-
-
-namesRecord t(namesRecord l) := transform
-    self.age := l.age +1;
-    self := l;
-end;
-
-output(project(myIndex(surname = 'Hawthorn'), t(left)));

rtl/eclrtl/rtlds.cpp

@@ -332,7 +332,6 @@ RtlLinkedDatasetBuilder::~RtlLinkedDatasetBuilder()
 
 void RtlLinkedDatasetBuilder::append(const void * source)
 {
-    flush();
     if (count < choosenLimit)
     {
         ensure(count+1);
@@ -343,20 +342,28 @@ void RtlLinkedDatasetBuilder::append(const void * source)
 
 void RtlLinkedDatasetBuilder::appendRows(size32_t num, byte * * rows)
 {
-    flush();
     if (num && (count < choosenLimit))
     {
-        unsigned numToAdd = (count + num < choosenLimit) ? num : choosenLimit - count;
-        ensure(count+numToAdd);
-        for (unsigned i=0; i < numToAdd; i++)
-            rowset[count+i] = (byte *)rowAllocator->linkRow(rows[i]);
-        count += numToAdd;
+        unsigned maxNumToAdd = (count + num < choosenLimit) ? num : choosenLimit - count;
+        unsigned numAdded = 0;
+        ensure(count+maxNumToAdd);
+        for (unsigned i=0; i < num; i++)
+        {
+            byte *row = rows[i];
+            if (row)
+            {
+                rowset[count+numAdded] = (byte *)rowAllocator->linkRow(row);
+                numAdded++;
+                if (numAdded == maxNumToAdd)
+                    break;
+            }
+        }
+        count += numAdded;
     }
 }
 
 void RtlLinkedDatasetBuilder::appendOwn(const void * row)
 {
-    //flush() must have been called before this... otherwise the order will be messed up
     assertex(!builder.exists());
     if (count < choosenLimit)
     {
@@ -371,7 +378,6 @@ void RtlLinkedDatasetBuilder::appendOwn(const void * row)
 
 byte * RtlLinkedDatasetBuilder::createRow()
 {
-    flush();
     if (count >= choosenLimit)
         return NULL;
     return builder.getSelf();
@@ -395,7 +401,6 @@ public:
 
 void RtlLinkedDatasetBuilder::cloneRow(size32_t len, const void * row)
 {
-    flush();
     if (count >= choosenLimit)
         return;
 
@@ -415,7 +420,6 @@ void RtlLinkedDatasetBuilder::cloneRow(size32_t len, const void * row)
 
 void RtlLinkedDatasetBuilder::deserializeRow(IOutputRowDeserializer & deserializer, IRowDeserializerSource & in)
 {
-    flush();
     builder.ensureRow();
     size32_t rowSize = deserializer.deserialize(builder, in);
     finalizeRow(rowSize);
@@ -439,11 +443,15 @@ inline void doSerializeRowset(IRowSerializerTarget & out, IOutputRowSerializer *
 {
     for (unsigned i=0; i < count; i++)
     {
-        serializer->serialize(out, rows[i]);
-        if (isGrouped)
+        byte *row = rows[i];
+        if (row)   // When serializing a dictionary, there may be nulls in the rowset. These can be skipped (we rehash on deserialize)
         {
-            byte eogPending = (i+1 < count) && (rows[i+1] == NULL);
-            out.put(1, &eogPending);
+            serializer->serialize(out, rows[i]);
+            if (isGrouped)
+            {
+                byte eogPending = (i+1 < count) && (rows[i+1] == NULL);
+                out.put(1, &eogPending);
+            }
         }
     }
 }
@@ -451,8 +459,6 @@ inline void doSerializeRowset(IRowSerializerTarget & out, IOutputRowSerializer *
 
 void RtlLinkedDatasetBuilder::deserialize(IOutputRowDeserializer & deserializer, IRowDeserializerSource & in, bool isGrouped)
 {
-    flush();
-
     offset_t marker = in.beginNested();
     doDeserializeRowset(*this, deserializer, in, marker, isGrouped);
 }
@@ -460,17 +466,11 @@ void RtlLinkedDatasetBuilder::deserialize(IOutputRowDeserializer & deserializer,
 
 void RtlLinkedDatasetBuilder::finalizeRows()
 {
-    flush();
     if (count != max)
         resize(count);
 }
 
 
-void RtlLinkedDatasetBuilder::flush()
-{
-//  builder.clear();
-}
-
 void RtlLinkedDatasetBuilder::finalizeRow(size32_t rowSize)
 {
     assertex(builder.exists());
@@ -511,9 +511,17 @@ void appendRowsToRowset(size32_t & targetCount, byte * * & targetRowset, IEngine
     {
         size32_t prevCount = targetCount;
         byte * * expandedRowset = rowAllocator->reallocRows(targetRowset, prevCount, prevCount+extraCount);
+        unsigned numAdded = 0;
         for (unsigned i=0; i < extraCount; i++)
-            expandedRowset[prevCount+i] = (byte *)rowAllocator->linkRow(extraRows[i]);
-        targetCount = prevCount + extraCount;
+        {
+            byte *extraRow = extraRows[i];
+            if (extraRow)
+            {
+                expandedRowset[prevCount+numAdded] = (byte *)rowAllocator->linkRow(extraRow);
+                numAdded++;
+            }
+        }
+        targetCount = prevCount + numAdded;
         targetRowset = expandedRowset;
     }
 }
@@ -558,6 +566,185 @@ void rtlSerializeGroupedRowset(IRowSerializerTarget & out, IOutputRowSerializer
 
 //---------------------------------------------------------------------------
 
+RtlLinkedDictionaryBuilder::RtlLinkedDictionaryBuilder(IEngineRowAllocator * _rowAllocator, IHThorHashLookupInfo *_hashInfo, unsigned _initialSize)
+  : builder(_rowAllocator, false)
+{
+    init(_rowAllocator, _hashInfo, _initialSize);
+}
+
+RtlLinkedDictionaryBuilder::RtlLinkedDictionaryBuilder(IEngineRowAllocator * _rowAllocator, IHThorHashLookupInfo *_hashInfo)
+  : builder(_rowAllocator, false)
+{
+    init(_rowAllocator, _hashInfo, 8);
+}
+
+void RtlLinkedDictionaryBuilder::init(IEngineRowAllocator * _rowAllocator, IHThorHashLookupInfo *_hashInfo, unsigned _initialSize)
+{
+    hash  = _hashInfo->queryHash();
+    compare  = _hashInfo->queryCompare();
+    initialSize = _initialSize;
+    rowAllocator = LINK(_rowAllocator);
+    table = NULL;
+    usedCount = 0;
+    usedLimit = 0;
+    tableSize = 0;
+}
+
+RtlLinkedDictionaryBuilder::~RtlLinkedDictionaryBuilder()
+{
+//    builder.clear();
+    if (table)
+        rowAllocator->releaseRowset(tableSize, table);
+    ::Release(rowAllocator);
+}
+
+void RtlLinkedDictionaryBuilder::append(const void * source)
+{
+    if (source)
+    {
+        appendOwn(rowAllocator->linkRow(source));
+    }
+}
+
+void RtlLinkedDictionaryBuilder::appendOwn(const void * source)
+{
+    if (source)
+    {
+        checkSpace();
+        unsigned rowidx = hash->hash(source) % tableSize;
+        loop
+        {
+            const void *entry = table[rowidx];
+            if (entry && compare->docompare(source, entry)==0)
+            {
+                rowAllocator->releaseRow(entry);
+                usedCount--;
+                entry = NULL;
+            }
+            if (!entry)
+            {
+                table[rowidx] = (byte *) source;
+                usedCount++;
+                break;
+            }
+            rowidx++;
+            if (rowidx==tableSize)
+                rowidx = 0;
+        }
+    }
+}
+
+void RtlLinkedDictionaryBuilder::checkSpace()
+{
+    if (!table)
+    {
+        table = rowAllocator->createRowset(initialSize);
+        tableSize = initialSize;
+        memset(table, 0, tableSize*sizeof(byte *));
+        usedLimit = (tableSize * 3) / 4;
+        usedCount = 0;
+    }
+    else if (usedCount > usedLimit)
+    {
+        // Rehash
+        byte * * oldTable = table;
+        unsigned oldSize = tableSize;
+        table = rowAllocator->createRowset(tableSize*2);
+        tableSize = tableSize*2; // Don't update until we have successfully allocated, so that we remain consistent if createRowset throws an exception.
+        memset(table, 0, tableSize * sizeof(byte *));
+        usedLimit = (tableSize * 3) / 4;
+        usedCount = 0;
+        unsigned i;
+        for (i = 0; i < oldSize; i++)
+        {
+            append(oldTable[i]);  // we link the rows here...
+        }
+        rowAllocator->releaseRowset(oldSize, oldTable);   // ... because this will release them
+    }
+}
+
+void RtlLinkedDictionaryBuilder::appendRows(size32_t num, byte * * rows)
+{
+    // MORE - if we know that the source is already a hashtable, we can optimize the add to an empty table...
+    for (unsigned i=0; i < num; i++)
+        append(rows[i]);
+}
+
+void RtlLinkedDictionaryBuilder::finalizeRow(size32_t rowSize)
+{
+    assertex(builder.exists());
+    const void * next = builder.finalizeRowClear(rowSize);
+    appendOwn(next);
+}
+
+void RtlLinkedDictionaryBuilder::cloneRow(size32_t len, const void * row)
+{
+    byte * self = builder.ensureCapacity(len, NULL);
+    memcpy(self, row, len);
+
+    IOutputMetaData * meta = rowAllocator->queryOutputMeta();
+    if (meta->getMetaFlags() & MDFneedserialize)
+    {
+        RtlChildRowLinkerWalker walker;
+        meta->walkIndirectMembers(self, walker);
+    }
+
+    finalizeRow(len);
+}
+
+extern ECLRTL_API unsigned __int64 rtlDictionaryCount(size32_t tableSize, byte **table)
+{
+    unsigned __int64 ret = 0;
+    for (size32_t i = 0; i < tableSize; i++)
+        if (table[i])
+            ret++;
+    return ret;
+}
+
+extern ECLRTL_API byte *rtlDictionaryLookup(IHThorHashLookupInfo &hashInfo, size32_t tableSize, byte **table, const byte *source, byte *defaultRow)
+{
+    if (!tableSize)
+        return (byte *) rtlLinkRow(defaultRow);
+
+    IHash *hash  = hashInfo.queryHash();
+    ICompare *compare  = hashInfo.queryCompare();
+    unsigned rowidx = hash->hash(source) % tableSize;
+    loop
+    {
+        const void *entry = table[rowidx];
+        if (!entry)
+            return (byte *) rtlLinkRow(defaultRow);
+        if (compare->docompare(source, entry)==0)
+            return (byte *) rtlLinkRow(entry);
+        rowidx++;
+        if (rowidx==tableSize)
+            rowidx = 0;
+    }
+}
+
+extern ECLRTL_API bool rtlDictionaryLookupExists(IHThorHashLookupInfo &hashInfo, size32_t tableSize, byte **table, const byte *source)
+{
+    if (!tableSize)
+        return false;
+
+    IHash *hash  = hashInfo.queryHash();
+    ICompare *compare  = hashInfo.queryCompare();
+    unsigned rowidx = hash->hash(source) % tableSize;
+    loop
+    {
+        const void *entry = table[rowidx];
+        if (!entry)
+            return false;
+        if (compare->docompare(source, entry)==0)
+            return true;
+        rowidx++;
+        if (rowidx==tableSize)
+            rowidx = 0;
+    }
+}
+
+//---------------------------------------------------------------------------
+
 //These definitions should be shared with thorcommon, but to do that
 //they would need to be moved to an rtlds.ipp header, which thorcommon then included.
 

rtl/eclrtl/rtlds_imp.hpp

@@ -363,7 +363,6 @@ public:
     void appendRows(size32_t num, byte * * rows);
     inline void appendEOG() { appendOwn(NULL); }
     byte * createRow();
-    void cancelRow();
     void cloneRow(size32_t len, const void * ptr);
     void deserialize(IOutputRowDeserializer & deserializer, IRowDeserializerSource & in, bool isGrouped);
     void deserializeRow(IOutputRowDeserializer & deserializer, IRowDeserializerSource & in);
@@ -382,9 +381,6 @@ public:
     inline byte * row() const { return builder.row(); }
 
 protected:
-    void flush();
-
-protected:
     IEngineRowAllocator * rowAllocator;
     RtlDynamicRowBuilder builder;
     byte * * rowset;
@@ -393,6 +389,57 @@ protected:
     size32_t choosenLimit;
 };
 
+class ECLRTL_API RtlLinkedDictionaryBuilder
+{
+public:
+    RtlLinkedDictionaryBuilder(IEngineRowAllocator * _rowAllocator, IHThorHashLookupInfo *_hashInfo, unsigned _initialTableSize);
+    RtlLinkedDictionaryBuilder(IEngineRowAllocator * _rowAllocator, IHThorHashLookupInfo *_hashInfo);
+    ~RtlLinkedDictionaryBuilder();
+
+    void append(const void * source);
+    void appendOwn(const void * source);
+    void appendRows(size32_t num, byte * * rows);
+
+    inline size32_t getcount() { return tableSize; }
+    inline byte * * linkrows() { return rtlLinkRowset(table); }
+
+    byte * createRow()         { return builder.getSelf(); }
+    void finalizeRow(size32_t len);
+    inline RtlDynamicRowBuilder & rowBuilder() { return builder; }
+    void cloneRow(size32_t len, const void * ptr);
+    /*
+        Not clear which if any of these we will want...
+    void deserialize(IOutputRowDeserializer & deserializer, IRowDeserializerSource & in, bool isGrouped);
+    void deserializeRow(IOutputRowDeserializer & deserializer, IRowDeserializerSource & in);
+    void expand(size32_t required);
+    void resize(size32_t required);
+    void finalizeRows();
+
+    inline void ensure(size32_t required) { if (required > max) expand(required); }
+    */
+    inline byte * * queryrows() { return table; }
+    inline byte * row() const { return builder.row(); }
+
+protected:
+    void checkSpace();
+    void init(IEngineRowAllocator * _rowAllocator, IHThorHashLookupInfo *_hashInfo, unsigned _initialTableSize);
+
+protected:
+    IEngineRowAllocator *rowAllocator;
+    IHash *hash;
+    ICompare *compare;
+    RtlDynamicRowBuilder builder;
+    byte * * table;
+    size32_t usedCount;
+    size32_t usedLimit;
+    size32_t initialSize;
+    size32_t tableSize;
+};
+
+extern ECLRTL_API unsigned __int64 rtlDictionaryCount(size32_t tableSize, byte **table);
+extern ECLRTL_API byte *rtlDictionaryLookup(IHThorHashLookupInfo &hashInfo, size32_t tableSize, byte **table, const byte *source, byte *defaultRow);
+extern ECLRTL_API bool rtlDictionaryLookupExists(IHThorHashLookupInfo &hashInfo, size32_t tableSize, byte **table, const byte *source);
+
 extern ECLRTL_API void appendRowsToRowset(size32_t & targetCount, byte * * & targetRowset, IEngineRowAllocator * rowAllocator, size32_t count, byte * * rows);
 
 

rtl/eclrtl/rtlfield.cpp

@@ -834,7 +834,8 @@ size32_t RtlDatasetTypeInfo::toXML(const byte * self, const byte * selfrow, cons
         for (unsigned i= 0; i < thisCount; i++)
         {
             const byte * row = rows[i];
-            child->toXML(row, row, field, target);
+            if (row)
+                child->toXML(row, row, field, target);
         }
         thisSize = sizeof(size32_t) + sizeof(void * *);
     }

rtl/include/eclhelper.hpp

@@ -2683,6 +2683,14 @@ struct IHThorExternalArg : public IHThorArg
     virtual IThorExternalRowProcessor * createProcessor() = 0;
 };
 
+//------------------------- Dictionary stuff -------------------------
+
+interface IHThorHashLookupInfo
+{
+    virtual IHash * queryHash() = 0;
+    virtual ICompare * queryCompare() = 0;
+};
+
 
 //------------------------- Other stuff -------------------------
 

testing/ecl/dict1.ecl

@@ -0,0 +1,23 @@
+
+d := nofold(DATASET(
+  [{1, {'Richard' => 50}},
+   {1, {'Richard' => 100}},
+   {1, {'R2ichard' => 100}},
+   {1, {'Ri3chard' => 100}},
+   {1, {'Ric4hard' => 100}},
+   {1, {'Rich5ard' => 100}},
+   {1, {'Richa6rd' => 100}},
+   {1, {'Richar7d' => 100}},
+   {1, {'Richar8d' => 100}},
+   {1, {'Richar9d' => 100}},
+   {1, {'Richard' => 100}},
+   {1, {'Richard' => 100}}
+  ], { unsigned top, DICTIONARY({STRING8 name => UNSIGNED iq}) nest}));
+//d := nofold(DATASET([{1, [{'Richard' , 50}]}, {1, [{'Richard' , 100}]}], { unsigned top, DATASET({STRING8 name , UNSIGNED iq}) nest}));
+
+d t(d L, d R) := TRANSFORM
+  SELF.nest := L.nest + R.nest;
+  SELF := R;
+END;
+
+rollup(d, LEFT.top=RIGHT.top, t(LEFT, RIGHT));

testing/ecl/dict2.ecl

@@ -0,0 +1,49 @@
+// Valid...
+d1 := dictionary([{5 => 'Richard'}], { integer id => string name });
+d2 := dictionary([{5,2 => 'Richard'}], { integer id1, integer id2 => string name });
+d3 := dictionary([], { integer id => string name });
+d4 := dictionary([], { integer id, string name });
+d5 := dictionary([{5, 'Richard'}], { integer id, string name });
+
+d1n := nofold(d1);
+d2n := nofold(d2);
+d3n := nofold(d3);
+d4n := nofold(d4);
+d5n := nofold(d5);
+
+d1[5].name = 'Richard';
+5 in d1;
+d1[1].name = '';
+1 not in d1;
+count(d1) = 1;
+d1n[5].name = 'Richard';
+5 in d1n;
+d1n[1].name = '';
+1 not in d1n;
+count(d1n) = 1;
+
+d2[5,2].name = 'Richard';
+d2[5,1].name = '';
+ROW({5,2}, { integer id1, integer id2} ) in d2;
+ROW({5,1}, { integer id1, integer id2} ) not in d2;
+count(d2) = 1;
+d2n[5,2].name = 'Richard';
+d2n[5,1].name = '';
+ROW({5,2}, { integer id1, integer id2} ) in d2n;
+ROW({5,1}, { integer id1, integer id2} ) not in d2n;
+count(d2n) = 1;
+
+5 not in d3;
+count(d3) = 0;
+5 not in d3n;
+count(d3n) = 0;
+
+ds6 := dataset([{5, 'Richard'}], { integer id, string name });
+d6 := DICTIONARY(ds6, { id => name });
+
+d7 := IF(nofold(true), d5);
+d7n := IF(nofold(true), d5, d4);
+ROW({5,'Richard'}, { integer id, string name} ) in d7;
+ROW({5,'Richard'}, { integer id, string name} ) in d7n;
+ROW({52,'Richard'}, { integer id, string name} ) not in d7;
+ROW({51,'Richard'}, { integer id, string name} ) not in d7n;

testing/ecl/dict3.ecl

@@ -0,0 +1,5 @@
+ds6 := nofold(dataset([{5, 'Richard'}], { integer id, string name }));
+d6 := DICTIONARY(ds6, { id => name });
+
+5 in d6;
+count(d6)=1;

testing/ecl/key/dict1.xml

@@ -0,0 +1,3 @@
+<Dataset name='Result 1'>
+ <Row><top>1</top><nest><Row><name>Richar9d</name><iq>100</iq></Row><Row><name>R2ichard</name><iq>100</iq></Row><Row><name>Rich5ard</name><iq>100</iq></Row><Row><name>Richar8d</name><iq>100</iq></Row><Row><name>Ri3chard</name><iq>100</iq></Row><Row><name>Richard </name><iq>100</iq></Row><Row><name>Richa6rd</name><iq>100</iq></Row><Row><name>Ric4hard</name><iq>100</iq></Row><Row><name>Richar7d</name><iq>100</iq></Row></nest></Row>
+</Dataset>

testing/ecl/key/dict2.xml

@@ -0,0 +1,72 @@
+<Dataset name='Result 1'>
+ <Row><Result_1>true</Result_1></Row>
+</Dataset>
+<Dataset name='Result 2'>
+ <Row><Result_2>true</Result_2></Row>
+</Dataset>
+<Dataset name='Result 3'>
+ <Row><Result_3>true</Result_3></Row>
+</Dataset>
+<Dataset name='Result 4'>
+ <Row><Result_4>true</Result_4></Row>
+</Dataset>
+<Dataset name='Result 5'>
+ <Row><Result_5>true</Result_5></Row>
+</Dataset>
+<Dataset name='Result 6'>
+ <Row><Result_6>true</Result_6></Row>
+</Dataset>
+<Dataset name='Result 7'>
+ <Row><Result_7>true</Result_7></Row>
+</Dataset>
+<Dataset name='Result 8'>
+ <Row><Result_8>true</Result_8></Row>
+</Dataset>
+<Dataset name='Result 9'>
+ <Row><Result_9>true</Result_9></Row>
+</Dataset>
+<Dataset name='Result 10'>
+ <Row><Result_10>true</Result_10></Row>
+</Dataset>
+<Dataset name='Result 11'>
+ <Row><Result_11>true</Result_11></Row>
+</Dataset>
+<Dataset name='Result 12'>
+ <Row><Result_12>true</Result_12></Row>
+</Dataset>
+<Dataset name='Result 13'>
+ <Row><Result_13>true</Result_13></Row>
+</Dataset>
+<Dataset name='Result 14'>
+ <Row><Result_14>true</Result_14></Row>
+</Dataset>
+<Dataset name='Result 15'>
+ <Row><Result_15>true</Result_15></Row>
+</Dataset>
+<Dataset name='Result 16'>
+ <Row><Result_16>true</Result_16></Row>
+</Dataset>
+<Dataset name='Result 17'>
+ <Row><Result_17>true</Result_17></Row>
+</Dataset>
+<Dataset name='Result 18'>
+ <Row><Result_18>true</Result_18></Row>
+</Dataset>
+<Dataset name='Result 19'>
+ <Row><Result_19>true</Result_19></Row>
+</Dataset>
+<Dataset name='Result 20'>
+ <Row><Result_20>true</Result_20></Row>
+</Dataset>
+<Dataset name='Result 21'>
+ <Row><Result_21>true</Result_21></Row>
+</Dataset>
+<Dataset name='Result 22'>
+ <Row><Result_22>true</Result_22></Row>
+</Dataset>
+<Dataset name='Result 23'>
+ <Row><Result_23>true</Result_23></Row>
+</Dataset>
+<Dataset name='Result 24'>
+ <Row><Result_24>true</Result_24></Row>
+</Dataset>

testing/ecl/key/dict3.xml

@@ -0,0 +1,6 @@
+<Dataset name='Result 1'>
+ <Row><Result_1>true</Result_1></Row>
+</Dataset>
+<Dataset name='Result 2'>
+ <Row><Result_2>true</Result_2></Row>
+</Dataset>