Merge pull request #3794 from richardkchapman/dictionary_support

HPCC-8329 Dictionary support

Reviewed-by: Gavin Halliday <ghalliday@hpccsystems.com>

common/commonext/commonext.cpp

@@ -201,6 +201,8 @@ MODULE_INIT(INIT_PRIORITY_STANDARD)
     kindArray[TAKexternalprocess] = "externalprocess";
     kindArray[TAKwhen_action] = "when_action";
     kindArray[TAKshuffle] = "shuffle";
+    kindArray[TAKdictionaryworkunitwrite] = "dictionaryworkunitwrite";
+    kindArray[TAKdictionaryresultwrite] = "dictionaryresultwrite";
 
 //Non standard
     kindArray[TAKcountdisk] = "countdisk";

common/thorhelper/thorcommon.cpp

@@ -769,6 +769,8 @@ extern const char * getActivityText(ThorActivityKind kind)
     case TAKexternalprocess:        return "User Proceess";
     case TAKwhen_action:            return "When";
     case TAKshuffle:                return "Shuffle";
+    case TAKdictionaryworkunitwrite:return "Dictionary Write";
+    case TAKdictionaryresultwrite:  return "Dictionary Result";
     }
     throwUnexpected();
 }
@@ -850,6 +852,8 @@ extern bool isActivitySink(ThorActivityKind kind)
     case TAKparallel:
     case TAKsequential:
     case TAKwhen_action:
+    case TAKdictionaryworkunitwrite:
+    case TAKdictionaryresultwrite:
         return true;
     }
     return false;

common/thorhelper/thorcommon.hpp

@@ -411,10 +411,14 @@ public:
     {
         return ctx->getRowAllocator(meta, activityId);
     }
-    virtual void getResultRowset(size32_t & tcount, byte * * & tgt, const char * name, unsigned sequence, IEngineRowAllocator * _rowAllocator, IOutputRowDeserializer * deserializer, bool isGrouped, IXmlToRowTransformer * xmlTransformer, ICsvToRowTransformer * csvTransformer) 
+    virtual void getResultRowset(size32_t & tcount, byte * * & tgt, const char * name, unsigned sequence, IEngineRowAllocator * _rowAllocator, IOutputRowDeserializer * deserializer, bool isGrouped, IXmlToRowTransformer * xmlTransformer, ICsvToRowTransformer * csvTransformer)
     {
         ctx->getResultRowset(tcount, tgt, name, sequence, _rowAllocator, deserializer, isGrouped, xmlTransformer, csvTransformer);
     }
+    virtual void getResultDictionary(size32_t & tcount, byte * * & tgt, IEngineRowAllocator * _rowAllocator, const char * name, unsigned sequence, IOutputRowDeserializer * deserializer, IXmlToRowTransformer * xmlTransformer, ICsvToRowTransformer * csvTransformer, IHThorHashLookupInfo * hasher)
+    {
+        ctx->getResultDictionary(tcount, tgt, _rowAllocator, name, sequence, deserializer, xmlTransformer, csvTransformer, hasher);
+    }
     virtual void getRowXML(size32_t & lenResult, char * & result, IOutputMetaData & info, const void * row, unsigned flags)
     {
         convertRowToXML(lenResult, result, info, row, flags);

ecl/eclagent/eclagent.cpp

@@ -1053,6 +1053,18 @@ void EclAgent::getResultRowset(size32_t & tcount, byte * * & tgt, const char * s
     );
 }
 
+void EclAgent::getResultDictionary(size32_t & tcount, byte * * & tgt, IEngineRowAllocator * _rowAllocator, const char * stepname, unsigned sequence, IOutputRowDeserializer * deserializer, IXmlToRowTransformer * xmlTransformer, ICsvToRowTransformer * csvTransformer, IHThorHashLookupInfo * hasher)
+{
+    tcount = 0;
+    tgt = NULL;
+    PROTECTED_GETRESULT(stepname, sequence, "Rowset", "rowset",
+        MemoryBuffer datasetBuffer;
+        MemoryBuffer2IDataVal result(datasetBuffer);
+        r->getResultRaw(result, NULL, NULL);
+        rtlDictionary2RowsetX(tcount, tgt, _rowAllocator, deserializer, datasetBuffer.length(), datasetBuffer.toByteArray());
+    );
+}
+
 const void * EclAgent::fromXml(IEngineRowAllocator * rowAllocator, size32_t len, const char * utf8, IXmlToRowTransformer * xmlTransformer, bool stripWhitespace)
 {
     return createRowFromXml(rowAllocator, len, utf8, xmlTransformer, stripWhitespace);

ecl/eclagent/eclagent.ipp

@@ -483,6 +483,7 @@ public:
     virtual unsigned getResultHash(const char * name, unsigned sequence);
     virtual void getExternalResultRaw(unsigned & tlen, void * & tgt, const char * wuid, const char * stepname, unsigned sequence, IXmlToRowTransformer * xmlTransformer, ICsvToRowTransformer * csvTransformer);
     virtual void getResultRowset(size32_t & tcount, byte * * & tgt, const char * name, unsigned sequence, IEngineRowAllocator * _rowAllocator, IOutputRowDeserializer * deserializer, bool isGrouped, IXmlToRowTransformer * xmlTransformer, ICsvToRowTransformer * csvTransformer);
+    virtual void getResultDictionary(size32_t & tcount, byte * * & tgt, IEngineRowAllocator * _rowAllocator, const char * name, unsigned sequence, IOutputRowDeserializer * deserializer, IXmlToRowTransformer * xmlTransformer, ICsvToRowTransformer * csvTransformer, IHThorHashLookupInfo * hasher);
     virtual char *getDaliServers();
     virtual char *getJobName();
     virtual char *getJobOwner();
@@ -849,6 +850,10 @@ public:
     {
         queryResult(id)->getLinkedResult(count, ret);
     }
+    virtual void getDictionaryResult(unsigned & count, byte * * & ret, unsigned id)
+    {
+        queryResult(id)->getLinkedResult(count, ret);
+    }
 
 protected:
     void ensureAtleast(unsigned id);
@@ -1006,6 +1011,7 @@ public:
 
     virtual void getResult(unsigned & len, void * & data, unsigned id);
     virtual void getLinkedResult(unsigned & count, byte * * & ret, unsigned id);
+    virtual void getDictionaryResult(size32_t & tcount, byte * * & tgt, unsigned id);
     inline unsigned __int64 queryId() const
     {
         return id;

ecl/eclagent/eclgraph.cpp

@@ -98,6 +98,8 @@ static IHThorActivity * createActivity(IAgentContext & agent, unsigned activityI
         return createGroupActivity(agent, activityId, subgraphId, (IHThorGroupArg &)arg, kind);
     case TAKworkunitwrite:
         return createWorkUnitWriteActivity(agent, activityId, subgraphId, (IHThorWorkUnitWriteArg &)arg, kind);
+    case TAKdictionaryworkunitwrite:
+        return createDictionaryWorkUnitWriteActivity(agent, activityId, subgraphId, (IHThorDictionaryWorkUnitWriteArg &)arg, kind);
     case TAKfunnel:
         return createConcatActivity(agent, activityId, subgraphId, (IHThorFunnelArg &)arg, kind);
     case TAKapply:
@@ -1068,6 +1070,12 @@ void EclSubGraph::getLinkedResult(unsigned & count, byte * * & ret, unsigned id)
 }
 
 
+void EclSubGraph::getDictionaryResult(unsigned & count, byte * * & ret, unsigned id)
+{
+    localResults->queryResult(id)->getLinkedResult(count, ret);
+}
+
+
 EclGraphElement * EclSubGraph::idToActivity(unsigned id)
 {
     ForEachItemIn(idx, elements)

ecl/hql/hqlattr.cpp

@@ -275,9 +275,7 @@ unsigned getOperatorMetaFlags(node_operator op)
     case no_temprow:
 
 //Dictionaries
-    case no_userdictionary:
-    case no_newuserdictionary:
-    case no_inlinedictionary:
+    case no_createdictionary:
 
 //Datasets [see also selection operators]
     case no_rollup:
@@ -622,6 +620,7 @@ unsigned getOperatorMetaFlags(node_operator op)
     case no_unused40: case no_unused41: case no_unused42: case no_unused43: case no_unused44: case no_unused45: case no_unused46: case no_unused47: case no_unused48: case no_unused49:
     case no_unused50: case no_unused52:
     case no_unused80: case no_unused83:
+    case no_unused100: case no_unused101:
     case no_is_null:
     case no_position:
     case no_current_time:

ecl/hql/hqlexpr.cpp

@@ -1446,7 +1446,7 @@ const char *getOpString(node_operator op)
     case no_debug_option_value: return "__DEBUG__";
     case no_dataset_alias: return "TABLE";
     case no_childquery: return "no_childquery";
-    case no_inlinedictionary: case no_userdictionary: case no_newuserdictionary: return "DICTIONARY";
+    case no_createdictionary: return "DICTIONARY";
     case no_chooseds: return "CHOOSE";
 
     case no_unused6:
@@ -1458,6 +1458,8 @@ const char *getOpString(node_operator op)
     case no_unused80:
     case no_unused81:
     case no_unused83:
+    case no_unused100:
+    case no_unused101:
         return "unused";
     /* if fail, use "hqltest -internal" to find out why. */
     default: assertex(false); return "???";
@@ -1816,8 +1818,6 @@ childDatasetType getChildDatasetType(IHqlExpression * expr)
     case no_transformascii:
     case no_selectfields:
     case no_newaggregate:
-    case no_userdictionary:
-    case no_newuserdictionary:
     case no_newusertable:
     case no_usertable:
     case no_alias_project:
@@ -2107,7 +2107,6 @@ inline unsigned doGetNumChildTables(IHqlExpression * dataset)
     case no_compound_inline:
     case no_transformascii:
     case no_transformebcdic:
-    case no_newuserdictionary:
     case no_newusertable:
     case no_aggregate:
     case no_usertable:
@@ -2613,7 +2612,6 @@ IHqlExpression * queryNewColumnProvider(IHqlExpression * expr)
     case no_createrow:
     case no_typetransfer:
         return expr->queryChild(0);
-    case no_userdictionary:
     case no_usertable:
     case no_selectfields:
     case no_transformebcdic:
@@ -2631,7 +2629,6 @@ IHqlExpression * queryNewColumnProvider(IHqlExpression * expr)
     case no_newkeyindex:
     case no_aggregate:
     case no_newaggregate:
-    case no_newuserdictionary:
     case no_newusertable:
     case no_normalize:
     case no_xmlparse:
@@ -2899,6 +2896,7 @@ IHqlExpression * ensureExprType(IHqlExpression * expr, ITypeInfo * type, node_op
     case type_groupedtable:
         if (recordTypesMatch(type, exprType))
             return LINK(expr);
+        assertex(!expr->isDictionary());
         break;
     case type_scope:
     case type_function:
@@ -3592,6 +3590,12 @@ switch (op)
             }
             break;
         }
+    case no_indict:
+        assertex(queryChild(1)->isDictionary());
+        break;
+    case no_countdict:
+        assertex(queryChild(0)->isDictionary());
+        break;
     }
 
 #ifdef _DEBUG
@@ -9943,10 +9947,8 @@ IHqlExpression *createDictionary(node_operator op, HqlExprArray & parms)
 
     switch (op)
     {
-    case no_newuserdictionary:
-    case no_userdictionary:
-    case no_inlinedictionary:
-        type.setown(makeDictionaryType(makeRowType(createRecordType(&parms.item(1)))));
+    case no_createdictionary:
+        type.setown(makeDictionaryType(makeRowType(createRecordType(&parms.item(0)))));
         break;
     case no_select:
         type.set(parms.item(1).queryType());
@@ -9957,6 +9959,9 @@ IHqlExpression *createDictionary(node_operator op, HqlExprArray & parms)
     case no_if:
         type.set(parms.item(1).queryType());  // It's an error if they don't match, caught elsewhere
         break;
+    case no_chooseds:
+        type.set(parms.item(1).queryType());  // It's an error if they don't match, caught elsewhere
+        break;
     case no_case:
         //following is wrong, but they get removed pretty quickly so I don't really care
         type.set(parms.item(1).queryType());
@@ -9968,6 +9973,9 @@ IHqlExpression *createDictionary(node_operator op, HqlExprArray & parms)
     case no_null:
     case no_fail:
     case no_anon:
+    case no_workunit_dataset:
+    case no_getgraphresult:
+    case no_getresult:
     {
         IHqlExpression * record = &parms.item(0);
         IHqlExpression * metadata = queryProperty(_metadata_Atom, parms);
@@ -9993,6 +10001,7 @@ IHqlExpression *createDictionary(node_operator op, HqlExprArray & parms)
     case no_alias:
     case no_translated:
     case no_catch:
+    case no_colon:
         type.set(parms.item(0).queryType());
         break;
     default:
@@ -12955,6 +12964,9 @@ static IHqlExpression * doAttachWorkflowOwn(IHqlExpression * value, IHqlExpressi
         
         if (value->queryType()->getTypeCode() == type_row)
             return createRow(no_colon, value, LINK(workflow));
+
+        if (value->isDictionary())
+            return createDictionary(no_colon, value, LINK(workflow));
     }
 
     //If a string value is stored, its type is a string of unknown length 
@@ -14875,7 +14887,6 @@ extern HQL_API bool hasUnknownTransform(IHqlExpression * expr)
         if (expr->hasProperty(mergeTransformAtom))
             return true;
         break;
-    case no_inlinedictionary:
     case no_inlinetable:
         {
             IHqlExpression * transforms = expr->queryChild(0);
@@ -14940,7 +14951,7 @@ bool transformHasSkipAttr(IHqlExpression * transform)
 
 bool isPureInlineDataset(IHqlExpression * expr)
 {
-    assertex(expr->getOperator() == no_inlinetable || expr->getOperator() == no_inlinedictionary);
+    assertex(expr->getOperator() == no_inlinetable);
     IHqlExpression * values = expr->queryChild(0);
     ForEachChild(i, values)
     {

ecl/hql/hqlexpr.hpp

@@ -357,12 +357,12 @@ enum _node_operator {
         no_dataset_from_transform,
         no_childquery,
         no_unknown,
-        no_inlinedictionary,
+        no_createdictionary,
         no_indict,
         no_countdict,
         no_any,
-        no_userdictionary,
-        no_newuserdictionary,
+    no_unused100,
+    no_unused101,
     no_unused25,
     no_unused28,  
     no_unused29,

ecl/hql/hqlfold.cpp

@@ -2702,7 +2702,7 @@ IHqlExpression * foldConstantOperator(IHqlExpression * expr, unsigned foldOption
                         if (alreadyDone.find(*condValue) == NotFound)
                         {
                             alreadyDone.append(*condValue);
-                            args2.append(*createValue(no_mapto, LINK(condValue), LINK(mapValue)));
+                            args2.append(*createValue(no_mapto, mapValue->getType(), LINK(condValue), LINK(mapValue)));
                         }
                     }
                 }
@@ -3002,13 +3002,7 @@ IHqlExpression * foldConstantOperator(IHqlExpression * expr, unsigned foldOption
         {
             IHqlExpression * child = expr->queryChild(0);
             node_operator childOp = child->getOperator();
-            switch (childOp)
-            {
-            case no_inlinedictionary:
-                if (isPureInlineDataset(child))
-                    return createConstant(expr->queryType()->castFrom(false, (__int64)child->queryChild(0)->numChildren()));
-                break;
-            }
+            //MORE: Can't really optimize count of a dictionary since the input dataset may contain duplicates which will be removed.
             break;
         }
     case no_countlist:
@@ -3579,7 +3573,7 @@ IHqlExpression * NullFolderMixin::foldNullDataset(IHqlExpression * expr)
             break;
         }
     case no_newusertable:
-        if (isNullProject(expr, false))
+        if (isNullProject(expr, false, false))
             return removeParentNode(expr);
         if (isNull(child))
         {
@@ -3766,7 +3760,7 @@ IHqlExpression * NullFolderMixin::foldNullDataset(IHqlExpression * expr)
     case no_hqlproject:
     case no_projectrow:
         {
-            if (isNullProject(expr, false))
+            if (isNullProject(expr, false, false))
                 return removeParentNode(expr);
             if (isNull(child))
                 return replaceWithNull(expr);
@@ -5584,6 +5578,7 @@ HqlConstantPercolator * CExprFolderTransformer::gatherConstants(IHqlExpression *
     case no_outofline:
     case no_owned_ds:
     case no_dataset_alias:
+    case no_createdictionary:
         exprMapping.set(gatherConstants(expr->queryChild(0)));
         break;
     case no_normalizegroup:
@@ -5594,12 +5589,6 @@ HqlConstantPercolator * CExprFolderTransformer::gatherConstants(IHqlExpression *
         //all bets are off.
         break;
 
-    case no_newuserdictionary:
-    case no_userdictionary:
-    case no_inlinedictionary:
-    case no_selectmap:
-        // MORE - maybe should be something here?
-        break;
 
     case no_selectnth:
         {
@@ -5709,6 +5698,7 @@ HqlConstantPercolator * CExprFolderTransformer::gatherConstants(IHqlExpression *
             exprMapping.set(gatherConstants(expr->queryChild(0)));
         break;
 
+    case no_selectmap:
     case no_select:
     case no_record:
         break;

ecl/hql/hqlgram.y

@@ -7096,25 +7096,29 @@ simpleDictionary
                             HqlExprArray args;
                             args.append(*LINK(dataset));
                             args.append(*LINK(record));
-                            $$.setExpr(createDictionary(no_userdictionary, args));
-                            parser->checkProjectedFields($$.queryExpr(), $5);
-                            $$.setPosition($1);
+                            OwnedHqlExpr ds = createDataset(no_usertable, args);
+                            parser->checkProjectedFields(ds, $5);
+                            $$.setExpr(createDictionary(no_createdictionary, ds.getClear()), $1);
+                        }
+    | DICTIONARY '(' startTopFilter ')' endTopFilter
+                        {
+                            OwnedHqlExpr ds = $3.getExpr();
+                            $$.setExpr(createDictionary(no_createdictionary, ds.getClear()), $1);
                         }
-
     | DICTIONARY '(' '[' ']' ',' recordDef ')'
                         {
                             HqlExprArray values;  // Empty list
                             OwnedHqlExpr table = createDataset(no_temptable, createValue(no_recordlist, NULL, values), $6.getExpr());
-                            $$.setExpr(convertTempTableToInlineDictionary(parser->errorHandler, $4.pos, table));
-                            $$.setPosition($1);
+                            OwnedHqlExpr ds = convertTempTableToInlineTable(parser->errorHandler, $4.pos, table);
+                            $$.setExpr(createDictionary(no_createdictionary, ds.getClear()), $1);
                         }
     | DICTIONARY '(' '[' beginList inlineDatasetValueList ']' ',' recordDef ')'
                         {
                             HqlExprArray values;
                             parser->endList(values);
                             OwnedHqlExpr table = createDataset(no_temptable, createValue(no_recordlist, NULL, values), $8.getExpr());
-                            $$.setExpr(convertTempTableToInlineDictionary(parser->errorHandler, $5.pos, table));
-                            $$.setPosition($1);
+                            OwnedHqlExpr ds = convertTempTableToInlineTable(parser->errorHandler, $5.pos, table);
+                            $$.setExpr(createDictionary(no_createdictionary, ds.getClear()), $1);
                         }
     | '(' dictionary  ')'  {
                             $$.setExpr($2.getExpr());
@@ -7140,9 +7144,137 @@ simpleDictionary
                             OwnedHqlExpr ds = parser->processIfProduction($3, $5, NULL);
                             $$.setExpr(ds.getClear(), $1);
                         }
-// MORE - should do CASE and MAP
+    | MAP '(' mapDictionarySpec ',' dictionary ')'
+                        {
+                            HqlExprArray args;
+                            IHqlExpression * elseDict = $5.getExpr();
+                            $3.unwindCommaList(args);
+                            ForEachItemIn(idx, args)
+                            {
+                                IHqlExpression * cur = args.item(idx).queryChild(1);
+                                parser->checkRecordTypes(cur, elseDict, $5);
+                            }
+                            args.append(*elseDict);
+                            $$.setExpr(::createDictionary(no_map, args));
+                            $$.setPosition($1);
+                        }
+    | MAP '(' mapDictionarySpec ')'
+                        {
+                            HqlExprArray args;
+                            $3.unwindCommaList(args);
+                            IHqlExpression * elseDict;
+                            if (args.ordinality())
+                                elseDict = createNullExpr(&args.item(0));
+                            else
+                                elseDict = createDictionary(no_null, LINK(queryNullRecord()));
+                            ForEachItemIn(idx, args)
+                            {
+                                IHqlExpression * cur = args.item(idx).queryChild(1);
+                                parser->checkRecordTypes(cur, elseDict, $1);
+                            }
+                            args.append(*elseDict);
+                            $$.setExpr(::createDictionary(no_map, args));
+                            $$.setPosition($1);
+                        }
+    | CASE '(' expression ',' beginList caseDictionarySpec ',' dictionary ')'
+                        {
+                            parser->normalizeExpression($3, type_scalar, false);
+                            HqlExprArray args;
+                            IHqlExpression * elseDict = $8.getExpr();
+                            parser->endList(args);
+                            parser->checkCaseForDuplicates(args, $6);
+                            ForEachItemIn(idx, args)
+                            {
+                                IHqlExpression * cur = args.item(idx).queryChild(1);
+                                parser->checkRecordTypes(cur, elseDict, $8);
+                            }
+                            args.add(*$3.getExpr(),0);
+                            args.append(*elseDict);
+                            $$.setExpr(::createDataset(no_case, args));
+                            $$.setPosition($1);
+                        }
+    | CASE '(' expression ',' beginList caseDictionarySpec ')'
+                        {
+                            parser->normalizeExpression($3, type_scalar, false);
+                            HqlExprArray args;
+                            parser->endList(args);
+                            IHqlExpression * elseDict;
+                            if (args.ordinality())
+                                elseDict = createNullExpr(&args.item(0));
+                            else
+                                elseDict = createDictionary(no_null, LINK(queryNullRecord()));
+                            parser->checkCaseForDuplicates(args, $6);
+                            ForEachItemIn(idx, args)
+                            {
+                                IHqlExpression * cur = args.item(idx).queryChild(1);
+                                parser->checkRecordTypes(cur, elseDict, $6);
+                            }
+                            args.add(*$3.getExpr(),0);
+                            args.append(*elseDict);
+                            $$.setExpr(::createDictionary(no_case, args));
+                            $$.setPosition($1);
+                        }
+    | CASE '(' expression ',' beginList dictionary ')'
+                        {
+                            parser->normalizeExpression($3, type_scalar, false);
+                            // change error to warning.
+                            parser->reportWarning(WRN_CASENOCONDITION, $1.pos, "CASE does not have any conditions");
+                            HqlExprArray list;
+                            parser->endList(list);
+                            $3.release();
+                            $$.setExpr($6.getExpr(), $1);
+                        }
+    | FAIL '(' dictionary failDatasetParam ')'
+                        {
+                            OwnedHqlExpr dict = $3.getExpr();
+                            //Actually allow a sequence of arbitrary actions....
+                            $$.setExpr(createDictionary(no_fail, LINK(dict->queryRecord()), $4.getExpr()));
+                            $$.setPosition($1);
+                        }
+    | TOK_ERROR '(' dictionary failDatasetParam ')'
+                        {
+                            OwnedHqlExpr dict = $3.getExpr();
+                            //Actually allow a sequence of arbitrary actions....
+                            $$.setExpr(createDictionary(no_fail, LINK(dict->queryRecord()), $4.getExpr()));
+                            $$.setPosition($1);
+                        }
+    | CHOOSE '(' expression ',' dictionaryList ')'
+                        {
+                            parser->normalizeExpression($3, type_int, false);
+                            OwnedHqlExpr values = $5.getExpr();
+                            HqlExprArray args;
+                            values->unwindList(args, no_comma);
+
+                            IHqlExpression * compareDict = NULL;
+                            ForEachItemIn(idx, args)
+                            {
+                                IHqlExpression * cur = &args.item(idx);
+                                if (cur->queryRecord())
+                                {
+                                    if (compareDict)
+                                    {
+                                        parser->checkRecordTypes(cur, compareDict, $5);
+                                    }
+                                    else
+                                        compareDict = cur;
+                                }
+                            }
+
+                            args.add(*$3.getExpr(), 0);
+                            $$.setExpr(createDictionary(no_chooseds, args), $1); // no_choosedict ?
+                        }
     ;
 
+dictionaryList
+    : dictionary
+    | dictionary ',' dictionaryList
+                        {
+                            $$.setExpr(createComma($1.getExpr(), $3.getExpr()));
+                            $$.setPosition($1);
+                        }
+    ;
+
+
 scopedDictionaryId
     : globalScopedDictionaryId
     | dotScope DICTIONARY_ID leaveScope
@@ -7157,7 +7289,6 @@ scopedDictionaryId
                                 $$.setExpr(e2);
                             }
                         }
-/*
     | dictionaryFunction '('
                         {
                             parser->beginFunctionCall($1);
@@ -7166,7 +7297,6 @@ scopedDictionaryId
                         {
                             $$.setExpr(parser->bindParameters($1, $4.getExpr()));
                         }
-*/
     ;
 
 globalScopedDictionaryId
@@ -10659,6 +10789,20 @@ datasetFunction
                         }
     ;
 
+dictionaryFunction
+    : DICTIONARY_FUNCTION
+    | moduleScopeDot DICTIONARY_FUNCTION leaveScope
+                        {
+                            $1.release();
+                            $$.setExpr($2.getExpr());
+                        }
+    | startCompoundExpression reqparmdef beginInlineFunctionToken optDefinitions RETURN dictionary ';' endInlineFunctionToken
+                        {
+                            Owned<ITypeInfo> retType = $1.getType();
+                            $$.setExpr(parser->leaveLamdaExpression($6), $8);
+                        }
+    ;
+
 scopeFunction
     : SCOPE_FUNCTION
     | moduleScopeDot SCOPE_FUNCTION leaveScope
@@ -11075,6 +11219,24 @@ mapDatasetItem
                         }
     ;
 
+mapDictionarySpec
+    : mapDictionaryItem
+    | mapDictionarySpec ',' mapDictionaryItem
+                        {
+                            ITypeInfo *type = parser->checkType($1, $3);
+                            $$.setExpr(createValue(no_comma, type, $1.getExpr(), $3.getExpr()));
+                        }
+    ;
+
+mapDictionaryItem
+    : booleanExpr GOESTO dictionary
+                        {
+                            IHqlExpression *e3 = $3.getExpr();
+                            $$.setExpr(createValue(no_mapto, e3->getType(), $1.getExpr(), e3));
+                            $$.setPosition($3);
+                        }
+    ;
+
 caseDatasetSpec
     : caseDatasetItem
     | caseDatasetSpec ',' caseDatasetItem
@@ -11092,6 +11254,23 @@ caseDatasetItem
                         }
     ;
 
+caseDictionarySpec
+    : caseDictionaryItem
+    | caseDictionarySpec ',' caseDictionaryItem
+    ;
+
+caseDictionaryItem
+    : expression GOESTO dictionary
+                        {
+                            parser->normalizeExpression($1);
+                            parser->applyDefaultPromotions($1, true);
+                            IHqlExpression *e3 = $3.getExpr();
+                            parser->addListElement(createValue(no_mapto, e3->getType(), $1.getExpr(), e3));
+                            $$.clear();
+                            $$.setPosition($3);
+                        }
+    ;
+
 mapDatarowSpec
     : mapDatarowItem
     | mapDatarowSpec ',' mapDatarowItem

ecl/hql/hqlgram2.cpp

@@ -3654,6 +3654,14 @@ ITypeInfo *HqlGram::checkPromoteIfType(attribute &a1, attribute &a2)
         checkDatarow(a2);
         return NULL;
     }
+    if (a1.isDictionary() || a2.isDictionary())
+    {
+        OwnedHqlExpr right = a2.getExpr();
+        a2.setExpr(checkEnsureRecordsMatch(a1.queryExpr(), right, a2, true));
+        checkDictionary(a1);
+        checkDictionary(a2);
+        return NULL;
+    }
 
     checkCompatible(a1.queryExprType(), a2.queryExprType(), a2);
     ITypeInfo *t1 = a1.queryExprType();
@@ -8987,6 +8995,7 @@ void HqlGram::defineSymbolProduction(attribute & nameattr, attribute & paramattr
                 case type_record:
                 case type_row:
                 case type_transform:
+                case type_dictionary:
                     break;
                 default:
                     expr.setown(forceEnsureExprType(expr, type));
@@ -10362,6 +10371,7 @@ void HqlGram::simplifyExpected(int *expected)
                        TOXML, '@', SECTION, EVENTEXTRA, EVENTNAME, __SEQUENCE__, IFF, OMITTED, GETENV, __DEBUG__, __STAND_ALONE__, 0);
     simplify(expected, DATA_CONST, REAL_CONST, STRING_CONST, INTEGER_CONST, UNICODE_CONST, 0);
     simplify(expected, VALUE_MACRO, DEFINITIONS_MACRO, 0);
+    simplify(expected, DICTIONARY_ID, DICTIONARY_FUNCTION, DICTIONARY, 0);
     simplify(expected, VALUE_ID, DATASET_ID, DICTIONARY_ID, RECORD_ID, ACTION_ID, UNKNOWN_ID, SCOPE_ID, VALUE_FUNCTION, DATAROW_FUNCTION, DATASET_FUNCTION, DICTIONARY_FUNCTION, LIST_DATASET_FUNCTION, LIST_DATASET_ID, ALIEN_ID, TYPE_ID, SET_TYPE_ID, TRANSFORM_ID, TRANSFORM_FUNCTION, RECORD_FUNCTION, FEATURE_ID, EVENT_ID, EVENT_FUNCTION, SCOPE_FUNCTION, ENUM_ID, PATTERN_TYPE_ID, 0);
     simplify(expected, LIBRARY, LIBRARY, SCOPE_FUNCTION, STORED, PROJECT, INTERFACE, MODULE, 0);
     simplify(expected, MATCHROW, MATCHROW, LEFT, RIGHT, IF, IFF, ROW, HTTPCALL, SOAPCALL, PROJECT, GLOBAL, NOFOLD, NOHOIST, ALLNODES, THISNODE, SKIP, DATAROW_FUNCTION, TRANSFER, RIGHT_NN, FROMXML, 0);

ecl/hql/hqlir.cpp

@@ -814,11 +814,9 @@ static const char * getOperatorText(node_operator op)
     DUMP_CASE(no,dataset_alias);
     DUMP_CASE(no,childquery);
     DUMP_CASE(no,selectmap);
-    DUMP_CASE(no,inlinedictionary);
+    DUMP_CASE(no,createdictionary);
     DUMP_CASE(no,indict);
     DUMP_CASE(no,countdict);
-    DUMP_CASE(no,userdictionary);
-    DUMP_CASE(no,newuserdictionary);
     DUMP_CASE(no,chooseds);
 
     case no_unused6:
@@ -827,6 +825,8 @@ static const char * getOperatorText(node_operator op)
     case no_unused30: case no_unused31: case no_unused32: case no_unused33: case no_unused34: case no_unused35: case no_unused36: case no_unused37: case no_unused38:
     case no_unused40: case no_unused41: case no_unused42: case no_unused43: case no_unused44: case no_unused45: case no_unused46: case no_unused47: case no_unused48: case no_unused49:
     case no_unused50: case no_unused52:
+    case no_unused100:
+    case no_unused101:
         return "unused";
     }
     return "unknown_op";

ecl/hql/hqlpmap.cpp

@@ -972,10 +972,11 @@ static bool isTrivialTransform(IHqlExpression * expr, IHqlExpression * selector)
     return true;
 }
 
-bool isNullProject(IHqlExpression * expr, bool canLoseFieldsFromEnd)
+bool isNullProject(IHqlExpression * expr, bool canIgnorePayload, bool canLoseFieldsFromEnd)
 {
     IHqlExpression * ds = expr->queryChild(0);
-    if (!recordTypesMatchIgnorePayload(expr, ds))
+    bool matches = canIgnorePayload ? recordTypesMatchIgnorePayload(expr, ds) : recordTypesMatch(expr, ds);
+    if (!matches)
     {
         if (canLoseFieldsFromEnd)
         {
@@ -999,7 +1000,6 @@ bool isSimpleProject(IHqlExpression * expr)
     case no_projectrow:
         selector.setown(createSelector(no_left, ds, querySelSeq(expr)));
         break;
-    case no_newuserdictionary:
     case no_newusertable:
          if (isAggregateDataset(expr))
              return false;

ecl/hql/hqlpmap.hpp

@@ -175,7 +175,7 @@ extern HQL_API void replaceSelectors(HqlExprArray & out, unsigned first, IHqlExp
 extern HQL_API IHqlExpression * scopedReplaceSelector(IHqlExpression * expr, IHqlExpression * oldDataset, IHqlExpression * newDataset);
 extern HQL_API IHqlExpression * replaceSelfRefSelector(IHqlExpression * expr, IHqlExpression * newDataset);
 
-extern HQL_API bool isNullProject(IHqlExpression * expr, bool canLoseFieldsFromEnd);
+extern HQL_API bool isNullProject(IHqlExpression * expr, bool canIgnorePayload, bool canLoseFieldsFromEnd);
 extern HQL_API bool isSimpleProject(IHqlExpression * expr);                             // Restriction or rearrangement only
 extern HQL_API bool leftRecordIsSubsetOfRight(IHqlExpression * left, IHqlExpression * right);
 extern HQL_API IHqlExpression * transformTrivialSelectProject(IHqlExpression * select);

ecl/hql/hqltrans.cpp

@@ -3375,7 +3375,7 @@ void ScopedTransformer::analyseChildren(IHqlExpression * expr)
     case no_setgraphresult:
     case no_setgraphloopresult:
     case no_extractresult:
-    case no_newuserdictionary:
+    case no_createdictionary:
         {
             IHqlExpression * dataset = expr->queryChild(0);
             pushScope();
@@ -3753,7 +3753,7 @@ IHqlExpression * ScopedTransformer::createTransformed(IHqlExpression * expr)
     case no_setgraphresult:
     case no_setgraphloopresult:
     case no_extractresult:
-    case no_newuserdictionary:
+    case no_createdictionary:
         {
             IHqlExpression * dataset = expr->queryChild(0);
             pushScope();

ecl/hql/hqlutil.cpp

@@ -3250,7 +3250,7 @@ IHqlExpression * createGetResultFromSetResult(IHqlExpression * setResult, ITypeI
     case type_groupedtable:
         return createDataset(no_getresult, LINK(queryOriginalRecord(valueType)), createComma(LINK(seqAttr), createAttribute(groupedAtom), LINK(aliasAttr)));
     case type_dictionary:
-        return createDictionary(no_getresult, LINK(queryOriginalRecord(valueType)), createComma(LINK(seqAttr), createAttribute(groupedAtom), LINK(aliasAttr)));
+        return createDictionary(no_workunit_dataset, LINK(queryOriginalRecord(valueType)), createComma(LINK(seqAttr), LINK(aliasAttr)));
     case type_row:
     case type_record:
          return createRow(no_getresult, LINK(queryOriginalRecord(valueType)), createComma(LINK(seqAttr), LINK(aliasAttr)));
@@ -4569,6 +4569,10 @@ static bool splitDatasetAttribute(SharedHqlExpr & dataset, SharedHqlExpr & attri
     node_operator leftOp = left->getOperator();
     if ((leftOp !=no_select) || expr->hasProperty(newAtom))
     {
+        //Ensure selections from dictionaries do not have a separate activity for the row lookup.
+        if (leftOp == no_selectmap)
+            return false;
+
         IHqlExpression * lhs = LINK(left);
         IHqlExpression * field = expr->queryChild(1);
         if (lhs->isDataset()) lhs = createRow(no_activerow, lhs);
@@ -4620,7 +4624,7 @@ IHqlExpression * createSetResult(HqlExprArray & args)
         args.add(*attribute.getClear(), 1);
         return createValue(no_extractresult, makeVoidType(), args);
     }
-    if (value->isDataset())
+    if (value->isDataset() || value->isDictionary())
         return createValue(no_output, makeVoidType(), args);
     return createValue(no_setresult, makeVoidType(), args);
 }
@@ -5403,7 +5407,7 @@ IHqlExpression * convertTempRowToCreateRow(IErrorReceiver * errors, ECLlocation
     return expr->cloneAllAnnotations(ret);
 }
 
-static IHqlExpression * convertTempTableToInline(IErrorReceiver * errors, ECLlocation & location, IHqlExpression * expr, bool isDictionary)
+static IHqlExpression * convertTempTableToInline(IErrorReceiver * errors, ECLlocation & location, IHqlExpression * expr)
 {
     IHqlExpression * oldValues = expr->queryChild(0);
     IHqlExpression * record = expr->queryChild(1);
@@ -5440,18 +5444,13 @@ static IHqlExpression * convertTempTableToInline(IErrorReceiver * errors, ECLloc
     HqlExprArray children;
     children.append(*createValue(no_transformlist, makeNullType(), transforms));
     children.append(*LINK(record));
-    OwnedHqlExpr ret = isDictionary ? createDictionary(no_inlinedictionary, children) : createDataset(no_inlinetable, children);
+    OwnedHqlExpr ret = createDataset(no_inlinetable, children);
     return expr->cloneAllAnnotations(ret);
 }
 
 IHqlExpression * convertTempTableToInlineTable(IErrorReceiver * errors, ECLlocation & location, IHqlExpression * expr)
 {
-    return convertTempTableToInline(errors, location, expr, false);
-}
-
-IHqlExpression * convertTempTableToInlineDictionary(IErrorReceiver * errors, ECLlocation & location, IHqlExpression * expr)
-{
-    return convertTempTableToInline(errors, location, expr, true);
+    return convertTempTableToInline(errors, location, expr);
 }
 
 void setPayloadAttribute(HqlExprArray &args)

ecl/hql/hqlutil.hpp

@@ -454,7 +454,6 @@ extern HQL_API IHqlExpression * createINDictExpr(IErrorReceiver * errors, ECLloc
 extern HQL_API IHqlExpression *createINDictRow(IErrorReceiver * errors, ECLlocation & location, IHqlExpression *row, IHqlExpression *dict);
 extern HQL_API IHqlExpression * convertTempRowToCreateRow(IErrorReceiver * errors, ECLlocation & location, IHqlExpression * expr);
 extern HQL_API IHqlExpression * convertTempTableToInlineTable(IErrorReceiver * errors, ECLlocation & location, IHqlExpression * expr);
-extern HQL_API IHqlExpression * convertTempTableToInlineDictionary(IErrorReceiver * errors, ECLlocation & location, IHqlExpression * expr);
 extern HQL_API void setPayloadAttribute(HqlExprArray &args);
 
 extern HQL_API bool areTypesComparable(ITypeInfo * leftType, ITypeInfo * rightType);

ecl/hqlcpp/hqlcatom.cpp

@@ -275,6 +275,7 @@ _ATOM forceAllCheckAtom;
 _ATOM freeAtom;
 _ATOM freeExceptionAtom;
 _ATOM getBytesFromBuilderAtom;
+_ATOM getChildQueryDictionaryResultAtom;
 _ATOM getChildQueryLinkedResultAtom;
 _ATOM getChildQueryResultAtom;
 _ATOM getClusterSizeAtom;
@@ -288,6 +289,7 @@ _ATOM getFailMessageAtom;
 _ATOM getFilePositionAtom;
 _ATOM getGraphLoopCounterAtom;
 _ATOM getIsValidAtom;
+_ATOM getLocalDictionaryResultAtom;
 _ATOM getLocalFailMessageAtom;
 _ATOM getLocalFilePositionAtom;
 _ATOM getLocalLinkedResultAtom;
@@ -311,6 +313,7 @@ _ATOM getResultBoolAtom;
 _ATOM getResultDataAtom;
 _ATOM getResultDatasetAtom;
 _ATOM getResultDecimalAtom;
+_ATOM getResultDictionaryAtom;
 _ATOM getResultHashAtom;
 _ATOM getResultIntAtom;
 _ATOM getResultQStringAtom;
@@ -983,6 +986,7 @@ MODULE_INIT(INIT_PRIORITY_HQLATOM-1)
     MAKEATOM(free);
     MAKEATOM(freeException);
     MAKEATOM(getBytesFromBuilder);
+    MAKEATOM(getChildQueryDictionaryResult);
     MAKEATOM(getChildQueryLinkedResult);
     MAKEATOM(getChildQueryResult);
     MAKEATOM(getClusterSize);
@@ -996,6 +1000,7 @@ MODULE_INIT(INIT_PRIORITY_HQLATOM-1)
     MAKEATOM(getFilePosition);
     MAKEATOM(getGraphLoopCounter);
     MAKEATOM(getIsValid);
+    MAKEATOM(getLocalDictionaryResult);
     MAKEATOM(getLocalFailMessage);
     MAKEATOM(getLocalFilePosition);
     MAKEATOM(getLocalLinkedResult);
@@ -1019,6 +1024,7 @@ MODULE_INIT(INIT_PRIORITY_HQLATOM-1)
     MAKEATOM(getResultData);
     MAKEATOM(getResultDataset);
     MAKEATOM(getResultDecimal);
+    MAKEATOM(getResultDictionary);
     MAKEATOM(getResultHash);
     MAKEATOM(getResultInt);
     MAKEATOM(getResultQString);

ecl/hqlcpp/hqlcatom.hpp

@@ -276,6 +276,7 @@ extern _ATOM freeAtom;
 extern _ATOM freeExceptionAtom;
 extern _ATOM getBytesFromBuilderAtom;
 extern _ATOM getClusterSizeAtom;
+extern _ATOM getChildQueryDictionaryResultAtom;
 extern _ATOM getChildQueryLinkedResultAtom;
 extern _ATOM getChildQueryResultAtom;
 extern _ATOM getDatasetHashAtom;
@@ -288,6 +289,7 @@ extern _ATOM getFailMessageAtom;
 extern _ATOM getFilePositionAtom;
 extern _ATOM getGraphLoopCounterAtom;
 extern _ATOM getIsValidAtom;
+extern _ATOM getLocalDictionaryResultAtom;
 extern _ATOM getLocalFailMessageAtom;
 extern _ATOM getLocalFilePositionAtom;
 extern _ATOM getLocalLinkedResultAtom;
@@ -311,6 +313,7 @@ extern _ATOM getResultBoolAtom;
 extern _ATOM getResultDataAtom;
 extern _ATOM getResultDatasetAtom;
 extern _ATOM getResultDecimalAtom;
+extern _ATOM getResultDictionaryAtom;
 extern _ATOM getResultHashAtom;
 extern _ATOM getResultIntAtom;
 extern _ATOM getResultQStringAtom;

ecl/hqlcpp/hqlcpp.cpp

@@ -920,6 +920,12 @@ IHqlExpression * CHqlBoundExpr::getTranslatedExpr() const
     return createValue(no_translated, LINK(type), args);
 }
 
+IHqlExpression * CHqlBoundExpr::getComplexExpr() const
+{
+    assertex(count && expr);
+    return createValue(no_complex, expr->getType(), LINK(count), LINK(expr));
+}
+
 IHqlExpression * CHqlBoundExpr::getIsAll() const
 {
     if (isAll)

ecl/hqlcpp/hqlcpp.ipp

@@ -246,7 +246,9 @@ public:
     void clear()                                { expr.clear(); length.clear(); }
     bool exists() const                         { return (expr != NULL); }
     IHqlExpression * getIsAll() const;
+    IHqlExpression * getComplexExpr() const;
     IHqlExpression * getTranslatedExpr() const;
+
     ITypeInfo * queryType() const               { return expr->queryType(); }
     void set(const CHqlBoundExpr & src)         { expr.set(src.expr); length.set(src.length); count.set(src.count); isAll.set(src.isAll); }
     void setFromTarget(const CHqlBoundTarget & target);
@@ -1184,6 +1186,7 @@ public:
 
     void buildDatasetAssignChoose(BuildCtx & ctx, const CHqlBoundTarget & target, IHqlExpression * expr);
     void buildDatasetAssignIf(BuildCtx & ctx, const CHqlBoundTarget & target, IHqlExpression * expr);
+
     BoundRow * buildDatasetIterateSelectN(BuildCtx & ctx, IHqlExpression * expr, bool needToBreak);
     BoundRow * buildDatasetIterateChoosen(BuildCtx & ctx, IHqlExpression * expr, bool needToBreak);
     BoundRow * buildDatasetIterateLimit(BuildCtx & ctx, IHqlExpression * expr, bool needToBreak);
@@ -1365,6 +1368,7 @@ public:
     ABoundActivity * doBuildActivityDedup(BuildCtx & ctx, IHqlExpression * expr);
     ABoundActivity * doBuildActivityDefineSideEffect(BuildCtx & ctx, IHqlExpression * expr);
     ABoundActivity * doBuildActivityDenormalize(BuildCtx & ctx, IHqlExpression * expr);
+    ABoundActivity * doBuildActivityDictionaryWorkunitWrite(BuildCtx & ctx, IHqlExpression * expr, bool isRoot);
     ABoundActivity * doBuildActivityDiskAggregate(BuildCtx & ctx, IHqlExpression * expr);
     ABoundActivity * doBuildActivityDiskGroupAggregate(BuildCtx & ctx, IHqlExpression * expr);
     ABoundActivity * doBuildActivityDiskNormalize(BuildCtx & ctx, IHqlExpression * expr);
@@ -1433,6 +1437,7 @@ public:
     ABoundActivity * doBuildActivitySelectNth(BuildCtx & ctx, IHqlExpression * expr);
     ABoundActivity * doBuildActivitySequentialParallel(BuildCtx & ctx, IHqlExpression * expr, bool isRoot);
     ABoundActivity * doBuildActivitySerialize(BuildCtx & ctx, IHqlExpression * expr);
+    ABoundActivity * doBuildActivitySetGraphDictionaryResult(BuildCtx & ctx, IHqlExpression * expr, bool isRoot);
     ABoundActivity * doBuildActivitySetGraphResult(BuildCtx & ctx, IHqlExpression * expr, bool isRoot);
     ABoundActivity * doBuildActivitySetGraphLoopResult(BuildCtx & ctx, IHqlExpression * expr);
     ABoundActivity * doBuildActivitySetResult(BuildCtx & ctx, IHqlExpression * expr, bool isRoot);
@@ -1574,7 +1579,8 @@ public:
     void buildConnectOrders(BuildCtx & ctx, ABoundActivity * slaveActivity, ABoundActivity * masterActivity);
     void buildDedupFilterFunction(BuildCtx & ctx, HqlExprArray & equalities, HqlExprArray & conds, IHqlExpression * dataset, IHqlExpression * selSeq);
     void buildDedupSerializeFunction(BuildCtx & ctx, const char * funcName, IHqlExpression * srcDataset, IHqlExpression * tgtDataset, HqlExprArray & srcValues, HqlExprArray & tgtValues, IHqlExpression * selSeq);
-    void buildDictionaryHashClass(BuildCtx &ctx, IHqlExpression *record, IHqlExpression *dictionary, StringBuffer &lookupHelperName);
+    void buildDictionaryHashClass(IHqlExpression *record, IHqlExpression *dictionary, StringBuffer &lookupHelperName);
+    void buildDictionaryHashMember(BuildCtx & ctx, IHqlExpression *dictionary, const char * memberName);
     void buildHashClass(BuildCtx & ctx, const char * name, IHqlExpression * orderExpr, const DatasetReference & dataset);
     void buildHashOfExprsClass(BuildCtx & ctx, const char * name, IHqlExpression * cond, const DatasetReference & dataset, bool compareToSelf);
     void buildInstancePrefix(ActivityInstance * instance);

ecl/hqlcpp/hqlcppds.cpp

@@ -1242,7 +1242,7 @@ IHqlExpression * ChildGraphExprBuilder::addDataset(IHqlExpression * expr)
     IHqlExpression * recordCountAttr = queryRecordCountInfo(expr);
     if (recordCountAttr)
         args.append(*LINK(recordCountAttr));
-    OwnedHqlExpr ret = createDataset(no_getgraphresult, args);
+    OwnedHqlExpr ret = expr->isDictionary() ? createDictionary(no_getgraphresult, args) : createDataset(no_getgraphresult, args);
     if (expr->isDatarow())
         ret.setown(createRow(no_selectnth, LINK(ret), createComma(getSizetConstant(1), createAttribute(noBoundCheckAtom))));
     return ret.getClear();
@@ -1935,15 +1935,18 @@ void HqlCppTranslator::doBuildDataset(BuildCtx & ctx, IHqlExpression * expr, CHq
     case no_owned_ds:
         buildTempExpr(ctx, expr, tgt);
         return;
+    case no_fail:
+        doBuildStmtFail(ctx, expr->queryChild(1));
+        //fallthrough
     case no_null:
         {
             tgt.count.setown(getSizetConstant(0));
             tgt.length.setown(getSizetConstant(0));
             IHqlExpression * record = expr->queryRecord();
-            ITypeInfo * type = makeTableType(makeRowType(record->getType()), NULL, NULL, NULL);
+            Owned<ITypeInfo> type = makeTableType(makeRowType(record->getType()), NULL, NULL, NULL);
             if ((format == FormatLinkedDataset) || (format == FormatArrayDataset))
-                type = makeAttributeModifier(type, getLinkCountedAttr());
-            tgt.expr.setown(createValue(no_nullptr, makeReferenceModifier(type)));
+                type.setown(setLinkCountedAttr(type, true));
+            tgt.expr.setown(createValue(no_nullptr, makeReferenceModifier(type.getClear())));
             return;
         }
     case no_translated:
@@ -2077,6 +2080,25 @@ void HqlCppTranslator::doBuildDataset(BuildCtx & ctx, IHqlExpression * expr, CHq
             buildDataset(ctx, expr->queryChild(1), tgt, format);
             return;
         }
+    case no_createdictionary:
+        {
+            IHqlExpression * record = expr->queryRecord();
+            Owned<IHqlCppDatasetBuilder> builder = createLinkedDictionaryBuilder(record);
+
+            builder->buildDeclare(ctx);
+
+            buildDatasetAssign(ctx, builder, expr->queryChild(0));
+
+            builder->buildFinish(ctx, tgt);
+            ctx.associateExpr(expr, tgt);
+            return;
+        }
+    }
+
+    if (expr->isDictionary())
+    {
+        buildTempExpr(ctx, expr, tgt, format);
+        return;
     }
 
     bool singleRow = hasSingleRow(expr);
@@ -2140,22 +2162,15 @@ void HqlCppTranslator::doBuildDataset(BuildCtx & ctx, IHqlExpression * expr, CHq
 
             if (format == FormatLinkedDataset || format == FormatArrayDataset)
             {
-                if (expr->isDictionary())
+                IHqlExpression * choosenLimit = NULL;
+                if ((op == no_choosen) && !isChooseNAllLimit(expr->queryChild(1)) && !queryRealChild(expr, 2))
                 {
-                    builder.setown(createLinkedDictionaryBuilder(record));
+                    choosenLimit = expr->queryChild(1);
+                    expr = expr->queryChild(0);
                 }
-                else
-                {
-                    IHqlExpression * choosenLimit = NULL;
-                    if ((op == no_choosen) && !isChooseNAllLimit(expr->queryChild(1)) && !queryRealChild(expr, 2))
-                    {
-                        choosenLimit = expr->queryChild(1);
-                        expr = expr->queryChild(0);
-                    }
 
-                    //MORE: Extract limit and choosen and pass as parameters
-                    builder.setown(createLinkedDatasetBuilder(record, choosenLimit));
-                }
+                //MORE: Extract limit and choosen and pass as parameters
+                builder.setown(createLinkedDatasetBuilder(record, choosenLimit));
             }
             else if ((op == no_choosen) && !isChooseNAllLimit(expr->queryChild(1)) && !queryRealChild(expr, 2))
             {
@@ -2202,6 +2217,9 @@ void HqlCppTranslator::buildDatasetAssign(BuildCtx & ctx, const CHqlBoundTarget
     node_operator op = expr->getOperator();
     switch (op)
     {
+    case no_fail:
+        doBuildStmtFail(ctx, expr->queryChild(1));
+        return;
     case no_call:
     case no_externalcall:
         if (!hasStreamedModifier(expr->queryType()))
@@ -2220,14 +2238,20 @@ void HqlCppTranslator::buildDatasetAssign(BuildCtx & ctx, const CHqlBoundTarget
     case no_null:
         {
             CHqlBoundExpr bound;
-            buildDataset(ctx, expr, bound, isArrayRowset(expr->queryType()) ? FormatLinkedDataset : FormatBlockedDataset);
-            OwnedHqlExpr translated = bound.getTranslatedExpr();
-            if (target.count) ctx.addAssign(target.count, bound.count);
-            if (target.length) ctx.addAssign(target.length, bound.length);
-            ctx.addAssign(target.expr, bound.expr);
+            buildDataset(ctx, expr, bound, isArrayRowset(target.queryType()) ? FormatLinkedDataset : FormatBlockedDataset);
+            if (hasWrapperModifier(target.queryType()))
+            {
+                OwnedHqlExpr complex = bound.getComplexExpr();
+                ctx.addAssign(target.expr, complex);
+            }
+            else
+            {
+                if (target.count) ctx.addAssign(target.count, bound.count);
+                if (target.length) ctx.addAssign(target.length, bound.length);
+                ctx.addAssign(target.expr, bound.expr);
+            }
             return;
         }
-    case no_inlinedictionary:
     case no_inlinetable:
         {
             //This will typically generate a loop.  If few items then it is more efficient to expand the assigns/clones out.
@@ -2347,6 +2371,17 @@ void HqlCppTranslator::buildDatasetAssign(BuildCtx & ctx, const CHqlBoundTarget
             }
             break;
         }
+    case no_createdictionary:
+        {
+            IHqlExpression * record = expr->queryRecord();
+            Owned<IHqlCppDatasetBuilder> builder = createLinkedDictionaryBuilder(record);
+            builder->buildDeclare(ctx);
+
+            buildDatasetAssign(ctx, builder, expr->queryChild(0));
+
+            builder->buildFinish(ctx, target);
+            return;
+        }
     }
 
     if (!canAssignInline(&ctx, expr) && (op != no_translated))
@@ -2768,7 +2803,7 @@ void HqlCppTranslator::buildDatasetAssignProject(BuildCtx & ctx, IHqlCppDatasetB
 
     if (sourceCursor)
     {
-        if (isNullProject(expr, false))
+        if (isNullProject(expr, true, false))
         {
             if (target->buildLinkRow(iterctx, sourceCursor))
                 return;
@@ -2787,7 +2822,6 @@ void HqlCppTranslator::buildDatasetAssignProject(BuildCtx & ctx, IHqlCppDatasetB
         case no_hqlproject:
             doBuildRowAssignProject(iterctx, targetRef, expr);
             break;
-        case no_newuserdictionary:
         case no_newusertable:
             doBuildRowAssignUserTable(iterctx, targetRef, expr);
             break;
@@ -2933,6 +2967,9 @@ void HqlCppTranslator::buildDatasetAssign(BuildCtx & ctx, IHqlCppDatasetBuilder
     BuildCtx subctx(ctx);
     switch (expr->getOperator())
     {
+    case no_fail:
+        doBuildStmtFail(ctx, expr->queryChild(1));
+        return;
     case no_addfiles:
         buildDatasetAssign(subctx, target, expr->queryChild(0));
         buildDatasetAssign(subctx, target, expr->queryChild(1));
@@ -2941,7 +2978,6 @@ void HqlCppTranslator::buildDatasetAssign(BuildCtx & ctx, IHqlCppDatasetBuilder
         buildDatasetAssignTempTable(subctx, target, expr);
         //MORE: Create rows and assign each one in turn.  Could possibly be done with a different dataset selector
         return;
-    case no_inlinedictionary:
     case no_inlinetable:
         buildDatasetAssignInlineTable(subctx, target, expr);
         return;
@@ -2989,7 +3025,6 @@ void HqlCppTranslator::buildDatasetAssign(BuildCtx & ctx, IHqlCppDatasetBuilder
         return;
     case no_hqlproject:
     case no_newusertable:
-    case no_newuserdictionary:
         buildDatasetAssignProject(subctx, target, expr);
         return;
     case no_compound_childread:
@@ -4366,7 +4401,7 @@ IReferenceSelector * HqlCppTranslator::buildDatasetIndex(BuildCtx & ctx, IHqlExp
             specialCase = !isMultiLevelDatasetSelector(expr, false);
             break;
         case no_if:
-        case no_inlinedictionary:
+        case no_createdictionary:
         case no_inlinetable:
         case no_join:
             //Always creates a temporary, so don't use an iterator
@@ -4416,23 +4451,20 @@ IReferenceSelector * HqlCppTranslator::buildDatasetSelectMap(BuildCtx & ctx, IHq
     if (match)
         return createReferenceSelector(static_cast<BoundRow *>(match));
 
-    OwnedHqlExpr dataset = normalizeAnyDatasetAliases(expr->queryChild(0));
-    BoundRow * row = NULL;
-    assertex(canProcessInline(&ctx, expr));
+    OwnedHqlExpr dictionary = normalizeAnyDatasetAliases(expr->queryChild(0));
+
+    //MORE: This should really be a createDictionarySelector call.
+    Owned<IHqlCppDatasetCursor> cursor = createDatasetSelector(ctx, dictionary);
+    BoundRow * row = cursor->buildSelectMap(ctx, expr);
 
     if (!row)
     {
-        Owned<IHqlCppDatasetCursor> cursor = createDatasetSelector(ctx, dataset);
-        row = cursor->buildSelectMap(ctx, expr);
-
-        if (!row)
-        {
-            CHqlBoundExpr boundCleared;
-            buildDefaultRow(ctx, dataset, boundCleared);
-            OwnedHqlExpr defaultRowPtr = getPointer(boundCleared.expr);
-            row = bindRow(ctx, expr, defaultRowPtr);
-        }
+        CHqlBoundExpr boundCleared;
+        buildDefaultRow(ctx, dictionary, boundCleared);
+        OwnedHqlExpr defaultRowPtr = getPointer(boundCleared.expr);
+        row = bindRow(ctx, expr, defaultRowPtr);
     }
+
     return createReferenceSelector(row);
 }
 
@@ -4463,6 +4495,8 @@ IHqlExpression * HqlCppTranslator::buildGetLocalResult(BuildCtx & ctx, IHqlExpre
         HqlExprArray args;
         args.append(*LINK(matchedResults->queryExpr()));
         args.append(*LINK(resultNum));
+        if (expr->isDictionary())
+            return bindFunctionCall(getChildQueryDictionaryResultAtom, args, exprType);
         if (preferLinkedRows)
             return bindFunctionCall(getChildQueryLinkedResultAtom, args, exprType);
         return bindFunctionCall(getChildQueryResultAtom, args, exprType);
@@ -4485,6 +4519,8 @@ IHqlExpression * HqlCppTranslator::buildGetLocalResult(BuildCtx & ctx, IHqlExpre
     HqlExprArray args;
     args.append(*LINK(retInstanceExpr));
     args.append(*LINK(resultNum));
+    if (expr->isDictionary())
+        return bindFunctionCall(getLocalDictionaryResultAtom, args, exprType);
     if (preferLinkedRows)
         return bindFunctionCall(getLocalLinkedResultAtom, args, exprType);
     return bindFunctionCall(getLocalResultAtom, args, exprType);
@@ -4585,10 +4621,51 @@ ABoundActivity * HqlCppTranslator::doBuildActivityGetGraphResult(BuildCtx & ctx,
     return instance->getBoundActivity();
 }
 
+ABoundActivity * HqlCppTranslator::doBuildActivitySetGraphDictionaryResult(BuildCtx & ctx, IHqlExpression * expr, bool isRoot)
+{
+    IHqlExpression * dictionary = expr->queryChild(0);
+    IHqlExpression * dataset = dictionary->queryChild(0);
+    IHqlExpression * graphId = expr->queryChild(1);
+    IHqlExpression * resultNum = expr->queryChild(2);
+    bool isSpill = expr->hasProperty(_spill_Atom);
+
+    ABoundActivity * parentActivity = activeActivities.ordinality() ? &activeActivities.tos() : NULL;
+    Owned<ABoundActivity> boundDataset = buildCachedActivity(ctx, dataset);
+
+    Owned<ActivityInstance> instance = new ActivityInstance(*this, ctx, TAKdictionaryresultwrite, expr, "DictionaryResultWrite");
+
+    buildActivityFramework(instance, isRoot && !isSpill);
+
+    buildInstancePrefix(instance);
+
+    doBuildUnsignedFunction(instance->classctx, "querySequence", resultNum);
+    doBuildBoolFunction(instance->classctx, "usedOutsideGraph", !isSpill);
+
+    if (parentActivity && !insideRemoteGraph(ctx) && !isSpill)
+    {
+        addDependency(ctx, instance->queryBoundActivity(), parentActivity, childAtom, "Child");
+    }
+
+    buildDictionaryHashMember(instance->createctx, dictionary, "queryHashLookupInfo");
+
+    instance->addAttributeBool("_isSpill", isSpill);
+    if (targetRoxie())
+        addGraphIdAttribute(instance, ctx, graphId);
+
+    buildInstanceSuffix(instance);
+
+    buildConnectInputOutput(ctx, instance, boundDataset, 0, 0);
+    associateRemoteResult(*instance, graphId, resultNum);
+
+    return instance->getBoundActivity();
+}
 
 ABoundActivity * HqlCppTranslator::doBuildActivitySetGraphResult(BuildCtx & ctx, IHqlExpression * expr, bool isRoot)
 {
     IHqlExpression * dataset = expr->queryChild(0);
+    if (dataset->isDictionary())
+        return doBuildActivitySetGraphDictionaryResult(ctx, expr, isRoot);
+
     IHqlExpression * graphId = expr->queryChild(1);
     IHqlExpression * resultNum = expr->queryChild(2);
     bool isSpill = expr->hasProperty(_spill_Atom);

ecl/hqlcpp/hqlcppsys.ecl

@@ -585,6 +585,8 @@ const char * cppSystemText[]  = {
     "   evaluateChildQueryInstance(unsigned4 lenExtract, row parentExtract) : method,entrypoint='evaluate';",       // actually returns something el
     "   dataset getChildQueryResult(unsigned4 id)   : method,pure,entrypoint='getResult';",
     "   _linkcounted_ dataset getChildQueryLinkedResult(unsigned4 id)   : method,allocator(false),pure,entrypoint='getLinkedResult';",
+    "   _linkcounted_ dictionary getChildQueryDictionaryResult(unsigned4 id)   : method,allocator(false),pure,entrypoint='getDictionaryResult';",
+    
     //MORE: Should this be utf8?
     "   varstring getenv(const varstring name, const varstring defaultValue) : pure,ctxmethod,entrypoint='getEnv';",
 
@@ -643,6 +645,7 @@ const char * cppSystemText[]  = {
     "   set of any getResultSet(const varstring stepname, unsigned4 sequence, boolean xmltransformer, boolean csvtransformer) : ctxmethod,pure,entrypoint='getResultSet',newset;",
 
     "   _linkcounted_ dataset getResultRowset(const varstring stepname, unsigned4 sequence, boolean _allocator, boolean _deserializer, boolean isGrouped, boolean xmltransformer, boolean csvtransformer) : ctxmethod,allocator(false),pure,entrypoint='getResultRowset';",
+    "   linkcounted dictionary getResultDictionary(const varstring stepname, unsigned4 sequence, boolean _deserializer, boolean xmltransformer, boolean csvtransformer, boolean hasher) : ctxmethod,pure,entrypoint='getResultDictionary';",
 
     //Don't make these pure because they may change over time.
     "   boolean isResult(const varstring stepname, unsigned4 sequence) : gctxmethod,entrypoint='isResult';",
@@ -738,6 +741,7 @@ const char * cppSystemText[]  = {
 
     "   dataset getLocalResult(unsigned4 id) :  method,pure,entrypoint='getResult';",
     "   _linkcounted_ dataset getLocalLinkedResult(unsigned4 id) : method,allocator(false),pure,entrypoint='getLinkedResult';",
+    "   linkcounted dictionary getLocalDictionaryResult(unsigned4 id) : method,allocator(false),pure,entrypoint='getDictionaryResult';",
     "   unsigned4 getGraphLoopCounter() : ctxmethod,entrypoint='getGraphLoopCounter';",
 
     "   _linkcounted_ row(dummyRecord) finalizeRowClear(unsigned4 _size) : omethod,entrypoint='finalizeRowClear';",

ecl/hqlcpp/hqlcse.cpp

@@ -1217,6 +1217,7 @@ static bool canHoistInvariant(IHqlExpression * expr)
     {
     case no_list:
     case no_datasetlist:
+    case no_createdictionary:
         return false;       // probably don't want to hoist these
     }
     return true;

ecl/hqlcpp/hqlcset.cpp

@@ -723,7 +723,7 @@ BoundRow * InlineLinkedDictionaryCursor::buildSelectMap(BuildCtx & ctx, IHqlExpr
 
     StringBuffer lookupHelperName;
     OwnedHqlExpr dict = createDictionary(no_null, LINK(record));
-    translator.buildDictionaryHashClass(ctx, record, dict, lookupHelperName);
+    translator.buildDictionaryHashClass(record, dict, lookupHelperName);
     CHqlBoundTarget target;
     target.expr.set(tempRow->queryBound());
 
@@ -745,7 +745,7 @@ void InlineLinkedDictionaryCursor::buildInDataset(BuildCtx & ctx, IHqlExpression
 
     StringBuffer lookupHelperName;
     OwnedHqlExpr dict = createDictionary(no_null, LINK(record));
-    translator.buildDictionaryHashClass(ctx, record, dict, lookupHelperName);
+    translator.buildDictionaryHashClass(record, dict, lookupHelperName);
 
     HqlExprArray args;
     args.append(*createQuoted(lookupHelperName, makeBoolType()));
@@ -1799,7 +1799,7 @@ void LinkedDictionaryBuilder::buildDeclare(BuildCtx & ctx)
 
     StringBuffer lookupHelperName;
     OwnedHqlExpr dict = createDictionary(no_null, record.getLink()); // MORE - is the actual dict not available?
-    translator.buildDictionaryHashClass(ctx, record, dict, lookupHelperName);
+    translator.buildDictionaryHashClass(record, dict, lookupHelperName);
 
     decl.append("RtlLinkedDictionaryBuilder ").append(instanceName).append("(");
     decl.append(allocatorName).append(", &").append(lookupHelperName);

ecl/hqlcpp/hqlhtcpp.cpp

@@ -4692,11 +4692,21 @@ void HqlCppTranslator::buildGetResultInfo(BuildCtx & ctx, IHqlExpression * expr,
         {
             OwnedHqlExpr record;
             bool ensureSerialized = true;
-            if (ttc != type_set)
+            if (ttc == type_dictionary)
+            {
+                record.set(::queryRecord(type));
+
+                //NB: The result type will be overridden when this function is bound
+                ensureSerialized = false;
+                args.append(*createRowSerializer(ctx, record, deserializerAtom));
+                overrideType.setown(setLinkCountedAttr(type, true));
+                func = getResultDictionaryAtom;
+            }
+            else if (ttc != type_set)
             {
                 overrideType.set(type);
                 record.set(::queryRecord(type));
-                //NB: The result type (including grouping) will be overridden then this function is bound
+                //NB: The result type (including grouping) will be overridden when this function is bound
                 func = getResultDatasetAtom;
                 bool defaultLCR = targetAssign ? hasLinkedRow(targetAssign->queryType()) : options.tempDatasetsUseLinkedRows;
                 if (hasLinkCountedModifier(type) || defaultLCR)
@@ -4753,6 +4763,14 @@ void HqlCppTranslator::buildGetResultInfo(BuildCtx & ctx, IHqlExpression * expr,
                 args.append(*createQuoted("0", makeBoolType()));
                 args.append(*createQuoted("0", makeBoolType()));
             }
+            if (ttc == type_dictionary)
+            {
+                StringBuffer lookupHelperName;
+                buildDictionaryHashClass(expr->queryRecord(), expr, lookupHelperName);
+
+                lookupHelperName.insert(0, "&");    // MORE: Should this be passed by reference instead - it isn't optional
+                args.append(*createQuoted(lookupHelperName.str(), makeBoolType()));
+            }
             break;
         }
     case type_string:
@@ -5298,7 +5316,7 @@ void HqlCppTranslator::buildHashOfExprsClass(BuildCtx & ctx, const char * name,
 }
 
 
-void HqlCppTranslator::buildDictionaryHashClass(BuildCtx &ctx, IHqlExpression *record, IHqlExpression *dictionary, StringBuffer &funcName)
+void HqlCppTranslator::buildDictionaryHashClass(IHqlExpression *record, IHqlExpression *dictionary, StringBuffer &funcName)
 {
     BuildCtx declarectx(*code, declareAtom);
     OwnedHqlExpr attr = createAttribute(lookupAtom, LINK(record));
@@ -5314,7 +5332,7 @@ void HqlCppTranslator::buildDictionaryHashClass(BuildCtx &ctx, IHqlExpression *r
         beginNestedClass(classctx, lookupHelperName, "IHThorHashLookupInfo");
         HqlExprArray keyedFields;
         IHqlExpression * payload = record->queryProperty(_payload_Atom);
-        unsigned payloadSize = payload ? getIntValue(payload->queryChild(0)) : 0;
+        unsigned payloadSize = payload ? (unsigned)getIntValue(payload->queryChild(0)) : 0;
         unsigned max = record->numChildren() - payloadSize;
         for (unsigned idx = 0; idx < max; idx++)
         {
@@ -5340,6 +5358,17 @@ void HqlCppTranslator::buildDictionaryHashClass(BuildCtx &ctx, IHqlExpression *r
     }
 }
 
+void HqlCppTranslator::buildDictionaryHashMember(BuildCtx & ctx, IHqlExpression *dictionary, const char * memberName)
+{
+    StringBuffer lookupHelperName;
+    buildDictionaryHashClass(dictionary->queryRecord(), dictionary, lookupHelperName);
+
+    BuildCtx funcctx(ctx);
+    StringBuffer s;
+    s.append("virtual IHThorHashLookupInfo * ").append(memberName).append("() { return &").append(lookupHelperName).append("; }");
+    funcctx.addQuoted(s);
+}
+
 
 //---------------------------------------------------------------------------
 
@@ -6330,7 +6359,7 @@ ABoundActivity * HqlCppTranslator::buildActivity(BuildCtx & ctx, IHqlExpression
                 result = doBuildActivitySelectNth(ctx, expr);
                 break;
             case no_selectmap:
-                UNIMPLEMENTED;
+                result = doBuildActivityCreateRow(ctx, expr, false);
                 break;
             case no_join:
             case no_selfjoin:
@@ -9963,6 +9992,8 @@ ABoundActivity * HqlCppTranslator::doBuildActivityOutput(BuildCtx & ctx, IHqlExp
     IHqlExpression * dataset  = expr->queryChild(0);
     IHqlExpression * rawFilename = queryRealChild(expr, 1);
 
+    if (dataset->isDictionary())
+        return doBuildActivityDictionaryWorkunitWrite(ctx, expr, isRoot);
     if (!rawFilename)
         return doBuildActivityOutputWorkunit(ctx, expr, isRoot);
 
@@ -10676,8 +10707,6 @@ void HqlCppTranslator::buildXmlSerializeUsingMeta(BuildCtx & ctx, IHqlExpression
 
 //-------------------------------------------------------------------------------------------------------------------
 
-//-------------------------------------------------------------------------------------------------------------------
-
 ABoundActivity * HqlCppTranslator::doBuildActivityOutputWorkunit(BuildCtx & ctx, IHqlExpression * expr, bool isRoot)
 {
     IHqlExpression * dataset = expr->queryChild(0);
@@ -10806,6 +10835,59 @@ void HqlCppTranslator::doBuildStmtOutput(BuildCtx & ctx, IHqlExpression * expr)
 }
 
 
+//-------------------------------------------------------------------------------------------------------------------
+
+ABoundActivity * HqlCppTranslator::doBuildActivityDictionaryWorkunitWrite(BuildCtx & ctx, IHqlExpression * expr, bool isRoot)
+{
+    IHqlExpression * dictionary = expr->queryChild(0);
+    IHqlExpression * record = dictionary->queryRecord();
+    IHqlExpression * seq = querySequence(expr);
+    IHqlExpression * name = queryResultName(expr);
+    int sequence = (int)getIntValue(seq, ResultSequenceInternal);
+
+    assertex(dictionary->getOperator() == no_createdictionary);
+    IHqlExpression * dataset = dictionary->queryChild(0);
+
+    Owned<ABoundActivity> boundDataset = buildCachedActivity(ctx, dataset);
+
+    StringBuffer graphLabel;
+    Owned<ActivityInstance> instance = new ActivityInstance(*this, ctx, TAKdictionaryworkunitwrite, expr, "DictionaryWorkUnitWrite");
+
+    graphLabel.append(getActivityText(instance->kind)).append("\n");
+    getStoredDescription(graphLabel, seq, name, true);
+    instance->graphLabel.set(graphLabel.str());
+    buildActivityFramework(instance, isRoot && !isInternalSeq(seq));
+
+    buildInstancePrefix(instance);
+
+    noteResultDefined(ctx, instance, seq, name, isRoot);
+
+    //virtual unsigned getFlags()
+    StringBuffer flags;
+
+    doBuildSequenceFunc(instance->classctx, seq, true);
+    if (name)
+    {
+        BuildCtx namectx(instance->startctx);
+        namectx.addQuotedCompound("virtual const char * queryName()");
+        buildReturn(namectx, name, constUnknownVarStringType);
+    }
+
+    //Owned<IWUResult> result = createDatasetResultSchema(seq, name, record, true, false);
+    buildDictionaryHashMember(instance->createctx, dictionary, "queryHashLookupInfo");
+
+    if (flags.length())
+        doBuildUnsignedFunction(instance->classctx, "getFlags", flags.str()+1);
+
+
+    buildInstanceSuffix(instance);
+
+    buildConnectInputOutput(ctx, instance, boundDataset, 0, 0);
+    associateRemoteResult(*instance, seq, name);
+    return instance->getBoundActivity();
+}
+
+
 //---------------------------------------------------------------------------
 
 

ecl/hqlcpp/hqlinline.cpp

@@ -211,14 +211,13 @@ static unsigned calcInlineFlags(BuildCtx * ctx, IHqlExpression * expr)
                 return flags;
             return RETevaluate|HEFspillinline;
         }
+    case no_fail:
+        return RETevaluate;
     case no_catchds:
         return 0;       // for the moment always do this out of line 
     case no_table:
         return 0;
-    case no_newuserdictionary:
-    case no_userdictionary:
-        return RETassign;
-    case no_inlinedictionary:
+    case no_createdictionary:
         return RETassign;
     case no_owned_ds:
         {

ecl/hqlcpp/hqliproj.cpp

@@ -938,6 +938,11 @@ static int compareHqlExprPtr(IInterface * * left, IInterface * * right)
     return *left == *right ? 0 : *left < *right ? -1 : +1;
 }
 
+inline bool hasActivityType(IHqlExpression * expr)
+{
+    return (expr->isDataset() || expr->isDatarow() || expr->isDictionary());
+}
+
 //------------------------------------------------------------------------
 
 ImplicitProjectInfo::ImplicitProjectInfo(IHqlExpression * _original, ProjectExprKind _kind) : NewTransformInfo(_original), kind(_kind)
@@ -1322,7 +1327,7 @@ void ImplicitProjectTransformer::analyseExpr(IHqlExpression * expr)
         case no_evaluate:
             throwUnexpected();
         case no_select:
-            if (expr->isDataset() || expr->isDatarow())
+            if (hasActivityType(expr))
             {
                 //MORE: These means that selects from a parent dataset don't project down the parent dataset.
                 //I'm not sure how big an issue that would be.
@@ -1361,7 +1366,7 @@ void ImplicitProjectTransformer::analyseExpr(IHqlExpression * expr)
             allowActivity = true;
             return;
         case no_thor:
-            if (expr->isDataset() || expr->isDatarow())
+            if (hasActivityType(expr))
             {
                 assertex(extra->activityKind() == SimpleActivity);
                 Parent::analyseExpr(expr);
@@ -1459,7 +1464,7 @@ void ImplicitProjectTransformer::analyseExpr(IHqlExpression * expr)
                 unsigned first = 0;
                 unsigned last = numArgs;
                 unsigned start = 0;
-                if (!expr->isAction() && !expr->isDataset() && !expr->isDatarow())
+                if (!expr->isAction() && !expr->isDataset() && !expr->isDatarow() && !expr->isDictionary())
                 {
                     switch (op)
                     {
@@ -1627,7 +1632,7 @@ void ImplicitProjectTransformer::analyseExpr(IHqlExpression * expr)
     }
 
     IHqlExpression * record = expr->queryRecord();
-    if (record && !isPatternType(type) && !expr->isTransform() && !expr->isDictionary())
+    if (record && !isPatternType(type) && !expr->isTransform())
     {
         assertex(complexExtra);
         complexExtra->setOriginalRecord(queryBodyComplexExtra(record));
@@ -1912,7 +1917,6 @@ const SelectUsedArray & ImplicitProjectTransformer::querySelectsUsed(IHqlExpress
     return extra->querySelectsUsed(); 
 }
 
-
 ProjectExprKind ImplicitProjectTransformer::getProjectExprKind(IHqlExpression * expr)
 {
     switch (expr->getOperator())
@@ -1920,7 +1924,7 @@ ProjectExprKind ImplicitProjectTransformer::getProjectExprKind(IHqlExpression *
     case no_evaluate:
         throwUnexpected();
     case no_select:
-        if (expr->isDataset() || expr->isDatarow())
+        if (hasActivityType(expr))
             return SourceActivity;
         if (isNewSelector(expr))
             return ScalarSelectActivity;
@@ -1932,21 +1936,21 @@ ProjectExprKind ImplicitProjectTransformer::getProjectExprKind(IHqlExpression *
     case no_attr_link:
         return NonActivity;
     case no_typetransfer:
-        if (expr->isDataset() || expr->isDatarow())
+        if (hasActivityType(expr))
             return SourceActivity;
         return NonActivity;
     case no_thor:
-        if (expr->isDataset() || expr->isDatarow())
+        if (hasActivityType(expr))
             return SimpleActivity;
         return NonActivity;
     case no_compound:
-        if (expr->isDataset())
+        if (expr->isDataset() || expr->isDictionary())
             return SimpleActivity;
         if (expr->isDatarow())
             return ComplexNonActivity;
         return NonActivity;
     case no_executewhen:
-        if (expr->isDataset() || expr->isDatarow())
+        if (hasActivityType(expr))
             return SimpleActivity;
         return NonActivity;
     case no_subgraph:
@@ -1979,8 +1983,8 @@ ProjectExprKind ImplicitProjectTransformer::getProjectExprKind(IHqlExpression *
     case no_inlinetable:
     case no_dataset_from_transform:
         return CreateRecordSourceActivity;
-    case no_inlinedictionary:
-        return NonActivity;
+    case no_createdictionary:
+        return FixedInputActivity;
     case no_indict:
     case no_selectmap:
         return FixedInputActivity;
@@ -2005,7 +2009,7 @@ ProjectExprKind ImplicitProjectTransformer::getProjectExprKind(IHqlExpression *
         return AnyTypeActivity;
     case no_skip:
     case no_fail:
-        if (expr->isDataset() || expr->isDatarow())
+        if (hasActivityType(expr))
             return AnyTypeActivity;
         return NonActivity;
     case no_table:
@@ -2061,19 +2065,23 @@ ProjectExprKind ImplicitProjectTransformer::getProjectExprKind(IHqlExpression *
     case no_getgraphloopresult:
     case no_rows:
         return SourceActivity;
+    case no_getresult:
+        if (hasActivityType(expr))
+            return SourceActivity;
+        return NonActivity;
     case no_allnodes:
     case no_httpcall:
     case no_soapcall:
     case no_newsoapcall:
     case no_libraryinput:
     case no_thisnode:
-        if (expr->isDataset() || expr->isDatarow())
+        if (hasActivityType(expr))
             return SourceActivity;
         return NonActivity;
     case no_pipe:
     case no_nofold:
     case no_nohoist:
-        if (expr->isDataset() || expr->isDatarow())
+        if (hasActivityType(expr))
             return FixedInputActivity;
         return NonActivity;
     case no_soapcall_ds:
@@ -2103,9 +2111,7 @@ ProjectExprKind ImplicitProjectTransformer::getProjectExprKind(IHqlExpression *
     case no_selfjoin:
         return CreateRecordActivity;
     case no_if:
-        if (expr->isDataset())
-            return PassThroughActivity;
-        if (expr->isDatarow())
+        if (hasActivityType(expr))
             return PassThroughActivity;
         return NonActivity;
     case no_addfiles:
@@ -2134,7 +2140,7 @@ ProjectExprKind ImplicitProjectTransformer::getProjectExprKind(IHqlExpression *
         return SinkActivity;
     case no_call:
     case no_externalcall:
-        if (expr->isDataset() || expr->isDatarow())
+        if (hasActivityType(expr))
             return SourceActivity;
         //MORE: What about parameters??
         return NonActivity;
@@ -2166,6 +2172,7 @@ ProjectExprKind ImplicitProjectTransformer::getProjectExprKind(IHqlExpression *
     case type_groupedtable:
         break;
     case type_dictionary:
+        return FixedInputActivity;
     case type_transform:
         return NonActivity;
     default:

ecl/hqlcpp/hqlresource.cpp

@@ -1224,7 +1224,10 @@ IHqlExpression * ResourcerInfo::createSpilledRead(IHqlExpression * spillReason)
             args.append(*LINK(recordCountAttr));
         if (options->targetThor() && original->isDataset() && !options->isChildQuery)
             args.append(*createAttribute(_distributed_Atom));
-        dataset.setown(createDataset(no_getgraphresult, args));
+        if (original->isDictionary())
+            dataset.setown(createDictionary(no_getgraphresult, args));
+        else
+            dataset.setown(createDataset(no_getgraphresult, args));
         loseDistribution = false;
     }
     else if (useGlobalResult())
@@ -1236,7 +1239,10 @@ IHqlExpression * ResourcerInfo::createSpilledRead(IHqlExpression * spillReason)
         IHqlExpression * recordCountAttr = queryRecordCountInfo(original);
         if (recordCountAttr)
             args.append(*LINK(recordCountAttr));
-        dataset.setown(createDataset(no_workunit_dataset, args));
+        if (original->isDictionary())
+            dataset.setown(createDictionary(no_workunit_dataset, args));
+        else
+            dataset.setown(createDataset(no_workunit_dataset, args));
     }
     else
     {
@@ -1733,7 +1739,7 @@ bool ResourcerInfo::isSpilledWrite()
 
 IHqlExpression * ResourcerInfo::wrapRowOwn(IHqlExpression * expr)
 {
-    if (!original->isDataset())
+    if (!original->isDataset() && !original->isDictionary())
         expr = createRow(no_selectnth, expr, getSizetConstant(1));
     return expr;
 }
@@ -2181,7 +2187,7 @@ protected:
                     if (isCompoundAggregate(ds))
                         break;
 
-                    if (!expr->isDatarow() && !expr->isDataset())
+                    if (!expr->isDatarow() && !expr->isDataset() && !expr->isDictionary())
                     {
                         if (queryHoistDataset(ds))
                         {
@@ -2215,7 +2221,7 @@ protected:
             {
                 IHqlExpression * rhs = expr->queryChild(1);
                 //if rhs is a new, evaluatable, dataset then we want to add it
-                if (rhs->isDataset() && isEvaluateable(rhs))
+                if ((rhs->isDataset() || rhs->isDictionary()) && isEvaluateable(rhs))
                 {
                     if (queryNoteDataset(rhs))
                         return;
@@ -2229,7 +2235,7 @@ protected:
             return;
         case no_globalscope:
         case no_evalonce:
-            if (expr->isDataset() || expr->isDatarow())
+            if (expr->isDataset() || expr->isDatarow() || expr->isDictionary())
                 noteDataset(expr, expr->queryChild(0), true);
             else
                 noteScalar(expr, expr->queryChild(0));
@@ -2246,6 +2252,11 @@ protected:
         case no_getgraphloopresult:
             noteDataset(expr, expr, true);
             return;
+        case no_createdictionary:
+            if (isEvaluateable(expr))
+                noteDataset(expr, expr, true);
+            return;
+
         case no_selectnth:
             if (expr->queryChild(1)->isConstant())
             {
@@ -2310,20 +2321,20 @@ protected:
             {
                 IHqlExpression * cond = expr->queryChild(0);
                 analyseExpr(cond);
-                if (expr->isDataset() || expr->isDatarow())
+                if (expr->isDataset() || expr->isDatarow() || expr->isDictionary())
                     conditionalDepth++;
                 doAnalyseChildren(expr, 1);
-                if (expr->isDataset() || expr->isDatarow())
+                if (expr->isDataset() || expr->isDatarow() || expr->isDictionary())
                     conditionalDepth--;
                 break;
             }
         case no_mapto:
             {
                 analyseExpr(expr->queryChild(0));
-                if (expr->isDataset() || expr->isDatarow())
+                if (expr->isDataset() || expr->isDatarow() || expr->isDictionary())
                     conditionalDepth++;
                 analyseExpr(expr->queryChild(1));
-                if (expr->isDataset() || expr->isDatarow())
+                if (expr->isDataset() || expr->isDatarow() || expr->isDictionary())
                     conditionalDepth--;
                 break;
             }
@@ -2455,6 +2466,9 @@ protected:
         //depends on at least the following...
         //a) cost of serializing v cost of re-evaluating (which can depend on the engine).
         //b) How many times it will be evaluated in the child context
+        if (ds->getOperator() == no_createdictionary)
+            return true;
+
         if (isInlineTrivialDataset(ds))
             return false;
 
@@ -4200,7 +4214,7 @@ IHqlExpression * EclResourcer::replaceResourcedReferences(ResourcerInfo * info,
                 replacement.setown(createResourced(&cur, NULL, false, false));
 
             IHqlExpression * original = &info->originalChildDependents.item(i);
-            if (!original->isDataset() && !original->isDatarow())
+            if (!original->isDataset() && !original->isDatarow() && !original->isDictionary())
                 replacement.setown(getScalarReplacement(original, replacement));
 
             replacements.append(*replacement.getClear());

ecl/hqlcpp/hqlttcpp.cpp

@@ -10229,7 +10229,7 @@ IHqlExpression * HqlTreeNormalizer::convertSelectToProject(IHqlExpression * newR
     unsigned numChildren = expr->numChildren();
     for (unsigned idx = 2; idx < numChildren; idx++)
         args.append(*transform(expr->queryChild(idx)));
-    OwnedHqlExpr project = expr->isDictionary() ? createDictionary(no_newuserdictionary, args) : createDataset(no_newusertable, args);
+    OwnedHqlExpr project = createDataset(no_newusertable, args);
     return expr->cloneAllAnnotations(project);
 }
 
@@ -11333,11 +11333,11 @@ IHqlExpression * HqlTreeNormalizer::createTransformedBody(IHqlExpression * expr)
     case no_constant:
         return LINK(expr);          // avoid creating an array in default code...
     case no_case:
-        if (isVoidOrDatasetOrList(expr))
+        if (isVoidOrDatasetOrList(expr) || expr->isDictionary())
             return transformCaseToChoose(expr);
         break;
     case no_map:
-        if (isVoidOrDatasetOrList(expr))
+        if (isVoidOrDatasetOrList(expr) || expr->isDictionary())
             return transformMap(expr);
         break;
     case no_transform:
@@ -11361,7 +11361,6 @@ IHqlExpression * HqlTreeNormalizer::createTransformedBody(IHqlExpression * expr)
             }
             return Parent::createTransformed(cleaned);
         }
-    case no_userdictionary:
     case no_usertable:
     case no_selectfields:
         {

ecl/hthor/hthor.cpp

@@ -5884,6 +5884,61 @@ void CHThorWorkUnitWriteActivity::execute()
 
 //=====================================================================================================
 
+CHThorDictionaryWorkUnitWriteActivity::CHThorDictionaryWorkUnitWriteActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorDictionaryWorkUnitWriteArg &_arg, ThorActivityKind _kind)
+ : CHThorActivityBase(_agent, _activityId, _subgraphId, _arg, _kind), helper(_arg)
+{
+}
+
+void CHThorDictionaryWorkUnitWriteActivity::execute()
+{
+    int sequence = helper.getSequence();
+    const char *storedName = helper.queryName();
+    assertex(storedName && *storedName);
+    assertex(sequence < 0);
+
+    RtlLinkedDictionaryBuilder builder(rowAllocator, helper.queryHashLookupInfo());
+    loop
+    {
+        const void *row = input->nextInGroup();
+        if (!row)
+        {
+            row = input->nextInGroup();
+            if (!row)
+                break;
+        }
+        builder.appendOwn(row);
+        processed++;
+    }
+    size32_t usedCount = rtlDictionaryCount(builder.getcount(), builder.queryrows());
+
+    size32_t outputLimit = agent.queryWorkUnit()->getDebugValueInt("outputLimit", defaultWorkUnitWriteLimit) * 0x100000;
+    MemoryBuffer rowdata;
+    CThorDemoRowSerializer out(rowdata);
+    Owned<IOutputRowSerializer> serializer = input->queryOutputMeta()->createRowSerializer(agent.queryCodeContext(), activityId);
+    rtlSerializeDictionary(out, serializer, builder.getcount(), builder.queryrows());
+    if(outputLimit && (rowdata.length()  > outputLimit))
+    {
+        StringBuffer errMsg("Dictionary too large to output to workunit (limit ");
+        errMsg.append(outputLimit/0x100000).append(") megabytes, in result (");
+        const char *name = helper.queryName();
+        if (name)
+            errMsg.append("name=").append(name);
+        else
+            errMsg.append("sequence=").append(helper.getSequence());
+        errMsg.append(")");
+        throw MakeStringException(0, "%s", errMsg.str());
+    }
+
+    WorkunitUpdate w = agent.updateWorkUnit();
+    Owned<IWUResult> result = updateWorkUnitResult(w, helper.queryName(), helper.getSequence());
+    result->setResultRaw(rowdata.length(), rowdata.toByteArray(), ResultFormatRaw);
+    result->setResultStatus(ResultStatusCalculated);
+    result->setResultRowCount(usedCount);
+    result->setResultTotalRowCount(usedCount); // Is this right??
+}
+
+//=====================================================================================================
+
 CHThorCountActivity::CHThorCountActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorCountArg &_arg, ThorActivityKind _kind)
  : CHThorActivityBase(_agent, _activityId, _subgraphId, _arg, _kind), helper(_arg)
 {
@@ -9767,6 +9822,7 @@ MAKEFACTORY_ARG(SelfJoin, Join);
 MAKEFACTORY_ARG(LookupJoin, HashJoin);
 MAKEFACTORY(AllJoin);
 MAKEFACTORY(WorkUnitWrite);
+MAKEFACTORY(DictionaryWorkUnitWrite);
 MAKEFACTORY(FirstN);
 MAKEFACTORY(Count);
 MAKEFACTORY(TempTable);

ecl/hthor/hthor.hpp

@@ -104,6 +104,7 @@ extern HTHOR_API IHThorActivity *createSelfJoinActivity(IAgentContext &, unsigne
 extern HTHOR_API IHThorActivity *createLookupJoinActivity(IAgentContext &, unsigned _activityId, unsigned _subgraphId, IHThorHashJoinArg & arg, ThorActivityKind kind);
 extern HTHOR_API IHThorActivity *createAllJoinActivity(IAgentContext &, unsigned _activityId, unsigned _subgraphId, IHThorAllJoinArg & arg, ThorActivityKind kind);
 extern HTHOR_API IHThorActivity *createWorkUnitWriteActivity(IAgentContext &, unsigned _activityId, unsigned _subgraphId, IHThorWorkUnitWriteArg &arg, ThorActivityKind kind);
+extern HTHOR_API IHThorActivity *createDictionaryWorkUnitWriteActivity(IAgentContext &, unsigned _activityId, unsigned _subgraphId, IHThorDictionaryWorkUnitWriteArg &arg, ThorActivityKind kind);
 extern HTHOR_API IHThorActivity *createFirstNActivity(IAgentContext &, unsigned _activityId, unsigned _subgraphId, IHThorFirstNArg &arg, ThorActivityKind kind);
 extern HTHOR_API IHThorActivity *createCountActivity(IAgentContext &, unsigned _activityId, unsigned _subgraphId, IHThorCountArg &arg, ThorActivityKind kind);
 extern HTHOR_API IHThorActivity *createTempTableActivity(IAgentContext &, unsigned _activityId, unsigned _subgraphId, IHThorTempTableArg &arg, ThorActivityKind kind);

ecl/hthor/hthor.ipp

@@ -1468,6 +1468,18 @@ public:
     virtual void execute();
 };
 
+class CHThorDictionaryWorkUnitWriteActivity : public CHThorActivityBase
+{
+    IHThorDictionaryWorkUnitWriteArg &helper;
+
+public:
+    IMPLEMENT_SINKACTIVITY;
+
+    CHThorDictionaryWorkUnitWriteActivity (IAgentContext &agent, unsigned _activityId, unsigned _subgraphId, IHThorDictionaryWorkUnitWriteArg &_arg, ThorActivityKind _kind);
+    virtual void execute();
+    virtual bool needsAllocator() const { return true; }
+};
+
 class CHThorCountActivity : public CHThorActivityBase
 {
     IHThorCountArg &helper;

githooks/commit-msg

@@ -14,7 +14,7 @@ the --oneline version of the generated changelog.\n
 Use git commit -s -t $1 to reopen previous message for editing\n"
 
 
-(head -1 "$1" | grep -E -q "^HPCC-[0-9]+ ") || {
+(head -1 "$1" | grep -E -q "^HPCC-[0-9]+ |^Merge ") || {
   echo >&2 "\n ERROR: Missing Jira issue on beginning of first line (HPCC-XXXX)"
   echo >&2 $usage
   exit 1
@@ -26,7 +26,7 @@ Use git commit -s -t $1 to reopen previous message for editing\n"
   exit 1
 }
 
-(head -1 "$1" | grep -E -q "^HPCC-[0-9]+ [^a-z]") || {
+(head -1 "$1" | grep -E -q "^HPCC-[0-9]+ [^a-z]|^Merge ") || {
   echo >&2 "\n ERROR: Short description should start with a capital"
   echo >&2 $usage
   exit 1

roxie/ccd/CMakeLists.txt

@@ -79,6 +79,10 @@ include_directories (
 
 ADD_DEFINITIONS( -D_USRDLL -DCCD_EXPORTS -DSTARTQUERY_EXPORTS )
 
+    if (CMAKE_COMPILER_IS_CLANGXX)
+      SET (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-switch-enum -Wno-format-security")
+    endif()
+
 HPCC_ADD_LIBRARY( ccd SHARED ${SRCS} )
 install ( TARGETS ccd RUNTIME DESTINATION ${EXEC_DIR} LIBRARY DESTINATION ${LIB_DIR} ARCHIVE DESTINATION componentfiles/cl/lib )
 

roxie/ccd/ccdactivities.cpp

@@ -491,6 +491,7 @@ public:
     virtual UChar *getResultVarUnicode(const char * name, unsigned sequence) { throwUnexpected(); }
     virtual unsigned getResultHash(const char * name, unsigned sequence) { throwUnexpected(); }
     virtual void getResultRowset(size32_t & tcount, byte * * & tgt, const char * name, unsigned sequence, IEngineRowAllocator * _rowAllocator, IOutputRowDeserializer * deserializer, bool isGrouped, IXmlToRowTransformer * xmlTransformer, ICsvToRowTransformer * csvTransformer) { throwUnexpected(); }
+    virtual void getResultDictionary(size32_t & tcount, byte * * & tgt, IEngineRowAllocator * _rowAllocator, const char * name, unsigned sequence, IOutputRowDeserializer * deserializer, IXmlToRowTransformer * xmlTransformer, ICsvToRowTransformer * csvTransformer, IHThorHashLookupInfo * hasher) { throwUnexpected(); }
 
     // Not yet thought about these....
 

roxie/ccd/ccdmain.cpp

@@ -538,6 +538,7 @@ int STARTQUERY_API start_query(int argc, const char *argv[])
             topology->setPropInt("RoxieServerProcess/@port", port);
             topology->setProp("@daliServers", globals->queryProp("--daliServers"));
             topology->setProp("@traceLevel", globals->queryProp("--traceLevel"));
+            topology->setProp("@memTraceLevel", globals->queryProp("--memTraceLevel"));
         }
 
         topology->getProp("@name", roxieName);

roxie/ccd/ccdquery.cpp

@@ -412,6 +412,8 @@ protected:
             return createRoxieServerParseActivityFactory(id, subgraphId, *this, helperFactory, kind, this);
         case TAKworkunitwrite:
             return createRoxieServerWorkUnitWriteActivityFactory(id, subgraphId, *this, helperFactory, kind, usageCount(node), isRootAction(node));
+        case TAKdictionaryworkunitwrite:
+            return createRoxieServerWorkUnitWriteDictActivityFactory(id, subgraphId, *this, helperFactory, kind, usageCount(node), isRootAction(node));
         case TAKpiperead:
             return createRoxieServerPipeReadActivityFactory(id, subgraphId, *this, helperFactory, kind);
         case TAKpipethrough:

roxie/ccd/ccdserver.cpp

@@ -19247,6 +19247,8 @@ public:
         }
     }
 
+    virtual bool needsAllocator() const { return true; }
+
     virtual void onExecute() 
     {
         int sequence = helper.getSequence();
@@ -19272,18 +19274,14 @@ public:
             rowSerializer.setown(rowAllocator->createRowSerializer(ctx->queryCodeContext()));
         }
         __int64 initialProcessed = processed;
-        ConstPointerArray *lResult = NULL;
-        if (saveInContext)
-            lResult = new ConstPointerArray;
+        RtlLinkedDatasetBuilder builder(rowAllocator);
         loop
         {
             const void *row = input->nextInGroup();
-            if (lResult)
+            if (saveInContext)
             {
-                if (row) 
-                    LinkRoxieRow(row);
                 if (row || grouped)
-                    lResult->append(row);
+                    builder.append(row);
             }
             if (grouped && (processed != initialProcessed))
             {
@@ -19304,12 +19302,8 @@ public:
                 row = input->nextInGroup();
                 if (!row)
                     break;
-                if (lResult)
-                {
-                    if (row) 
-                        LinkRoxieRow(row);
-                    lResult->append(row);
-                }
+                if (saveInContext)
+                    builder.append(row);
             }
             processed++;
             if (serverContext->outputResultsToWorkUnit())
@@ -19346,8 +19340,8 @@ public:
             }
             ReleaseRoxieRow(row);
         }
-        if (lResult)
-            serverContext->appendResultDeserialized(storedName, sequence, lResult, (helper.getFlags() & POFextend) != 0, LINK(meta.queryOriginal()));
+        if (saveInContext)
+            serverContext->appendResultDeserialized(storedName, sequence, builder.getcount(), builder.linkrows(), (helper.getFlags() & POFextend) != 0, LINK(meta.queryOriginal()));
         if (serverContext->outputResultsToWorkUnit())
             serverContext->appendResultRawContext(storedName, sequence, result.length(), result.toByteArray(), processed, (helper.getFlags() & POFextend) != 0, false); // MORE - shame to do extra copy...
     }
@@ -19363,7 +19357,7 @@ public:
     {
         isReread = usageCount > 0;
         Owned<IHThorWorkUnitWriteArg> helper = (IHThorWorkUnitWriteArg *) helperFactory();
-        isInternal = (helper->getSequence()==-3);
+        isInternal = (helper->getSequence()==ResultSequenceInternal);
     }
 
     virtual IRoxieServerActivity *createActivity(IProbeManager *_probeManager) const
@@ -19379,6 +19373,81 @@ IRoxieServerActivityFactory *createRoxieServerWorkUnitWriteActivityFactory(unsig
 
 }
 
+//=====================================================================================================
+
+class CRoxieServerWorkUnitWriteDictActivity : public CRoxieServerInternalSinkActivity
+{
+    IHThorDictionaryWorkUnitWriteArg &helper;
+    IRoxieServerContext *serverContext;
+
+public:
+    CRoxieServerWorkUnitWriteDictActivity(const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager)
+        : CRoxieServerInternalSinkActivity(_factory, _probeManager), helper((IHThorDictionaryWorkUnitWriteArg &)basehelper)
+    {
+        serverContext = NULL;
+    }
+
+    virtual void onCreate(IRoxieSlaveContext *_ctx, IHThorArg *_colocalParent)
+    {
+        CRoxieServerInternalSinkActivity::onCreate(_ctx, _colocalParent);
+        serverContext = ctx->queryServerContext();
+        if (!serverContext)
+        {
+            throw MakeStringException(ROXIE_PIPE_ERROR, "Write Dictionary activity cannot be executed in slave context");
+        }
+    }
+
+    virtual bool needsAllocator() const { return true; }
+
+    virtual void onExecute()
+    {
+        int sequence = helper.getSequence();
+        const char *storedName = helper.queryName();
+        assertex(storedName && *storedName);
+        assertex(sequence < 0);
+
+        __int64 initialProcessed = processed;
+        RtlLinkedDictionaryBuilder builder(rowAllocator, helper.queryHashLookupInfo());
+        loop
+        {
+            const void *row = input->nextInGroup();
+            if (!row)
+            {
+                row = input->nextInGroup();
+                if (!row)
+                    break;
+            }
+            builder.appendOwn(row);
+            processed++;
+        }
+        serverContext->appendResultDeserialized(storedName, sequence, builder.getcount(), builder.linkrows(), (helper.getFlags() & POFextend) != 0, LINK(meta.queryOriginal()));
+    }
+};
+
+class CRoxieServerWorkUnitWriteDictActivityFactory : public CRoxieServerInternalSinkFactory
+{
+
+public:
+    CRoxieServerWorkUnitWriteDictActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, unsigned _usageCount, bool _isRoot)
+        : CRoxieServerInternalSinkFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _usageCount, _isRoot)
+    {
+        Owned<IHThorDictionaryWorkUnitWriteArg> helper = (IHThorDictionaryWorkUnitWriteArg *) helperFactory();
+        isInternal = (helper->getSequence()==ResultSequenceInternal);
+    }
+
+    virtual IRoxieServerActivity *createActivity(IProbeManager *_probeManager) const
+    {
+        return new CRoxieServerWorkUnitWriteDictActivity(this, _probeManager);
+    }
+
+};
+
+IRoxieServerActivityFactory *createRoxieServerWorkUnitWriteDictActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, unsigned _usageCount, bool _isRoot)
+{
+    return new CRoxieServerWorkUnitWriteDictActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _usageCount, _isRoot);
+
+}
+
 //=================================================================================
 
 class CRoxieServerRemoteResultActivity : public CRoxieServerInternalSinkActivity
@@ -19406,7 +19475,7 @@ public:
         : CRoxieServerInternalSinkFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind, _usageCount, _isRoot)
     {
         Owned<IHThorRemoteResultArg> helper = (IHThorRemoteResultArg *) helperFactory();
-        isInternal = (helper->getSequence()==-3);
+        isInternal = (helper->getSequence()==ResultSequenceInternal);
     }
 
     virtual IRoxieServerActivity *createActivity(IProbeManager *_probeManager) const
@@ -25164,6 +25233,10 @@ public:
     {
         select(id).getLinkedResult(count, ret);
     }
+    virtual void getDictionaryResult(unsigned & count, byte * * & ret, unsigned id)
+    {
+        select(id).getLinkedResult(count, ret);
+    }
     void setResult(unsigned id, IGraphResult * result)
     {
         CriticalBlock procedure(cs);
@@ -27056,6 +27129,10 @@ public:
     {
         results->getLinkedResult(count, ret, id);
     }
+    virtual void getDictionaryResult(unsigned & count, byte * * & ret, unsigned id)
+    {
+        results->getLinkedResult(count, ret, id);
+    }
     virtual void setResult(unsigned id, IGraphResult * result)
     {
         results->setResult(id, result);
@@ -28171,6 +28248,7 @@ public:
     virtual UChar *getResultVarUnicode(const char * name, unsigned sequence) { throwUnexpected(); }
     virtual unsigned getResultHash(const char * name, unsigned sequence) { throwUnexpected(); }
     virtual void getResultRowset(size32_t & tcount, byte * * & tgt, const char * name, unsigned sequence, IEngineRowAllocator * _rowAllocator, IOutputRowDeserializer * deserializer, bool isGrouped, IXmlToRowTransformer * xmlTransformer, ICsvToRowTransformer * csvTransformer) { throwUnexpected(); }
+    virtual void getResultDictionary(size32_t & tcount, byte * * & tgt, IEngineRowAllocator * _rowAllocator, const char * name, unsigned sequence, IOutputRowDeserializer * deserializer, IXmlToRowTransformer * xmlTransformer, ICsvToRowTransformer * csvTransformer, IHThorHashLookupInfo * hasher) { throwUnexpected(); }
 
     virtual void printResults(IXmlWriter *output, const char *name, unsigned sequence) { throwUnexpected(); }
 
@@ -28412,13 +28490,13 @@ public:
         if (!type || stricmp(type, "temporary"))
         {
             output->outputBeginNested("Temporary", true);
-            ctx->printResults(output, name, (unsigned) -3);
+            ctx->printResults(output, name, (unsigned) ResultSequenceInternal);
             output->outputEndNested("Temporary");
         }
         if (!type || stricmp(type, "global"))
         {
             output->outputBeginNested("Global", true);
-            ctx->printResults(output, name, (unsigned) -1);
+            ctx->printResults(output, name, (unsigned) ResultSequenceStored);
             output->outputEndNested("Global");
         }
         output->outputEndNested("Variables");
@@ -28426,21 +28504,26 @@ public:
 
 };
 
+typedef byte *row_t;
+typedef row_t * rowset_t;
+
 class DeserializedDataReader : public CInterface, implements IWorkUnitRowReader
 {
-    const ConstPointerArray &data;
+    const rowset_t data;
+    size32_t count;
     unsigned idx;
 public:
     IMPLEMENT_IINTERFACE;
-    DeserializedDataReader(const ConstPointerArray &_data) : data(_data)
+    DeserializedDataReader(size32_t _count, rowset_t _data)
+    : data(_data), count(_count)
     {
         idx = 0;
     }
     virtual const void * nextInGroup() 
     {
-        if (data.isItem(idx))
+        if (idx < count)
         {
-            const void *row = data.item(idx);
+            const void *row = data[idx];
             if (row)
                 LinkRoxieRow(row);
             idx++;
@@ -28448,11 +28531,19 @@ public:
         }
         return NULL;
     }
+    virtual void getResultRowset(size32_t & tcount, byte * * & tgt)
+    {
+        tcount = count;
+        if (data)
+            rtlLinkRowset(data);
+        tgt = data;
+    }
 };
 
 class CDeserializedResultStore : public CInterface, implements IDeserializedResultStore 
 {
-    PointerArrayOf<ConstPointerArray> stored;
+    PointerArrayOf<row_t> stored;
+    UnsignedArray counts;
     PointerIArrayOf<IOutputMetaData> metas;
     mutable SpinLock lock;
 public:
@@ -28461,63 +28552,48 @@ public:
     {
         ForEachItemIn(idx, stored)
         {
-            ConstPointerArray *rows = stored.item(idx);
+            rowset_t rows = stored.item(idx);
             if (rows)
             {
-                ReleaseRoxieRowSet(*rows);
-                delete rows;
+                rtlReleaseRowset(counts.item(idx), (byte**) rows);
             }
         }
     }
-    virtual int addResult(ConstPointerArray *data, IOutputMetaData *meta)
+    virtual int addResult(size32_t count, rowset_t data, IOutputMetaData *meta)
     {
         SpinBlock b(lock);
         stored.append(data);
+        counts.append(count);
         metas.append(meta);
         return stored.ordinality()-1;
     }
-    virtual int appendResult(int oldId, ConstPointerArray *data, IOutputMetaData *meta)
+    virtual void queryResult(int id, size32_t &count, rowset_t &data) const
     {
-        SpinBlock b(lock);
-        ConstPointerArray *oldData = stored.item(oldId);
-        ConstPointerArray *newData = new ConstPointerArray;
-        ForEachItemIn(idx, *oldData)
-        {
-            const void * row = oldData->item(idx);
-            if (row)
-                LinkRoxieRow(row);
-            newData->append(row);
-        }
-        ForEachItemIn(idx2, *data)
-        {
-            const void * row = data->item(idx2);
-            newData->append(row);       // don't link these rows, because...
-        }
-        delete data;                    // ...then we don't need to release them when we delete this array
-        // Note that we don't delete the previons ConstPointerArray yet, as it could be in use. 
-        return addResult(newData, meta);
+        count = counts.item(id);
+        data = stored.item(id);
     }
     virtual IWorkUnitRowReader *createDeserializedReader(int id) const
     {
-        return new DeserializedDataReader(*stored.item(id));
+        return new DeserializedDataReader(counts.item(id), stored.item(id));
     }
     virtual void serialize(unsigned & tlen, void * & tgt, int id, ICodeContext *codectx) const
     {
         IOutputMetaData *meta = metas.item(id);
-        ConstPointerArray *data = stored.item(id);
+        rowset_t data = stored.item(id);
+        size32_t count = counts.item(id);
 
         MemoryBuffer result;
-        Owned<IOutputRowSerializer> rowSerializer = meta->createRowSerializer(codectx, 0); // NOTE - we don't have a maningful activity id. Only used for error reporting.
+        Owned<IOutputRowSerializer> rowSerializer = meta->createRowSerializer(codectx, 0); // NOTE - we don't have a meaningful activity id. Only used for error reporting.
         bool grouped = meta->isGrouped();
-        ForEachItemIn(idx, *data)
+        for (size32_t idx = 0; idx<count; idx++)
         {
-            const void *row = data->item(idx);
+            const byte *row = data[idx];
             if (grouped && idx)
                 result.append(row == NULL);
             if (row)
             {
                 CThorDemoRowSerializer serializerTarget(result);
-                rowSerializer->serialize(serializerTarget, (const byte *) row);
+                rowSerializer->serialize(serializerTarget, row);
             }
         }
         tlen = result.length();
@@ -29058,7 +29134,7 @@ public:
             }
         }
     }
-    virtual void appendResultDeserialized(const char *name, unsigned sequence, ConstPointerArray *data, bool extend, IOutputMetaData *meta)
+    virtual void appendResultDeserialized(const char *name, unsigned sequence, size32_t count, rowset_t data, bool extend, IOutputMetaData *meta)
     {
         CriticalBlock b(contextCrit);
         IPropertyTree &ctx = useContext(sequence);
@@ -29066,16 +29142,26 @@ public:
         IPropertyTree *val = ctx.queryPropTree(name);
         if (extend && val)
         {
-            int old = val->getPropInt("@id", -1);
-            assertex(old!=-1);
-            val->setPropInt("@id", resultStore.appendResult(old, data, meta));
+            int oldId = val->getPropInt("@id", -1);
+            const char * oldFormat = val->queryProp("@format");
+            assertex(oldId != -1);
+            assertex(oldFormat && strcmp(oldFormat, "deserialized")==0);
+            size32_t oldCount;
+            rowset_t oldData;
+            resultStore.queryResult(oldId, oldCount, oldData);
+            Owned<IEngineRowAllocator> allocator = createRoxieRowAllocator(*rowManager, meta, 0, 0, roxiemem::RHFnone);
+            RtlLinkedDatasetBuilder builder(allocator);
+            builder.appendRows(oldCount, oldData);
+            builder.appendRows(count, data);
+            rtlReleaseRowset(count, data);
+            val->setPropInt("@id", resultStore.addResult(builder.getcount(), builder.linkrows(), meta));
         }
         else
         {
             if (!val)
                 val = ctx.addPropTree(name, createPTree());
             val->setProp("@format", "deserialized");
-            val->setPropInt("@id", resultStore.addResult(data, meta));
+            val->setPropInt("@id", resultStore.addResult(count, data, meta));
         }
 
     }
@@ -29680,24 +29766,8 @@ public:
     {
         try
         {
-            bool atEOG = true;
             Owned<IWorkUnitRowReader> wuReader = getWorkunitRowReader(stepname, sequence, xmlTransformer, _rowAllocator, isGrouped);
-            RtlLinkedDatasetBuilder builder(_rowAllocator);
-            loop
-            {
-                const void *ret = wuReader->nextInGroup();
-                if (!ret)
-                {
-                    if (atEOG || !isGrouped)
-                        break;
-                    atEOG = true;
-                }
-                else
-                    atEOG = false;
-                builder.appendOwn(ret);
-            }
-            tcount = builder.getcount();
-            tgt = builder.linkrows();
+            wuReader->getResultRowset(tcount, tgt);
         }
         catch (IException * e)
         {
@@ -29712,6 +29782,25 @@ public:
         }
     }
 
+    virtual void getResultDictionary(size32_t & tcount, byte * * & tgt, IEngineRowAllocator * _rowAllocator, const char * stepname, unsigned sequence, IOutputRowDeserializer * deserializer, IXmlToRowTransformer * xmlTransformer, ICsvToRowTransformer * csvTransformer, IHThorHashLookupInfo * hasher)
+    {
+        try
+        {
+            Owned<IWorkUnitRowReader> wuReader = getWorkunitRowReader(stepname, sequence, xmlTransformer, _rowAllocator, false);
+            wuReader->getResultRowset(tcount, tgt);
+        }
+        catch (IException * e)
+        {
+            StringBuffer text;
+            e->errorMessage(text);
+            e->Release();
+            throw MakeStringException(ROXIE_DATA_ERROR, "Failed to retrieve data value \"%s\".  [%s]", stepname, text.str());
+        }
+        catch (...)
+        {
+            throw MakeStringException(ROXIE_DATA_ERROR, "Failed to retrieve data value \"%s\"", stepname);
+        }
+    }
     virtual void getResultSet(bool & tisAll, unsigned & tlen, void * & tgt, const char *stepname, unsigned sequence, IXmlToRowTransformer * xmlTransformer, ICsvToRowTransformer * csvTransformer)
     {
         try
@@ -29754,7 +29843,43 @@ public:
         }
     }
 
-    class RawDataReader : public CInterface, implements IWorkUnitRowReader
+    class WorkUnitRowReaderBase : public CInterface, implements IWorkUnitRowReader
+    {
+    protected:
+        Linked<IEngineRowAllocator> rowAllocator;
+        bool isGrouped;
+
+    public:
+        IMPLEMENT_IINTERFACE;
+        WorkUnitRowReaderBase(IEngineRowAllocator *_rowAllocator, bool _isGrouped)
+            : rowAllocator(_rowAllocator), isGrouped(_isGrouped)
+        {
+
+        }
+
+        virtual void getResultRowset(size32_t & tcount, byte * * & tgt)
+        {
+            bool atEOG = true;
+            RtlLinkedDatasetBuilder builder(rowAllocator);
+            loop
+            {
+                const void *ret = nextInGroup();
+                if (!ret)
+                {
+                    if (atEOG || !isGrouped)
+                        break;
+                    atEOG = true;
+                }
+                else
+                    atEOG = false;
+                builder.appendOwn(ret);
+            }
+            tcount = builder.getcount();
+            tgt = builder.linkrows();
+        }
+    };
+
+    class RawDataReader : public WorkUnitRowReaderBase
     {
     protected:
         const IRoxieContextLogger &logctx;
@@ -29764,8 +29889,6 @@ public:
         CThorStreamDeserializerSource rowSource;
         bool eof;
         bool eogPending;
-        bool isGrouped;
-        Linked<IEngineRowAllocator> rowAllocator;
         Owned<IOutputRowDeserializer> rowDeserializer;
 
         virtual bool nextBlock(unsigned & tlen, void * & tgt, void * & base) = 0;
@@ -29783,9 +29906,8 @@ public:
         }
 
     public:
-        IMPLEMENT_IINTERFACE;
         RawDataReader(CRoxieServerContext *parent, IEngineRowAllocator *_rowAllocator, bool _isGrouped)
-            : logctx(*parent), rowAllocator(_rowAllocator), isGrouped(_isGrouped)
+            : WorkUnitRowReaderBase(_rowAllocator, _isGrouped), logctx(*parent)
         {
             eof = false;
             eogPending = false;
@@ -29928,17 +30050,16 @@ public:
         }
     };
 
-    class InlineXmlDataReader : public CInterface, implements IWorkUnitRowReader
+    class InlineXmlDataReader : public WorkUnitRowReaderBase
     {
         Linked<IPropertyTree> xml;
         Owned <XmlColumnProvider> columns;
         Owned<IPropertyTreeIterator> rows;
         IXmlToRowTransformer &rowTransformer;
-        Linked<IEngineRowAllocator> rowAllocator;
     public:
         IMPLEMENT_IINTERFACE;
-        InlineXmlDataReader(IXmlToRowTransformer &_rowTransformer, IPropertyTree *_xml, IEngineRowAllocator *_rowAllocator) 
-            : xml(_xml), rowTransformer(_rowTransformer), rowAllocator(_rowAllocator)
+        InlineXmlDataReader(IXmlToRowTransformer &_rowTransformer, IPropertyTree *_xml, IEngineRowAllocator *_rowAllocator, bool _isGrouped)
+            : WorkUnitRowReaderBase(_rowAllocator, _isGrouped), xml(_xml), rowTransformer(_rowTransformer)
         {
             columns.setown(new XmlDatasetColumnProvider);
             rows.setown(xml->getElements("Row")); // NOTE - the 'hack for Gordon' as found in thorxmlread is not implemented here. Does it need to be ?
@@ -29988,7 +30109,7 @@ public:
                     if (!format || strcmp(format, "xml") == 0)
                     {
                         if (xmlTransformer)
-                            return new InlineXmlDataReader(*xmlTransformer, val, rowAllocator);
+                            return new InlineXmlDataReader(*xmlTransformer, val, rowAllocator, isGrouped);
                     }
                     else if (strcmp(format, "raw") == 0)
                     {

roxie/ccd/ccdserver.hpp

@@ -86,12 +86,13 @@ interface IRoxieServerLoopResultProcessor;
 interface IWorkUnitRowReader : public IInterface
 {
     virtual const void * nextInGroup() = 0;
+    virtual void getResultRowset(size32_t & tcount, byte * * & tgt) = 0;
 };
 
 interface IDeserializedResultStore : public IInterface
 {
-    virtual int addResult(ConstPointerArray *data, IOutputMetaData *meta) = 0;
-    virtual int appendResult(int oldId, ConstPointerArray *data, IOutputMetaData *meta) = 0;
+    virtual int addResult(size32_t count, byte **data, IOutputMetaData *meta) = 0;
+    virtual void queryResult(int id, size32_t &count, byte ** &data) const = 0;
     virtual IWorkUnitRowReader *createDeserializedReader(int id) const = 0;
     virtual void serialize(unsigned & tlen, void * & tgt, int id, ICodeContext *ctx) const = 0;
 };
@@ -161,7 +162,7 @@ interface IRoxieServerContext : extends IInterface
     virtual IGlobalCodeContext *queryGlobalCodeContext() = 0;
     virtual FlushingStringBuffer *queryResult(unsigned sequence) = 0;
     virtual void setResultXml(const char *name, unsigned sequence, const char *xml) = 0;
-    virtual void appendResultDeserialized(const char *name, unsigned sequence, ConstPointerArray *data, bool extend, IOutputMetaData *meta) = 0;
+    virtual void appendResultDeserialized(const char *name, unsigned sequence, size32_t count, byte **data, bool extend, IOutputMetaData *meta) = 0;
     virtual void appendResultRawContext(const char *name, unsigned sequence, int len, const void * data, int numRows, bool extend, bool saveInContext) = 0;
     virtual IWorkUnitRowReader *getWorkunitRowReader(const char * name, unsigned sequence, IXmlToRowTransformer * xmlTransformer, IEngineRowAllocator *rowAllocator, bool isGrouped) = 0;
     virtual roxiemem::IRowManager &queryRowManager() = 0;
@@ -487,6 +488,7 @@ extern IRoxieServerActivityFactory *createRoxieServerCaseActivityFactory(unsigne
 extern IRoxieServerActivityFactory *createRoxieServerIfActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, bool _graphInvariant);
 extern IRoxieServerActivityFactory *createRoxieServerParseActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, IResourceContext *rc);
 extern IRoxieServerActivityFactory *createRoxieServerWorkUnitWriteActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, unsigned _usageCount, bool _isRoot);
+extern IRoxieServerActivityFactory *createRoxieServerWorkUnitWriteDictActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, unsigned _usageCount, bool _isRoot);
 extern IRoxieServerActivityFactory *createRoxieServerRemoteResultActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, unsigned _usageCount, bool _isRoot);
 extern IRoxieServerActivityFactory *createRoxieServerXmlParseActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind);
 extern IRoxieServerActivityFactory *createRoxieServerDiskReadActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, const RemoteActivityId &_remoteId, IPropertyTree &_graphNode);

roxie/roxiemem/roxiemem.cpp

@@ -1065,12 +1065,20 @@ private:
         const char * ptr = (const char *)_ptr;
         const char *baseptr = ptr - chunkHeaderSize;
         ChunkHeader * header = (ChunkHeader *)baseptr;
+#ifdef _DEBUG
+        if (memTraceLevel >= 100)
+        {
+            DBGLOG("%s: Pointer %p %x", reason, ptr, *(unsigned *) baseptr);
+            if (memTraceLevel >= 1000)
+                PrintStackReport();
+        }
+#endif
         if ((header->allocatorId & ~ACTIVITY_MASK) != ACTIVITY_MAGIC)
         {
-            DBGLOG("%s: Attempt to free invalid pointer %p %x", reason, ptr, *(unsigned *) baseptr);
+            DBGLOG("%s: Invalid pointer %p %x", reason, ptr, *(unsigned *) baseptr);
             PrintStackReport();
             PrintMemoryReport();
-            HEAPERROR("Attempt to free invalid pointer");
+            HEAPERROR("Invalid pointer");
         }
     }
 };

rtl/eclrtl/rtlds.cpp

@@ -334,7 +334,7 @@ void RtlLinkedDatasetBuilder::append(const void * source)
     if (count < choosenLimit)
     {
         ensure(count+1);
-        rowset[count] = (byte *)rowAllocator->linkRow(source);
+        rowset[count] = source ? (byte *)rowAllocator->linkRow(source) : NULL;
         count++;
     }
 }
@@ -742,6 +742,57 @@ extern ECLRTL_API bool rtlDictionaryLookupExists(IHThorHashLookupInfo &hashInfo,
     }
 }
 
+extern ECLRTL_API void rtlSerializeDictionary(IRowSerializerTarget & out, IOutputRowSerializer * serializer, size32_t count, byte * * rows)
+{
+    out.put(sizeof(count), &count);
+    size32_t idx = 0;
+    while (idx < count)
+    {
+        byte numRows = 0;
+        while (numRows < 255 && idx+numRows < count && rows[idx+numRows] != NULL)
+            numRows++;
+        out.put(1, &numRows);
+        for (int i = 0; i < numRows; i++)
+        {
+            byte *nextrec = rows[idx+i];
+            assert(nextrec);
+            serializer->serialize(out, nextrec);
+        }
+        idx += numRows;
+        byte numNulls = 0;
+        while (numNulls < 255 && idx+numNulls < count && rows[idx+numNulls] == NULL)
+            numNulls++;
+        out.put(1, &numNulls);
+        idx += numNulls;
+    }
+}
+
+extern ECLRTL_API void rtlDictionary2RowsetX(size32_t & count, byte * * & rowset, IEngineRowAllocator * rowAllocator, IOutputRowDeserializer * deserializer, size32_t lenSrc, const void * src)
+{
+    RtlLinkedDatasetBuilder builder(rowAllocator);
+    Owned<ISerialStream> stream = createMemorySerialStream(src, lenSrc);
+    CThorStreamDeserializerSource source(stream);
+
+    size32_t totalRows;
+    source.read(sizeof(totalRows), &totalRows);
+    builder.ensure(totalRows);
+    byte nullsPending = 0;
+    byte rowsPending = 0;
+    while (!source.finishedNested(lenSrc))
+    {
+        source.read(1, &rowsPending);
+        for (int i = 0; i < rowsPending; i++)
+            builder.deserializeRow(*deserializer, source);
+        source.read(1, &nullsPending);
+        for (int i = 0; i < nullsPending; i++)
+            builder.append(NULL);
+    }
+
+    count = builder.getcount();
+    assertex(count==totalRows);
+    rowset = builder.linkrows();
+}
+
 //---------------------------------------------------------------------------
 
 //These definitions should be shared with thorcommon, but to do that
@@ -833,6 +884,17 @@ extern ECLRTL_API void rtlGroupedRowset2DatasetX(unsigned & tlen, void * & tgt,
     doRowset2DatasetX(tlen, tgt, serializer, count, rows, true);
 }
 
+extern ECLRTL_API void rtlRowset2DictionaryX(unsigned & tlen, void * & tgt, IOutputRowSerializer * serializer, size32_t count, byte * * rows)
+{
+    MemoryBuffer rowdata;
+    CThorRtlRowSerializer out(rowdata);
+    rtlSerializeDictionary(out, serializer, count, rows);
+
+    rtlFree(tgt);
+    tlen = rowdata.length();
+    tgt = rowdata.detach();
+}
+
 void deserializeRowsetX(size32_t & count, byte * * & rowset, IEngineRowAllocator * _rowAllocator, IOutputRowDeserializer * deserializer, MemoryBuffer &in)
 {
     Owned<ISerialStream> stream = createMemoryBufferSerialStream(in);

rtl/eclrtl/rtlds_imp.hpp

@@ -455,6 +455,10 @@ extern ECLRTL_API void rtlGroupedDataset2RowsetX(size32_t & count, byte * * & ro
 extern ECLRTL_API void rtlRowset2DatasetX(unsigned & tlen, void * & tgt, IOutputRowSerializer * serializer, size32_t count, byte * * rows);
 extern ECLRTL_API void rtlGroupedRowset2DatasetX(unsigned & tlen, void * & tgt, IOutputRowSerializer * serializer, size32_t count, byte * * rows);
 
+extern ECLRTL_API void rtlSerializeDictionary(IRowSerializerTarget & out, IOutputRowSerializer * serializer, size32_t count, byte * * rows);
+extern ECLRTL_API void rtlDictionary2RowsetX(size32_t & count, byte * * & rowset, IEngineRowAllocator * rowAllocator, IOutputRowDeserializer * deserializer, size32_t lenSrc, const void * src);
+extern ECLRTL_API void rtlRowset2DictionaryX(unsigned & tlen, void * & tgt, IOutputRowSerializer * serializer, size32_t count, byte * * rows);
+
 extern ECLRTL_API size32_t rtlDeserializeRow(size32_t lenOut, void * out, IOutputRowDeserializer * deserializer, const void * src);
 extern ECLRTL_API size32_t rtlSerializeRow(size32_t lenOut, void * out, IOutputRowSerializer * serializer, const void * src);
 extern ECLRTL_API size32_t rtlDeserializeToBuilder(ARowBuilder & builder, IOutputRowDeserializer * deserializer, const void * src);

rtl/include/eclhelper.hpp

@@ -464,6 +464,7 @@ interface IEclGraphResults : public IInterface
 {
     virtual void getResult(size32_t & retSize, void * & ret, unsigned id) = 0;
     virtual void getLinkedResult(unsigned & count, byte * * & ret, unsigned id) = 0;
+    virtual void getDictionaryResult(size32_t & tcount, byte * * & tgt, unsigned id) = 0;
 };
 
 //Provided by engine=>can extent
@@ -477,6 +478,7 @@ interface ILocalGraph : public IInterface
 {
     virtual void getResult(unsigned & len, void * & data, unsigned id) = 0;
     virtual void getLinkedResult(unsigned & count, byte * * & ret, unsigned id) = 0;
+    virtual void getDictionaryResult(size32_t & tcount, byte * * & tgt, unsigned id) = 0;
 };
 
 //NB: New methods must always be added at the end of this interface to retain backward compatibility
@@ -485,6 +487,7 @@ interface IDebuggableContext;
 interface IDistributedFileTransaction;
 interface IUserDescriptor;
 interface IHThorArg;
+interface IHThorHashLookupInfo;
 
 interface ICodeContext : public IResourceContext
 {
@@ -559,6 +562,7 @@ interface ICodeContext : public IResourceContext
     virtual void getRowXML(size32_t & lenResult, char * & result, IOutputMetaData & info, const void * row, unsigned flags) = 0;
     virtual void addWuAssertFailure(unsigned code, const char * text, const char * filename, unsigned lineno, unsigned column, bool isAbort) = 0;
     virtual const void * fromXml(IEngineRowAllocator * _rowAllocator, size32_t len, const char * utf8, IXmlToRowTransformer * xmlTransformer, bool stripWhitespace) = 0;
+    virtual void getResultDictionary(size32_t & tcount, byte * * & tgt, IEngineRowAllocator * _rowAllocator, const char * name, unsigned sequence, IOutputRowDeserializer * deserializer, IXmlToRowTransformer * xmlTransformer, ICsvToRowTransformer * csvTransformer, IHThorHashLookupInfo * hasher) = 0;
 };
 
 
@@ -814,6 +818,8 @@ enum ThorActivityKind
     TAKexternalsource,
     TAKexternalsink,
     TAKexternalprocess,
+    TAKdictionaryworkunitwrite,
+    TAKdictionaryresultwrite,
 
     TAKlast
 };
@@ -959,6 +965,8 @@ enum ActivityInterfaceEnum
     TAIshuffleextra_1,
     TAIhashdeduparg_2,
     TAIsoapcallextra_2,
+    TAIdictionaryworkunitwritearg_1,
+    TAIdictionaryresultwritearg_1,
 
 //Should remain as last of all meaningful tags, but before aliases
     TAImax,
@@ -2704,6 +2712,21 @@ interface IHThorHashLookupInfo
 };
 
 
+struct IHThorDictionaryWorkUnitWriteArg : public IHThorArg
+{
+    virtual int getSequence() = 0;
+    virtual const char * queryName() = 0;
+    virtual unsigned getFlags() = 0;
+    virtual IHThorHashLookupInfo * queryHashLookupInfo() = 0;
+};
+
+struct IHThorDictionaryResultWriteArg : public IHThorArg
+{
+    virtual unsigned querySequence() = 0;
+    virtual bool usedOutsideGraph() = 0;
+    virtual IHThorHashLookupInfo * queryHashLookupInfo() = 0;
+};
+
 //------------------------- Other stuff -------------------------
 
 struct IGlobalCodeContext

rtl/include/eclhelper_base.hpp

@@ -1934,6 +1934,50 @@ class CThorXmlWorkunitWriteArg : public CThorArg, implements IHThorXmlWorkunitWr
     virtual unsigned getFlags()                             { return 0; }
 };
 
+class CThorDictionaryWorkUnitWriteArg : public CThorArg, implements IHThorDictionaryWorkUnitWriteArg
+{
+    virtual void Link() const { RtlCInterface::Link(); }
+    virtual bool Release() const { return RtlCInterface::Release(); }
+    virtual void onCreate(ICodeContext * _ctx, IHThorArg *, MemoryBuffer * in) { ctx = _ctx; }
+    virtual IOutputMetaData * queryOutputMeta() { return NULL; }
+
+    virtual IInterface * selectInterface(ActivityInterfaceEnum which)
+    {
+        switch (which)
+        {
+        case TAIarg:
+        case TAIdictionaryworkunitwritearg_1:
+            return static_cast<IHThorDictionaryWorkUnitWriteArg *>(this);
+        }
+        return NULL;
+    }
+
+    virtual int getSequence()                               { return -3; }
+    virtual const char * queryName()                        { return NULL; }
+    virtual unsigned getFlags()                             { return 0; }
+};
+
+class CThorDictionaryResultWriteArg : public CThorArg, implements IHThorDictionaryResultWriteArg
+{
+    virtual void Link() const { RtlCInterface::Link(); }
+    virtual bool Release() const { return RtlCInterface::Release(); }
+    virtual void onCreate(ICodeContext * _ctx, IHThorArg *, MemoryBuffer * in) { ctx = _ctx; }
+    virtual IOutputMetaData * queryOutputMeta() { return NULL; }
+
+    virtual IInterface * selectInterface(ActivityInterfaceEnum which)
+    {
+        switch (which)
+        {
+        case TAIarg:
+        case TAIdictionaryresultwritearg_1:
+            return static_cast<IHThorDictionaryResultWriteArg *>(this);
+        }
+        return NULL;
+    }
+
+    virtual bool usedOutsideGraph() { return true; }
+};
+
 class CThorHashDistributeArg : public CThorArg, implements IHThorHashDistributeArg
 {
     virtual void Link() const { RtlCInterface::Link(); }

testing/ecl/dict10.ecl

@@ -0,0 +1,36 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+squareRecord := RECORD
+  unsigned number;
+  unsigned square;
+END;
+
+squareRecord makeSquare(unsigned value) := TRANSFORM
+  SELF.number := value;
+  SELF.square := value * value;
+END;
+
+numSquares := 10000;
+squares := DATASET(numSquares, makeSquare(COUNTER-1));
+
+squareRoots := DICTIONARY(squares, { UNSIGNED value := square => UNSIGNED root := number; });
+
+
+searchValue := 100 : STORED('search');
+
+OUTPUT(searchValue IN squareRoots);

testing/ecl/dict11.ecl

@@ -0,0 +1,40 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+squareRecord := RECORD
+  unsigned number;
+  unsigned square;
+END;
+
+squareRecord makeSquare(unsigned value) := TRANSFORM
+  SELF.number := value;
+  SELF.square := value * value;
+END;
+
+numSquares := 10000;
+squares := DATASET(numSquares, makeSquare(COUNTER-1));
+
+squareRoots := DICTIONARY(squares, { UNSIGNED value := square => UNSIGNED root := number; });
+
+
+inRec := { unsigned value; };
+outRec := { unsigned value; unsigned root; };
+
+searchValues := DATASET([1,2,3,4,5,99,100,32768,65536,-1], inRec) : STORED('search');
+
+p := PROJECT(searchValues, TRANSFORM(outRec, SELF.root := squareRoots[LEFT.value].root; SELF := LEFT));
+OUTPUT(p);

testing/ecl/dict12.ecl

@@ -0,0 +1,40 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+squareRecord := RECORD
+  unsigned number;
+  unsigned square;
+END;
+
+squareRecord makeSquare(unsigned value) := TRANSFORM
+  SELF.number := value;
+  SELF.square := value * value;
+END;
+
+numSquares := 20000;
+squares := DATASET(numSquares, makeSquare(COUNTER-1));
+
+squareRoots := DICTIONARY(squares, { UNSIGNED value := square => UNSIGNED root := number; });
+
+
+inRec := { unsigned value; };
+outRec := { unsigned value; unsigned root; };
+
+searchValues := DATASET([1,2,3,4,5,99,100,32768,65536,-1], inRec) : STORED('search');
+
+p := SORT(searchValues, squareRoots[value].root);
+OUTPUT(p);

testing/ecl/dict13.ecl

@@ -0,0 +1,38 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+squareRecord := RECORD
+  unsigned number;
+  unsigned square;
+END;
+
+squareRecord makeSquare(unsigned value, unsigned delta) := TRANSFORM
+  SELF.number := value;
+  SELF.square := value * value + delta;
+END;
+
+numSquares := 200;
+squares := DATASET(numSquares, makeSquare((COUNTER-1) DIV 2,(COUNTER-1) % 2));
+
+squareRoots := DICTIONARY(squares, { UNSIGNED value := square => UNSIGNED root := number; });
+
+//Check duplicate entries supplied to a dictionary always pick the first item
+
+values := TABLE(NOFOLD(squares(square = number * number)), { unsigned value := square });
+
+p := TABLE(values, { value, squareRoots[value].root });
+OUTPUT(p);

testing/ecl/dict14.ecl

@@ -0,0 +1,36 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+squareRecord := RECORD
+  unsigned number;
+  unsigned square;
+END;
+
+squareRecord makeSquare(unsigned value, unsigned delta) := TRANSFORM
+  SELF.number := value;
+  SELF.square := value * value + delta;
+END;
+
+numSquares := 200;
+squares := DATASET(numSquares, makeSquare((COUNTER-1) DIV 2,(COUNTER-1) % 2));
+
+squareRoots := DICTIONARY(squares, { UNSIGNED value := square => UNSIGNED root := number; });
+
+//Use a dictionary as a dataset
+values := NOFOLD(DATASET(squareRoots)(value = root * root));
+
+OUTPUT(values);

testing/ecl/dict15.ecl

@@ -0,0 +1,46 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//UseStandardFiles
+//Find all anagrams of a word, that match the list of known words
+
+allWordsDs := DEDUP(SORTED(TS_wordIndex), word);
+
+knownWords := DICTIONARY(allWordsDs, { word });
+
+string searchWord := 'gabs' : stored('word');
+
+R := RECORD
+  STRING SoFar;
+  STRING Rest;
+END;
+
+Initial := DATASET([{'',searchWord}],R);
+
+R Pluck1(DATASET(R) infile) := FUNCTION
+  R TakeOne(R le, UNSIGNED1 c) := TRANSFORM
+    SELF.SoFar := le.SoFar + le.Rest[c];
+    SELF.Rest := le.Rest[..c-1]+le.Rest[c+1..]; // Boundary Conditions handled automatically
+  END;
+  RETURN NORMALIZE(infile,LENGTH(LEFT.Rest),TakeOne(LEFT,COUNTER));
+END;
+
+Processed := LOOP(Initial,LENGTH(searchWord),Pluck1(ROWS(LEFT)));
+
+Anagrams := TABLE(Processed, { string word := SoFar; });
+
+OUTPUT(Anagrams(Word in knownWords));

testing/ecl/dict15a.ecl

@@ -0,0 +1,47 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//UseStandardFiles
+//Find all anagrams of a word, that match the list of known words
+
+allWordsDs := DEDUP(SORTED(TS_wordIndex), word);
+
+knownWords := DICTIONARY(allWordsDs, { word });
+
+string searchWord := 'gabs' : stored('word');
+
+R := RECORD
+  STRING Word;
+END;
+
+findAnagrams(string searchWord) := FUNCTION
+  Initial := DATASET([{searchWord}],R);
+    
+  R Pluck1(DATASET(R) infile, unsigned4 numDone) := FUNCTION
+    R TakeOne(R le, UNSIGNED1 c) := TRANSFORM
+      SELF.Word := le.Word[1..numDone] + le.Word[c] + le.Word[numDone+1..c-1]+le.Word[c+1..]; // Boundary Conditions handled automatically
+    END;
+    RETURN NORMALIZE(infile,LENGTH(LEFT.Word)-numDone,TakeOne(LEFT,numDone+COUNTER));
+  END;
+    
+  Anagrams := LOOP(Initial,LENGTH(searchWord),Pluck1(ROWS(LEFT),COUNTER-1));
+    
+  RETURN Anagrams(Word in knownWords);
+END;
+
+OUTPUT(findAnagrams(searchWord));
+

testing/ecl/dict15b.ecl

@@ -0,0 +1,51 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//UseStandardFiles
+//Find all anagrams of a word, that match the list of known words
+
+allWordsDs := DEDUP(SORTED(TS_wordIndex(word[1]='t'), word), word);  // Restrict to words starting T for speed
+
+knownWords := DICTIONARY(allWordsDs, { word });
+
+R := RECORD
+  STRING Word;
+END;
+
+findAnagrams(string searchWord) := FUNCTION
+  trimmedWord := TRIM(searchWord);
+  Initial := DATASET([{trimmedWord}],R);
+    
+  R Pluck1(DATASET(R) infile, unsigned4 numDone) := FUNCTION
+    R TakeOne(R le, UNSIGNED1 c) := TRANSFORM
+      SELF.Word := le.Word[1..numDone] + le.Word[c] + le.Word[numDone+1..c-1]+le.Word[c+1..]; // Boundary Conditions handled automatically
+    END;
+    RETURN NORMALIZE(infile,LENGTH(LEFT.Word)-numDone,TakeOne(LEFT,numDone+COUNTER));
+  END;
+    
+  anagrams := LOOP(Initial,LENGTH(trimmedWord),Pluck1(ROWS(LEFT),COUNTER-1));
+    
+  uniqueAnagrams := DEDUP(anagrams, Word, HASH);
+  RETURN uniqueAnagrams(Word in knownWords);
+END;
+
+shortWords := TABLE(allWordsDs, { Word })(LENGTH(TRIM(Word)) <= 5);
+
+//BUG: Without the NOFOLD the code generator merges the projects, and introduces an ambiguous dataset
+moreThanOne := NOFOLD(shortWords)(count(findAnagrams(Word))>1);
+
+OUTPUT(moreThanOne);

testing/ecl/dict15c.ecl

@@ -0,0 +1,57 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//UseStandardFiles
+//Find all anagrams of a word, that match the list of known words
+
+#option ('showMetaIngraph', true);
+
+allWordsDs := DEDUP(TS_wordIndex, word, HASH);
+
+knownWords := DICTIONARY(allWordsDs, { word });
+
+R := RECORD
+  STRING Word;
+END;
+
+findAnagrams(string searchWord) := FUNCTION
+  trimmedWord := TRIM(searchWord);
+  Initial := DATASET([{trimmedWord}],R);
+    
+  R Pluck1(DATASET(R) infile, unsigned4 numDone) := FUNCTION
+    R TakeOne(R le, UNSIGNED1 c) := TRANSFORM
+      SELF.Word := le.Word[1..numDone] + le.Word[c] + le.Word[numDone+1..c-1]+le.Word[c+1..]; // Boundary Conditions handled automatically
+    END;
+    RETURN NORMALIZE(infile,LENGTH(LEFT.Word)-numDone,TakeOne(LEFT,numDone+COUNTER));
+  END;
+    
+  anagrams := LOOP(Initial,LENGTH(trimmedWord),Pluck1(ROWS(LEFT),COUNTER-1));
+   
+  uniqueAnagrams := DEDUP(anagrams, Word, HASH);
+   
+  RETURN uniqueAnagrams(Word in knownWords);
+END;
+
+shortWords := TABLE(allWordsDs, { Word })(LENGTH(TRIM(Word)) <= 6); 
+
+//Find all words that have anagrams 
+//BUG: Without the NOFOLD the code generator merges the projects, and introduces an ambiguous dataset
+withAnagrams := TABLE(NOFOLD(shortWords), { word, anagrams := findAnagrams(Word) });
+
+moreThanOne := withAnagrams(count(anagrams)>1);
+
+OUTPUT(moreThanOne);

testing/ecl/dict16.ecl

@@ -0,0 +1,37 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+idRec := { unsigned id; };
+
+inRecord := RECORD
+  unsigned search;
+  DATASET(idRec) ids;
+END;
+
+inRecord processIds(inRecord l) := TRANSFORM
+  myDict := DICTIONARY([{l.search-1},{l.search},{l.search+1}], idRec);
+  SELF.ids := l.ids(id IN myDict);
+  SELF := l;
+END;
+
+ds := NOFOLD(DATASET([
+    {1,[{2},{3},{4}]},
+    {3,[{2},{3},{4}]},
+    {10,[{4},{8},{11}, {10}, {7}]},
+    {11,[{4},{8},{11}, {10}, {7}]}], inRecord));
+    
+output(PROJECT(ds, processIds(LEFT)));

testing/ecl/dict17.ecl

@@ -0,0 +1,37 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+idRec := { unsigned id; };
+
+inRecord := RECORD
+  unsigned search;
+  DATASET(idRec) ids;
+END;
+
+inRecord processIds(inRecord l) := TRANSFORM
+  myDict := DICTIONARY([{l.search-1},{l.search},{l.search+1}], idRec);
+  SELF.ids := SORT(l.ids, id)(id IN myDict);
+  SELF := l;
+END;
+
+ds := NOFOLD(DATASET([
+    {1,[{2},{3},{4}]},
+    {3,[{2},{3},{4}]},
+    {10,[{4},{8},{11}, {10}, {7}]},
+    {11,[{4},{8},{11}, {10}, {7}]}], inRecord));
+    
+output(PROJECT(ds, processIds(LEFT)));

testing/ecl/dict3a.ecl

@@ -0,0 +1,5 @@
+ds6 := nofold(dataset([{5 => 'Richard'}], { integer id => string name }));
+d6 := DICTIONARY(ds6);
+
+5 in d6;
+count(d6)=1;

testing/ecl/dict_case.ecl

@@ -0,0 +1,40 @@
+resistorCodesEN := dataset([{0, 'Black'},
+                            {1, 'Brown'},
+                            {2, 'Red'},
+                            {3, 'Orange'},
+                            {4, 'Yellow'},
+                            {5, 'Green'},
+                            {6, 'Blue'},
+                            {7, 'Violet'},
+                            {8, 'Grey'},
+                            {9, 'White'}], {unsigned1 value, string color});
+
+resistorCodesFR := dataset([{0, 'Noir'},
+                            {1, 'Marron'},
+                            {2, 'Rouge'},
+                            {3, 'Orange'},
+                            {4, 'Jaune'},
+                            {5, 'Vert'},
+                            {6, 'Bleu'},
+                            {7, 'Violet'},
+                            {8, 'Gris'},
+                            {9, 'Blanc'}], {unsigned1 value, string color});
+
+string2 lang := 'EN' : STORED('lang');
+
+color2codeEN := DICTIONARY(resistorCodesEN, { color => value});
+code2colorEN := DICTIONARY(resistorCodesEN, { value => color});
+color2codeFR := DICTIONARY(resistorCodesFR, { color => value});
+code2colorFR := DICTIONARY(resistorCodesFR, { value => color});
+
+color2code := CASE(lang, 'EN'=>color2codeEN, 'FR'=>color2codeFR, ERROR(color2codeEN, 'Unknown language ' + lang));
+
+integer pow10(integer val) := CASE(val, 0=>1, 1=>10, 2=>100, ERROR('Out of range'));
+
+bands := DATASET([{'Red'},{'Red'},{'Red'}], {string band}) : STORED('bands');
+
+getBandValue(string band) := IF(band IN color2code, color2Code[band].value, ERROR('Unrecognised band ' + band));
+
+value := (getBandValue(bands[1].band)*10 + getBandValue(bands[2].band)) * pow10(getBandValue(bands[3].band));
+
+output(if(count(bands)=3, value, ERROR('3 bands expected')));

testing/ecl/dict_choose.ecl

@@ -0,0 +1,40 @@
+resistorCodesEN := dataset([{0, 'Black'},
+                            {1, 'Brown'},
+                            {2, 'Red'},
+                            {3, 'Orange'},
+                            {4, 'Yellow'},
+                            {5, 'Green'},
+                            {6, 'Blue'},
+                            {7, 'Violet'},
+                            {8, 'Grey'},
+                            {9, 'White'}], {unsigned1 value, string color});
+
+resistorCodesFR := dataset([{0, 'Noir'},
+                            {1, 'Marron'},
+                            {2, 'Rouge'},
+                            {3, 'Orange'},
+                            {4, 'Jaune'},
+                            {5, 'Vert'},
+                            {6, 'Bleu'},
+                            {7, 'Violet'},
+                            {8, 'Gris'},
+                            {9, 'Blanc'}], {unsigned1 value, string color});
+
+integer lang := 1 : STORED('lang');
+
+color2codeEN := DICTIONARY(resistorCodesEN, { color => value});
+code2colorEN := DICTIONARY(resistorCodesEN, { value => color});
+color2codeFR := DICTIONARY(resistorCodesFR, { color => value});
+code2colorFR := DICTIONARY(resistorCodesFR, { value => color});
+
+color2code := CHOOSE(lang, color2codeEN, color2codeFR);
+
+integer pow10(integer val) := CASE(val, 0=>1, 1=>10, 2=>100, ERROR('Out of range'));
+
+bands := DATASET([{'Red'},{'Red'},{'Red'}], {string band}) : STORED('bands');
+
+getBandValue(string band) := IF(band IN color2code, color2Code[band].value, ERROR('Unrecognised band ' + band));
+
+value := (getBandValue(bands[1].band)*10 + getBandValue(bands[2].band)) * pow10(getBandValue(bands[3].band));
+
+output(if(count(bands)=3, value, ERROR('3 bands expected')));

testing/ecl/dict_func.ecl

@@ -0,0 +1,41 @@
+resistorCodesEN := dataset([{0, 'Black'},
+                            {1, 'Brown'},
+                            {2, 'Red'},
+                            {3, 'Orange'},
+                            {4, 'Yellow'},
+                            {5, 'Green'},
+                            {6, 'Blue'},
+                            {7, 'Violet'},
+                            {8, 'Grey'},
+                            {9, 'White'}], {unsigned1 value, string color});
+
+resistorCodesFR := dataset([{0, 'Noir'},
+                            {1, 'Marron'},
+                            {2, 'Rouge'},
+                            {3, 'Orange'},
+                            {4, 'Jaune'},
+                            {5, 'Vert'},
+                            {6, 'Bleu'},
+                            {7, 'Violet'},
+                            {8, 'Gris'},
+                            {9, 'Blanc'}], {unsigned1 value, string color});
+
+string2 lang := 'EN' : STORED('lang');
+
+color2codeEN := DICTIONARY(resistorCodesEN, { color => value});
+code2colorEN := DICTIONARY(resistorCodesEN, { value => color});
+color2codeFR := DICTIONARY(resistorCodesFR, { color => value});
+code2colorFR := DICTIONARY(resistorCodesFR, { value => color});
+
+typeof(color2codeEN) _color2code(string language) := IF (language='FR', color2codeFR, color2codeEN);
+color2code := _color2code(lang);
+
+integer pow10(integer val) := CASE(val, 0=>1, 1=>10, 2=>100, ERROR('Out of range'));
+
+bands := DATASET([{'Red'},{'Red'},{'Red'}], {string band}) : STORED('bands');
+
+getBandValue(string band) := IF(band IN color2code, color2Code[band].value, ERROR('Unrecognised band ' + band));
+
+value := (getBandValue(bands[1].band)*10 + getBandValue(bands[2].band)) * pow10(getBandValue(bands[3].band));
+
+output(if(count(bands)=3, value, ERROR('3 bands expected')));

testing/ecl/dict_if.ecl

@@ -0,0 +1,40 @@
+resistorCodesEN := dataset([{0, 'Black'},
+                            {1, 'Brown'},
+                            {2, 'Red'},
+                            {3, 'Orange'},
+                            {4, 'Yellow'},
+                            {5, 'Green'},
+                            {6, 'Blue'},
+                            {7, 'Violet'},
+                            {8, 'Grey'},
+                            {9, 'White'}], {unsigned1 value, string color});
+
+resistorCodesFR := dataset([{0, 'Noir'},
+                            {1, 'Marron'},
+                            {2, 'Rouge'},
+                            {3, 'Orange'},
+                            {4, 'Jaune'},
+                            {5, 'Vert'},
+                            {6, 'Bleu'},
+                            {7, 'Violet'},
+                            {8, 'Gris'},
+                            {9, 'Blanc'}], {unsigned1 value, string color});
+
+string2 lang := 'EN' : STORED('lang');
+
+color2codeEN := DICTIONARY(resistorCodesEN, { color => value});
+code2colorEN := DICTIONARY(resistorCodesEN, { value => color});
+color2codeFR := DICTIONARY(resistorCodesFR, { color => value});
+code2colorFR := DICTIONARY(resistorCodesFR, { value => color});
+
+typeof(color2codeEN) color2code := IF(lang='FR', color2codeFR, color2codeEN);
+
+integer pow10(integer val) := CASE(val, 0=>1, 1=>10, 2=>100, ERROR('Out of range'));
+
+bands := DATASET([{'Red'},{'Red'},{'Red'}], {string band}) : STORED('bands');
+
+getBandValue(string band) := IF(band IN color2code, color2Code[band].value, ERROR('Unrecognised band ' + band));
+
+value := (getBandValue(bands[1].band)*10 + getBandValue(bands[2].band)) * pow10(getBandValue(bands[3].band));
+
+output(if(count(bands)=3, value, ERROR('3 bands expected')));

testing/ecl/dict_map.ecl

@@ -0,0 +1,40 @@
+resistorCodesEN := dataset([{0, 'Black'},
+                            {1, 'Brown'},
+                            {2, 'Red'},
+                            {3, 'Orange'},
+                            {4, 'Yellow'},
+                            {5, 'Green'},
+                            {6, 'Blue'},
+                            {7, 'Violet'},
+                            {8, 'Grey'},
+                            {9, 'White'}], {unsigned1 value, string color});
+
+resistorCodesFR := dataset([{0, 'Noir'},
+                            {1, 'Marron'},
+                            {2, 'Rouge'},
+                            {3, 'Orange'},
+                            {4, 'Jaune'},
+                            {5, 'Vert'},
+                            {6, 'Bleu'},
+                            {7, 'Violet'},
+                            {8, 'Gris'},
+                            {9, 'Blanc'}], {unsigned1 value, string color});
+
+string2 lang := 'EN' : STORED('lang');
+
+color2codeEN := DICTIONARY(resistorCodesEN, { color => value});
+code2colorEN := DICTIONARY(resistorCodesEN, { value => color});
+color2codeFR := DICTIONARY(resistorCodesFR, { color => value});
+code2colorFR := DICTIONARY(resistorCodesFR, { value => color});
+
+color2code := MAP(lang='EN'=>color2codeEN, lang='FR'=>color2codeFR, ERROR(color2codeEN, 'Unknown language ' + lang));
+
+integer pow10(integer val) := CASE(val, 0=>1, 1=>10, 2=>100, ERROR('Out of range'));
+
+bands := DATASET([{'Red'},{'Red'},{'Red'}], {string band}) : STORED('bands');
+
+getBandValue(string band) := IF(band IN color2code, color2Code[band].value, ERROR('Unrecognised band ' + band));
+
+value := (getBandValue(bands[1].band)*10 + getBandValue(bands[2].band)) * pow10(getBandValue(bands[3].band));
+
+output(if(count(bands)=3, value, ERROR('3 bands expected')));

testing/ecl/dict_once.ecl

@@ -0,0 +1,23 @@
+resistorCodes := dataset([{0, 'Black'},
+                          {1, 'Brown'},
+                          {2, 'Red'},
+                          {3, 'Orange'},
+                          {4, 'Yellow'},
+                          {5, 'Green'},
+                          {6, 'Blue'},
+                          {7, 'Violet'},
+                          {8, 'Grey'},
+                          {9, 'White'}], {unsigned1 value, string color});
+
+color2code := DICTIONARY(resistorCodes, { color => value}) : ONCE;
+code2color := DICTIONARY(resistorCodes, { value => color}) : ONCE;
+
+integer pow10(integer val) := CASE(val, 0=>1, 1=>10, 2=>100, ERROR('Out of range'));
+
+bands := DATASET([{'Red'},{'Red'},{'Red'}], {string band}) : STORED('bands');
+
+getBandValue(string band) := IF(band IN color2code, color2Code[band].value, ERROR('Unrecognised band ' + band));
+
+value := (getBandValue(bands[1].band)*10 + getBandValue(bands[2].band)) * pow10(getBandValue(bands[3].band));
+
+output(if(count(bands)=3, value, ERROR('3 bands expected')));

testing/ecl/ds_map.ecl

@@ -0,0 +1,35 @@
+resistorCodesEN := dataset([{0, 'Black'},
+                            {1, 'Brown'},
+                            {2, 'Red'},
+                            {3, 'Orange'},
+                            {4, 'Yellow'},
+                            {5, 'Green'},
+                            {6, 'Blue'},
+                            {7, 'Violet'},
+                            {8, 'Grey'},
+                            {9, 'White'}], {unsigned1 value, string color});
+
+resistorCodesFR := dataset([{0, 'Noir'},
+                            {1, 'Marron'},
+                            {2, 'Rouge'},
+                            {3, 'Orange'},
+                            {4, 'Jaune'},
+                            {5, 'Vert'},
+                            {6, 'Bleu'},
+                            {7, 'Violet'},
+                            {8, 'Gris'},
+                            {9, 'Blanc'}], {unsigned1 value, string color});
+
+string2 lang := 'EN' : STORED('lang');
+
+color2code := MAP(lang='EN'=>resistorCodesEN, lang='FR'=>resistorCodesFR, ERROR(resistorCodesEN, 'Unknown language ' + lang));
+
+integer pow10(integer val) := CASE(val, 0=>1, 1=>10, 2=>100, ERROR('Out of range'));
+
+bands := DATASET([{'Red'},{'Red'},{'Red'}], {string band}) : STORED('bands');
+
+getBandValue(string band) := IF(exists(color2code(color=band)), color2code(color=band)[1].value, ERROR('Unrecognised band ' + band));
+
+value := (getBandValue(bands[1].band)*10 + getBandValue(bands[2].band)) * pow10(getBandValue(bands[3].band));
+
+output(if(count(bands)=3, value, ERROR('3 bands expected')));

testing/ecl/key/dict10.xml

@@ -0,0 +1,3 @@
+<Dataset name='Result 1'>
+ <Row><Result_1>true</Result_1></Row>
+</Dataset>

testing/ecl/key/dict11.xml

@@ -0,0 +1,12 @@
+<Dataset name='Result 1'>
+ <Row><value>1</value><root>1</root></Row>
+ <Row><value>2</value><root>0</root></Row>
+ <Row><value>3</value><root>0</root></Row>
+ <Row><value>4</value><root>2</root></Row>
+ <Row><value>5</value><root>0</root></Row>
+ <Row><value>99</value><root>0</root></Row>
+ <Row><value>100</value><root>10</root></Row>
+ <Row><value>32768</value><root>0</root></Row>
+ <Row><value>65536</value><root>256</root></Row>
+ <Row><value>18446744073709551615</value><root>0</root></Row>
+</Dataset>

testing/ecl/key/dict12.xml

@@ -0,0 +1,12 @@
+<Dataset name='Result 1'>
+ <Row><value>2</value></Row>
+ <Row><value>3</value></Row>
+ <Row><value>5</value></Row>
+ <Row><value>99</value></Row>
+ <Row><value>32768</value></Row>
+ <Row><value>18446744073709551615</value></Row>
+ <Row><value>1</value></Row>
+ <Row><value>4</value></Row>
+ <Row><value>100</value></Row>
+ <Row><value>65536</value></Row>
+</Dataset>

testing/ecl/key/dict13.xml

@@ -0,0 +1,102 @@
+<Dataset name='Result 1'>
+ <Row><value>0</value><root>0</root></Row>
+ <Row><value>1</value><root>1</root></Row>
+ <Row><value>4</value><root>2</root></Row>
+ <Row><value>9</value><root>3</root></Row>
+ <Row><value>16</value><root>4</root></Row>
+ <Row><value>25</value><root>5</root></Row>
+ <Row><value>36</value><root>6</root></Row>
+ <Row><value>49</value><root>7</root></Row>
+ <Row><value>64</value><root>8</root></Row>
+ <Row><value>81</value><root>9</root></Row>
+ <Row><value>100</value><root>10</root></Row>
+ <Row><value>121</value><root>11</root></Row>
+ <Row><value>144</value><root>12</root></Row>
+ <Row><value>169</value><root>13</root></Row>
+ <Row><value>196</value><root>14</root></Row>
+ <Row><value>225</value><root>15</root></Row>
+ <Row><value>256</value><root>16</root></Row>
+ <Row><value>289</value><root>17</root></Row>
+ <Row><value>324</value><root>18</root></Row>
+ <Row><value>361</value><root>19</root></Row>
+ <Row><value>400</value><root>20</root></Row>
+ <Row><value>441</value><root>21</root></Row>
+ <Row><value>484</value><root>22</root></Row>
+ <Row><value>529</value><root>23</root></Row>
+ <Row><value>576</value><root>24</root></Row>
+ <Row><value>625</value><root>25</root></Row>
+ <Row><value>676</value><root>26</root></Row>
+ <Row><value>729</value><root>27</root></Row>
+ <Row><value>784</value><root>28</root></Row>
+ <Row><value>841</value><root>29</root></Row>
+ <Row><value>900</value><root>30</root></Row>
+ <Row><value>961</value><root>31</root></Row>
+ <Row><value>1024</value><root>32</root></Row>
+ <Row><value>1089</value><root>33</root></Row>
+ <Row><value>1156</value><root>34</root></Row>
+ <Row><value>1225</value><root>35</root></Row>
+ <Row><value>1296</value><root>36</root></Row>
+ <Row><value>1369</value><root>37</root></Row>
+ <Row><value>1444</value><root>38</root></Row>
+ <Row><value>1521</value><root>39</root></Row>
+ <Row><value>1600</value><root>40</root></Row>
+ <Row><value>1681</value><root>41</root></Row>
+ <Row><value>1764</value><root>42</root></Row>
+ <Row><value>1849</value><root>43</root></Row>
+ <Row><value>1936</value><root>44</root></Row>
+ <Row><value>2025</value><root>45</root></Row>
+ <Row><value>2116</value><root>46</root></Row>
+ <Row><value>2209</value><root>47</root></Row>
+ <Row><value>2304</value><root>48</root></Row>
+ <Row><value>2401</value><root>49</root></Row>
+ <Row><value>2500</value><root>50</root></Row>
+ <Row><value>2601</value><root>51</root></Row>
+ <Row><value>2704</value><root>52</root></Row>
+ <Row><value>2809</value><root>53</root></Row>
+ <Row><value>2916</value><root>54</root></Row>
+ <Row><value>3025</value><root>55</root></Row>
+ <Row><value>3136</value><root>56</root></Row>
+ <Row><value>3249</value><root>57</root></Row>
+ <Row><value>3364</value><root>58</root></Row>
+ <Row><value>3481</value><root>59</root></Row>
+ <Row><value>3600</value><root>60</root></Row>
+ <Row><value>3721</value><root>61</root></Row>
+ <Row><value>3844</value><root>62</root></Row>
+ <Row><value>3969</value><root>63</root></Row>
+ <Row><value>4096</value><root>64</root></Row>
+ <Row><value>4225</value><root>65</root></Row>
+ <Row><value>4356</value><root>66</root></Row>
+ <Row><value>4489</value><root>67</root></Row>
+ <Row><value>4624</value><root>68</root></Row>
+ <Row><value>4761</value><root>69</root></Row>
+ <Row><value>4900</value><root>70</root></Row>
+ <Row><value>5041</value><root>71</root></Row>
+ <Row><value>5184</value><root>72</root></Row>
+ <Row><value>5329</value><root>73</root></Row>
+ <Row><value>5476</value><root>74</root></Row>
+ <Row><value>5625</value><root>75</root></Row>
+ <Row><value>5776</value><root>76</root></Row>
+ <Row><value>5929</value><root>77</root></Row>
+ <Row><value>6084</value><root>78</root></Row>
+ <Row><value>6241</value><root>79</root></Row>
+ <Row><value>6400</value><root>80</root></Row>
+ <Row><value>6561</value><root>81</root></Row>
+ <Row><value>6724</value><root>82</root></Row>
+ <Row><value>6889</value><root>83</root></Row>
+ <Row><value>7056</value><root>84</root></Row>
+ <Row><value>7225</value><root>85</root></Row>
+ <Row><value>7396</value><root>86</root></Row>
+ <Row><value>7569</value><root>87</root></Row>
+ <Row><value>7744</value><root>88</root></Row>
+ <Row><value>7921</value><root>89</root></Row>
+ <Row><value>8100</value><root>90</root></Row>
+ <Row><value>8281</value><root>91</root></Row>
+ <Row><value>8464</value><root>92</root></Row>
+ <Row><value>8649</value><root>93</root></Row>
+ <Row><value>8836</value><root>94</root></Row>
+ <Row><value>9025</value><root>95</root></Row>
+ <Row><value>9216</value><root>96</root></Row>
+ <Row><value>9409</value><root>97</root></Row>
+ <Row><value>9604</value><root>98</root></Row>
+ <Row><value>9801</value><root>99</root></Row>
+</Dataset>

testing/ecl/key/dict15.xml

@@ -0,0 +1,3 @@
+<Dataset name='Result 1'>
+ <Row><word>bags</word></Row>
+</Dataset>

testing/ecl/key/dict15a.xml

@@ -0,0 +1,3 @@
+<Dataset name='Result 1'>
+ <Row><word>bags</word></Row>
+</Dataset>

testing/ecl/key/dict15b.xml

@@ -0,0 +1,4 @@
+<Dataset name='Result 1'>
+ <Row><word>there               </word></Row>
+ <Row><word>three               </word></Row>
+</Dataset>

testing/ecl/key/dict15c.xml

@@ -0,0 +1,12 @@
+<Dataset name='Result 1'>
+ <Row><word>course              </word><anagrams><Row><word>course</word></Row><Row><word>source</word></Row></anagrams></Row>
+ <Row><word>gch01               </word><anagrams><Row><word>gch01</word></Row><Row><word>gch10</word></Row></anagrams></Row>
+ <Row><word>gch02               </word><anagrams><Row><word>gch02</word></Row><Row><word>gch20</word></Row></anagrams></Row>
+ <Row><word>gch10               </word><anagrams><Row><word>gch10</word></Row><Row><word>gch01</word></Row></anagrams></Row>
+ <Row><word>gch20               </word><anagrams><Row><word>gch20</word></Row><Row><word>gch02</word></Row></anagrams></Row>
+ <Row><word>how                 </word><anagrams><Row><word>how</word></Row><Row><word>who</word></Row></anagrams></Row>
+ <Row><word>source              </word><anagrams><Row><word>source</word></Row><Row><word>course</word></Row></anagrams></Row>
+ <Row><word>there               </word><anagrams><Row><word>there</word></Row><Row><word>three</word></Row></anagrams></Row>
+ <Row><word>three               </word><anagrams><Row><word>three</word></Row><Row><word>there</word></Row></anagrams></Row>
+ <Row><word>who                 </word><anagrams><Row><word>who</word></Row><Row><word>how</word></Row></anagrams></Row>
+</Dataset>

testing/ecl/key/dict16.xml

@@ -0,0 +1,6 @@
+<Dataset name='Result 1'>
+ <Row><search>1</search><ids><Row><id>2</id></Row></ids></Row>
+ <Row><search>3</search><ids><Row><id>2</id></Row><Row><id>3</id></Row><Row><id>4</id></Row></ids></Row>
+ <Row><search>10</search><ids><Row><id>11</id></Row><Row><id>10</id></Row></ids></Row>
+ <Row><search>11</search><ids><Row><id>11</id></Row><Row><id>10</id></Row></ids></Row>
+</Dataset>

testing/ecl/key/dict3a.xml

@@ -0,0 +1,6 @@
+<Dataset name='Result 1'>
+ <Row><Result_1>true</Result_1></Row>
+</Dataset>
+<Dataset name='Result 2'>
+ <Row><Result_2>true</Result_2></Row>
+</Dataset>

testing/ecl/key/dict_case.xml

@@ -0,0 +1,3 @@
+<Dataset name='Result 1'>
+ <Row><Result_1>2200</Result_1></Row>
+</Dataset>

testing/ecl/key/dict_choose.xml

@@ -0,0 +1,3 @@
+<Dataset name='Result 1'>
+ <Row><Result_1>2200</Result_1></Row>
+</Dataset>

testing/ecl/key/dict_func.xml

@@ -0,0 +1,3 @@
+<Dataset name='Result 1'>
+ <Row><Result_1>2200</Result_1></Row>
+</Dataset>

testing/ecl/key/dict_if.xml

@@ -0,0 +1,3 @@
+<Dataset name='Result 1'>
+ <Row><Result_1>2200</Result_1></Row>
+</Dataset>

testing/ecl/key/dict_map.xml

@@ -0,0 +1,3 @@
+<Dataset name='Result 1'>
+ <Row><Result_1>2200</Result_1></Row>
+</Dataset>

testing/ecl/key/dict_once.xml

@@ -0,0 +1,3 @@
+<Dataset name='Result 1'>
+ <Row><Result_1>2200</Result_1></Row>
+</Dataset>

testing/ecl/key/ds_map.xml

@@ -0,0 +1,3 @@
+<Dataset name='Result 1'>
+ <Row><Result_1>2200</Result_1></Row>
+</Dataset>

thorlcr/graph/thgraph.cpp

@@ -1870,6 +1870,12 @@ void CGraphBase::getResult(size32_t & len, void * & data, unsigned id)
     result->getResult(len, data);
 }
 
+void CGraphBase::getDictionaryResult(unsigned & count, byte * * & ret, unsigned id)
+{
+    Owned<IThorResult> result = getResult(id, true); // will get collated distributed result
+    result->getLinkedResult(count, ret);
+}
+
 void CGraphBase::getLinkedResult(unsigned & count, byte * * & ret, unsigned id)
 {
     Owned<IThorResult> result = getResult(id, true); // will get collated distributed result

thorlcr/graph/thgraph.hpp

@@ -503,6 +503,8 @@ class graph_decl CGraphBase : public CInterface, implements ILocalGraph, impleme
         virtual const IContextLogger &queryContextLogger() const { return ctx->queryContextLogger(); }
         virtual IEngineRowAllocator * getRowAllocator(IOutputMetaData * meta, unsigned activityId) const { return ctx->getRowAllocator(meta, activityId); }
         virtual void getResultRowset(size32_t & tcount, byte * * & tgt, const char * name, unsigned sequence, IEngineRowAllocator * _rowAllocator, IOutputRowDeserializer * deserializer, bool isGrouped, IXmlToRowTransformer * xmlTransformer, ICsvToRowTransformer * csvTransformer) { ctx->getResultRowset(tcount, tgt, name, sequence, _rowAllocator, deserializer, isGrouped, xmlTransformer, csvTransformer); }
+        virtual void getResultDictionary(size32_t & tcount, byte * * & tgt,IEngineRowAllocator * _rowAllocator,  const char * name, unsigned sequence, IOutputRowDeserializer * deserializer, IXmlToRowTransformer * xmlTransformer, ICsvToRowTransformer * csvTransformer, IHThorHashLookupInfo * hasher) { ctx->getResultDictionary(tcount, tgt, _rowAllocator, name, sequence, deserializer, xmlTransformer, csvTransformer, hasher); }
+
         virtual void getRowXML(size32_t & lenResult, char * & result, IOutputMetaData & info, const void * row, unsigned flags) { convertRowToXML(lenResult, result, info, row, flags); }
         virtual unsigned getGraphLoopCounter() const
         {
@@ -737,6 +739,7 @@ public:
 
 // ILocalGraph
     virtual void getResult(size32_t & len, void * & data, unsigned id);
+    virtual void getDictionaryResult(unsigned & count, byte * * & ret, unsigned id);
     virtual void getLinkedResult(unsigned & count, byte * * & ret, unsigned id);
 
 // IThorChildGraph
@@ -1070,6 +1073,7 @@ protected:
         virtual void serialize(MemoryBuffer &mb) { throw MakeStringException(0, "Graph Result %d accessed before it is created", id); }
         virtual void getResult(size32_t & retSize, void * & ret) { throw MakeStringException(0, "Graph Result %d accessed before it is created", id); }
         virtual void getLinkedResult(unsigned & count, byte * * & ret) { throw MakeStringException(0, "Graph Result %d accessed before it is created", id); }
+        virtual void getDictionaryResult(unsigned & count, byte * * & ret) { throw MakeStringException(0, "Graph Result %d accessed before it is created", id); }
     };
     IArrayOf<IThorResult> results;
     CriticalSection cs;
@@ -1127,6 +1131,11 @@ public:
         Owned<IThorResult> result = getResult(id, true);
         result->getLinkedResult(count, ret);
     }
+    virtual void getDictionaryResult(unsigned & count, byte * * & ret, unsigned id)
+    {
+        Owned<IThorResult> result = getResult(id, true);
+        result->getLinkedResult(count, ret);
+    }
     virtual void setOwner(activity_id _ownerId) { ownerId = _ownerId; }
     virtual activity_id queryOwnerId() const { return ownerId; }
 };

thorlcr/graph/thgraphmaster.cpp

@@ -1002,6 +1002,18 @@ public:
             rtlDataset2RowsetX(tcount, tgt, _rowAllocator, deserializer, datasetBuffer.length(), datasetBuffer.toByteArray(), isGrouped);
         );
     }
+    virtual void getResultDictionary(size32_t & tcount, byte * * & tgt, IEngineRowAllocator * _rowAllocator, const char * stepname, unsigned sequence, IOutputRowDeserializer * deserializer, IXmlToRowTransformer * xmlTransformer, ICsvToRowTransformer * csvTransformer, IHThorHashLookupInfo * hasher)
+    {
+        tgt = NULL;
+        PROTECTED_GETRESULT(stepname, sequence, "Dictionary", "dictionary",
+            MemoryBuffer datasetBuffer;
+            MemoryBuffer2IDataVal result(datasetBuffer);
+            Owned<IXmlToRawTransformer> rawXmlTransformer = createXmlRawTransformer(xmlTransformer);
+            Owned<ICsvToRawTransformer> rawCsvTransformer = createCsvRawTransformer(csvTransformer);
+            r->getResultRaw(result, rawXmlTransformer, rawCsvTransformer);
+            throwUnexpected();
+        );
+    }
     virtual void getExternalResultRaw(unsigned & tlen, void * & tgt, const char * wuid, const char * stepname, unsigned sequence, IXmlToRowTransformer * xmlTransformer, ICsvToRowTransformer * csvTransformer)
     {
         tgt = NULL;

thorlcr/graph/thgraphslave.cpp

@@ -978,6 +978,7 @@ public:
     }
     virtual void getResultStringF(unsigned tlen, char * tgt, const char * name, unsigned sequence) { throwUnexpected(); }
     virtual void getResultRowset(size32_t & tcount, byte * * & tgt, const char * name, unsigned sequence, IEngineRowAllocator * _rowAllocator, IOutputRowDeserializer * deserializer, bool isGrouped, IXmlToRowTransformer * xmlTransformer, ICsvToRowTransformer * csvTransformer) { throwUnexpected(); }
+    virtual void getResultDictionary(size32_t & tcount, byte * * & tgt, IEngineRowAllocator * _rowAllocator, const char * name, unsigned sequence, IOutputRowDeserializer * deserializer, IXmlToRowTransformer * xmlTransformer, ICsvToRowTransformer * csvTransformer, IHThorHashLookupInfo * hasher) { throwUnexpected(); }
     virtual void addWuAssertFailure(unsigned code, const char * text, const char * filename, unsigned lineno, unsigned column, bool isAbort)
     {
         DBGLOG("%s", text);