WIP: DICTIONARY support - lookup support

Call rtl helper function to lookup values in a dictionary.
Add _linkcounted_ support on row parameters, and use it to pass
default row to hash lookup function.

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

common/deftype/deftype.cpp

@@ -1332,6 +1332,18 @@ bool CRowTypeInfo::assignableFrom(ITypeInfo *t2)
 
 //===========================================================================
 
+bool CDictionaryTypeInfo::assignableFrom(ITypeInfo *t2)
+{
+    if (getTypeCode()==t2->getTypeCode())
+    {
+        ITypeInfo *c1 = queryChildType();
+        ITypeInfo *c2 = t2->queryChildType();
+        if (c1==NULL || c2==NULL || c1->assignableFrom(c2))
+            return true;
+    }
+    return false;
+}
+
 StringBuffer & CDictionaryTypeInfo::getECLType(StringBuffer & out)
 {
     ITypeInfo * recordType = ::queryRecordType(this);

common/deftype/deftype.ipp

@@ -642,6 +642,7 @@ public:
     virtual bool isScalar()                                 { return false; }
     virtual const char *queryTypeName()                     { return "dictionary"; }
     virtual StringBuffer &getECLType(StringBuffer & out);
+    virtual bool assignableFrom(ITypeInfo *t2);
 
     virtual void serialize(MemoryBuffer &tgt);
 };

ecl/hql/hqlexpr.cpp

@@ -6139,7 +6139,7 @@ bool CHqlRecord::assignableFrom(ITypeInfo * source)
 
     case type_record:
         {
-            if (recordTypesMatch(source, this))
+            if  (numChildren() == 0 || recordTypesMatch(source, this))
                 return true;
 
             //Record inheritance.  If the first entry in the source record is also a record, then check if compatible.
@@ -8806,7 +8806,7 @@ IHqlExpression * CHqlParameter::makeParameter(_ATOM _name, unsigned _idx, ITypeI
     switch (tc)
     {
     case type_dictionary:
-        UNIMPLEMENTED;
+        e = new CHqlDictionaryParameter(_name, _idx, _type);
         break;
     case type_table:
     case type_groupedtable:
@@ -10491,6 +10491,10 @@ static void normalizeCallParameters(HqlExprArray & resolvedActuals, IHqlExpressi
                     }
                 }
                 break;
+            case type_dictionary:
+                // MORE - needs some code
+                // For now, never cast
+                break;
             case type_row:
             case type_transform:
             case type_function:

ecl/hql/hqlexpr.ipp

@@ -1326,6 +1326,33 @@ public:
     virtual IHqlSimpleScope* querySimpleScope() { return CHqlParameter::querySimpleScope(); }
 };
 
+class CHqlDictionaryParameter : public CHqlParameter, implements IHqlDataset
+{
+public:
+    IMPLEMENT_IINTERFACE_USING(CHqlParameter)
+
+    CHqlDictionaryParameter(_ATOM name, unsigned idx, ITypeInfo *type)
+     : CHqlParameter(name, idx, type) { }
+
+//IHqlExpression
+    virtual bool assignableFrom(ITypeInfo * source) { type_t tc = source->getTypeCode(); return tc==type_dictionary; }
+    virtual IHqlDataset *queryDataset() { return this; }
+
+//CHqlParameter
+
+    //virtual IHqlSimpleScope *querySimpleScope();
+
+//IHqlDataset
+    virtual IHqlDataset* queryTable() { return this; }
+    virtual IHqlDataset * queryRootTable() { return this; }
+    virtual IHqlExpression * queryContainer() { return NULL; }
+
+    virtual bool isAggregate() { return false; }
+
+//Overlapped methods
+    virtual IHqlSimpleScope* querySimpleScope() { return CHqlParameter::querySimpleScope(); }
+};
+
 class CHqlScopeParameter : public CHqlScope
 {
 protected:

ecl/hql/hqlgram.y

@@ -775,6 +775,12 @@ explicitDictionaryType
                             $$.setType(makeDictionaryType(makeRowType(recordType)));
                             $$.setPosition($1);
                         }
+    | _LINKCOUNTED_ explicitDictionaryType
+                        {
+                            Owned<ITypeInfo> dsType = $2.getType();
+                            $$.setType(setLinkCountedAttr(dsType, true));
+                            $$.setPosition($1);
+                        }
     ;
 
 
@@ -838,11 +844,17 @@ paramType
                             parser->setTemplateAttribute();
                         }
     | explicitDatasetType
+    | explicitDictionaryType
     | ROW               {
                             IHqlExpression* record = queryNullRecord();
                             $$.setType(makeRowType(record->getType()));
                             $$.setPosition($1);
                         }
+    | _LINKCOUNTED_ ROW {
+                            IHqlExpression* record = queryNullRecord();
+                            $$.setType(setLinkCountedAttr(makeRowType(record->getType()), true));
+                            $$.setPosition($1);
+                        }
     | abstractModule
                         {
                             OwnedHqlExpr scope = $1.getExpr();
@@ -1208,6 +1220,7 @@ knownId
 knownFunction1
     : DATAROW_FUNCTION
     | DATASET_FUNCTION
+    | DICTIONARY_FUNCTION
     | VALUE_FUNCTION
     | ACTION_FUNCTION
     | PATTERN_FUNCTION
@@ -3724,6 +3737,7 @@ funcRetType
     | propType
     | setType
     | explicitDatasetType
+    | explicitDictionaryType
     | transformType
  // A plain record would be better, but that then causes a s/r error in knownOrUnknownId because scope
     | ROW '(' recordDef ')'     
@@ -6793,7 +6807,8 @@ dataRow
                         }
     | dictionary '[' expressionList ']'
                         {
-                            $$.setExpr(createRow(no_selectmap, $1.getExpr(), $3.getExpr()));
+                            OwnedHqlExpr row = createValue(no_rowvalue, makeNullType(), $3.getExpr());
+                            $$.setExpr(createSelectMapRow(parser->errorHandler, $3.pos, $1.getExpr(), row.getClear()));
                         }
     | dataSet '[' NOBOUNDCHECK expression ']'
                         {   
@@ -10486,6 +10501,7 @@ actualValue
                             $$.setExpr(parser->bindFieldMap(expr,map));
                         }
     | dataRow
+    | dictionary
     | TOK_PATTERN pattern
                         {   $$.setExpr($2.getExpr()); }
     | TOKEN pattern     {   $$.setExpr($2.getExpr()); }

ecl/hql/hqlutil.cpp

@@ -5301,11 +5301,37 @@ void TempTableTransformer::reportWarning(IHqlExpression * location, int code,con
     errors->reportWarning(code, errorMsg.str(), where->sourcePath->str(), where->lineno, where->column, where->position);
 }
 
+IHqlExpression *getDictionaryKeyRecord(IHqlExpression *record)
+{
+    // MORE - should probably use an attr to cache this?
+    IHqlExpression * payload = record ? record->queryProperty(_payload_Atom) : NULL;
+    unsigned payloadSize = payload ? getIntValue(payload->queryChild(0)) : 0;
+    unsigned max = record->numChildren() - payloadSize;
+    IHqlExpression *newrec = createRecord();
+    HqlExprArray fields;
+    for (unsigned idx = 0; idx < max; idx++)
+    {
+        IHqlExpression *child = record->queryChild(idx);
+        if (!child->isAttribute())  // Strip off the payload attribute
+            newrec->addOperand(LINK(child));
+    }
+    return newrec->closeExpr();
+}
+
+IHqlExpression * createSelectMapRow(IErrorReceiver * errors, ECLlocation & location, IHqlExpression * dict, IHqlExpression *values)
+{
+    OwnedHqlExpr record = getDictionaryKeyRecord(dict->queryRecord());
+    TempTableTransformer transformer(errors, location);
+    OwnedHqlExpr newTransform = transformer.createTempTableTransform(values, record);
+    return createRow(no_selectmap, dict, createRow(no_createrow, newTransform.getClear()));
+}
+
+
 IHqlExpression * convertTempRowToCreateRow(IErrorReceiver * errors, ECLlocation & location, IHqlExpression * expr)
 {
     IHqlExpression * oldValues = expr->queryChild(0);
     IHqlExpression * record = expr->queryChild(1);
-    OwnedHqlExpr values = normalizeListCasts(oldValues);
+    OwnedHqlExpr values = normalizeListCasts(oldValues); // ??? not used
 
     TempTableTransformer transformer(errors, location);
     OwnedHqlExpr newTransform = transformer.createTempTableTransform(oldValues, record);

ecl/hql/hqlutil.hpp

@@ -447,6 +447,7 @@ extern HQL_API IHqlExpression * extractCppBodyAttrs(unsigned len, const char * v
 extern HQL_API unsigned cleanupEmbeddedCpp(unsigned len, char * buffer);
 extern HQL_API bool isNullList(IHqlExpression * expr);
 
+extern HQL_API IHqlExpression * createSelectMapRow(IErrorReceiver * errors, ECLlocation & location, IHqlExpression * dict, IHqlExpression *values);
 extern HQL_API IHqlExpression * convertTempRowToCreateRow(IErrorReceiver * errors, ECLlocation & location, IHqlExpression * expr);
 extern HQL_API IHqlExpression * convertTempTableToInlineTable(IErrorReceiver * errors, ECLlocation & location, IHqlExpression * expr);
 extern HQL_API IHqlExpression * convertTempTableToInlineDictionary(IErrorReceiver * errors, ECLlocation & location, IHqlExpression * expr);

ecl/hqlcpp/hqlcatom.cpp

@@ -239,6 +239,7 @@ _ATOM deserializerSkipVUniAtom;
 _ATOM destroyRegexAtom;
 _ATOM destroyWRegexAtom;
 _ATOM destructMetaMemberAtom;
+_ATOM dictionaryLookupAtom;
 _ATOM doNotifyAtom;
 _ATOM doNotifyTargetAtom;
 _ATOM ebcdic2asciiAtom;
@@ -944,6 +945,7 @@ MODULE_INIT(INIT_PRIORITY_HQLATOM-1)
     MAKEATOM(destroyRegex);
     MAKEATOM(destroyWRegex);
     MAKEATOM(destructMetaMember);
+    MAKEATOM(dictionaryLookup);
     MAKEATOM(doNotify);
     MAKEATOM(doNotifyTarget);
     MAKEATOM(ebcdic2ascii);

ecl/hqlcpp/hqlcatom.hpp

@@ -239,6 +239,7 @@ extern _ATOM deserializerSkipVUniAtom;
 extern _ATOM destroyRegexAtom;
 extern _ATOM destroyWRegexAtom;
 extern _ATOM destructMetaMemberAtom;
+extern _ATOM dictionaryLookupAtom;
 extern _ATOM doNotifyAtom;
 extern _ATOM doNotifyTargetAtom;
 extern _ATOM ebcdic2asciiAtom;

ecl/hqlcpp/hqlcpp.cpp

@@ -5804,8 +5804,16 @@ void HqlCppTranslator::doBuildCall(BuildCtx & ctx, const CHqlBoundTarget * tgt,
             }
         case type_row:
             {
-                Owned<IReferenceSelector> selector = buildNewRow(ctx, castParam);
-                selector->buildAddress(ctx, bound);
+                if (hasLinkCountedModifier(argType))
+                {
+                    doBuildAliasValue(ctx, castParam, bound);
+//                    buildTempExpr(ctx, castParam, bound, FormatLinkedDataset);
+                }
+                else
+                {
+                    Owned<IReferenceSelector> selector = buildNewRow(ctx, castParam);
+                    selector->buildAddress(ctx, bound);
+                }
     //          buildExpr(ctx, castParam, bound);       // more this needs more work I think
                 break;
             }

ecl/hqlcpp/hqlcppsys.ecl

@@ -796,6 +796,9 @@ const char * cppSystemText[]  = {
     "   unsigned4 serializerBeginNested() : omethod,entrypoint='beginNested';",
     "   serializerEndNested(unsigned4 pos) : omethod,entrypoint='endNested';",
 
+    // Dictionary support
+    " _linkcounted_ row(dummyRecord) dictionaryLookup(boolean meta, _linkcounted_ dictionary dict, row key, _linkcounted_ row defaultrow) : include,entrypoint='rtlDictionaryLookup';",
+
     "   END;",
     NULL };
 

ecl/hqlcpp/hqlcset.cpp

@@ -703,24 +703,27 @@ InlineLinkedDictionaryCursor::InlineLinkedDictionaryCursor(HqlCppTranslator & _t
 {
 }
 
-BoundRow * InlineLinkedDictionaryCursor::buildSelectMap(BuildCtx & ctx, IHqlExpression * indexExpr)
+BoundRow * InlineLinkedDictionaryCursor::buildSelectMap(BuildCtx & ctx, IHqlExpression * mapExpr)
 {
-    OwnedHqlExpr index = foldHqlExpression(indexExpr->queryChild(1));
-
-    StringBuffer s, rowName;
-    OwnedHqlExpr row = createRow(ctx, "row", rowName, false); // MORE - what should this be?
-
-    CHqlBoundExpr boundIndex;
-    BuildCtx subctx(ctx);
-    translator.buildExpr(ctx, index, boundIndex);
+    Owned<BoundRow> tempRow = translator.declareLinkedRow(ctx, mapExpr, false);
+    IHqlExpression *record = ds->queryRecord();
 
-    OwnedHqlExpr address = getPointer(boundDs.expr);
-    OwnedHqlExpr indexedValue = createValue(no_selectmap, row->getType(), LINK(address), LINK(boundIndex.expr));
-    subctx.addAssign(row, indexedValue);
+    StringBuffer lookupHelperName;
+    OwnedHqlExpr dict = createDictionary(no_null, LINK(record));
+    translator.buildDictionaryHashClass(ctx, record, dict, lookupHelperName);
+    CHqlBoundTarget target;
+    target.expr.set(tempRow->queryBound());
 
-    //MORE: Should mark as linked if it is.
-    BoundRow * cursor = translator.bindRow(ctx, indexExpr, row);
-    return cursor;
+    HqlExprArray args;
+    args.append(*createQuoted(lookupHelperName, makeBoolType()));
+    args.append(*LINK(mapExpr->queryChild(0)));
+    args.append(*LINK(mapExpr->queryChild(1)));
+    args.append(*::createRow(no_null, LINK(record)));
+    Owned<ITypeInfo> resultType = makeReferenceModifier(makeAttributeModifier(makeRowType(record->getType()), getLinkCountedAttr()));
+    OwnedHqlExpr call = translator.bindFunctionCall(dictionaryLookupAtom, args, resultType);
+    translator.buildExprAssign(ctx, target, call);
+
+    return tempRow.getClear();
 }
 
 //---------------------------------------------------------------------------

ecl/hqlcpp/hqlinline.cpp

@@ -75,7 +75,7 @@ static unsigned calcInlineFlags(BuildCtx * ctx, IHqlExpression * expr)
     //But it would be really good if the code could be made context independent - then it could go in hqlattr and be cached.
     if (ctx)
     {
-        if (expr->isDataset())
+        if (expr->isDataset() || expr->isDictionary())
         {
             if (ctx->queryMatchExpr(expr))
                 return RETevaluate;
@@ -216,6 +216,8 @@ static unsigned calcInlineFlags(BuildCtx * ctx, IHqlExpression * expr)
         return 0;       // for the moment always do this out of line 
     case no_table:
         return 0;
+    case no_inlinedictionary:
+        return RETassign;
     case no_owned_ds:
         {
             unsigned childFlags = getInlineFlags(ctx, expr->queryChild(0));

rtl/eclrtl/rtlds.cpp

@@ -659,6 +659,24 @@ void RtlLinkedDictionaryBuilder::finalizeRow(size32_t rowSize)
     appendOwn(next);
 }
 
+extern ECLRTL_API byte *rtlDictionaryLookup(IHThorHashLookupInfo &hashInfo, size32_t tableSize, byte **table, const byte *source, byte *defaultRow)
+{
+    IHash *hash  = hashInfo.queryHash();
+    ICompare *compare  = hashInfo.queryCompare();
+    unsigned rowidx = hash->hash(source) % tableSize;
+    loop
+    {
+        const void *entry = table[rowidx];
+        if (!entry)
+            return (byte *) rtlLinkRow(defaultRow);
+        if (compare->docompare(source, entry)==0)
+            return (byte *) rtlLinkRow(entry);
+        rowidx++;
+        if (rowidx==tableSize)
+            rowidx = 0;
+    }
+}
+
 //---------------------------------------------------------------------------
 
 //These definitions should be shared with thorcommon, but to do that

rtl/eclrtl/rtlds_imp.hpp

@@ -436,6 +436,8 @@ protected:
     size32_t tableSize;
 };
 
+extern ECLRTL_API byte *rtlDictionaryLookup(IHThorHashLookupInfo &hashInfo, size32_t tableSize, byte **table, const byte *source, byte *defaultRow);
+
 extern ECLRTL_API void appendRowsToRowset(size32_t & targetCount, byte * * & targetRowset, IEngineRowAllocator * rowAllocator, size32_t count, byte * * rows);