HPCC-8808 Fix issues with truncation of lookup values

Refactor the dictionary lookup code so that the search rows use full precision
fields, to avoid false positives when the search keys are larger than the
range that can be represented by the key fields.

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

common/deftype/deftype.cpp

@@ -3159,6 +3159,28 @@ ITypeInfo * getStretchedType(unsigned newLen, ITypeInfo * type)
     return NULL;
 }
 
+ITypeInfo * getMaxLengthType(ITypeInfo * type)
+{
+    switch (type->getTypeCode())
+    {
+    case type_boolean:
+        return LINK(type);
+    case type_int:
+        return makeIntType(8, type->isSigned());
+    case type_string:
+    case type_varstring:
+    case type_unicode:
+    case type_varunicode:
+    case type_utf8:
+    case type_qstring:
+    case type_data:
+        return getStretchedType(UNKNOWN_LENGTH, type);
+    default:
+        throw MakeStringException(99, "Internal error: getMaxLengthType");
+    }
+    return NULL;
+}
+
 ITypeInfo * getAsciiType(ITypeInfo * type)
 {
     ICharsetInfo * charset = type->queryCharset();

common/deftype/deftype.hpp

@@ -285,6 +285,7 @@ extern DEFTYPE_API ITranslationInfo * queryDefaultTranslation(ICharsetInfo * tgt
 //---------------------------------------------------------------------------
 
 extern DEFTYPE_API ITypeInfo * getStretchedType(unsigned newLen, ITypeInfo * type);
+extern DEFTYPE_API ITypeInfo * getMaxLengthType(ITypeInfo * type);
 extern DEFTYPE_API ITypeInfo * getNumericType(ITypeInfo * type);
 extern DEFTYPE_API ITypeInfo * getStringType(ITypeInfo * type);
 extern DEFTYPE_API ITypeInfo * getVarStringType(ITypeInfo * type);

ecl/hql/hqlutil.cpp

@@ -5413,9 +5413,34 @@ IHqlExpression *getDictionaryKeyRecord(IHqlExpression *record)
     return newrec->closeExpr();
 }
 
+IHqlExpression *getDictionarySearchRecord(IHqlExpression *record)
+{
+    // MORE - should probably use an attr to cache this?
+    IHqlExpression * payload = record->queryProperty(_payload_Atom);
+    unsigned payloadSize = payload ? getIntValue(payload->queryChild(0)) : 0;
+    unsigned max = record->numChildren() - payloadSize;
+    IHqlExpression *newrec = createRecord();
+    for (unsigned idx = 0; idx < max; idx++)
+    {
+        IHqlExpression *child = record->queryChild(idx);
+        if (!child->isAttribute() || child->queryName()!=_payload_Atom)  // Strip off the payload attribute
+        {
+            if (child->getOperator()==no_field)
+            {
+                ITypeInfo *fieldType = child->queryType();
+                Owned<ITypeInfo> stretched = getMaxLengthType(fieldType);
+                newrec->addOperand(createField(child->queryName(), stretched.getClear(), NULL, NULL));
+            }
+            else
+                newrec->addOperand(LINK(child));
+        }
+    }
+    return newrec->closeExpr();
+}
+
 IHqlExpression * createSelectMapRow(IErrorReceiver * errors, ECLlocation & location, IHqlExpression * dict, IHqlExpression *values)
 {
-    OwnedHqlExpr record = getDictionaryKeyRecord(dict->queryRecord());
+    OwnedHqlExpr record = getDictionarySearchRecord(dict->queryRecord());
     TempTableTransformer transformer(errors, location, true);
     OwnedHqlExpr newTransform = transformer.createTempTableTransform(values, record);
     return createRow(no_selectmap, dict, createRow(no_createrow, newTransform.getClear()));
@@ -5423,7 +5448,7 @@ IHqlExpression * createSelectMapRow(IErrorReceiver * errors, ECLlocation & locat
 
 IHqlExpression *createINDictExpr(IErrorReceiver * errors, ECLlocation & location, IHqlExpression *expr, IHqlExpression *dict)
 {
-    OwnedHqlExpr record = getDictionaryKeyRecord(dict->queryRecord());
+    OwnedHqlExpr record = getDictionarySearchRecord(dict->queryRecord());
     TempTableTransformer transformer(errors, location, true);
     OwnedHqlExpr newTransform = transformer.createTempTableTransform(expr, record);
     return createBoolExpr(no_indict, createRow(no_createrow, newTransform.getClear()), dict);
@@ -5431,10 +5456,10 @@ IHqlExpression *createINDictExpr(IErrorReceiver * errors, ECLlocation & location
 
 IHqlExpression *createINDictRow(IErrorReceiver * errors, ECLlocation & location, IHqlExpression *row, IHqlExpression *dict)
 {
-    OwnedHqlExpr record = getDictionaryKeyRecord(dict->queryRecord());
-    if (!record->queryType()->assignableFrom(row->queryType()))
-        errors->reportError(ERR_TYPE_DIFFER, "Type mismatch", location.sourcePath->str(), location.lineno, location.column, location.position);
-    return createBoolExpr(no_indict, row, dict);
+    OwnedHqlExpr record = getDictionarySearchRecord(dict->queryRecord());
+    Owned<ITypeInfo> rowType = makeRowType(record->getType());
+    OwnedHqlExpr castRow = ensureExprType(row, rowType);
+    return createBoolExpr(no_indict, castRow.getClear(), dict);
 }
 
 IHqlExpression * convertTempRowToCreateRow(IErrorReceiver * errors, ECLlocation & location, IHqlExpression * expr)

ecl/hql/hqlutil.hpp

@@ -451,6 +451,8 @@ extern HQL_API IHqlExpression * extractCppBodyAttrs(unsigned len, const char * v
 extern HQL_API unsigned cleanupEmbeddedCpp(unsigned len, char * buffer);
 extern HQL_API bool isNullList(IHqlExpression * expr);
 
+extern HQL_API IHqlExpression *getDictionaryKeyRecord(IHqlExpression *record);
+extern HQL_API IHqlExpression *getDictionarySearchRecord(IHqlExpression *record);
 extern HQL_API IHqlExpression * createSelectMapRow(IErrorReceiver * errors, ECLlocation & location, IHqlExpression * dict, IHqlExpression *values);
 extern HQL_API IHqlExpression * createINDictExpr(IErrorReceiver * errors, ECLlocation & location, IHqlExpression *expr, IHqlExpression *dict);
 extern HQL_API IHqlExpression *createINDictRow(IErrorReceiver * errors, ECLlocation & location, IHqlExpression *row, IHqlExpression *dict);

ecl/hqlcpp/hqlhtcpp.cpp

@@ -5390,21 +5390,43 @@ void HqlCppTranslator::buildDictionaryHashClass(IHqlExpression *record, StringBu
         classctx.setNextPriority(RowMetaPrio);
 
         beginNestedClass(classctx, lookupHelperName, "IHThorHashLookupInfo");
-        HqlExprArray keyedFields;
-        OwnedHqlExpr dictionary = createDataset(no_null, LINK(record));
-        IHqlExpression * payload = record->queryProperty(_payload_Atom);
-        unsigned payloadSize = payload ? (unsigned)getIntValue(payload->queryChild(0)) : 0;
-        unsigned max = record->numChildren() - payloadSize;
-        for (unsigned idx = 0; idx < max; idx++)
-        {
-            IHqlExpression *child = record->queryChild(idx);
+        OwnedHqlExpr searchRecord = getDictionarySearchRecord(record);
+        OwnedHqlExpr keyRecord = getDictionaryKeyRecord(record);
+
+        HqlExprArray keyedSourceFields;
+        HqlExprArray keyedDictFields;
+        OwnedHqlExpr source = createDataset(no_null, LINK(searchRecord));
+        DatasetReference sourceRef(source, no_none, NULL);
+        OwnedHqlExpr dict = createDataset(no_null, LINK(record));
+        DatasetReference dictRef(dict, no_none, NULL);
+
+        ForEachChild(idx, searchRecord)
+        {
+            IHqlExpression *child = searchRecord->queryChild(idx);
             if (!child->isAttribute())
-                keyedFields.append(*createSelectExpr(LINK(dictionary->queryNormalizedSelector()), LINK(child)));
+                keyedSourceFields.append(*createSelectExpr(LINK(source->queryNormalizedSelector()), LINK(child)));
         }
-        OwnedHqlExpr keyedList = createValueSafe(no_sortlist, makeSortListType(NULL), keyedFields);
-        DatasetReference dictRef(dictionary, no_none, NULL);
-        buildHashOfExprsClass(classctx, "Hash", keyedList, dictRef, false);
-        buildCompareMember(classctx, "Compare", keyedList, dictRef);
+        ForEachChild(idx2, keyRecord)
+        {
+            IHqlExpression *child = keyRecord->queryChild(idx2);
+            if (!child->isAttribute())
+                keyedDictFields.append(*createSelectExpr(LINK(dict->queryNormalizedSelector()), LINK(child)));
+        }
+        OwnedHqlExpr keyedSourceList = createValueSafe(no_sortlist, makeSortListType(NULL), keyedSourceFields);
+        OwnedHqlExpr keyedDictList = createValueSafe(no_sortlist, makeSortListType(NULL), keyedDictFields);
+
+        buildHashOfExprsClass(classctx, "HashLookup", keyedSourceList, sourceRef, false);
+        buildHashOfExprsClass(classctx, "Hash", keyedDictList, dictRef, false);
+
+        OwnedHqlExpr seq = createDummySelectorSequence();
+        OwnedHqlExpr leftSelect = createSelector(no_left, source, seq);
+        OwnedHqlExpr rightSelect = createSelector(no_right, dict, seq);
+        IHqlExpression * left = sourceRef.mapCompound(keyedSourceList, leftSelect);
+        IHqlExpression * right = dictRef.mapCompound(keyedDictList, rightSelect);
+        OwnedHqlExpr compare = createValue(no_order, LINK(signedType), left, right);
+
+        buildCompareMemberLR(classctx, "CompareLookup", compare, source, dict, seq);
+        buildCompareMember(classctx, "Compare", keyedDictList, dictRef);
         endNestedClass();
 
         if (queryOptions().spanMultipleCpp)

rtl/eclrtl/rtlds.cpp

@@ -667,8 +667,8 @@ extern ECLRTL_API byte *rtlDictionaryLookup(IHThorHashLookupInfo &hashInfo, size
     if (!tableSize)
         return (byte *) rtlLinkRow(defaultRow);
 
-    IHash *hash  = hashInfo.queryHash();
-    ICompare *compare  = hashInfo.queryCompare();
+    IHash *hash  = hashInfo.queryHashLookup();
+    ICompare *compare  = hashInfo.queryCompareLookup();
     unsigned rowidx = hash->hash(source) % tableSize;
     loop
     {
@@ -688,8 +688,8 @@ extern ECLRTL_API bool rtlDictionaryLookupExists(IHThorHashLookupInfo &hashInfo,
     if (!tableSize)
         return false;
 
-    IHash *hash  = hashInfo.queryHash();
-    ICompare *compare  = hashInfo.queryCompare();
+    IHash *hash  = hashInfo.queryHashLookup();
+    ICompare *compare  = hashInfo.queryCompareLookup();
     unsigned rowidx = hash->hash(source) % tableSize;
     loop
     {

rtl/include/eclhelper.hpp

@@ -2717,6 +2717,8 @@ interface IHThorHashLookupInfo
 {
     virtual IHash * queryHash() = 0;
     virtual ICompare * queryCompare() = 0;
+    virtual IHash * queryHashLookup() = 0;
+    virtual ICompare * queryCompareLookup() = 0;
 };
 
 

testing/ecl/dict_field.ecl

@@ -0,0 +1,76 @@
+resistorCodes := dataset([{0, 'Black'},
+                            {1, 'Brown'},
+                            {2, 'Red'},
+                            {3, 'Orange'},
+                            {4, 'Yellow'},
+                            {5, 'Green'},
+                            {6, 'Blue'},
+                            {7, 'Violet'},
+                            {8, 'Grey'},
+                            {9, 'White'}], {unsigned1 value, string color});
+
+color2code := DICTIONARY(resistorCodes, { STRING fcolor := color => value});
+colorn2code := DICTIONARY(resistorCodes, { STRING6 ncolor := color => value});
+colorv2code := DICTIONARY(resistorCodes, { VARSTRING vcolor := color => value});
+
+ucolor2code := DICTIONARY(resistorCodes, { UNICODE fcolor := color => value});
+ucolorn2code := DICTIONARY(resistorCodes, { UNICODE6 ncolor := color => value});
+ucolorv2code := DICTIONARY(resistorCodes, { VARUNICODE vcolor := color => value});
+ucoloru2code := DICTIONARY(resistorCodes, { UTF8 ucolor := color => value});
+
+code1ToColor := DICTIONARY(resistorCodes, { value => color});
+
+string c1 := 'White' : STORED('color1');
+string c2 := 'White ' : STORED('color2');
+string c3 := 'Whit' : STORED('color3');
+string c4 := 'White or maybe not' : STORED('color4');
+
+unicode u1 := u'White' : STORED('ucolor1');
+unicode u2 := u'White ' : STORED('ucolor2');
+unicode u3 := u'Whit' : STORED('ucolor3');
+unicode u4 := u'White with a hint of yellow' : STORED('ucolor4');
+
+c1 in color2code;
+c2 in color2code;
+c3 not in color2code;
+c4 not in color2code;
+
+c1 in colorn2code;
+c2 in colorn2code;
+c3 not in colorn2code;
+c4 not in colorn2code;
+
+c1 in colorv2code;
+c2 in colorv2code;
+c3 not in colorv2code;
+c4 not in colorv2code;
+
+1 in code1ToColor;
+257 not in code1ToColor;
+
+'---';
+
+ROW({u1}, { unicode fcolor} ) IN ucolor2code;
+ROW({u1}, { unicode6 ncolor} ) IN ucolorn2code;
+ROW({u1}, { varunicode vcolor} ) IN ucolorv2code;
+ROW({u1}, { utf8 ucolor} ) IN ucoloru2code;
+
+u1 in ucolor2code;
+u2 in ucolor2code;
+u3 not in ucolor2code;
+u4 not in ucolor2code;
+
+u1 in ucolorn2code;
+u2 in ucolorn2code;
+u3 not in ucolorn2code;
+u4 not in ucolorn2code;
+
+u1 in ucolorv2code;
+u2 in ucolorv2code;
+u3 not in ucolorv2code;
+u4 not in ucolorv2code;
+
+u1 in ucoloru2code;
+u2 in ucoloru2code;
+u3 not in ucoloru2code;
+u4 not in ucoloru2code;

testing/ecl/dict_field2.ecl

@@ -0,0 +1,72 @@
+resistorCodes := dataset([{0, 'Black'},
+                            {1, 'Brown'},
+                            {2, 'Red'},
+                            {3, 'Orange'},
+                            {4, 'Yellow'},
+                            {5, 'Green'},
+                            {6, 'Blue'},
+                            {7, 'Violet'},
+                            {8, 'Grey'},
+                            {9, 'White'}], {unsigned1 value, string color});
+
+color2code := DICTIONARY(resistorCodes, { STRING fcolor := color => value});
+colorn2code := DICTIONARY(resistorCodes, { STRING6 ncolor := color => value});
+colorv2code := DICTIONARY(resistorCodes, { VARSTRING vcolor := color => value});
+
+ucolor2code := DICTIONARY(resistorCodes, { UNICODE fcolor := color => value});
+ucolorn2code := DICTIONARY(resistorCodes, { UNICODE6 ncolor := color => value});
+ucolorv2code := DICTIONARY(resistorCodes, { VARUNICODE vcolor := color => value});
+ucoloru2code := DICTIONARY(resistorCodes, { UTF8 ucolor := color => value});
+
+code1ToColor := DICTIONARY(resistorCodes, { value => color});
+
+string c1 := 'White' : STORED('color1');
+string c2 := 'White ' : STORED('color2');
+string c3 := 'Whit' : STORED('color3');
+string c4 := 'White or maybe not' : STORED('color4');
+
+unicode u1 := u'White' : STORED('ucolor1');
+unicode u2 := u'White ' : STORED('ucolor2');
+unicode u3 := u'Whit' : STORED('ucolor3');
+unicode u4 := u'White with a hint of yellow' : STORED('ucolor4');
+
+
+color2code[c1].value;
+color2code[c2].value;
+color2code[c3].value;
+color2code[c4].value;
+
+colorn2code[c1].value;
+colorn2code[c2].value;
+colorn2code[c3].value;
+colorn2code[c4].value;
+
+colorv2code[c1].value;
+colorv2code[c2].value;
+colorv2code[c3].value;
+colorv2code[c4].value;
+
+code1ToColor[1].value;
+code1ToColor[257].value;
+
+'---';
+
+ucolor2code[u1].value;
+ucolor2code[u2].value;
+ucolor2code[u3].value;
+ucolor2code[u4].value;
+
+ucolorn2code[u1].value;
+ucolorn2code[u2].value;
+ucolorn2code[u3].value;
+ucolorn2code[u4].value;
+
+ucolorv2code[u1].value;
+ucolorv2code[u2].value;
+ucolorv2code[u3].value;
+ucolorv2code[u4].value;
+
+ucoloru2code[u1].value;
+ucoloru2code[u2].value;
+ucoloru2code[u3].value;
+ucoloru2code[u4].value;

testing/ecl/key/dict_field.xml

@@ -0,0 +1,105 @@
+<Dataset name='Result 1'>
+ <Row><Result_1>true</Result_1></Row>
+</Dataset>
+<Dataset name='Result 2'>
+ <Row><Result_2>true</Result_2></Row>
+</Dataset>
+<Dataset name='Result 3'>
+ <Row><Result_3>true</Result_3></Row>
+</Dataset>
+<Dataset name='Result 4'>
+ <Row><Result_4>true</Result_4></Row>
+</Dataset>
+<Dataset name='Result 5'>
+ <Row><Result_5>true</Result_5></Row>
+</Dataset>
+<Dataset name='Result 6'>
+ <Row><Result_6>true</Result_6></Row>
+</Dataset>
+<Dataset name='Result 7'>
+ <Row><Result_7>true</Result_7></Row>
+</Dataset>
+<Dataset name='Result 8'>
+ <Row><Result_8>true</Result_8></Row>
+</Dataset>
+<Dataset name='Result 9'>
+ <Row><Result_9>true</Result_9></Row>
+</Dataset>
+<Dataset name='Result 10'>
+ <Row><Result_10>true</Result_10></Row>
+</Dataset>
+<Dataset name='Result 11'>
+ <Row><Result_11>true</Result_11></Row>
+</Dataset>
+<Dataset name='Result 12'>
+ <Row><Result_12>true</Result_12></Row>
+</Dataset>
+<Dataset name='Result 13'>
+ <Row><Result_13>true</Result_13></Row>
+</Dataset>
+<Dataset name='Result 14'>
+ <Row><Result_14>true</Result_14></Row>
+</Dataset>
+<Dataset name='Result 15'>
+ <Row><Result_15>---</Result_15></Row>
+</Dataset>
+<Dataset name='Result 16'>
+ <Row><Result_16>true</Result_16></Row>
+</Dataset>
+<Dataset name='Result 17'>
+ <Row><Result_17>true</Result_17></Row>
+</Dataset>
+<Dataset name='Result 18'>
+ <Row><Result_18>true</Result_18></Row>
+</Dataset>
+<Dataset name='Result 19'>
+ <Row><Result_19>true</Result_19></Row>
+</Dataset>
+<Dataset name='Result 20'>
+ <Row><Result_20>true</Result_20></Row>
+</Dataset>
+<Dataset name='Result 21'>
+ <Row><Result_21>true</Result_21></Row>
+</Dataset>
+<Dataset name='Result 22'>
+ <Row><Result_22>true</Result_22></Row>
+</Dataset>
+<Dataset name='Result 23'>
+ <Row><Result_23>true</Result_23></Row>
+</Dataset>
+<Dataset name='Result 24'>
+ <Row><Result_24>true</Result_24></Row>
+</Dataset>
+<Dataset name='Result 25'>
+ <Row><Result_25>true</Result_25></Row>
+</Dataset>
+<Dataset name='Result 26'>
+ <Row><Result_26>true</Result_26></Row>
+</Dataset>
+<Dataset name='Result 27'>
+ <Row><Result_27>true</Result_27></Row>
+</Dataset>
+<Dataset name='Result 28'>
+ <Row><Result_28>true</Result_28></Row>
+</Dataset>
+<Dataset name='Result 29'>
+ <Row><Result_29>true</Result_29></Row>
+</Dataset>
+<Dataset name='Result 30'>
+ <Row><Result_30>true</Result_30></Row>
+</Dataset>
+<Dataset name='Result 31'>
+ <Row><Result_31>true</Result_31></Row>
+</Dataset>
+<Dataset name='Result 32'>
+ <Row><Result_32>true</Result_32></Row>
+</Dataset>
+<Dataset name='Result 33'>
+ <Row><Result_33>true</Result_33></Row>
+</Dataset>
+<Dataset name='Result 34'>
+ <Row><Result_34>true</Result_34></Row>
+</Dataset>
+<Dataset name='Result 35'>
+ <Row><Result_35>true</Result_35></Row>
+</Dataset>

testing/ecl/key/dict_field2.xml

@@ -0,0 +1,93 @@
+<Dataset name='Result 1'>
+ <Row><Result_1>9</Result_1></Row>
+</Dataset>
+<Dataset name='Result 2'>
+ <Row><Result_2>9</Result_2></Row>
+</Dataset>
+<Dataset name='Result 3'>
+ <Row><Result_3>0</Result_3></Row>
+</Dataset>
+<Dataset name='Result 4'>
+ <Row><Result_4>0</Result_4></Row>
+</Dataset>
+<Dataset name='Result 5'>
+ <Row><Result_5>9</Result_5></Row>
+</Dataset>
+<Dataset name='Result 6'>
+ <Row><Result_6>9</Result_6></Row>
+</Dataset>
+<Dataset name='Result 7'>
+ <Row><Result_7>0</Result_7></Row>
+</Dataset>
+<Dataset name='Result 8'>
+ <Row><Result_8>0</Result_8></Row>
+</Dataset>
+<Dataset name='Result 9'>
+ <Row><Result_9>9</Result_9></Row>
+</Dataset>
+<Dataset name='Result 10'>
+ <Row><Result_10>9</Result_10></Row>
+</Dataset>
+<Dataset name='Result 11'>
+ <Row><Result_11>0</Result_11></Row>
+</Dataset>
+<Dataset name='Result 12'>
+ <Row><Result_12>0</Result_12></Row>
+</Dataset>
+<Dataset name='Result 13'>
+ <Row><Result_13>1</Result_13></Row>
+</Dataset>
+<Dataset name='Result 14'>
+ <Row><Result_14>0</Result_14></Row>
+</Dataset>
+<Dataset name='Result 15'>
+ <Row><Result_15>---</Result_15></Row>
+</Dataset>
+<Dataset name='Result 16'>
+ <Row><Result_16>9</Result_16></Row>
+</Dataset>
+<Dataset name='Result 17'>
+ <Row><Result_17>9</Result_17></Row>
+</Dataset>
+<Dataset name='Result 18'>
+ <Row><Result_18>0</Result_18></Row>
+</Dataset>
+<Dataset name='Result 19'>
+ <Row><Result_19>0</Result_19></Row>
+</Dataset>
+<Dataset name='Result 20'>
+ <Row><Result_20>9</Result_20></Row>
+</Dataset>
+<Dataset name='Result 21'>
+ <Row><Result_21>9</Result_21></Row>
+</Dataset>
+<Dataset name='Result 22'>
+ <Row><Result_22>0</Result_22></Row>
+</Dataset>
+<Dataset name='Result 23'>
+ <Row><Result_23>0</Result_23></Row>
+</Dataset>
+<Dataset name='Result 24'>
+ <Row><Result_24>9</Result_24></Row>
+</Dataset>
+<Dataset name='Result 25'>
+ <Row><Result_25>9</Result_25></Row>
+</Dataset>
+<Dataset name='Result 26'>
+ <Row><Result_26>0</Result_26></Row>
+</Dataset>
+<Dataset name='Result 27'>
+ <Row><Result_27>0</Result_27></Row>
+</Dataset>
+<Dataset name='Result 28'>
+ <Row><Result_28>9</Result_28></Row>
+</Dataset>
+<Dataset name='Result 29'>
+ <Row><Result_29>9</Result_29></Row>
+</Dataset>
+<Dataset name='Result 30'>
+ <Row><Result_30>0</Result_30></Row>
+</Dataset>
+<Dataset name='Result 31'>
+ <Row><Result_31>0</Result_31></Row>
+</Dataset>