HPCC-8744 Optimize dictionary lookups of single fixed length fields

Added stringN to list of optimized cases, and other minor cleanup.

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

ecl/hql/hqlutil.cpp

@@ -5405,7 +5405,6 @@ void TempTableTransformer::reportWarning(IHqlExpression * location, int code,con
 
 IHqlExpression *getDictionaryKeyRecord(IHqlExpression *record)
 {
-    // MORE - should probably use an attr to cache this?
     IHqlExpression * payload = record->queryProperty(_payload_Atom);
     unsigned payloadSize = payload ? getIntValue(payload->queryChild(0)) : 0;
     unsigned max = record->numChildren() - payloadSize;

ecl/hqlcpp/hqlcppsys.ecl

@@ -822,17 +822,19 @@ const char * cppSystemText[]  = {
     "    integer8 dictionaryCount(_linkcounted_ dictionary dict) : eclrtl,include,pure,entrypoint='rtlDictionaryCount';",
     "   _linkcounted_ row(dummyRecord) dictionaryLookup(IHThorHashLookupInfo meta, _linkcounted_ dictionary dict, row key, _linkcounted_ row defaultrow) : eclrtl,include,pure,entrypoint='rtlDictionaryLookup';",
     "   _linkcounted_ row(dummyRecord) dictionaryLookupString(_linkcounted_ dictionary dict, const string key, _linkcounted_ row defaultrow) : eclrtl,include,pure,entrypoint='rtlDictionaryLookupString';",
+    "   _linkcounted_ row(dummyRecord) dictionaryLookupStringN(_linkcounted_ dictionary dict, const unsigned4 size, const string key, _linkcounted_ row defaultrow) : eclrtl,include,pure,entrypoint='rtlDictionaryLookupStringN';",
     "   _linkcounted_ row(dummyRecord) dictionaryLookupSigned(_linkcounted_ dictionary dict, const integer key, _linkcounted_ row defaultrow) : eclrtl,include,pure,entrypoint='rtlDictionaryLookupSigned';",
     "   _linkcounted_ row(dummyRecord) dictionaryLookupUnsigned(_linkcounted_ dictionary dict, const unsigned key, _linkcounted_ row defaultrow) : eclrtl,include,pure,entrypoint='rtlDictionaryLookupUnsigned';",
-    "   _linkcounted_ row(dummyRecord) dictionaryLookupSignedN(_linkcounted_ dictionary dict, const integer4 size, const integer key, _linkcounted_ row defaultrow) : eclrtl,include,pure,entrypoint='rtlDictionaryLookupSignedN';",
-    "   _linkcounted_ row(dummyRecord) dictionaryLookupUnsignedN(_linkcounted_ dictionary dict, const integer4 size, const unsigned key, _linkcounted_ row defaultrow) : eclrtl,include,pure,entrypoint='rtlDictionaryLookupUnsignedN';",
+    "   _linkcounted_ row(dummyRecord) dictionaryLookupSignedN(_linkcounted_ dictionary dict, const unsigned4 size, const integer key, _linkcounted_ row defaultrow) : eclrtl,include,pure,entrypoint='rtlDictionaryLookupSignedN';",
+    "   _linkcounted_ row(dummyRecord) dictionaryLookupUnsignedN(_linkcounted_ dictionary dict, const unsigned4 size, const unsigned key, _linkcounted_ row defaultrow) : eclrtl,include,pure,entrypoint='rtlDictionaryLookupUnsignedN';",
 
     "   boolean dictionaryLookupExists(boolean meta, _linkcounted_ dictionary dict, row key) : eclrtl,include,pure,entrypoint='rtlDictionaryLookupExists';",
     "   boolean dictionaryLookupExistsString(_linkcounted_ dictionary dict, const string key) : eclrtl,include,pure,entrypoint='rtlDictionaryLookupExistsString';",
+    "   boolean dictionaryLookupExistsStringN(_linkcounted_ dictionary dict, const unsigned4 size, const string key) : eclrtl,include,pure,entrypoint='rtlDictionaryLookupExistsStringN';",
     "   boolean dictionaryLookupExistsSigned(_linkcounted_ dictionary dict, const integer key) : eclrtl,include,pure,entrypoint='rtlDictionaryLookupExistsSigned';",
     "   boolean dictionaryLookupExistsUnsigned(_linkcounted_ dictionary dict, const unsigned key) : eclrtl,include,pure,entrypoint='rtlDictionaryLookupExistsUnsigned';",
-    "   boolean dictionaryLookupExistsSignedN(_linkcounted_ dictionary dict, const integer4 size, const integer key) : eclrtl,include,pure,entrypoint='rtlDictionaryLookupExistsSignedN';",
-    "   boolean dictionaryLookupExistsUnsignedN(_linkcounted_ dictionary dict, const integer4 size, const unsigned key) : eclrtl,include,pure,entrypoint='rtlDictionaryLookupExistsUnsignedN';",
+    "   boolean dictionaryLookupExistsSignedN(_linkcounted_ dictionary dict, const unsigned4 size, const integer key) : eclrtl,include,pure,entrypoint='rtlDictionaryLookupExistsSignedN';",
+    "   boolean dictionaryLookupExistsUnsignedN(_linkcounted_ dictionary dict, const unsigned4 size, const unsigned key) : eclrtl,include,pure,entrypoint='rtlDictionaryLookupExistsUnsignedN';",
 
     // Marshalling parameters to external languages
     "   bindBooleanParam(const varstring name, boolean val) : method,entrypoint='bindBooleanParam';",

ecl/hqlcpp/hqlcset.cpp

@@ -738,20 +738,24 @@ BoundRow * InlineLinkedDictionaryCursor::buildSelectMap(BuildCtx & ctx, IHqlExpr
         switch (type->getTypeCode())
         {
         case type_string:
-            if (type->getStringLen()==UNKNOWN_LENGTH && isAscii(type))
-                optimizedLookupFunc.append("dictionaryLookupString");
-            break;
-        case type_int:
-            if (!type->isSwappedEndian())
+            if (isAscii(type))
             {
-                optimizedLookupFunc.appendf("dictionaryLookup%s", type->isSigned() ? "Signed" : "Unsigned");
-                if (type->getSize() != 8)
+                optimizedLookupFunc.append("dictionaryLookupString");
+                if (type->getStringLen()!=UNKNOWN_LENGTH)
                 {
                     optimizedLookupFunc.append("N");
-                    args.append(*createConstant((int) type->getSize()));
+                    args.append(*getSizetConstant(type->getStringLen()));
                 }
             }
             break;
+        case type_int:
+            optimizedLookupFunc.appendf("dictionaryLookup%s", type->isSigned() ? "Signed" : "Unsigned");
+            if (type->getSize() != 8)
+            {
+                optimizedLookupFunc.append("N");
+                args.append(*getSizetConstant(type->getSize()));
+            }
+            break;
         }
     }
     if (optimizedLookupFunc.length())
@@ -798,20 +802,24 @@ void InlineLinkedDictionaryCursor::buildInDataset(BuildCtx & ctx, IHqlExpression
         switch (type->getTypeCode())
         {
         case type_string:
-            if (type->getStringLen()==UNKNOWN_LENGTH && isAscii(type))
-                optimizedLookupFunc.append("dictionaryLookupExistsString");
-            break;
-        case type_int:
-            if (!type->isSwappedEndian())
+            if (isAscii(type))
             {
-                optimizedLookupFunc.appendf("dictionaryLookupExists%s", type->isSigned() ? "Signed" : "Unsigned");
-                if (type->getSize() != 8)
+                optimizedLookupFunc.append("dictionaryLookupExistsString");
+                if (type->getStringLen()!=UNKNOWN_LENGTH)
                 {
                     optimizedLookupFunc.append("N");
-                    args.append(*createConstant((int) type->getSize()));
+                    args.append(*getSizetConstant(type->getStringLen()));
                 }
             }
             break;
+        case type_int:
+            optimizedLookupFunc.appendf("dictionaryLookupExists%s", type->isSigned() ? "Signed" : "Unsigned");
+            if (type->getSize() != 8)
+            {
+                optimizedLookupFunc.append("N");
+                args.append(*getSizetConstant(type->getSize()));
+            }
+            break;
         }
     }
     if (optimizedLookupFunc.length())

rtl/eclrtl/rtlds.cpp

@@ -704,6 +704,25 @@ extern ECLRTL_API byte *rtlDictionaryLookupString(size32_t tableSize, byte **tab
     }
 }
 
+extern ECLRTL_API byte *rtlDictionaryLookupStringN(size32_t tableSize, byte **table, size32_t N, size32_t searchLen, const char *searchFor, byte *defaultRow)
+{
+    if (!tableSize)
+        return (byte *) rtlLinkRow(defaultRow);
+    unsigned hash = rtlHash32Data(rtlTrimStrLen(searchLen, searchFor), searchFor, HASH32_INIT);
+    unsigned rowidx = hash % tableSize;
+    loop
+    {
+        const char *entry = (const char *) table[rowidx];
+        if (!entry)
+            return (byte *) rtlLinkRow(defaultRow);
+        if (rtlCompareStrStr(searchLen, searchFor, N, entry)==0)
+            return (byte *) rtlLinkRow(entry);
+        rowidx++;
+        if (rowidx==tableSize)
+            rowidx = 0;
+    }
+}
+
 extern ECLRTL_API byte *rtlDictionaryLookupSigned(size32_t tableSize, byte **table, __int64 searchFor, byte *defaultRow)
 {
     if (!tableSize)
@@ -742,7 +761,7 @@ extern ECLRTL_API byte *rtlDictionaryLookupUnsigned(size32_t tableSize, byte **t
     }
 }
 
-extern ECLRTL_API byte *rtlDictionaryLookupSignedN(size32_t tableSize, byte **table, int size, __int64 searchFor, byte *defaultRow)
+extern ECLRTL_API byte *rtlDictionaryLookupSignedN(size32_t tableSize, byte **table, size32_t size, __int64 searchFor, byte *defaultRow)
 {
     if (!tableSize)
         return (byte *) rtlLinkRow(defaultRow);
@@ -761,7 +780,7 @@ extern ECLRTL_API byte *rtlDictionaryLookupSignedN(size32_t tableSize, byte **ta
     }
 }
 
-extern ECLRTL_API byte *rtlDictionaryLookupUnsignedN(size32_t tableSize, byte **table, int size, __uint64 searchFor, byte *defaultRow)
+extern ECLRTL_API byte *rtlDictionaryLookupUnsignedN(size32_t tableSize, byte **table, size32_t size, __uint64 searchFor, byte *defaultRow)
 {
     if (!tableSize)
         return (byte *) rtlLinkRow(defaultRow);
@@ -822,6 +841,25 @@ extern ECLRTL_API bool rtlDictionaryLookupExistsString(size32_t tableSize, byte
     }
 }
 
+extern ECLRTL_API bool rtlDictionaryLookupExistsStringN(size32_t tableSize, byte **table, size32_t N, size32_t searchLen, const char *searchFor)
+{
+    if (!tableSize)
+        return false;
+    unsigned hash = rtlHash32Data(rtlTrimStrLen(searchLen, searchFor), searchFor, HASH32_INIT);
+    unsigned rowidx = hash % tableSize;
+    loop
+    {
+        const char *entry = (const char *) table[rowidx];
+        if (!entry)
+            return false;
+        if (rtlCompareStrStr(searchLen, searchFor, N, entry)==0)
+            return true;
+        rowidx++;
+        if (rowidx==tableSize)
+            rowidx = 0;
+    }
+}
+
 extern ECLRTL_API bool rtlDictionaryLookupExistsSigned(size32_t tableSize, byte **table, __int64 searchFor)
 {
     if (!tableSize)
@@ -860,7 +898,7 @@ extern ECLRTL_API bool rtlDictionaryLookupExistsUnsigned(size32_t tableSize, byt
     }
 }
 
-extern ECLRTL_API bool rtlDictionaryLookupExistsSignedN(size32_t tableSize, byte **table, int size, __int64 searchFor)
+extern ECLRTL_API bool rtlDictionaryLookupExistsSignedN(size32_t tableSize, byte **table, size32_t size, __int64 searchFor)
 {
     if (!tableSize)
         return false;
@@ -879,7 +917,7 @@ extern ECLRTL_API bool rtlDictionaryLookupExistsSignedN(size32_t tableSize, byte
     }
 }
 
-extern ECLRTL_API bool rtlDictionaryLookupExistsUnsignedN(size32_t tableSize, byte **table, int size, __uint64 searchFor)
+extern ECLRTL_API bool rtlDictionaryLookupExistsUnsignedN(size32_t tableSize, byte **table, size32_t size, __uint64 searchFor)
 {
     if (!tableSize)
         return false;

rtl/eclrtl/rtlds_imp.hpp

@@ -439,17 +439,19 @@ protected:
 extern ECLRTL_API unsigned __int64 rtlDictionaryCount(size32_t tableSize, byte **table);
 extern ECLRTL_API byte *rtlDictionaryLookup(IHThorHashLookupInfo &hashInfo, size32_t tableSize, byte **table, const byte *source, byte *defaultRow);
 extern ECLRTL_API byte *rtlDictionaryLookupString(size32_t tableSize, byte **table, size32_t len, const char *source, byte *defaultRow);
+extern ECLRTL_API byte *rtlDictionaryLookupStringN(size32_t tableSize, byte **table, size32_t N, size32_t len, const char *source, byte *defaultRow);
 extern ECLRTL_API byte *rtlDictionaryLookupSigned(size32_t tableSize, byte **table, __int64 source, byte *defaultRow);
 extern ECLRTL_API byte *rtlDictionaryLookupUnsigned(size32_t tableSize, byte **table, __uint64 source, byte *defaultRow);
-extern ECLRTL_API byte *rtlDictionaryLookupSignedN(size32_t tableSize, byte **table, int size, __uint64 source, byte *defaultRow);
-extern ECLRTL_API byte *rtlDictionaryLookupUnsignedN(size32_t tableSize, byte **table, int size, __uint64 source, byte *defaultRow);
+extern ECLRTL_API byte *rtlDictionaryLookupSignedN(size32_t tableSize, byte **table, size32_t size, __uint64 source, byte *defaultRow);
+extern ECLRTL_API byte *rtlDictionaryLookupUnsignedN(size32_t tableSize, byte **table, size32_t size, __uint64 source, byte *defaultRow);
 
 extern ECLRTL_API bool rtlDictionaryLookupExists(IHThorHashLookupInfo &hashInfo, size32_t tableSize, byte **table, const byte *source);
 extern ECLRTL_API bool rtlDictionaryLookupExistsString(size32_t tableSize, byte **table, size32_t len, const char *source);
+extern ECLRTL_API bool rtlDictionaryLookupExistsStringN(size32_t tableSize, byte **table, size32_t N, size32_t len, const char *source);
 extern ECLRTL_API bool rtlDictionaryLookupExistsSigned(size32_t tableSize, byte **table, __int64 source);
 extern ECLRTL_API bool rtlDictionaryLookupExistsUnsigned(size32_t tableSize, byte **table, __uint64 source);
-extern ECLRTL_API bool rtlDictionaryLookupExistsSignedN(size32_t tableSize, byte **table, int size, __uint64 source);
-extern ECLRTL_API bool rtlDictionaryLookupExistsUnsignedN(size32_t tableSize, byte **table, int size, __uint64 source);
+extern ECLRTL_API bool rtlDictionaryLookupExistsSignedN(size32_t tableSize, byte **table, size32_t size, __uint64 source);
+extern ECLRTL_API bool rtlDictionaryLookupExistsUnsignedN(size32_t tableSize, byte **table, size32_t size, __uint64 source);
 
 extern ECLRTL_API void appendRowsToRowset(size32_t & targetCount, byte * * & targetRowset, IEngineRowAllocator * rowAllocator, size32_t count, byte * * rows);