Merge pull request #465 from ghalliday/bug85564

BUG: #85564 Resolve problems with HASH(data)

Reviewed-By: Richard Chapman <rchapman@hpccsystems.com>

common/deftype/deftype.cpp

@@ -2442,8 +2442,8 @@ static ITypeInfo * getPromotedData(ITypeInfo * left, ITypeInfo * right)
 {
     unsigned lLen = left->getStringLen();
     unsigned rLen = right->getStringLen();
-    if (lLen < rLen) lLen = rLen;
-    return makeDataType(lLen);
+    assertex(lLen != rLen);
+    return makeDataType(UNKNOWN_LENGTH);
 }
 
 static ITypeInfo * makeUnknownLengthDecimal(bool isCompare)
@@ -2654,7 +2654,7 @@ ITypeInfo * getPromotedMulDivType(ITypeInfo * lType, ITypeInfo * rType)
 
 ITypeInfo * getPromotedCompareType(ITypeInfo * left, ITypeInfo * right)
 {
-    ITypeInfo * promoted = getPromotedType(left, right, true);
+    Owned<ITypeInfo> promoted = getPromotedType(left, right, true);
     if (left != right)
     {
         type_t ptc = promoted->getTypeCode();
@@ -2667,13 +2667,11 @@ ITypeInfo * getPromotedCompareType(ITypeInfo * left, ITypeInfo * right)
                     if ((left->queryCollation() == right->queryCollation()) && 
                         (left->queryCharset() == right->queryCharset()))
                     {
-                        promoted->Release();
-                        promoted = getStretchedType(UNKNOWN_LENGTH, left);
+                        promoted.setown(getStretchedType(UNKNOWN_LENGTH, left));
                     }
                 }
             }
             break;
-
         case type_unicode:
         case type_utf8:
             {
@@ -2681,7 +2679,7 @@ ITypeInfo * getPromotedCompareType(ITypeInfo * left, ITypeInfo * right)
             break;
         }
     }
-    return promoted;
+    return promoted.getClear();
 }
 
 static bool preservesValue(ITypeInfo * after, ITypeInfo * before, bool preserveInformation)

common/deftype/defvalue.cpp

@@ -281,15 +281,13 @@ unsigned VarStringValue::getHash(unsigned initval)
 int VarStringValue::compare(IValue *_to)
 {
     assertThrow(_to->getTypeCode()==type->getTypeCode());
-    // MORE - should maybe allow, with space padding?
     VarStringValue *to = (VarStringValue *) _to;
-    return strcmp(val.get(), to->val.get());
+    return rtlCompareVStrVStr(val.get(), to->val.get());
 }
 
 int VarStringValue::compare(const void *mem)
 {
-    return strcmp(val.get(), (const char *)mem);
-//  return stricmp(val.get(), (const char *)mem);
+    return rtlCompareVStrVStr(val.get(), (const char *)mem);
 }
 
 IValue *VarStringValue::castTo(ITypeInfo *t)
@@ -496,7 +494,20 @@ IValue *StringValue::castTo(ITypeInfo *t)
     if (tc == type_any)
         return LINK(this);
 
-    return doCastTo(type->getStringLen(), (const char *)val.get(), t);
+    const char * str = (const char *)val.get();
+    if (tc == type_data)
+        return t->castFrom(type->getSize(), str);        //NB: Must not go through translation in default case
+
+    ICharsetInfo * srcCharset = type->queryCharset();
+    Owned<ICharsetInfo> asciiCharset = getCharset(asciiAtom);
+    if (queryDefaultTranslation(asciiCharset, srcCharset))
+    {
+        Owned<ITypeInfo> asciiType = getAsciiType(type);
+        Owned<IValue> asciiValue = createStringValue(str, LINK(asciiType), type->getStringLen(), srcCharset);
+        return asciiValue->castTo(t);
+    }
+
+    return doCastTo(type->getStringLen(), str, t);
 }
 
 const void *StringValue::queryValue() const
@@ -2929,37 +2940,38 @@ IValue * substringValue(IValue * v, IValue * lower, IValue * higher)
 
 IValue * trimStringValue(IValue * v, char typecode)
 {
-    if(isUnicodeType(v->queryType()))
+    ITypeInfo * type = v->queryType();
+    type_t tc = type->getTypeCode();
+    if(isUnicodeType(type))
     {
         unsigned tlen = 0;
         rtlDataAttr resultstr;
-        bool isVarUnicode = (v->queryType()->getTypeCode() == type_varunicode);
-        if(isVarUnicode)
+        unsigned len = type->getStringLen();
+        if (tc == type_utf8)
         {
-            unsigned bufflen = v->queryType()->getStringLen()+1;
-            rtlDataAttr buff(bufflen*2);
-            v->getUCharStringValue(bufflen, buff.getdata());
-            UChar const * str = buff.getustr();
+            char const * str = (char const *)v->queryValue();
             switch(typecode) 
             {
             case 'A':
-                rtlTrimVUnicodeAll(tlen, resultstr.refustr(), str);
+                rtlTrimUtf8All(tlen, resultstr.refstr(), len, str);
                 break;
             case 'B':
-                rtlTrimVUnicodeBoth(tlen, resultstr.refustr(), str);
+                rtlTrimUtf8Both(tlen, resultstr.refstr(), len, str);
                 break;
             case 'L':
-                rtlTrimVUnicodeLeft(tlen, resultstr.refustr(), str);
+                rtlTrimUtf8Left(tlen, resultstr.refstr(), len, str);
                 break;
             default:
-                rtlTrimVUnicodeRight(tlen, resultstr.refustr(), str);
+                rtlTrimUtf8Right(tlen, resultstr.refstr(), len, str);
                 break;
             }
+
+            ITypeInfo * newtype = makeUtf8Type(tlen, type->queryLocale());
+            return createUtf8Value(tlen, resultstr.getstr(), newtype);
         }
         else
         {
             UChar const * str = (UChar const *)v->queryValue();
-            unsigned len = v->queryType()->getStringLen();
             switch(typecode) 
             {
             case 'A':
@@ -2975,75 +2987,40 @@ IValue * trimStringValue(IValue * v, char typecode)
                 rtlTrimUnicodeRight(tlen, resultstr.refustr(), len, str);
                 break;
             }
-        }
-#if 0
-        //It seems trim always returns a unicode result...
-        if(isVarUnicode)
-            return createVarUnicodeValue(tlen+1, resultstr.getustr(), v->queryType());
-        else
-#endif
-        {
+
             ITypeInfo * newtype = makeUnicodeType(tlen, v->queryType()->queryLocale());
             return createUnicodeValue(resultstr.getustr(), tlen, newtype);
         }
     }
     else
     {
-        Owned<ITypeInfo> st = getStringType(v->queryType());
-        Owned<IValue> sv = v->castTo(st);
-        bool isVarString = (st->getTypeCode() == type_varstring);
+        Owned<ITypeInfo> st = getStringType(type);
+        Owned<ITypeInfo> asciiType = getAsciiType(st);
+        Owned<IValue> sv = v->castTo(asciiType);
         StringBuffer s;
         sv->getStringValue(s);
-        char const * str = s.str();
         unsigned tlen = 0;
         rtlDataAttr resultstr;
-        if (isVarString) 
-        {
-            switch(typecode) 
-            {
-            case 'A':
-                rtlTrimVAll(tlen, resultstr.refstr(), str);
-                break;
-            case 'B':
-                rtlTrimVBoth(tlen, resultstr.refstr(), str);
-                break;
-            case 'L':
-                rtlTrimVLeft(tlen, resultstr.refstr(), str);
-                break;
-            default:
-                rtlTrimVRight(tlen, resultstr.refstr(), str);
-                break;
-            }
-        }
-        else 
-        {
-            unsigned len = s.length();
-            switch(typecode) 
-            {
-            case 'A':
-                rtlTrimAll(tlen, resultstr.refstr(), len, str);
-                break;
-            case 'B':
-                rtlTrimBoth(tlen, resultstr.refstr(), len, str);
-                break;
-            case 'L':
-                rtlTrimLeft(tlen, resultstr.refstr(), len, str);
-                break;
-            default:
-                rtlTrimRight(tlen, resultstr.refstr(), len, str);
-                break;
-            }
-        }
-#if 0
-        //It seems trim always returns a string result...
-        if (isVarString)
-            return createVarStringValue(tlen + 1, resultstr, st);
-        else
-#endif
+        unsigned len = s.length();
+        char const * str = s.str();
+        switch(typecode)
         {
-            ITypeInfo * newtype = makeStringType(tlen, LINK(st->queryCharset()), LINK(st->queryCollation()));
-            return createStringValue(resultstr.getstr(), newtype);
+        case 'A':
+            rtlTrimAll(tlen, resultstr.refstr(), len, str);
+            break;
+        case 'B':
+            rtlTrimBoth(tlen, resultstr.refstr(), len, str);
+            break;
+        case 'L':
+            rtlTrimLeft(tlen, resultstr.refstr(), len, str);
+            break;
+        default:
+            rtlTrimRight(tlen, resultstr.refstr(), len, str);
+            break;
         }
+
+        ITypeInfo * newtype = makeStringType(tlen, LINK(asciiType->queryCharset()), LINK(asciiType->queryCollation()));
+        return createStringValue(resultstr.getstr(), newtype);
     }
 }
 
@@ -3242,7 +3219,7 @@ IValue * logicalOrValues(unsigned num, IValue * * values)
 
 int orderValues(IValue * left, IValue * right)
 {
-    Owned<ITypeInfo> pt = getPromotedType(left->queryType(), right->queryType());
+    Owned<ITypeInfo> pt = getPromotedCompareType(left->queryType(), right->queryType());
     Owned<IValue> lv = left->castTo(pt);
     Owned<IValue> rv = right->castTo(pt);
     return lv->compare(rv);

ecl/hql/hqlutil.cpp

@@ -3182,11 +3182,7 @@ IHqlExpression * createTrimExpr(IHqlExpression * value, IHqlExpression * flags)
     else if (isUnicodeType(srcType))
         tgtType = makeUnicodeType(UNKNOWN_LENGTH, srcType->queryLocale());
     else
-    {
-        ICharsetInfo * charset = srcType->queryCharset();
-        ICollationInfo * collation = srcType->queryCollation();
-        tgtType = makeStringType(UNKNOWN_LENGTH, LINK(charset),LINK(collation));
-    }
+        tgtType = makeStringType(UNKNOWN_LENGTH, getCharset(asciiAtom), getCollation(asciiAtom));
 
     HqlExprArray args;
     args.append(*LINK(expr));

ecl/hqlcpp/hqlcatom.cpp

@@ -107,6 +107,7 @@ _ATOM compareUnicodeUnicodeAtom;
 _ATOM compareUnicodeUnicodeStrengthAtom;
 _ATOM compareUtf8Utf8Atom;
 _ATOM compareUtf8Utf8StrengthAtom;
+_ATOM compareVStrVStrAtom;
 _ATOM compareVUnicodeVUnicodeAtom;
 _ATOM compareVUnicodeVUnicodeStrengthAtom;
 _ATOM concatAtom;
@@ -591,7 +592,6 @@ _ATOM str2StrAtom;
 _ATOM str2StrXAtom;
 _ATOM str2VStrAtom;
 _ATOM str2VStrXAtom;
-_ATOM strcmpAtom;
 _ATOM strcpyAtom;
 _ATOM strlenAtom;
 _ATOM subDataFTAtom;
@@ -810,6 +810,7 @@ MODULE_INIT(INIT_PRIORITY_HQLATOM)
     MAKEATOM(compareUnicodeUnicodeStrength);
     MAKEATOM(compareUtf8Utf8);
     MAKEATOM(compareUtf8Utf8Strength);
+    MAKEATOM(compareVStrVStr);
     MAKEATOM(compareVUnicodeVUnicode);
     MAKEATOM(compareVUnicodeVUnicodeStrength);
     MAKEATOM(concat);
@@ -1322,7 +1323,6 @@ MODULE_INIT(INIT_PRIORITY_HQLATOM)
     MAKEATOM(str2StrX);
     MAKEATOM(str2VStr);
     MAKEATOM(str2VStrX);
-    MAKEATOM(strcmp);
     MAKEATOM(strcpy);
     MAKEATOM(strlen);
     MAKEATOM(subDataFT);

ecl/hqlcpp/hqlcatom.hpp

@@ -108,6 +108,7 @@ extern _ATOM compareUnicodeUnicodeAtom;
 extern _ATOM compareUnicodeUnicodeStrengthAtom;
 extern _ATOM compareUtf8Utf8Atom;
 extern _ATOM compareUtf8Utf8StrengthAtom;
+extern _ATOM compareVStrVStrAtom;
 extern _ATOM compareVUnicodeVUnicodeAtom;
 extern _ATOM compareVUnicodeVUnicodeStrengthAtom;
 extern _ATOM concatAtom;
@@ -591,7 +592,6 @@ extern _ATOM str2StrAtom;
 extern _ATOM str2StrXAtom;
 extern _ATOM str2VStrAtom;
 extern _ATOM str2VStrXAtom;
-extern _ATOM strcmpAtom;
 extern _ATOM strcpyAtom;
 extern _ATOM strlenAtom;
 extern _ATOM subDataFTAtom;

ecl/hqlcpp/hqlcpp.cpp

@@ -4681,7 +4681,7 @@ void HqlCppTranslator::doBuildExprCompare(BuildCtx & ctx, IHqlExpression * expr,
                 args.append(*getElementPointer(lhs.expr));
                 args.append(*getElementPointer(rhs.expr));
                 
-                orderExpr.setown(bindTranslatedFunctionCall(strcmpAtom, args));
+                orderExpr.setown(bindTranslatedFunctionCall(compareVStrVStrAtom, args));
                 break;
             }
         case type_decimal:
@@ -8035,7 +8035,7 @@ void HqlCppTranslator::doBuildAssignCompareElement(BuildCtx & ctx, EvaluateCompa
                 args.append(*getElementPointer(lhs.expr));
                 args.append(*getElementPointer(rhs.expr));
                 
-                op = bindTranslatedFunctionCall(strcmpAtom, args);
+                op = bindTranslatedFunctionCall(compareVStrVStrAtom, args);
                 break;
             }
         case type_decimal:
@@ -8520,11 +8520,15 @@ void HqlCppTranslator::doBuildAssignHashElement(BuildCtx & ctx, HashCodeCreator
     CHqlBoundExpr bound;
     OwnedHqlExpr length;
     OwnedHqlExpr ptr;
+    bool alreadyTrimmedRight = (elem->getOperator() == no_trim) && (elem->hasProperty(rightAtom) || !elem->hasProperty(leftAtom));
+    //If this hash is generated internally (e.g., for a dedup) and fixed length, then can simplify the hash calculation
+    bool canOptimizeHash = (creator.optimize() && isFixedSize(type));
+    bool optimizeTrim = alreadyTrimmedRight || canOptimizeHash;
     switch (type->getTypeCode())
     {
         case type_string:
             {
-                if (!creator.optimize())
+                if (!optimizeTrim)
                 {
                     OwnedHqlExpr trimmed = createValue(no_trim, getStretchedType(UNKNOWN_LENGTH, type), LINK(elem));
                     buildCachedExpr(ctx, trimmed, bound);
@@ -8539,7 +8543,7 @@ void HqlCppTranslator::doBuildAssignHashElement(BuildCtx & ctx, HashCodeCreator
 
         case type_unicode:
             {
-                if (!creator.optimize())
+                if (!optimizeTrim)
                 {
                     OwnedHqlExpr trimmed = createValue(no_trim, getStretchedType(UNKNOWN_LENGTH, type), LINK(elem));
                     buildCachedExpr(ctx, trimmed, bound);
@@ -8560,7 +8564,7 @@ void HqlCppTranslator::doBuildAssignHashElement(BuildCtx & ctx, HashCodeCreator
 
         case type_utf8:
             {
-                if (!creator.optimize())
+                if (!optimizeTrim)
                 {
                     OwnedHqlExpr trimmed = createValue(no_trim, getStretchedType(UNKNOWN_LENGTH, type), LINK(elem));
                     buildCachedExpr(ctx, trimmed, bound);
@@ -8581,11 +8585,35 @@ void HqlCppTranslator::doBuildAssignHashElement(BuildCtx & ctx, HashCodeCreator
 
 
         case type_data:
+            {
+                buildCachedExpr(ctx, elem, bound);
+
+                length.setown(getBoundLength(bound));
+                ptr.setown(getElementPointer(bound.expr));
+                break;
+            }
         case type_qstring:
-            buildCachedExpr(ctx, elem, bound);
-            length.setown(getBoundSize(bound));
-            ptr.setown(getElementPointer(bound.expr));
-            break;
+            {
+                LinkedHqlExpr exprToHash = elem;
+                if (!canOptimizeHash)
+                {
+                    //Always convert to a string so the hash is compatible with a string.
+                    OwnedHqlExpr cast = ensureExprType(elem, unknownStringType);
+                    if (alreadyTrimmedRight)
+                    {
+                        exprToHash.set(cast);
+                    }
+                    else
+                    {
+                        OwnedHqlExpr trimmed = createValue(no_trim, LINK(unknownStringType), LINK(cast));
+                        exprToHash.setown(foldHqlExpression(trimmed));
+                    }
+                }
+                buildCachedExpr(ctx, exprToHash, bound);
+                length.setown(getBoundSize(bound));
+                ptr.setown(getElementPointer(bound.expr));
+                break;
+            }
 
         case type_varstring:
             buildCachedExpr(ctx, elem, bound);
@@ -9503,9 +9531,10 @@ void HqlCppTranslator::doBuildExprTrim(BuildCtx & ctx, IHqlExpression * expr, CH
     bool hasLeft = expr->hasProperty(leftAtom);
     bool hasRight = expr->hasProperty(rightAtom);
     
+    type_t btc = bound.expr->queryType()->getTypeCode();
     if(hasAll || hasLeft) 
     {
-        if (bound.expr->queryType()->getTypeCode() == type_varstring)
+        if (btc == type_varstring)
         {
             if(hasAll) {
                 func = trimVAllAtom;
@@ -9517,7 +9546,7 @@ void HqlCppTranslator::doBuildExprTrim(BuildCtx & ctx, IHqlExpression * expr, CH
                 func = trimVLeftAtom;
             }
         }
-        else if (bound.expr->queryType()->getTypeCode() == type_unicode)
+        else if (btc == type_unicode)
         {
             if(hasAll) {
                 func = trimUnicodeAllAtom;
@@ -9529,7 +9558,7 @@ void HqlCppTranslator::doBuildExprTrim(BuildCtx & ctx, IHqlExpression * expr, CH
                 func = trimUnicodeLeftAtom;
             }
         }
-        else if (bound.expr->queryType()->getTypeCode() == type_varunicode)
+        else if (btc == type_varunicode)
         {
             if(hasAll) {
                 func = trimVUnicodeAllAtom;
@@ -9541,7 +9570,7 @@ void HqlCppTranslator::doBuildExprTrim(BuildCtx & ctx, IHqlExpression * expr, CH
                 func = trimVUnicodeLeftAtom;
             }
         }
-        else if (bound.expr->queryType()->getTypeCode() == type_utf8)
+        else if (btc == type_utf8)
         {
             if(hasAll) {
                 func = trimUtf8AllAtom;
@@ -9571,23 +9600,23 @@ void HqlCppTranslator::doBuildExprTrim(BuildCtx & ctx, IHqlExpression * expr, CH
         buildExpr(ctx, call, tgt);
     }
     else {
-        if (bound.expr->queryType()->getTypeCode() == type_varstring)
+        if (btc == type_varstring)
         {
             args.append(*LINK(str));
             func = trimVStrLenAtom;
         }
-        else if (bound.expr->queryType()->getTypeCode() == type_unicode)
+        else if (btc == type_unicode)
         {
             args.append(*getBoundLength(bound));
             args.append(*LINK(str));
             func = trimUnicodeStrLenAtom;
         }
-        else if (bound.expr->queryType()->getTypeCode() == type_varunicode)
+        else if (btc == type_varunicode)
         {
             args.append(*LINK(str));
             func = trimVUnicodeStrLenAtom;
         }
-        else if (bound.expr->queryType()->getTypeCode() == type_utf8)
+        else if (btc == type_utf8)
         {
             args.append(*getBoundLength(bound));
             args.append(*LINK(str));

ecl/hqlcpp/hqlcpp.ipp

@@ -331,6 +331,8 @@ public:
     void setDefault(IHqlExpression * expr);
 
 protected:
+    bool canBuildStaticList(ITypeInfo * type) { return isFixedSize(type); }
+
     void buildChop3Map(BuildCtx & ctx, const CHqlBoundTarget & target, CHqlBoundExpr & test, IHqlExpression * temp, unsigned start, unsigned end);
     void buildChop3Map(BuildCtx & ctx, const CHqlBoundTarget & target, CHqlBoundExpr & test);
     void buildChop2Map(BuildCtx & ctx, const CHqlBoundTarget & target, CHqlBoundExpr & test, unsigned start, unsigned end);
@@ -372,6 +374,7 @@ protected:
     bool constantValues;
     OwnedITypeInfo resultType;
     OwnedITypeInfo indexType;
+    OwnedITypeInfo promotedElementType;
 };
 
 //===========================================================================

ecl/hqlcpp/hqlcppcase.cpp

@@ -200,7 +200,7 @@ bool HqlCppCaseInfo::buildAssign(BuildCtx & ctx, const CHqlBoundTarget & target)
                 if (condType->getTypeCode() != type_real)
                 {
                     OwnedHqlExpr search = test.getTranslatedExpr();
-                    if (constantValues && (condType->getTypeCode() == type_int) && (pairs.ordinality() > INTEGER_SEARCH_THRESHOLD))
+                    if (constantValues && (condType->getTypeCode() == type_int) && (pairs.ordinality() > INTEGER_SEARCH_THRESHOLD) && canBuildStaticList(resultType))
                         buildIntegerSearchMap(subctx, target, search);
                     else
                         buildSwitchMap(subctx, &target, test.expr);
@@ -434,6 +434,11 @@ void HqlCppCaseInfo::buildLoopChopMap(BuildCtx & ctx, const CHqlBoundTarget & ta
     //Declare a table that contains all the strings...
     ITypeInfo * compareType = queryCompareType();
     type_t ctc = compareType->getTypeCode();
+    if ((ctc == type_data) && !hasLibraryChop())
+    {
+         buildGeneralAssign(ctx, target);
+         return;
+    }
 
     OwnedHqlExpr values = createCompareList();
     CHqlBoundExpr boundTable;
@@ -925,7 +930,7 @@ IHqlExpression * HqlCppCaseInfo::createResultsExpr(IHqlExpression * matchVar, bo
     // easy way to create a value list...
     ITypeInfo * storeType = getArrayElementType(retType);
     OwnedHqlExpr newlist = createValue(no_list, makeSetType(storeType), values);
-    if (areConstant)
+    if (areConstant && canBuildStaticList(resultType))
     {
         IHqlExpression * index = adjustValue(matchVar, 1+firstMatchEntry);
         return createValue(no_index, LINK(retType), LINK(newlist), index, createAttribute(noBoundCheckAtom));
@@ -961,9 +966,22 @@ bool HqlCppCaseInfo::hasLibraryChop()
     ITypeInfo * compareType = queryCompareType();
     type_t ctc = compareType->getTypeCode();
 
-    return ((ctc == type_string) || (ctc == type_data) || (ctc == type_varstring) || (ctc == type_qstring) || (ctc == type_unicode) || (ctc == type_varunicode) || (ctc == type_utf8));
+    switch (ctc)
+    {
+    case type_data:
+        return canBuildStaticList(promotedElementType);
+    case type_string:
+    case type_varstring:
+    case type_qstring:
+    case type_unicode:
+    case type_varunicode:
+    case type_utf8:
+        return true;
+    }
+    return false;
 }
 
+
 void HqlCppCaseInfo::processBranches()
 {
     sortPairs();
@@ -991,6 +1009,7 @@ void HqlCppCaseInfo::promoteTypes()
 
         promoted.setown(::getPromotedECLType(promoted, type));
     }
+    promotedElementType.set(promoted);
 
     if (isStringType(promoted))
         promoted.setown(getStretchedType(UNKNOWN_LENGTH, promoted));
@@ -1021,7 +1040,7 @@ bool HqlCppCaseInfo::canBuildArrayLookup(const CHqlBoundExpr & test)
 
 bool HqlCppCaseInfo::queryBuildArrayLookup(BuildCtx & ctx, const CHqlBoundTarget & target, const CHqlBoundExpr & test)
 {
-    if (canBuildArrayLookup(test))
+    if (canBuildArrayLookup(test) && canBuildStaticList(resultType))
     {
         //MORE: Also support this for high density tables that don't start at 0... - checking upper and lower bounds
         ITypeInfo * condType = test.queryType()->queryPromotedType();

ecl/hqlcpp/hqlcppsys.ecl

@@ -209,6 +209,7 @@ const char * cppSystemText[]  = {
     "   unsigned4 rtlMin(unsigned4 idx, unsigned4 len) : eclrtl,pure,entrypoint='rtlMin',include;",
 
     "   integer4 compareStrStr(const string l, const string r) : eclrtl,pure,library='eclrtl',entrypoint='rtlCompareStrStr';",
+    "   integer4 compareVStrVStr(const varstring l, const varstring r) : eclrtl,pure,library='eclrtl',entrypoint='rtlCompareVStrVStr';",
     "   integer4 compareStrBlank(const string l) : eclrtl,pure,library='eclrtl',entrypoint='rtlCompareStrBlank';",
     "   integer4 compareDataData(const data l, const data r) : eclrtl,pure,library='eclrtl',entrypoint='rtlCompareDataData';",
     "   integer4 compareEStrEStr(const string l, const string r) : eclrtl,pure,library='eclrtl',entrypoint='rtlCompareEStrEStr';",
@@ -544,7 +545,6 @@ const char * cppSystemText[]  = {
     "   integer4 memcmp(const data1 target, const data1 src, unsigned4 len) : sys,pure,entrypoint='memcmp';",
     "   data1 memcpy(data1 target, const data1 src, unsigned4 len): sys,entrypoint='memcpy';",
     "   data1 memset(data1 target, integer4 fill, unsigned4 len) :  sys,entrypoint='memset';",
-    "   integer4 strcmp(const varstring1 target, const varstring1 src) :    sys,pure,entrypoint='strcmp';",
     "   strcpy(const varstring1 target, const varstring1 src) : sys,entrypoint='strcpy';",
     "   unsigned4 strlen(const varstring1 src) :    sys,pure,entrypoint='strlen';",
 

ecl/regress/bug85564.ecl

@@ -0,0 +1,84 @@
+/*##############################################################################
+
+    Copyright (C) 2011 HPCC Systems.
+
+    All rights reserved. This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as
+    published by the Free Software Foundation, either version 3 of the
+    License, or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+############################################################################## */
+
+output((string)trim((ebcdic string)'abcdef    '));
+output(trim(nofold((ebcdic string)'abcdef    ')));
+
+s1 := 'G';
+s2 := 'G ';
+d1 := x'34';
+d2 := x'3400';
+q1 := Q'G';
+q2 := Q'G ';
+u1 := U'G';
+u2 := U'G ';
+vu1 := (varunicode)U'G';
+vu2 := (varunicode)U'G ';
+v1 := V'G';
+v2 := V'G ';
+u81 := U8'G';
+u82 := U8'G ';
+
+dq1 := (>data<)Q'G';
+dq2 := (>data<)Q'G ';
+
+xs1 := 'G' : stored('xs1');
+xs2 := 'G ' : stored('xs2');
+xd1 := x'34' : stored('xd1');
+xd2 := x'3400' : stored('xd2');
+xq1 := Q'G' : stored('xq1');
+xq2 := Q'G ' : stored('xq2');
+xu1 := U'G' : stored('xu1');
+xu2 := U'G ' : stored('xu2');
+xu81 := U8'G' : stored('xu81');
+xu82 := U8'G ' : stored('xu82');
+varunicode xvu1 := U'G' : stored('xvu1');
+varunicode xvu2 := U'G ' : stored('xvu2');
+xv1 := 'G' : stored('xv1');
+xv2 := 'G ' : stored('xv2');
+
+compareValues(x, y, title) := MACRO
+    output('----- ' + title + ' -----');
+    output(hash(x));
+    output(hash(y));
+    output(hash(x) = hash(y));
+    output(x <=> y);
+    output(trim(x) <=> trim(y));
+ENDMACRO;
+
+compareValues(s1, s2, 's1 s2');
+compareValues(q1, q2, 'q1 q2');
+compareValues(v1, v2, 'v1 v2');
+compareValues(xs1, xs2, 'xs1 xs2');
+compareValues(xq1, xq2, 'xq1 xq2');
+compareValues(xv1, xv2, 'xv1 xv2');
+compareValues(s1, q1, 's1 q1');
+compareValues(s1, v2, 's1 v2');
+
+compareValues(d1, d2, 'd1 d2');
+compareValues(d2, d1, 'd2 d1');
+compareValues(xd1, xd2, 'xd1 xd2');
+compareValues(dq1, dq2, 'dq1 dq2');
+
+compareValues(u1, u2, 'u1 u2');
+compareValues(u81, u82, 'u81 u82');
+compareValues(xu1, xu2, 'xu1 xu2');
+compareValues(xu81, xu82, 'xu81 xu82');
+compareValues(vu1, vu2, 'vu1 vu2');
+compareValues(vu1, u2, 'vu1 u2');
+compareValues(xvu1, xvu2, 'xvu1 xvu2');

rtl/eclrtl/eclrtl.cpp

@@ -2435,6 +2435,11 @@ int rtlCompareStrStr(unsigned l1, const char * p1, unsigned l2, const char * p2)
     return diff;
 }
 
+int rtlCompareVStrVStr(const char * p1, const char * p2)
+{
+    return rtlCompareStrStr(strlen(p1), p1, strlen(p2), p2);
+}
+
 int rtlCompareStrBlank(unsigned l1, const char * p1)
 {
     while (l1--)
@@ -2454,16 +2459,10 @@ int rtlCompareDataData(unsigned l1, const void * p1, unsigned l2, const void * p
     int diff = memcmp(p1, p2, len);
     if (diff == 0)
     {
-        if (len != l1)
-        {
-            for (;(diff == 0) && (len != l1);len++)
-                diff = ((unsigned char *)p1)[len] - '\0';
-        }
-        else if (len != l2)
-        {
-            for (;(diff == 0) && (len != l2);len++)
-                diff = '\0' - ((unsigned char *)p2)[len];
-        }
+        if (l1 > l2)
+            diff = +1;
+        else if (l1 < l2)
+            diff = -1;
     }
     return diff;
 }
@@ -2506,12 +2505,12 @@ int rtlCompareUnicodeUnicodeStrength(unsigned l1, UChar const * p1, unsigned l2,
 
 int rtlCompareVUnicodeVUnicode(UChar const * p1, UChar const * p2, char const * locale)
 {
-    return ucol_strcoll(queryRTLLocale(locale)->queryCollator(), p1, rtlUnicodeStrlen(p1), p2, rtlUnicodeStrlen(p2));
+    return rtlCompareUnicodeUnicode(rtlUnicodeStrlen(p1), p1, rtlUnicodeStrlen(p2), p2, locale);
 }
 
 int rtlCompareVUnicodeVUnicodeStrength(UChar const * p1, UChar const * p2, char const * locale, unsigned strength)
 {
-    return ucol_strcoll(queryRTLLocale(locale)->queryCollator(strength), p1, rtlUnicodeStrlen(p1), p2, rtlUnicodeStrlen(p2));
+    return rtlCompareUnicodeUnicodeStrength(rtlUnicodeStrlen(p1), p1, rtlUnicodeStrlen(p2), p2, locale, strength);
 }
 
 void rtlKeyUnicodeX(unsigned & tlen, void * & tgt, unsigned slen, const UChar * src, const char * locale)
@@ -3465,17 +3464,18 @@ unsigned rtlHashString( unsigned length, const char *_k, unsigned initval)
 
 unsigned rtlHashUnicode(unsigned length, UChar const * k, unsigned initval)
 {
+    //Would make more sense to trim here.
     return rtlHashData(length*2, k, initval);
 }
 
 unsigned rtlHashVStr(const char * k, unsigned initval)
 {
-    return rtlHashData(strlen(k), k, initval);
+    return rtlHashData(rtlTrimVStrLen(k), k, initval);
 }
 
 unsigned rtlHashVUnicode(UChar const * k, unsigned initval)
 {
-    return rtlHashData(rtlUnicodeStrlen(k)*2, k, initval);
+    return rtlHashData(rtlTrimVUnicodeStrLen(k)*2, k, initval);
 }
 
 #define GETWORDNC(k,n) ((GETBYTE0(n)+GETBYTE1(n)+GETBYTE2(n)+GETBYTE3(n))&0xdfdfdfdf)

rtl/eclrtl/eclrtl.hpp

@@ -245,6 +245,7 @@ ECLRTL_API unsigned rtlTrimUtf8StrLen(size32_t l, const char * t);
 ECLRTL_API unsigned rtlTrimVStrLen(const char * t);
 ECLRTL_API unsigned rtlTrimVUnicodeStrLen(UChar const * t);
 ECLRTL_API int rtlCompareStrStr(unsigned l1, const char * p1, unsigned l2, const char * p2);
+ECLRTL_API int rtlCompareVStrVStr(const char * p1, const char * p2);
 ECLRTL_API int rtlCompareStrBlank(unsigned l1, const char * p1);
 ECLRTL_API int rtlCompareDataData(unsigned l1, const void * p1, unsigned l2, const void * p2);
 ECLRTL_API int rtlCompareEStrEStr(unsigned l1, const char * p1, unsigned l2, const char * p2);