Merge branch 'candidate-5.4.0'

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

common/fileview2/fvdatasource.hpp

@@ -31,6 +31,7 @@ enum
     FVFFdataset     = 5,
     FVFFset         = 6,
     FVFFvirtual     = 7,
+    FVFFblob        = 8
 };
 
 interface IFvDataSource;

common/fileview2/fvresultset.cpp

@@ -146,8 +146,9 @@ IFvDataSource * createFileDataSource(IDistributedFile * df, const char * logical
         const char * kind = properties.queryProp("@kind");
         if (kind && (stricmp(kind, "key") == 0))
         {
-            if (isSimplifiedRecord(diskRecord, true))
-                ds.setown(new IndexDataSource(logicalName, diskRecord, username, password));
+            OwnedHqlExpr indexRecord = annotateIndexBlobs(diskRecord);
+            if (isSimplifiedRecord(indexRecord, true))
+                ds.setown(new IndexDataSource(logicalName, indexRecord, username, password));
             else
                 throwError1(FVERR_ViewComplexKey, logicalName);
         }
@@ -656,6 +657,14 @@ void CResultSetMetaData::getXmlSchema(ISchemaBuilder & builder, bool useXPath) c
                 }
                 break;
             }
+        case FVFFblob: //for now FileViewer will output the string "[blob]"
+            {
+                Owned<ITypeInfo> stringType = makeStringType(UNKNOWN_LENGTH, NULL, NULL);
+                if (useXPath)
+                    fvSplitXPath(meta->queryXPath(idx), xname, name);
+                builder.addField(name, *stringType, idx < keyedCount);
+            }
+            break;
         default:
             {
                 ITypeInfo & type = *column.type;
@@ -1528,6 +1537,9 @@ void CResultSetCursor::writeXmlText(IXmlWriter &writer, int columnIndex, const c
     unsigned flags = column.flag;
     switch (flags)
     {
+    case FVFFblob:
+        writer.outputCString("[blob]", name);
+        return;
     case FVFFbeginif:
     case FVFFendif:
         return;
@@ -1759,6 +1771,7 @@ void CResultSetCursor::writeXmlRow(IXmlWriter &writer)
         case FVFFvirtual:
         case FVFFdataset:
         case FVFFset:
+        case FVFFblob:
             if (ignoreNesting == 0)
                 writeXmlText(writer, col);
             break;

common/fileview2/fvsource.cpp

@@ -249,7 +249,7 @@ void DataSourceMetaData::addFileposition()
     addVirtualField("__fileposition__", NULL, makeIntType(8, false));
 }
 
-void DataSourceMetaData::addSimpleField(const char * name, const char * xpath, ITypeInfo * type)
+void DataSourceMetaData::addSimpleField(const char * name, const char * xpath, ITypeInfo * type, unsigned flag)
 {
     ITypeInfo * promoted = type->queryPromotedType();
     unsigned size = promoted->getSize();
@@ -291,7 +291,7 @@ void DataSourceMetaData::addSimpleField(const char * name, const char * xpath, I
         minRecordSize += size;
     if (thisBits == 0)
         bitsRemaining = 0;
-    fields.append(*new DataSourceMetaItem(FVFFnone, name, xpath, type));
+    fields.append(*new DataSourceMetaItem(flag, name, xpath, type));
 }
 
 
@@ -369,9 +369,12 @@ void DataSourceMetaData::gatherFields(IHqlExpression * expr, bool isConditional,
             IHqlExpression * xpathAttr = expr->queryAttribute(xpathAtom);
             if (xpathAttr && xpathAttr->queryChild(0)->queryValue())
                 xpath = xpathAttr->queryChild(0)->queryValue()->getStringValue(xpathtext);
-
+            unsigned flag = FVFFnone;
             if (isKey() && expr->hasAttribute(blobAtom))
+            {
                 type.setown(makeIntType(8, false));
+                flag = FVFFblob;
+            }
             type_t tc = type->getTypeCode();
             if (tc == type_row)
             {
@@ -407,7 +410,7 @@ void DataSourceMetaData::gatherFields(IHqlExpression * expr, bool isConditional,
                 }
                 if (pMixedContent && xpath && !*xpath)
                     *pMixedContent = true;
-                addSimpleField(outname, xpath, type);
+                addSimpleField(outname, xpath, type, flag);
             }
             break;
         }

common/fileview2/fvsource.ipp

@@ -141,7 +141,7 @@ public:
     virtual size32_t getMinRecordSize() const;
 
 protected:
-    void addSimpleField(const char * name, const char * xpath, ITypeInfo * type);
+    void addSimpleField(const char * name, const char * xpath, ITypeInfo * type, unsigned flag=FVFFnone);
     void gatherFields(IHqlExpression * expr, bool isConditional, bool *pMixedContent);
     void gatherChildFields(IHqlExpression * expr, bool isConditional, bool *pMixedContent);
     void gatherAttributes();

ecl/hql/hqlexpr.cpp

@@ -14180,7 +14180,7 @@ static void simplifyFileViewRecordTypes(HqlExprArray & fields, IHqlExpression *
             bool forceSimplify = false;
             if (isKey)
             {
-                if (cur->hasAttribute(blobAtom))
+                if (cur->hasAttribute(blobAtom) && !cur->hasAttribute(_isBlobInIndex_Atom))
                     forceSimplify = true;
             }
             else
@@ -16072,6 +16072,27 @@ extern bool HQL_API extractVersion(unsigned & major, unsigned & minor, unsigned
     return true;
 }
 
+static HqlTransformerInfo cHqlBlobTransformerInfo("CHqlBlobTransformer");
+class CHqlBlobTransformer : public QuickHqlTransformer
+{
+public:
+    CHqlBlobTransformer() : QuickHqlTransformer(cHqlBlobTransformerInfo, NULL) {}
+
+    virtual IHqlExpression * createTransformed(IHqlExpression * expr)
+    {
+        OwnedHqlExpr transformed = QuickHqlTransformer::createTransformed(expr);
+        if ((expr->getOperator() == no_field) && expr->hasAttribute(blobAtom))
+            return appendOwnedOperand(transformed, createAttribute(_isBlobInIndex_Atom));
+        return transformed.getClear();
+    }
+};
+
+IHqlExpression * annotateIndexBlobs(IHqlExpression * expr)
+{
+    CHqlBlobTransformer transformer;
+    return transformer.transform(expr);
+}
+
 /*
 List of changes:
 

ecl/hql/hqlexpr.hpp

@@ -1862,4 +1862,7 @@ void exportSymbols(IPropertyTree* data, IHqlScope * scope, HqlLookupContext & ct
 //The following is only here to provide information about the source file being compiled when reporting leaks
 extern HQL_API void setActiveSource(const char * filename);
 
+extern HQL_API IHqlExpression * annotateIndexBlobs(IHqlExpression * expr);
+
+
 #endif

ecl/hqlcpp/hqlcerrors.hpp

@@ -217,6 +217,7 @@
 #define HQLERR_InconsistentEmbedded             4197
 #define HQLERR_UnsupportedRowDiffType           4198
 #define HQLERR_EmbedParamNotSupportedInOptions  4199
+#define HQLERR_InvalidXmlnsPrefix               4200
 
 //Warnings....
 #define HQLWRN_PersistDataNotLikely             4500
@@ -508,6 +509,7 @@
 #define HQLERR_StreamInputUsedDirectly_Text     "Library input used directly in a child query"
 #define HQLERR_UnsupportedRowDiffType_Text      "ROWDIFF: Does not support type '%s' for field %s"
 #define HQLERR_EmbedParamNotSupportedInOptions_Text   "Cannot use bound parameter in embed options - try adding a FUNCTION wrapper"
+#define HQLERR_InvalidXmlnsPrefix_Text          "Invalid XMLNS prefix: %s"
 
 //Warnings.
 #define HQLWRN_CannotRecreateDistribution_Text  "Cannot recreate the distribution for a persistent dataset"

ecl/hqlcpp/hqlhtcpp.cpp

@@ -5096,11 +5096,16 @@ void addDatasetResultXmlNamespaces(IWUResult &result, HqlExprArray &xmlnsAttrs,
         StringBuffer xmlnsURI;
         getUTF8Value(xmlnsPrefix, xmlns.queryChild(0));
         getUTF8Value(xmlnsURI, xmlns.queryChild(1));
+
         if (xmlnsURI.length())
         {
-            result.setResultXmlns(xmlnsPrefix, xmlnsURI);
             if (xmlnsPrefix.length() && declaredPrefixes.find(xmlnsPrefix)==NotFound)
+            {
+                if (!validateXMLTag(xmlnsPrefix))
+                    throwError1(HQLERR_InvalidXmlnsPrefix, xmlnsPrefix.str());
                 declaredPrefixes.append(xmlnsPrefix);
+            }
+            result.setResultXmlns(xmlnsPrefix, xmlnsURI);
         }
     }
     StringArray usedPrefixes;
@@ -10088,28 +10093,6 @@ protected:
 };
 
 
-static HqlTransformerInfo cHqlBlobTransformerInfo("CHqlBlobTransformer");
-class CHqlBlobTransformer : public QuickHqlTransformer
-{
-public:
-    CHqlBlobTransformer() : QuickHqlTransformer(cHqlBlobTransformerInfo, NULL) {}
-
-    virtual IHqlExpression * createTransformed(IHqlExpression * expr)
-    {
-        OwnedHqlExpr transformed = QuickHqlTransformer::createTransformed(expr);
-        if ((expr->getOperator() == no_field) && expr->hasAttribute(blobAtom))
-            return appendOwnedOperand(transformed, createAttribute(_isBlobInIndex_Atom));
-        return transformed.getClear();
-    }
-};
-
-IHqlExpression * annotateIndexBlobs(IHqlExpression * expr)
-{
-    CHqlBlobTransformer transformer;
-    return transformer.transform(expr);
-}
-
-
 IDefRecordElement * HqlCppTranslator::createMetaRecord(IHqlExpression * record)
 {
     TranslatorMaxSizeCallback callback(*this);

ecl/regress/errxmlns.ecl

@@ -0,0 +1,19 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2015 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+ds := dataset('x', { unsigned id; }, thor);
+output(ds,XMLNS(U8'Oh là là Straße','error bad xmlns prefix'));

initfiles/bin/init_thor

@@ -205,31 +205,35 @@ while [[ 1 ]]; do
         1)  log "Thormaster ($thorpid) Interrupted, Ctrl-C caught"
             killed
             ;;
-        # TEC_Idle, TEC_Watchdog, TEC_Swap, TEC_DaliDown
-        2|3|5|6)    [[ $errcode -eq 2 ]] && log "Thormaster ($thorpid) Idle"
-                    [[ $errcode -eq 3 ]] && log "Thormaster ($thorpid) Lost connection to slave(s)"
-                    [[ $errcode -eq 5 ]] && log "Thormaster ($thorpid) Swap node required"
-                    [[ $errcode -eq 6 ]] && log "Thormaster ($thorpid) Unable to connect to Dali"
-                    log "Stopping thorslave(s) for restart"
-                    kill_slaves
-                    if [[ 0 != $autoSwapNode ]]; then
-                        log "Running autoswap $THORNAME :: ($thorpid)"
-                        swapnode auto $DALISERVER $component
-                        errcode=$?
-                        if [[ 0 != ${errcode} ]]; then
-                            log "auto swap node failed, errcode=${errcode}"
-                            killed
-                        fi
-                    fi
-                # restarting thormaster
-                ;;
         # TEC_SlaveInit
         4)  log "Thormaster ($thorpid) Slaves failed to initialize"
             log "Shutting down"
             killed
             ;;
-        *)  log "Thormaster ($thorpid) Unknown error code.  Stopping"
-            killed
+        # TEC_Idle=2, TEC_Watchdog=3, TEC_Swap=5, TEC_DaliDown=6
+        *)  if [[ $errcode -eq 2 ]]; then
+                log "Thormaster ($thorpid) Idle"
+            elif [[ $errcode -eq 3 ]]; then
+                log "Thormaster ($thorpid) Lost connection to slave(s)"
+            elif [[ $errcode -eq 5 ]]; then
+                log "Thormaster ($thorpid) Swap node required"
+            elif [[ $errcode -eq 6 ]]; then
+                log "Thormaster ($thorpid) Unable to connect to Dali"
+            else
+                log "Thormaster ($thorpid) Unknown error code: [$errcode]"
+            fi
+            log "Stopping thorslave(s) for restart"
+            kill_slaves
+            if [[ 0 != $autoSwapNode ]]; then
+                log "Running autoswap $THORNAME :: ($thorpid)"
+                swapnode auto $DALISERVER $component
+                errcode=$?
+                if [[ 0 != ${errcode} ]]; then
+                    log "auto swap node failed, errcode=${errcode}"
+                    killed
+                fi
+            fi
+            # restarting thormaster
             ;;
         esac
     else

plugins/unicodelib/unicodelib.cpp

@@ -614,14 +614,6 @@ unsigned unicodeEditDistanceV4(UnicodeString & left, UnicodeString & right, unsi
     unsigned leftLen = left.length();
     unsigned rightLen = right.length();
 
-    // this shortcut is not applicable in the bi mode because unicode characters could take more than 2 UChars
-    if (!bi)
-    {
-        unsigned minED = (leftLen < rightLen)? rightLen - leftLen: leftLen - rightLen;
-        if (minED > radius)
-            return minED>255?255:minED;
-    }
-
     if (leftLen > 255)
         leftLen = 255;
 
@@ -640,9 +632,14 @@ unsigned unicodeEditDistanceV4(UnicodeString & left, UnicodeString & right, unsi
     if (leftCs.isInvalid() || rightCs.isInvalid())
         return DISTANCE_ON_ERROR;
 
+    // get Unicode character lengths
     leftLen = leftCs.length();
     rightLen = rightCs.length();
 
+    unsigned minED = (leftLen < rightLen)? rightLen - leftLen: leftLen - rightLen;
+    if (minED > radius)
+        return minED;
+
     /*
     This function applies two optimizations over the function above.
     a) Adding a character (next row) can at most decrease the edit distance by 1, so short circuit when