Merge pull request #4983 from afishbeck/empty-xpath2

HPCC-10032 Add schema generator support for xpath('')

Reviewed-by: Gavin Halliday <ghalliday@hpccsystems.com>

common/deftype/deftype.cpp

@@ -3588,7 +3588,17 @@ static bool alreadyHadSize(int size, IntArray &sizes)
     return false;
 }
 
-void XmlSchemaBuilder::addSchemaPrefix()
+StringBuffer &appendStartComplexType(StringBuffer &xml, bool hasMixedContent, unsigned *updatePos)
+{
+    xml.append("<xs:complexType");
+    if (hasMixedContent || updatePos)
+        xml.append(" mixed=\"").append(hasMixedContent ? '1' : '0').append('\"');
+    if (updatePos)
+        *updatePos = xml.length()-2;
+    return xml.append('>');
+}
+
+void XmlSchemaBuilder::addSchemaPrefix(bool hasMixedContent)
 {
     if (addHeader)
         xml.append("<?xml version=\"1.0\" encoding=\"UTF-8\"?>");
@@ -3597,9 +3607,8 @@ void XmlSchemaBuilder::addSchemaPrefix()
         "<xs:element name=\"Dataset\">"
             "<xs:complexType>"
                 "<xs:sequence minOccurs=\"0\" maxOccurs=\"unbounded\">\n"
-                    "<xs:element name=\"Row\">"
-                        "<xs:complexType>"
-                            "<xs:sequence>\n");
+                    "<xs:element name=\"Row\">");
+    appendStartComplexType(xml, hasMixedContent, NULL).append("<xs:sequence>\n");
     attributes.append(*new StringBufferItem);
 }
 
@@ -3750,6 +3759,9 @@ void XmlSchemaBuilder::addField(const char * name, ITypeInfo & type)
     if (xml.length() == 0)
         addSchemaPrefix();
 
+    if (!*name)
+        return;
+
     if (*name == '@')
     {
         if (attributes.length())
@@ -3764,10 +3776,13 @@ void XmlSchemaBuilder::addSetField(const char * name, const char * itemname, ITy
     if (xml.length() == 0)
         addSchemaPrefix();
 
+    if (!name || !*name) //xpath('') content inherited by parent
+        return;
+
     StringBuffer elementType;
     getXmlTypeName(elementType, *type.queryChildType());
 
-    if ((!itemname || !*itemname) && name) // xpaths 'Name', '/Name', and 'Name/' seem to be equivalent
+    if (!itemname || !*itemname) // xpaths 'Name', '/Name', and 'Name/' seem to be equivalent
     {
         itemname = name;
         name = NULL;
@@ -3789,12 +3804,12 @@ void XmlSchemaBuilder::addSetField(const char * name, const char * itemname, ITy
         xml.append("</xs:sequence></xs:complexType></xs:element>").newline();
 }
 
-void XmlSchemaBuilder::beginRecord(const char * name)
+void XmlSchemaBuilder::beginRecord(const char * name, bool hasMixedContent, unsigned *updatePos)
 {
     if (!name || !*name)
         return;
     if (xml.length() == 0)
-        addSchemaPrefix();
+        addSchemaPrefix(hasMixedContent);
 
     attributes.append(*new StringBufferItem);
 
@@ -3802,11 +3817,18 @@ void XmlSchemaBuilder::beginRecord(const char * name)
     if (optionalNesting)
         xml.append(" minOccurs=\"0\"");
     xml.append(">").newline();
-    xml.append("<xs:complexType><xs:sequence>").newline();
+    appendStartComplexType(xml, hasMixedContent, updatePos);
+    xml.append("<xs:sequence>").newline();
     nesting.append(optionalNesting);
     optionalNesting = 0;
 }
 
+void XmlSchemaBuilder::updateMixedRecord(unsigned updatePos, bool hasMixedContent)
+{
+    if (updatePos)
+        xml.setCharAt(updatePos, hasMixedContent ? '1' : '0');
+}
+
 void XmlSchemaBuilder::endRecord(const char * name)
 {
     if (!name || !*name)
@@ -3819,7 +3841,7 @@ void XmlSchemaBuilder::endRecord(const char * name)
     optionalNesting = nesting.pop();
 }
 
-bool XmlSchemaBuilder::beginDataset(const char * name, const char * row)
+bool XmlSchemaBuilder::beginDataset(const char * name, const char * row, bool hasMixedContent, unsigned *updatePos)
 {
     if (xml.length() == 0)
         addSchemaPrefix();
@@ -3838,7 +3860,11 @@ bool XmlSchemaBuilder::beginDataset(const char * name, const char * row)
         else if (optionalNesting)
             xml.append(" minOccurs=\"0\"");
         xml.append(">").newline();
-        xml.append("<xs:complexType>").newline();
+        if (row && *row)
+            appendStartComplexType(xml, false, NULL);
+        else
+            appendStartComplexType(xml, hasMixedContent, updatePos);
+        xml.newline();
     }
 
     xml.append("<xs:sequence");
@@ -3849,7 +3875,9 @@ bool XmlSchemaBuilder::beginDataset(const char * name, const char * row)
     if (row && *row)
     {
         attributes.append(*new StringBufferItem);
-        xml.append("<xs:element name=\"").append(row).append("\"><xs:complexType><xs:sequence>").newline();
+        xml.append("<xs:element name=\"").append(row).append("\">").newline();
+        appendStartComplexType(xml, hasMixedContent, updatePos);
+        xml.append("<xs:sequence>").newline();
     }
     nesting.append(optionalNesting);
     optionalNesting = 0;

common/deftype/deftype.hpp

@@ -365,8 +365,9 @@ public:
     virtual void addField(const char * name, ITypeInfo & type) = 0;
     virtual void addSetField(const char * name, const char * itemname, ITypeInfo & type) = 0;
     virtual void beginIfBlock() = 0;
-    virtual bool beginDataset(const char * name, const char * childname) = 0;
-    virtual void beginRecord(const char * name) = 0;
+    virtual bool beginDataset(const char * name, const char * childname, bool hasMixedContent, unsigned *updateMixed) = 0;
+    virtual void beginRecord(const char * name, bool hasMixedContent, unsigned *updateMixed) = 0;
+    virtual void updateMixedRecord(unsigned updateToken, bool hasMixedContent) = 0;
     virtual void endIfBlock() = 0;
     virtual void endDataset(const char * name, const char * childname) = 0;
     virtual void endRecord(const char * name) = 0;
@@ -381,8 +382,9 @@ public:
     virtual void addField(const char * name, ITypeInfo & type);
     virtual void addSetField(const char * name, const char * itemname, ITypeInfo & type);
     virtual void beginIfBlock()                                     { optionalNesting++; }
-    virtual bool beginDataset(const char * name, const char * childname);
-    virtual void beginRecord(const char * name);
+    virtual bool beginDataset(const char * name, const char * childname, bool hasMixedContent, unsigned *updateMixed);
+    virtual void beginRecord(const char * name, bool hasMixedContent, unsigned *updateMixed);
+    virtual void updateMixedRecord(unsigned updateToken, bool hasMixedContent);
     virtual void endIfBlock()                                       { optionalNesting--; }
     virtual void endDataset(const char * name, const char * childname);
     virtual void endRecord(const char * name);
@@ -392,7 +394,7 @@ public:
     void getXml(IStringVal & results);
 
 private:
-    void addSchemaPrefix();
+    void addSchemaPrefix(bool hasMixedContent=false);
     void addSchemaSuffix();
     void clear();
     void getXmlTypeName(StringBuffer & xmlType, ITypeInfo & type);

common/fileview2/fvdatasource.hpp

@@ -52,6 +52,8 @@ interface IFvDataSourceMetaData : extends IInterface
     virtual const char *queryXmlTag() const = 0;
     virtual const IntArray &queryAttrList() const = 0;
     virtual const IntArray &queryAttrList(unsigned column) const = 0;
+    virtual bool mixedContent(unsigned column) const = 0;
+    virtual bool mixedContent() const = 0;
 };
 
 IFvDataSourceMetaData * deserializeDataSourceMeta(MemoryBuffer & in);

common/fileview2/fvrelate.cpp

@@ -720,7 +720,7 @@ void ViewERdiagramVisitor::beginIfBlock()
     activeFieldCount++;
 }
 
-bool ViewERdiagramVisitor::beginDataset(const char * name, const char * childname)
+bool ViewERdiagramVisitor::beginDataset(const char * name, const char * childname, bool mixed, unsigned *updateMixed)
 {
     noteNextField();
     builder.noteField(activeFieldId.str(), name, "dataset");
@@ -731,7 +731,7 @@ bool ViewERdiagramVisitor::beginDataset(const char * name, const char * childnam
     return false;       // don't expand contents
 }
 
-void ViewERdiagramVisitor::beginRecord(const char * name)
+void ViewERdiagramVisitor::beginRecord(const char * name, bool mixedContent, unsigned *updateMixed)
 {
     noteNextField();
     builder.noteField(activeFieldId.str(), name, "record");

common/fileview2/fvrelate.ipp

@@ -340,8 +340,9 @@ public:
     virtual void addField(const char * name, ITypeInfo & type);
     virtual void addSetField(const char * name, const char * itemname, ITypeInfo & type);
     virtual void beginIfBlock();
-    virtual bool beginDataset(const char * name, const char * childname);
-    virtual void beginRecord(const char * name);
+    virtual bool beginDataset(const char * name, const char * childname, bool mixedContent, unsigned *updateMixed);
+    virtual void beginRecord(const char * name, bool mixedContent, unsigned *updateMixed);
+    virtual void updateMixedRecord(unsigned updateMixed, bool mixed){}
     virtual void endIfBlock();
     virtual void endDataset(const char * name, const char * childname);
     virtual void endRecord(const char * name);

common/fileview2/fvresultset.cpp

@@ -589,7 +589,7 @@ ITypeInfo * containsSingleSimpleFieldBlankXPath(IResultSetMetaData * meta)
 
 void fvSplitXPath(const char *xpath, StringBuffer &s, const char *&name, const char **childname=NULL)
 {
-    if (!xpath || !*xpath)
+    if (!xpath)
         return;
     const char * slash = strchr(xpath, '/');
     if (!slash)
@@ -628,7 +628,7 @@ void CResultSetMetaData::getXmlSchema(ISchemaBuilder & builder, bool useXPath) c
         case FVFFbeginrecord:
             if (useXPath)
                 fvSplitXPath(meta->queryXPath(idx), xname, name);
-            builder.beginRecord(name);
+            builder.beginRecord(name, meta->mixedContent(idx), NULL);
             break;
         case FVFFendrecord:
             if (useXPath)
@@ -643,9 +643,10 @@ void CResultSetMetaData::getXmlSchema(ISchemaBuilder & builder, bool useXPath) c
                 ITypeInfo * singleFieldType = (useXPath && name && *name && childname && *childname) ? containsSingleSimpleFieldBlankXPath(column.childMeta.get()) : NULL;
                 if (!singleFieldType || !builder.addSingleFieldDataset(name, childname, *singleFieldType))
                 {
-                    if (builder.beginDataset(name, childname))
+                    const CResultSetMetaData *childMeta = static_cast<const CResultSetMetaData *>(column.childMeta.get());
+                    if (builder.beginDataset(name, childname, childMeta->meta->mixedContent(), NULL))
                     {
-                        static_cast<const CResultSetMetaData *>(column.childMeta.get())->getXmlSchema(builder, useXPath);
+                        childMeta->getXmlSchema(builder, useXPath);
                     }
                     builder.endDataset(name, childname);
                 }

common/fileview2/fvsource.cpp

@@ -43,7 +43,7 @@ inline void appendNextXpathName(StringBuffer &s, const char *&xpath)
 
 void splitXmlTagNamesFromXPath(const char *xpath, StringAttr &inner, StringAttr *outer=NULL)
 {
-    if (!xpath || !xpath)
+    if (!xpath || !*xpath)
         return;
 
     StringBuffer s1;
@@ -75,11 +75,13 @@ DataSourceMetaItem::DataSourceMetaItem(unsigned _flags, const char * _name, cons
     type.set(_type);
     xpath.set(_xpath);
     splitXmlTagNamesFromXPath(_xpath, tagname);
+    hasMixedContent = false;
 }
 
 DataSourceMetaItem::DataSourceMetaItem(unsigned _flags, MemoryBuffer & in)
 {
     flags = _flags;
+    in.read(hasMixedContent);
     in.read(name);
     in.read(xpath);
     type.setown(deserializeType(in));
@@ -90,6 +92,7 @@ DataSourceMetaItem::DataSourceMetaItem(unsigned _flags, MemoryBuffer & in)
 void DataSourceMetaItem::serialize(MemoryBuffer & out) const
 {
     out.append(flags);
+    out.append(hasMixedContent);
     out.append(name);
     out.append(xpath);
     type->serialize(out);
@@ -128,7 +131,7 @@ DataSourceSetItem::DataSourceSetItem(const char * _name, const char * _xpath, IT
     xpath.set(_xpath);
     StringBuffer attr;
     splitXmlTagNamesFromXPath(_xpath, record.tagname, &tagname);
-    if (!record.tagname.length())
+    if (!_xpath)
         record.tagname.set("Item");
 }
 
@@ -136,7 +139,7 @@ DataSourceSetItem::DataSourceSetItem(unsigned flags, MemoryBuffer & in) : DataSo
 {
     createChild();
     splitXmlTagNamesFromXPath(xpath.get(), record.tagname, &tagname);
-    if (!record.tagname.length())
+    if (!record.tagname.get())
         record.tagname.set("Item");
 }
 
@@ -165,7 +168,7 @@ DataSourceMetaData::DataSourceMetaData(IHqlExpression * _record, byte _numFields
     maxRecordSize = ::getMaxRecordSize(_record, MAX_RECORD_SIZE);
     keyedSize = _keyedSize;
 
-    gatherFields(_record, false);
+    gatherFields(_record, false, &hasMixedContent);
     if (_isGrouped)
     {
         Owned<ITypeInfo> type = makeBoolType();
@@ -208,6 +211,7 @@ void DataSourceMetaData::init()
     isStoredFixedWidth = false;
     randomIsOk = false;
     numFieldsToIgnore = 0;
+    hasMixedContent = false;
 }
 
 DataSourceMetaData::DataSourceMetaData(MemoryBuffer & buffer)
@@ -215,6 +219,7 @@ DataSourceMetaData::DataSourceMetaData(MemoryBuffer & buffer)
     numVirtualFields = 0;
     buffer.read(numFieldsToIgnore);
     buffer.read(randomIsOk);
+    buffer.read(hasMixedContent);
     buffer.read(isStoredFixedWidth);
     buffer.read(minRecordSize);
     buffer.read(keyedSize);
@@ -329,12 +334,12 @@ void DataSourceMetaData::extractKeyedInfo(UnsignedArray & offsets, TypeInfoArray
 }
 
 //MORE: Really this should create no_selects for the sub records, but pass on that for the moment.
-void DataSourceMetaData::gatherFields(IHqlExpression * expr, bool isConditional)
+void DataSourceMetaData::gatherFields(IHqlExpression * expr, bool isConditional, bool *pMixedContent)
 {
     switch (expr->getOperator())
     {
     case no_record:
-        gatherChildFields(expr, isConditional);
+        gatherChildFields(expr, isConditional, pMixedContent);
         break;
     case no_ifblock:
         {
@@ -342,7 +347,7 @@ void DataSourceMetaData::gatherFields(IHqlExpression * expr, bool isConditional)
             OwnedITypeInfo voidType = makeVoidType();
             isStoredFixedWidth = false;
             fields.append(*new DataSourceMetaItem(FVFFbeginif, NULL, NULL, boolType));
-            gatherChildFields(expr->queryChild(1), true);
+            gatherChildFields(expr->queryChild(1), true, pMixedContent);
             fields.append(*new DataSourceMetaItem(FVFFendif, NULL, NULL, voidType));
             break;
         }
@@ -371,18 +376,26 @@ void DataSourceMetaData::gatherFields(IHqlExpression * expr, bool isConditional)
             if (tc == type_row)
             {
                 OwnedITypeInfo voidType = makeVoidType();
-                fields.append(*new DataSourceMetaItem(FVFFbeginrecord, outname, xpath, voidType));
-                gatherChildFields(expr->queryRecord(), isConditional);
+                Owned<DataSourceMetaItem> begin = new DataSourceMetaItem(FVFFbeginrecord, outname, xpath, voidType);
+                //inherit mixed content from child row with xpath('')
+                bool *pItemMixedContent = (pMixedContent && xpath && !*xpath) ? pMixedContent : &(begin->hasMixedContent);
+                fields.append(*begin.getClear());
+                gatherChildFields(expr->queryRecord(), isConditional, pItemMixedContent);
                 fields.append(*new DataSourceMetaItem(FVFFendrecord, outname, xpath, voidType));
             }
             else if ((tc == type_dictionary) || (tc == type_table) || (tc == type_groupedtable))
             {
                 isStoredFixedWidth = false;
-                fields.append(*new DataSourceDatasetItem(outname, xpath, expr));
+                Owned<DataSourceDatasetItem> ds = new DataSourceDatasetItem(outname, xpath, expr);
+                if (pMixedContent && xpath && !*xpath)
+                    *pMixedContent = ds->queryChildMeta()->hasMixedContent;
+                fields.append(*ds.getClear());
             }
             else if (tc == type_set)
             {
                 isStoredFixedWidth = false;
+                if (pMixedContent && xpath && !*xpath)
+                    *pMixedContent = true;
                 fields.append(*new DataSourceSetItem(outname, xpath, type));
             }
             else
@@ -392,6 +405,8 @@ void DataSourceMetaData::gatherFields(IHqlExpression * expr, bool isConditional)
                     IHqlAlienTypeInfo * alien = queryAlienType(type);
                     type.set(alien->queryPhysicalType());
                 }
+                if (pMixedContent && xpath && !*xpath)
+                    *pMixedContent = true;
                 addSimpleField(outname, xpath, type);
             }
             break;
@@ -400,11 +415,11 @@ void DataSourceMetaData::gatherFields(IHqlExpression * expr, bool isConditional)
 }
 
 
-void DataSourceMetaData::gatherChildFields(IHqlExpression * expr, bool isConditional)
+void DataSourceMetaData::gatherChildFields(IHqlExpression * expr, bool isConditional, bool *pMixedContent)
 {
     bitsRemaining = 0;
     ForEachChild(idx, expr)
-        gatherFields(expr->queryChild(idx), isConditional);
+        gatherFields(expr->queryChild(idx), isConditional, pMixedContent);
 }
 
 unsigned DataSourceMetaData::numKeyedColumns() const
@@ -450,6 +465,16 @@ unsigned DataSourceMetaData::queryFieldFlags(unsigned column) const
     return fields.item(column).flags;
 }
 
+bool DataSourceMetaData::mixedContent(unsigned column) const
+{
+    return fields.item(column).hasMixedContent;
+}
+
+bool DataSourceMetaData::mixedContent() const
+{
+    return hasMixedContent;
+}
+
 const char * DataSourceMetaData::queryName(unsigned column) const
 {
     return fields.item(column).name;
@@ -539,6 +564,7 @@ void DataSourceMetaData::serialize(MemoryBuffer & buffer) const
     //NB: Update NullDataSourceMeta if this changes....
     buffer.append(numFieldsToIgnore);
     buffer.append(randomIsOk);
+    buffer.append(hasMixedContent);
     buffer.append(isStoredFixedWidth);
     buffer.append(minRecordSize);
     buffer.append(keyedSize);

common/fileview2/fvsource.ipp

@@ -87,6 +87,7 @@ public:
     IntArray nestedAttributes;
     OwnedITypeInfo type;
     byte           flags;
+    bool hasMixedContent;
 };
 
 class DataSourceMetaData : public CInterface, implements IFvDataSourceMetaData, public IRecordSizeEx
@@ -111,6 +112,8 @@ public:
     virtual const char *queryXmlTag() const;
     virtual const IntArray &queryAttrList() const;
     virtual const IntArray &queryAttrList(unsigned column) const;
+    virtual bool mixedContent(unsigned column) const;
+    virtual bool mixedContent() const;
 
 
     virtual IFvDataSourceMetaData * queryChildMeta(unsigned column) const;
@@ -139,8 +142,8 @@ public:
 
 protected:
     void addSimpleField(const char * name, const char * xpath, ITypeInfo * type);
-    void gatherFields(IHqlExpression * expr, bool isConditional);
-    void gatherChildFields(IHqlExpression * expr, bool isConditional);
+    void gatherFields(IHqlExpression * expr, bool isConditional, bool *pMixedContent);
+    void gatherChildFields(IHqlExpression * expr, bool isConditional, bool *pMixedContent);
     void gatherAttributes();
     void gatherNestedAttributes(DataSourceMetaItem &rec, aindex_t &idx);
     void init();
@@ -155,6 +158,7 @@ protected:
     unsigned numVirtualFields;
     bool isStoredFixedWidth;
     bool randomIsOk;
+    bool hasMixedContent;
     byte numFieldsToIgnore;
     StringAttr tagname;
 };

ecl/hql/hqlutil.cpp

@@ -7473,7 +7473,9 @@ public:
     {
     }
 
-    void build(IHqlExpression * record) const;
+    void build(IHqlExpression * record, bool &hasMixedContent) const;
+    void build(IHqlExpression * record) const {bool mixed; build(record, mixed);}
+
 
 protected:
     void extractName(StringBuffer & name, StringBuffer * itemName, StringBuffer * valueName, IHqlExpression * field, const char * defaultItemName) const;
@@ -7485,7 +7487,7 @@ protected:
 
 
 
-void EclXmlSchemaBuilder::build(IHqlExpression * record) const
+void EclXmlSchemaBuilder::build(IHqlExpression * record, bool &hasMixedContent) const
 {
     StringBuffer name, childName;
     ForEachChild(i, record)
@@ -7502,15 +7504,22 @@ void EclXmlSchemaBuilder::build(IHqlExpression * record) const
                 case type_row:
                     {
                         extractName(name.clear(), NULL, NULL, cur, NULL);
-                        builder.beginRecord(name);
-                        build(cur->queryRecord());
+                        unsigned updateMixed=0;
+                        builder.beginRecord(name, false, &updateMixed);
+                        bool mixed = false;
+                        build(cur->queryRecord(), (name.length()) ? mixed : hasMixedContent);
+                        if (mixed)
+                            builder.updateMixedRecord(updateMixed, true);
                         builder.endRecord(name);
                         break;
                     }
                 case type_set:
                     {
                         extractName(name.clear(), &childName.clear(), NULL, cur, "Item");
-                        builder.addSetField(name, childName, *type);
+                        if (name.length())
+                            builder.addSetField(name, childName, *type);
+                        else
+                            hasMixedContent = true;
                         break;
                     }
                 case type_dictionary:
@@ -7521,8 +7530,14 @@ void EclXmlSchemaBuilder::build(IHqlExpression * record) const
                         ITypeInfo * singleFieldType = (useXPath && name.length() && childName.length()) ? containsSingleSimpleFieldBlankXPath(cur->queryRecord()) : NULL;
                         if (!singleFieldType || !builder.addSingleFieldDataset(name, childName, *singleFieldType))
                         {
-                            if (builder.beginDataset(name, childName))
-                                build(cur->queryRecord());
+                            unsigned updateMixed = 0;
+                            bool mixed = false;
+                            if (builder.beginDataset(name, childName, false, &updateMixed))
+                            {
+                                build(cur->queryRecord(), (name.length()) ? mixed : hasMixedContent);
+                                if (mixed)
+                                    builder.updateMixedRecord(updateMixed, true);
+                            }
                             builder.endDataset(name, childName);
                         }
                         break;
@@ -7531,18 +7546,21 @@ void EclXmlSchemaBuilder::build(IHqlExpression * record) const
                     type = queryAlienType(type)->queryLogicalType();
                 default:
                     extractName(name.clear(), NULL, NULL, cur, NULL);
-                    builder.addField(name, *type);
+                    if (name.length())
+                        builder.addField(name, *type);
+                    else
+                        hasMixedContent = true;
                     break;
                 }
                 break;
             }
         case no_ifblock:
             builder.beginIfBlock();
-            build(cur->queryChild(1));
+            build(cur->queryChild(1), hasMixedContent);
             builder.endIfBlock();
             break;
         case no_record:
-            build(cur);
+            build(cur, hasMixedContent);
             break;
         }
     }

esp/services/ws_ecl/ws_ecl_service.cpp

@@ -2603,7 +2603,7 @@ int CWsEclBinding::getWsEclDefinition(CHttpRequest* request, CHttpResponse* resp
 
             Owned<IPropertyTree> xsds_tree;
             if (xsds.length())
-                xsds_tree.setown(createPTreeFromXMLString(xsds.str()));
+                xsds_tree.setown(createPTreeFromXMLString(xsds.str(), ipt_ordered));
             if (xsds_tree)
             {
                 StringBuffer xpath;