Merge pull request #13459 from afishbeck/yamlRootArray

HPCC-23668 Improve JSON/YAML support for root level arrays

Reviewed-by: Gavin Halliday <ghalliday@hpccsystems.com>

system/jlib/jptree.cpp

@@ -5373,18 +5373,18 @@ IPropertyTree *createPTreeFromXMLString(unsigned len, const char *xml, byte flag
 //////////////////////////
 /////////////////////////
 
-inline bool checkSanitizedHide(const char *val)
+inline bool isHiddenWhenSanitized(const char *val)
 {
     if (!val || !*val)
         return false;
     return !(streq(val, "0") || streq(val, "1") || strieq(val, "true") || strieq(val, "false") || strieq(val, "yes") || strieq(val, "no"));
 }
 
-inline bool checkSanitizeFlagsAndHide(const char *val, byte flags, bool attribute)
+inline bool isSanitizedAndHidden(const char *val, byte flags, bool attribute)
 {
     bool sanitize = (attribute) ? ((flags & YAML_SanitizeAttributeValues)!=0) : ((flags & YAML_Sanitize)!=0);
     if (sanitize)
-        return checkSanitizedHide(val);
+        return isHiddenWhenSanitized(val);
     return false;
 }
 
@@ -5431,7 +5431,7 @@ static void _toXML(const IPropertyTree *tree, IIOStream &out, unsigned indent, u
                 const char *val = it->queryValue();
                 if (val)
                 {
-                    if (checkSanitizeFlagsAndHide(val, flags, true))
+                    if (isSanitizedAndHidden(val, flags, true))
                         writeCharsNToStream(out, '*', strlen(val));
                     else
                         encodeXML(val, out, ENCODE_NEWLINES, (unsigned)-1, true);
@@ -5494,7 +5494,7 @@ static void _toXML(const IPropertyTree *tree, IIOStream &out, unsigned indent, u
             // NOTE - we don't output anything for binary.... is that ok?
             if (thislevel)
             {
-                if (checkSanitizedHide(thislevel))
+                if (isHiddenWhenSanitized(thislevel))
                     writeCharsNToStream(out, '*', strlen(thislevel));
                 else
                     writeStringToStream(out, thislevel);
@@ -5692,7 +5692,10 @@ static void writeJSONBase64ValueToStream(IIOStream &out, const char *val, size32
         JBASE64_Encode(val, len, out, false);
     writeCharToStream(out, '"');
 }
-
+bool isRootArrayObjectHidden(bool root, const char *name, byte flags)
+{
+    return ((flags & JSON_HideRootArrayObject) && root && name && streq(name,"__array__"));
+}
 static void _toJSON(const IPropertyTree *tree, IIOStream &out, unsigned indent, byte flags, bool &delimit, bool root=false, bool isArrayItem=false)
 {
     Owned<IAttributeIterator> it = tree->getAttributes(true);
@@ -5716,30 +5719,34 @@ static void _toJSON(const IPropertyTree *tree, IIOStream &out, unsigned indent,
         writeCharsNToStream(out, ' ', indent);
     }
 
-    if (root || complex)
+    bool hiddenRootArrayObject = isRootArrayObjectHidden(root, name, flags);
+    if (!hiddenRootArrayObject)
     {
-        writeCharToStream(out, '{');
-        delimit = false;
-    }
+        if (root || complex)
+        {
+            writeCharToStream(out, '{');
+            delimit = false;
+        }
 
-    if (hasAttributes)
-    {
-        ForEach(*it)
+        if (hasAttributes)
         {
-            const char *key = it->queryName();
-            if (!isBinary || stricmp(key, "@xsi:type")!=0)
+            ForEach(*it)
             {
-                const char *val = it->queryValue();
-                if (val)
+                const char *key = it->queryName();
+                if (!isBinary || stricmp(key, "@xsi:type")!=0)
                 {
-                    writeJSONNameToStream(out, key, (flags & JSON_Format) ? indent+1 : 0, delimit);
-                    if (flags & JSON_SanitizeAttributeValues)
-                        writeJSONValueToStream(out, val, delimit, checkSanitizedHide(val));
-                    else
+                    const char *val = it->queryValue();
+                    if (val)
                     {
-                        StringBuffer encoded;
-                        encodeJSON(encoded, val);
-                        writeJSONValueToStream(out, encoded.str(), delimit);
+                        writeJSONNameToStream(out, key, (flags & JSON_Format) ? indent+1 : 0, delimit);
+                        if (flags & JSON_SanitizeAttributeValues)
+                            writeJSONValueToStream(out, val, delimit, isHiddenWhenSanitized(val));
+                        else
+                        {
+                            StringBuffer encoded;
+                            encodeJSON(encoded, val);
+                            writeJSONValueToStream(out, encoded.str(), delimit);
+                        }
                     }
                 }
             }
@@ -5748,30 +5755,33 @@ static void _toJSON(const IPropertyTree *tree, IIOStream &out, unsigned indent,
     MemoryBuffer thislevelbin;
     StringBuffer _thislevel;
     const char *thislevel = NULL; // to avoid uninitialized warning
-    bool isNull;
-    if (isBinary)
+    bool isNull = true;
+    if (!hiddenRootArrayObject)
     {
-        isNull = (!tree->getPropBin(NULL, thislevelbin))||(thislevelbin.length()==0);
-    }
-    else
-    {
-        if (tree->isCompressed(NULL))
+        if (isBinary)
         {
-            isNull = false; // can't be empty if compressed;
-            verifyex(tree->getProp(NULL, _thislevel));
-            thislevel = _thislevel.str();
+            isNull = (!tree->getPropBin(NULL, thislevelbin))||(thislevelbin.length()==0);
         }
         else
-            isNull = (NULL == (thislevel = tree->queryProp(NULL)));
-    }
+        {
+            if (tree->isCompressed(NULL))
+            {
+                isNull = false; // can't be empty if compressed;
+                verifyex(tree->getProp(NULL, _thislevel));
+                thislevel = _thislevel.str();
+            }
+            else
+                isNull = (NULL == (thislevel = tree->queryProp(NULL)));
+        }
 
-    if (isNull && !root && !complex)
-    {
-        writeJSONValueToStream(out, NULL, delimit);
-        return;
+        if (isNull && !root && !complex)
+        {
+            writeJSONValueToStream(out, NULL, delimit);
+            return;
+        }
     }
 
-    Owned<IPropertyTreeIterator> sub = tree->getElements("*", 0 != (flags & JSON_SortTags) ? iptiter_sort : iptiter_null);
+    Owned<IPropertyTreeIterator> sub = tree->getElements(hiddenRootArrayObject ? "__item__" : "*", 0 != (flags & JSON_SortTags) ? iptiter_sort : iptiter_null);
     //note that detection of repeating elements relies on the fact that ptree elements
     //of the same name will be grouped together
     bool repeatingElement = false;
@@ -5780,14 +5790,24 @@ static void _toJSON(const IPropertyTree *tree, IIOStream &out, unsigned indent,
     {
         Linked<IPropertyTree> element = &sub->query();
         const char *name = element->queryName();
-        if (sub->next() && !repeatingElement && streq(name, sub->query().queryName()))
+        sub->next();
+        if (!repeatingElement)
         {
-            if (flags & JSON_Format)
-                indent++;
-            writeJSONNameToStream(out, name, (flags & JSON_Format) ? indent : 0, delimit);
-            writeCharToStream(out, '[');
-            repeatingElement = true;
-            delimit = false;
+            if (hiddenRootArrayObject)
+            {
+                writeCharToStream(out, '[');
+                repeatingElement = true;
+                delimit = false;
+            }
+            else if (sub->isValid() && streq(name, sub->query().queryName()))
+            {
+                if (flags & JSON_Format)
+                    indent++;
+                writeJSONNameToStream(out, name, (flags & JSON_Format) ? indent : 0, delimit);
+                writeCharToStream(out, '[');
+                repeatingElement = true;
+                delimit = false;
+            }
         }
 
         _toJSON(element, out, indent+1, flags, delimit, false, repeatingElement);
@@ -5807,7 +5827,7 @@ static void _toJSON(const IPropertyTree *tree, IIOStream &out, unsigned indent,
     }
 
 
-    if (!isNull)
+    if (!hiddenRootArrayObject && !isNull)
     {
         if (complex)
             writeJSONNameToStream(out, isBinary ? "#valuebin" : "#value", (flags & JSON_Format) ? indent+1 : 0, delimit);
@@ -5815,19 +5835,22 @@ static void _toJSON(const IPropertyTree *tree, IIOStream &out, unsigned indent,
             writeJSONBase64ValueToStream(out, thislevelbin.toByteArray(), thislevelbin.length(), delimit, flags & XML_Sanitize);
         else
         {
-            writeJSONValueToStream(out, thislevel, delimit, checkSanitizeFlagsAndHide(thislevel, flags, false));
+            writeJSONValueToStream(out, thislevel, delimit, isSanitizedAndHidden(thislevel, flags, false));
         }
     }
 
-    if (root || complex)
+    if (!hiddenRootArrayObject)
     {
-        if (flags & JSON_Format)
+        if (root || complex)
         {
-            writeCharToStream(out, '\n');
-            writeCharsNToStream(out, ' ', indent);
+            if (flags & JSON_Format)
+            {
+                writeCharToStream(out, '\n');
+                writeCharsNToStream(out, ' ', indent);
+            }
+            writeCharToStream(out, '}');
+            delimit = true;
         }
-        writeCharToStream(out, '}');
-        delimit = true;
     }
 }
 
@@ -5845,6 +5868,21 @@ void toJSON(const IPropertyTree *tree, IIOStream &out, unsigned indent, byte fla
     _toJSON(tree, out, indent, flags, delimit, true);
 }
 
+void printJSON(const IPropertyTree *tree, unsigned indent, byte flags)
+{
+    StringBuffer json;
+    toJSON(tree, json, indent, flags);
+    printf("%s", json.str());
+}
+
+void dbglogJSON(const IPropertyTree *tree, unsigned indent, unsigned flags)
+{
+    StringBuffer json;
+    toJSON(tree, json, indent, flags);
+    DBGLOG("%s", json.str());
+}
+
+
 static inline void skipWS(const char *&xpath)
 {
     while (isspace(*xpath)) xpath++;
@@ -7912,7 +7950,6 @@ class CYAMLBufferReader : public CInterfaceOf<IPTreeReader>
 protected:
     Linked<IPTreeNotifyEvent> iEvent;
     yaml_parser_t parser;
-    StringAttr rootTag;
     PTreeReaderOptions readerOptions = ptr_none;
     bool noRoot = false;
 
@@ -7924,8 +7961,6 @@ public:
             throw makeStringException(99, "Filed to initialize libyaml parser");
         yaml_parser_set_input_string(&parser, (const unsigned char *)buf, bufLength);
         noRoot = 0 != ((unsigned)readerOptions & (unsigned)ptr_noRoot);
-        if (!noRoot)
-            rootTag.set("__object__"); //may support _array_ option later
     }
     ~CYAMLBufferReader()
     {
@@ -8093,18 +8128,18 @@ public:
                 //root content, the start of all mappings, should be only one at the root
                 if (content)
                     throw makeStringException(99, "YAML: Currently only support one content section (map) per stream");
-                loadMap(rootTag); //root map
+                loadMap(noRoot ? nullptr : "__object__"); //root map
                 content=true;
                 break;
             case YAML_SEQUENCE_START_EVENT:
                 //root content, sequence (array), should be only one at the root and can't mix with mappings
                 if (content)
                     throw makeStringException(99, "YAML: Currently only support one content section (sequence) per stream");
-                if (rootTag.length())
-                    iEvent->beginNode(rootTag, 0);
-                loadSequence("Row");
-                if (rootTag.length())
-                    iEvent->endNode(rootTag, 0, nullptr, false, parser.offset);
+                if (!noRoot)
+                    iEvent->beginNode("__array__", 0);
+                loadSequence("__item__");
+                if (!noRoot)
+                    iEvent->endNode("__array__", 0, nullptr, false, parser.offset);
                 content=true;
                 break;
             case YAML_STREAM_START_EVENT:
@@ -8321,46 +8356,55 @@ static void _toYAML(const IPropertyTree *tree, YAMLEmitter &yaml, byte flags, bo
     Owned<IAttributeIterator> it = tree->getAttributes(true);
     bool hasAttributes = it->first();
     bool complex = (hasAttributes || tree->hasChildren());
-    if (complex)
-        yaml.beginMap();
+    bool hiddenRootArrayObject = isRootArrayObjectHidden(root, name, flags);
 
-    if (hasAttributes)
+    if (!hiddenRootArrayObject)
     {
-        ForEach(*it)
+        if (complex)
+            yaml.beginMap();
+
+        if (hasAttributes)
         {
-            const char *key = it->queryName()+1;
-            const char *val = it->queryValue();
-            yaml.writeAttribute(key, val, checkSanitizeFlagsAndHide(val, flags, true));
+            ForEach(*it)
+            {
+                const char *key = it->queryName()+1;
+                const char *val = it->queryValue();
+                yaml.writeAttribute(key, val, isSanitizedAndHidden(val, flags, true));
+            }
         }
     }
     StringBuffer _content;
     const char *content = nullptr; // to avoid uninitialized warning
     bool isBinary = tree->isBinary(NULL);
-    bool isNull;
-    if (isBinary)
-    {
-        MemoryBuffer thislevelbin;
-        isNull = (!tree->getPropBin(NULL, thislevelbin))||(thislevelbin.length()==0);
-        if (!isNull)
-            JBASE64_Encode(thislevelbin.toByteArray(), thislevelbin.length(), _content, true);
-        content = _content.str();
-    }
-    else if (tree->isCompressed(NULL))
-    {
-        isNull = false; // can't be empty if compressed;
-        verifyex(tree->getProp(NULL, _content));
-        content = _content.str();
-    }
-    else
-        isNull = (NULL == (content = tree->queryProp(NULL)));
+    bool isNull = true;
 
-    if (isNull && !root && !complex)
+    if (!hiddenRootArrayObject)
     {
-        yaml.writeValue("null", false, false, false);
-        return;
+        if (isBinary)
+        {
+            MemoryBuffer thislevelbin;
+            isNull = (!tree->getPropBin(NULL, thislevelbin))||(thislevelbin.length()==0);
+            if (!isNull)
+                JBASE64_Encode(thislevelbin.toByteArray(), thislevelbin.length(), _content, true);
+            content = _content.str();
+        }
+        else if (tree->isCompressed(NULL))
+        {
+            isNull = false; // can't be empty if compressed;
+            verifyex(tree->getProp(NULL, _content));
+            content = _content.str();
+        }
+        else
+            isNull = (NULL == (content = tree->queryProp(NULL)));
+
+        if (isNull && !root && !complex)
+        {
+            yaml.writeValue("null", false, false, false);
+            return;
+        }
     }
 
-    Owned<IPropertyTreeIterator> sub = tree->getElements("*", 0 != (flags & YAML_SortTags) ? iptiter_sort : iptiter_null);
+    Owned<IPropertyTreeIterator> sub = tree->getElements(hiddenRootArrayObject ? "__item__" : "*", 0 != (flags & YAML_SortTags) ? iptiter_sort : iptiter_null);
     //note that detection of repeating elements relies on the fact that ptree elements
     //of the same name will be grouped together
     bool repeatingElement = false;
@@ -8369,10 +8413,19 @@ static void _toYAML(const IPropertyTree *tree, YAMLEmitter &yaml, byte flags, bo
     {
         Linked<IPropertyTree> element = &sub->query();
         const char *name = element->queryName();
-        if (sub->next() && !repeatingElement && streq(name, sub->query().queryName()))
+        sub->next();
+        if (!repeatingElement)
         {
-            yaml.beginSequence(name);
-            repeatingElement = true;
+            if (hiddenRootArrayObject)
+            {
+                yaml.beginSequence(nullptr);
+                repeatingElement = true;
+            }
+            else if (sub->isValid() && streq(name, sub->query().queryName()))
+            {
+                yaml.beginSequence(name);
+                repeatingElement = true;
+            }
         }
 
         _toYAML(element, yaml, flags, false, repeatingElement);
@@ -8389,10 +8442,10 @@ static void _toYAML(const IPropertyTree *tree, YAMLEmitter &yaml, byte flags, bo
         if (complex)
             yaml.writeName("^");
         //repeating/array/sequence items are implicitly elements, no need for tag
-        yaml.writeValue(content, isArrayItem ? false : true, checkSanitizeFlagsAndHide(content, flags, false), isBinary);
+        yaml.writeValue(content, isArrayItem ? false : true, isSanitizedAndHidden(content, flags, false), isBinary);
     }
 
-    if (complex)
+    if (!hiddenRootArrayObject && complex)
         yaml.endMap();
 }
 

system/jlib/jptree.hpp

@@ -258,11 +258,14 @@ jlib_decl void dbglogXML(const IPropertyTree *tree, unsigned indent = 0, unsigne
 
 #define JSON_SortTags XML_SortTags
 #define JSON_Format   0x02
+#define JSON_HideRootArrayObject 0x04
 #define JSON_Sanitize XML_Sanitize
 #define JSON_SanitizeAttributeValues XML_SanitizeAttributeValues
 
-jlib_decl StringBuffer &toJSON(const IPropertyTree *tree, StringBuffer &ret, unsigned indent = 0, byte flags=JSON_Format);
-jlib_decl void toJSON(const IPropertyTree *tree, IIOStream &out, unsigned indent = 0, byte flags=JSON_Format);
+jlib_decl StringBuffer &toJSON(const IPropertyTree *tree, StringBuffer &ret, unsigned indent = 0, byte flags=JSON_Format|JSON_HideRootArrayObject);
+jlib_decl void toJSON(const IPropertyTree *tree, IIOStream &out, unsigned indent = 0, byte flags=JSON_Format|JSON_HideRootArrayObject);
+jlib_decl void printJSON(const IPropertyTree *tree, unsigned indent = 0, byte flags=JSON_Format|JSON_HideRootArrayObject);
+jlib_decl void dbglogJSON(const IPropertyTree *tree, unsigned indent = 0, byte flags=JSON_Format|JSON_HideRootArrayObject);
 
 jlib_decl const char *splitXPath(const char *xpath, StringBuffer &head); // returns tail, fills 'head' with leading xpath
 jlib_decl bool validateXPathSyntax(const char *xpath, StringBuffer *error=NULL);
@@ -332,6 +335,7 @@ jlib_decl IPropertyTree *createPTreeFromYAMLString(const char *yaml, byte flags=
 jlib_decl IPropertyTree *createPTreeFromYAMLString(unsigned len, const char *yaml, byte flags=ipt_none, PTreeReaderOptions readFlags=ptr_ignoreWhiteSpace, IPTreeMaker *iMaker=NULL);
 jlib_decl IPropertyTree *createPTreeFromYAMLFile(const char *filename, byte flags=ipt_none, PTreeReaderOptions readFlags=ptr_ignoreWhiteSpace, IPTreeMaker *iMaker=NULL);
 
+#define YAML_HideRootArrayObject 0x04
 #define YAML_SortTags XML_SortTags
 #define YAML_Sanitize XML_Sanitize
 #define YAML_SanitizeAttributeValues XML_SanitizeAttributeValues
@@ -339,11 +343,11 @@ jlib_decl IPropertyTree *createPTreeFromYAMLFile(const char *filename, byte flag
 jlib_decl StringBuffer &toYAML(const IPropertyTree *tree, StringBuffer &ret, unsigned indent, byte flags);
 jlib_decl void toYAML(const IPropertyTree *tree, IIOStream &out, unsigned indent, byte flags);
 
-jlib_decl void saveYAML(const char *filename, const IPropertyTree *tree, unsigned indent = 0, unsigned flags=0);
-jlib_decl void saveYAML(IFile &ifile, const IPropertyTree *tree, unsigned indent = 0, unsigned=0);
-jlib_decl void saveYAML(IFileIO &ifileio, const IPropertyTree *tree, unsigned indent = 0, unsigned flags=0);
-jlib_decl void saveYAML(IIOStream &stream, const IPropertyTree *tree, unsigned indent = 0, unsigned flags=0);
-jlib_decl void printYAML(const IPropertyTree *tree, unsigned indent = 0, unsigned flags=0);
-jlib_decl void dbglogYAML(const IPropertyTree *tree, unsigned indent = 0, unsigned flags=0);
+jlib_decl void saveYAML(const char *filename, const IPropertyTree *tree, unsigned indent = 0, unsigned flags=YAML_HideRootArrayObject);
+jlib_decl void saveYAML(IFile &ifile, const IPropertyTree *tree, unsigned indent = 0, unsigned=YAML_HideRootArrayObject);
+jlib_decl void saveYAML(IFileIO &ifileio, const IPropertyTree *tree, unsigned indent = 0, unsigned flags=YAML_HideRootArrayObject);
+jlib_decl void saveYAML(IIOStream &stream, const IPropertyTree *tree, unsigned indent = 0, unsigned flags=YAML_HideRootArrayObject);
+jlib_decl void printYAML(const IPropertyTree *tree, unsigned indent = 0, unsigned flags=YAML_HideRootArrayObject);
+jlib_decl void dbglogYAML(const IPropertyTree *tree, unsigned indent = 0, unsigned flags=YAML_HideRootArrayObject);
 
 #endif

testing/unittests/jlibtests.cpp

@@ -1223,9 +1223,59 @@ class JlibIPTTest : public CppUnit::TestFixture
     CPPUNIT_TEST_SUITE(JlibIPTTest);
         CPPUNIT_TEST(test);
         CPPUNIT_TEST(testMarkup);
+        CPPUNIT_TEST(testRootArrayMarkup);
     CPPUNIT_TEST_SUITE_END();
 
 public:
+    void testRootArrayMarkup()
+    {
+        static constexpr const char * xmlMarkup = R"!!(<__array__>
+ <__item__ a="val1a" b="val2a"/>
+ <__item__ a="val1b" b="val2b"/>
+ <__item__ a="val1c" b="val2c"/>
+</__array__>
+)!!";
+
+        static constexpr const char * jsonMarkup = R"!!([
+ {
+  "@a": "val1a",
+  "@b": "val2a"
+ },
+ {
+  "@a": "val1b",
+  "@b": "val2b"
+ },
+ {
+  "@a": "val1c",
+  "@b": "val2c"
+ }
+])!!";
+
+        static constexpr const char * yamlMarkup = R"!!(- a: val1a
+  b: val2a
+- a: val1b
+  b: val2b
+- a: val1c
+  b: val2c
+)!!";
+
+        Owned<IPropertyTree> xml = createPTreeFromXMLString(xmlMarkup, ipt_none, ptr_ignoreWhiteSpace, nullptr);
+        Owned<IPropertyTree> yaml = createPTreeFromYAMLString(yamlMarkup, ipt_none, ptr_ignoreWhiteSpace, nullptr);
+        Owned<IPropertyTree> json = createPTreeFromJSONString(jsonMarkup, ipt_none, ptr_ignoreWhiteSpace, nullptr);
+
+        CPPUNIT_ASSERT(areMatchingPTrees(xml, json));
+        CPPUNIT_ASSERT(areMatchingPTrees(xml, yaml));
+
+        StringBuffer ml;
+        toXML(xml, ml, 0, XML_Format|XML_SortTags);
+        CPPUNIT_ASSERT(streq(ml, xmlMarkup));
+
+        toYAML(xml, ml.clear(), 0, YAML_SortTags|YAML_HideRootArrayObject);
+        CPPUNIT_ASSERT(streq(ml, yamlMarkup));
+
+        toJSON(xml, ml.clear(), 0, JSON_Format|JSON_SortTags|JSON_HideRootArrayObject);
+        CPPUNIT_ASSERT(streq(ml, jsonMarkup));
+    }
     void testMarkup()
     {
         static constexpr const char * xmlMarkup = R"!!(  <__object__ attr1="attrval1" attr2="attrval2">
@@ -1358,10 +1408,10 @@ subX:
         toXML(xml, ml, 2, XML_Format|XML_SortTags);
         CPPUNIT_ASSERT(streq(ml, xmlMarkup));
 
-        toYAML(yaml, ml.clear(), 2, YAML_SortTags);
+        toYAML(yaml, ml.clear(), 2, YAML_SortTags|YAML_HideRootArrayObject);
         CPPUNIT_ASSERT(streq(ml, yamlMarkup));
 
-        toJSON(json, ml.clear(), 2, JSON_Format|JSON_SortTags);
+        toJSON(json, ml.clear(), 2, JSON_Format|JSON_SortTags|JSON_HideRootArrayObject);
         CPPUNIT_ASSERT(streq(ml, jsonMarkup));
     }
     void test()