HPCC-19766 Initial esp.xsd

Add first pass of esp schema support

Signed-off-by: Ken Rowland <kenneth.rowland@lexisnexisrisk.com>

configuration/config2/CMakeLists.txt

@@ -36,6 +36,7 @@ SET(  SRCS
       EnvironmentEventHandlers.cpp
       Utils.cpp
       InsertableItem.cpp
+      ConfigPath.cpp
 )
 
 INCLUDE_DIRECTORIES(

configuration/config2/ConfigPath.cpp

@@ -0,0 +1,166 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2018 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#include "ConfigPath.hpp"
+#include "Exceptions.hpp"
+#include "Utils.hpp"
+
+
+bool ConfigPathItem::isValuePresentInValueList(const std::string val, bool returnTrueIfValueListEmpty) const
+{
+    if (m_attributeValues.empty())
+    {
+        return returnTrueIfValueListEmpty;
+    }
+
+    for (auto it=m_attributeValues.begin(); it!=m_attributeValues.end(); ++it)
+    {
+        if (*it == val)
+            return true;
+    }
+
+    return false;
+}
+
+
+std::shared_ptr<ConfigPathItem> ConfigPath::getNextPathItem()
+{
+    std::shared_ptr<ConfigPathItem> pPathItem;
+
+    //
+    // If notheing left, return an empty pointer
+    if (m_path.empty())
+    {
+        return pPathItem;
+    }
+
+    //
+    // There is more to do, allocate a pointer and get to work
+    pPathItem = std::make_shared<ConfigPathItem>();
+
+    //
+    // Root
+    if (m_path[0] == '/')
+    {
+        parsePathElement(1, pPathItem);
+        pPathItem->setIsRoot(true);
+        updatePath(1);
+    }
+
+    //
+    // Parent (MUST be done before current element test)
+    else if (m_path.substr(0,2) == "..")
+    {
+        pPathItem->setIsParentPathItemn(true);
+        updatePath(2);
+    }
+
+    //
+    // Current element?
+    else if (m_path[0] == '.')
+    {
+        pPathItem->setIsCurrentPathItem(true);
+        updatePath(1);
+    }
+
+    //
+    // Otherwise, need to parse it out
+    else
+    {
+        parsePathElement(0, pPathItem);
+        updatePath(0);
+    }
+
+    return pPathItem;
+}
+
+
+void ConfigPath::parsePathElement(std::size_t start, const std::shared_ptr<ConfigPathItem> &pPathItem)
+{
+    std::size_t slashPos = m_path.find_first_of('/', start);
+    std::size_t len = (slashPos != std::string::npos) ? (slashPos-start) : std::string::npos;
+    std::string element = m_path.substr(start, len);
+
+    //
+    // The attribute definition is enclosed by the brackets, extract it if present
+    std::string attr;
+    if (extractEnclosedString(element, attr, '[', ']', true))
+    {
+        //
+        // Make sure valid
+        if (attr.empty())
+        {
+            throw(ParseException("Bad path, missng attribute definition at or around: " + element));
+        }
+
+        //
+        // The attribute must begin with a '@' or '#'. The '#' is an extension for config manager that allows the selection
+        // of elements based on schema value properties as opposed to attribute values. So, for a '#', the name is that of a
+        // schema item property and not an attribute in the environment. Useful for selecting all elements of a particular
+        // type, for example, when elements have nothing in the environment that can be used to determine type such as ecl watch.
+        if (attr[0] == '@' || attr[0] == '#')
+        {
+            pPathItem->setIsSchemaItem(attr[0] == '#');
+
+            //
+            // Value present?
+            std::size_t equalPos = attr.find_first_of('=');
+            if (equalPos != std::string::npos)
+            {
+                pPathItem->setAttributeName(attr.substr(1, equalPos-1));
+                std::string valueStr;
+                extractEnclosedString(attr.substr(equalPos + 1), valueStr, '(', ')', true);
+                std::vector<std::string> values = splitString(valueStr, ",");
+                for (auto &valstr : values)
+                {
+                    std::string value;
+                    extractEnclosedString(valstr, value, '\'', '\'', false);
+                    pPathItem->addAttributeValue(value);
+                }
+            }
+            else
+            {
+                pPathItem->setAttributeName(attr.substr(1));
+            }
+        }
+        else
+        {
+            throw(ParseException("Bad attribute, expected '@' or '#' at or around: " + element));
+        }
+
+        std::size_t bracketPos = element.find_first_of('[');
+        pPathItem->setElementName(element.substr(0, bracketPos));
+    }
+    else
+    {
+        pPathItem->setElementName(element);
+    }
+}
+
+
+void ConfigPath::updatePath(std::size_t startPos)
+{
+    std::size_t nextPos = m_path.find_first_of('/', startPos);
+    if (nextPos != std::string::npos)
+    {
+        m_path = m_path.substr(nextPos+1);
+    }
+    else
+    {
+        m_path.clear();
+    }
+}

configuration/config2/ConfigPath.hpp

@@ -0,0 +1,86 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2018 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+
+#ifndef _CONFIG2_CONFIGPATH_HPP_
+#define _CONFIG2_CONFIGPATH_HPP_
+
+#include <string>
+#include <vector>
+#include <memory>
+
+class ConfigPathItem
+{
+    public:
+
+        ConfigPathItem() : m_isParentPathItem(false), m_isCurrentPathItem(false), m_isSchemaItem(false), m_isRoot(false) {}
+        ConfigPathItem(const std::string &elemName) : m_isParentPathItem(false), m_isCurrentPathItem(false), m_isSchemaItem(false), m_isRoot(false), m_elementName(elemName) {}
+
+        void setElementName(const std::string &elemName) { m_elementName = elemName; }
+        const std::string &getElementName() const { return m_elementName; }
+        void setAttributeName(const std::string &attrName) { m_attributeName = attrName; }
+        const std::string &getAttributeName() const { return m_attributeName; }
+        void addAttributeValue(const std::string &attrValue) { m_attributeValues.push_back(attrValue); }
+        bool hasAttributeValues() const { return m_attributeValues.size() > 0; }
+        std::vector<std::string> getAttributeValues() const { return m_attributeValues; }
+        bool isValuePresentInValueList(const std::string val, bool returnTrueIfValueListEmpty = true) const;
+        void setIsCurrentPathItem(bool currentItem) { m_isCurrentPathItem = currentItem; }
+        bool isCurrentPathItem() const { return m_isCurrentPathItem; }
+        void setIsParentPathItemn(bool parentElement) { m_isParentPathItem = parentElement; }
+        bool isParentPathtItem() const { return m_isParentPathItem; }
+        void setIsSchemaItem(bool isSchema) { m_isSchemaItem = isSchema; }
+        bool isSchemaItem() const { return m_isSchemaItem; }
+        void setIsRoot(bool root) { m_isRoot = root; }
+        bool isRoot() const { return m_isRoot; }
+
+
+    private:
+
+        bool m_isParentPathItem;
+        bool m_isCurrentPathItem;
+        bool m_isSchemaItem;
+        bool m_isRoot;
+        std::string m_elementName;
+        std::string m_attributeName;
+        std::vector<std::string> m_attributeValues;
+};
+
+
+
+class ConfigPath
+{
+    public:
+
+        ConfigPath(const std::string path) : m_path(path) {}
+        ~ConfigPath() {}
+        std::shared_ptr<ConfigPathItem> getNextPathItem();
+        bool isPathRemaining() const { return (!m_path.empty()); }
+
+
+    protected:
+
+        void updatePath(std::size_t pos);
+        void parsePathElement(std::size_t start, const std::shared_ptr<ConfigPathItem> &pPathItem);
+
+
+    private:
+
+        std::string m_path;
+};
+
+
+#endif

configuration/config2/EnvironmentMgr.cpp

@@ -33,7 +33,8 @@ EnvironmentMgr *getEnvironmentMgrInstance(const EnvironmentType envType)
 }
 
 
-EnvironmentMgr::EnvironmentMgr()
+EnvironmentMgr::EnvironmentMgr() :
+    m_message("Unknown error")
 {
     m_pSchema = std::make_shared<SchemaItem>("root");  // make the root
 }
@@ -47,13 +48,20 @@ bool EnvironmentMgr::loadSchema(const std::string &configPath, const std::string
         rc = m_pSchemaParser->parse(configPath, masterConfigFile, cfgParms);
         if (rc)
         {
-            // unique attribure value sets are global across a schema. Allocate one here and pass it in
-            // for use in building the necessary references and dependencies across the schema, then pass
-            // it to the post processing for finalization. Once referencs and dependencies are built, the
-            // attribute value sets are no longer needed.
-            std::map<std::string, std::vector<std::shared_ptr<SchemaValue>>> uniqueAttributeValueSets;
-            m_pSchema->processDefinedUniqueAttributeValueSets(uniqueAttributeValueSets);  // This must be done first
-            m_pSchema->postProcessConfig(uniqueAttributeValueSets);
+            try {
+                // unique attribure value sets are global across a schema. Allocate one here and pass it in
+                // for use in building the necessary references and dependencies across the schema, then pass
+                // it to the post processing for finalization. Once references and dependencies are built, the
+                // attribute value sets are no longer needed.
+                std::map<std::string, std::vector<std::shared_ptr<SchemaValue>>> uniqueAttributeValueSets;
+                m_pSchema->processDefinedUniqueAttributeValueSets(uniqueAttributeValueSets);  // This must be done first
+                m_pSchema->postProcessConfig(uniqueAttributeValueSets);
+            }
+            catch (ParseException &pe)
+            {
+                m_message = pe.what();
+                rc = false;
+            }
         }
     }
     return rc;
@@ -62,9 +70,12 @@ bool EnvironmentMgr::loadSchema(const std::string &configPath, const std::string
 
 std::string EnvironmentMgr::getLastSchemaMessage() const
 {
+    std::string msg;
     if (m_pSchemaParser)
-        return m_pSchemaParser->getLastMessage();
-    return "";
+        msg = m_pSchemaParser->getLastMessage();
+    if (msg.empty())
+        msg = m_message;
+    return msg;
 }
 
 

configuration/config2/EnvironmentNode.cpp

@@ -18,6 +18,7 @@
 #include "EnvironmentNode.hpp"
 #include "Exceptions.hpp"
 #include "Utils.hpp"
+#include "ConfigPath.hpp"
 
 void EnvironmentNode::addChild(std::shared_ptr<EnvironmentNode> pNode)
 {
@@ -356,133 +357,112 @@ void EnvironmentNode::initialize()
 
 void EnvironmentNode::fetchNodes(const std::string &path, std::vector<std::shared_ptr<EnvironmentNode>> &nodes) const
 {
-    //
-    // If path starts with / and we are not the root, get the root and do the find
-    if (path[0] == '/')
-    {
-        std::string remainingPath = path.substr(1);
-        std::string rootName = remainingPath;
-        std::shared_ptr<const EnvironmentNode> pRoot = getRoot();
-        size_t slashPos = path.find_first_of('/', 1);
-        if (slashPos != std::string::npos)
-        {
-            rootName = path.substr(1, slashPos-1);
-            remainingPath = path.substr(slashPos + 1);
-            size_t atPos = rootName.find_first_of('@');
-            if (atPos != std::string::npos)
-            {
-                rootName.erase(atPos, std::string::npos);
-            }
-        }
+    ConfigPath configPath(path);
+    doFetchNodes(configPath, nodes);
+}
 
-        if (pRoot->getName() == rootName)
-        {
-            pRoot->fetchNodes(remainingPath, nodes);
-        }
-    }
-    else if (path[0] == '.')
+
+void EnvironmentNode::doFetchNodes(ConfigPath &configPath, std::vector<std::shared_ptr<EnvironmentNode>> &nodes) const
+{
+    std::shared_ptr<ConfigPathItem> pPathItem = configPath.getNextPathItem();
+
+    if (pPathItem)
     {
-        //
-        // Parent ?
-        if (path[1] == '.')
+        if (pPathItem->isRoot())
         {
-            //
-            // Path must be at least 4 characters in length to support the leading ../<remaining path>
-            if (!m_pParent.expired() && path.length() >= 4)
+            std::shared_ptr<const EnvironmentNode> pRoot = getRoot();
+            if (pRoot->getName() == pPathItem->getElementName())
             {
-                m_pParent.lock()->fetchNodes(path.substr(3), nodes);  // note skipping over '..'
+                pRoot->doFetchNodes(configPath, nodes);
             }
             else
             {
-                throw new ParseException("Attempt to navigate to parent with no parent or path is incomplete");
+                throw new ParseException("Invalid root element name ('" + pPathItem->getElementName() + "') specified in path");
             }
         }
-        else
+        else if (pPathItem->isParentPathtItem())
         {
-            fetchNodes(path.substr(1), nodes); // do the find from here stripping the '.' indicator
+            getParent()->doFetchNodes(configPath, nodes);
         }
-    }
-
-    //
-    // Otherwise, start searching
-    else
-    {
-        std::string nodeName = path;
-        std::string remainingPath, searchAttrName, searchAttrValue;
-
-        //
-        // Get our portion of the path which is up to the next / or the remaining string and
-        // set the remaining portion of the path to search
-        size_t slashPos = nodeName.find_first_of('/');
-        if (slashPos != std::string::npos)
+        else if (pPathItem->isCurrentPathItem())
         {
-            remainingPath = nodeName.substr(slashPos + 1);
-            nodeName.erase(slashPos, std::string::npos);  // truncate
+            doFetchNodes(configPath, nodes);
         }
-
-        //
-        // Any attributes we need to look for?
-        size_t atPos = nodeName.find_first_of('@');
-        if (atPos != std::string::npos)
+        else
         {
-            searchAttrName = nodeName.substr(atPos + 1);
-            nodeName.erase(atPos, std::string::npos);
-            size_t equalPos = searchAttrName.find_first_of('=');
-            if (equalPos != std::string::npos)
+            //
+            // Get children nodes matching path element name (use this node if no element name)
+            std::vector<std::shared_ptr<EnvironmentNode>> childNodes;
+            if (pPathItem->getElementName().empty())
             {
-                searchAttrValue = searchAttrName.substr(equalPos + 1);
-                searchAttrName.erase(equalPos, std::string::npos);
+                childNodes.push_back(std::const_pointer_cast<EnvironmentNode>(shared_from_this()));
+            }
+            else
+            {
+                getChildren(childNodes, pPathItem->getElementName());
             }
-        }
-
-        //
-        // Now search the children for nodes matching the node name
-        std::vector<std::shared_ptr<EnvironmentNode>> childNodes;
-        getChildren(childNodes, nodeName);
 
-        //
-        // If there is an attribute specified, dig deeper
-        if (!searchAttrName.empty())
-        {
-            auto childNodeIt = childNodes.begin();
-            while (childNodeIt != childNodes.end())
+            //
+            // If there is an attribute and/or attribute values, search the child nodes from above
+            if (!pPathItem->getAttributeName().empty())
             {
-                std::shared_ptr<EnvironmentValue> pValue = (*childNodeIt)->getAttribute(searchAttrName);
-                if (pValue)
+                std::string attrName = pPathItem->getAttributeName();
+                auto childNodeIt = childNodes.begin();
+                while (childNodeIt != childNodes.end())
                 {
                     //
-                    // The attribute value must be present and, if necessary, must match the search value
-                    std::string curAttrValue = pValue->getValue();
-                    if (!curAttrValue.empty() && (searchAttrValue.empty() || (searchAttrValue == curAttrValue)))
+                    // If an attribute, then search attributes for those with the specified name, otherwise
+                    // get the schema item for this node and see if it has a property with the specified name.
+                    // In each case, if there is a value, check it too
+                    if (!pPathItem->isSchemaItem())
                     {
-                        ++childNodeIt;  // keep it
+                        std::shared_ptr<EnvironmentValue> pAttribute = (*childNodeIt)->getAttribute(attrName);
+                        if (pAttribute)
+                        {
+                            if (pPathItem->isValuePresentInValueList(pAttribute->getValue(), true))
+                            {
+                                ++childNodeIt;
+                            }
+                            else
+                            {
+                                childNodeIt = childNodes.erase(childNodeIt);
+                            }
+                        }
+                        else
+                        {
+                            childNodeIt = childNodes.erase(childNodeIt);
+                        }
                     }
                     else
                     {
-                        childNodeIt = childNodes.erase(childNodeIt);
+                        std::shared_ptr<SchemaItem> pSchemaItem = (*childNodeIt)->getSchemaItem();
+                        std::string propertyValue = (attrName == "itemType") ? pSchemaItem->getItemType() : pSchemaItem->getProperty(attrName);
+                        if (pPathItem->isValuePresentInValueList(propertyValue, true))
+                        {
+                            ++childNodeIt;
+                        }
+                        else
+                        {
+                            childNodeIt = childNodes.erase(childNodeIt);
+                        }
                     }
                 }
-                else
+            }
+
+            if (configPath.isPathRemaining())
+            {
+                //
+                // For all the matching nodes at this element, call each to continue the search
+                for (auto childNodeIt = childNodes.begin(); childNodeIt != childNodes.end(); ++childNodeIt)
                 {
-                    childNodeIt = childNodes.erase(childNodeIt);
+                    (*childNodeIt)->doFetchNodes(configPath, nodes);
                 }
             }
-        }
-
-        //
-        // If there is path remaining, call the children in childNodes with the remaining path, otherwise we've reached
-        // the end. Whatever nodes are in childNodes are appended to the input nodes vector for return
-        if (!remainingPath.empty())
-        {
-            for (auto childNodeIt = childNodes.begin(); childNodeIt != childNodes.end(); ++childNodeIt)
+            else
             {
-                (*childNodeIt)->fetchNodes(remainingPath, nodes);
+                nodes.insert(nodes.end(), childNodes.begin(), childNodes.end());
             }
         }
-        else
-        {
-            nodes.insert(nodes.end(), childNodes.begin(), childNodes.end());
-        }
     }
 }
 

configuration/config2/EnvironmentNode.hpp

@@ -27,6 +27,7 @@
 #include "InsertableItem.hpp"
 #include "NameValue.hpp"
 #include "platform.h"
+#include "ConfigPath.hpp"
 
 
 class DECL_EXPORT EnvironmentNode : public std::enable_shared_from_this<EnvironmentNode>
@@ -74,6 +75,11 @@ class DECL_EXPORT EnvironmentNode : public std::enable_shared_from_this<Environm
 
     protected:
 
+        void doFetchNodes(ConfigPath &configPath, std::vector<std::shared_ptr<EnvironmentNode>> &nodes) const;
+
+
+    protected:
+
         std::string m_name;
         std::shared_ptr<SchemaItem> m_pSchemaItem;
         std::weak_ptr<EnvironmentNode> m_pParent;

configuration/config2/SchemaItem.cpp

@@ -140,7 +140,7 @@ void SchemaItem::addSchemaType(const std::shared_ptr<SchemaItem> &pItem, const s
     }
     else
     {
-        throw(ParseException("Duplicate config type found: " + m_properties["name"]));
+        throw(ParseException("Element: " + getProperty("name") + ", duplicate config type found: " + typeName));
     }
 }
 
@@ -227,7 +227,7 @@ void SchemaItem::addAttribute(const std::shared_ptr<SchemaValue> &pCfgValue)
     auto retVal = m_attributes.insert({ pCfgValue->getName(), pCfgValue });
     if (!retVal.second)
     {
-        throw(ParseException("Duplicate attribute (" + pCfgValue->getName() + ") found for element " + m_properties["name"]));
+        throw(ParseException("Element: " + getProperty("name") + ", duplicate attribute (" + pCfgValue->getName() + ") found"));
     }
 }
 
@@ -293,7 +293,7 @@ void SchemaItem::processUniqueAttributeValueSetReferences(const std::map<std::st
             for (auto cfgIt = keyIt->second.begin(); cfgIt != keyIt->second.end(); ++cfgIt)
             {
                 std::shared_ptr<SchemaValue> pKeyRefAttribute = *cfgIt;     // this is the reference attribute from which attributeName must be a member
-                std::string cfgValuePath = ((setRefIt->second.m_elementPath != ".") ? setRefIt->second.m_elementPath : "") + "@" + setRefIt->second.m_attributeName;
+                std::string cfgValuePath = setRefIt->second.m_elementPath + "[@" + setRefIt->second.m_attributeName + "]";
                 std::vector<std::shared_ptr<SchemaValue>> cfgValues;
                 fetchSchemaValues(cfgValuePath, cfgValues);
                 if (!cfgValues.empty())
@@ -305,23 +305,34 @@ void SchemaItem::processUniqueAttributeValueSetReferences(const std::map<std::st
                 }
                 else
                 {
-                    throw(ParseException("Attribute " + (setRefIt->second.m_attributeName + " not found when adding keyRef for key " + (setRefIt->second.m_setName))));
+                    throw(ParseException("Element: " + getProperty("name") + ", Attribute '" + (setRefIt->second.m_attributeName + "' not found when adding keyRef for key " + (setRefIt->second.m_setName))));
                 }
             }
         }
         else
         {
-            throw(ParseException("Keyref to key '" + (setRefIt->second.m_setName + "' was not found")));
+            throw(ParseException("Element: " + getProperty("name") + ", Keyref to key '" + (setRefIt->second.m_setName + "' was not found")));
         }
     }
 }
 
 
-void SchemaItem::getChildren(std::vector<std::shared_ptr<SchemaItem>> &children) const
+void SchemaItem::getChildren(std::vector<std::shared_ptr<SchemaItem>> &children, const std::string &name) const
 {
-    for (auto it = m_children.begin(); it != m_children.end(); ++it)
+    if (name.empty())
     {
-        children.push_back(it->second);
+        for (auto it = m_children.begin(); it != m_children.end(); ++it)
+        {
+            children.push_back(it->second);
+        }
+    }
+    else
+    {
+        auto rangeIt = m_children.equal_range(name);
+        for (auto it=rangeIt.first; it!=rangeIt.second; ++it)
+        {
+            children.push_back(it->second);
+        }
     }
 }
 
@@ -364,62 +375,89 @@ void SchemaItem::resetEnvironment()
 }
 
 
+void SchemaItem::fetchSchemaValues(const std::string &path, std::vector<std::shared_ptr<SchemaValue>> &schemaValues) const
+{
+    ConfigPath configPath(path);
+    doFetchSchemaValues(configPath, schemaValues);
+}
+
+
 
-void SchemaItem::fetchSchemaValues(const std::string &path, std::vector<std::shared_ptr<SchemaValue>> &schemaValues)
+void SchemaItem::doFetchSchemaValues(ConfigPath &configPath, std::vector<std::shared_ptr<SchemaValue>> &schemaValues) const
 {
-    bool rootPath = path[0] == '/';   //todo: convert this to use the findSchemaRoot below
 
-    //
-    // If path is from the root, and we aren't the root, pass the request to our parent
-    if (rootPath && !m_pParent.expired())
+    std::shared_ptr<ConfigPathItem> pPathItem = configPath.getNextPathItem();
+
+    if (pPathItem)
     {
-        std::shared_ptr<SchemaItem> pParent = m_pParent.lock();
-        if (pParent)
+        if (pPathItem->isRoot())
         {
-            return pParent->fetchSchemaValues(path, schemaValues);
+            getSchemaRoot()->doFetchSchemaValues(configPath, schemaValues);
         }
-    }
-
-    //
-    // Break the path down and process it
-    size_t start = rootPath ? 1 : 0;    // skip leading slash if we are at the root
-    size_t end = path.find_first_of("/@", start);
-    if (end != std::string::npos)
-    {
-        std::string elem = path.substr(start, end - start);
-
-        if (rootPath)
+        else if (pPathItem->isParentPathtItem())
         {
-            if (m_properties["name"] == elem)
+            if (!m_pParent.expired())
             {
-                return fetchSchemaValues(path.substr(end + 1), schemaValues);
+                m_pParent.lock()->doFetchSchemaValues(configPath, schemaValues);
+            }
+        }
+        else if (pPathItem->isCurrentPathItem())
+        {
+            doFetchSchemaValues(configPath, schemaValues);
+        }
+        else
+        {
+            //
+            // Get the items to check for this path element. If there is no element name, then use
+            // this item, otherwise get children matching the element name
+            std::vector<std::shared_ptr<SchemaItem>> items;
+            if (pPathItem->getElementName().empty())
+            {
+                items.push_back(std::const_pointer_cast<SchemaItem>(shared_from_this()));
             }
             else
             {
-                throw(ParseException("Unable to find root element '" + elem + "' when searching path '" + path + "'"));
+                getChildren(items, pPathItem->getElementName());
             }
-        }
 
-        if (path[0] == '@')
-        {
-            std::string attrName = path.substr(1);
-            auto rangeIt = m_attributes.equal_range(attrName);
-            for (auto it = rangeIt.first; it != rangeIt.second; ++it)
+            if (!pPathItem->getAttributeName().empty())  // note that attribute values are not supported for schema xpaths
             {
-                schemaValues.push_back(it->second);
+                auto it = items.begin();
+                while (it != items.end())
+                {
+                    std::shared_ptr<SchemaValue> pAttribute = (*it)->getAttribute(pPathItem->getAttributeName());
+                    if (!pAttribute)
+                    {
+                        it = items.erase(it);
+                    }
+                    else
+                    {
+                        ++it;
+                    }
+                }
             }
-        }
 
-        else
-        {
-            auto rangeIt = m_children.equal_range(elem);
-            for (auto it = rangeIt.first; it != rangeIt.second; ++it)
+            //
+            // If path elements remain, fetch schema values from each match at this level, otherwise push the results to
+            // the result schema value vector
+            if (configPath.isPathRemaining())
+            {
+                //
+                // For all the matching nodes at this element, call each to continue the search
+                for (auto itemIt = items.begin(); itemIt != items.end(); ++itemIt)
+                {
+                    (*itemIt)->doFetchSchemaValues(configPath, schemaValues);
+                }
+            }
+            else
             {
-                it->second->fetchSchemaValues(path.substr(end + ((path[end] == '/') ? 1 : 0)), schemaValues);
+                for (auto itemIt = items.begin(); itemIt != items.end(); ++itemIt)
+                {
+                    schemaValues.push_back((*itemIt)->getAttribute(pPathItem->getAttributeName()));
+                }
             }
         }
     }
-
     return;
 }
 
@@ -444,7 +482,7 @@ void SchemaItem::processDefinedUniqueAttributeValueSets(std::map<std::string, st
         bool keyDefExists = it != uniqueAttributeValueSets.end();
         if (!keyDefExists || setIt->second.m_duplicateOk)
         {
-            std::string cfgValuePath = ((setIt->second.m_elementPath != ".") ? setIt->second.m_elementPath : "") + "@" + setIt->second.m_attributeName;
+            std::string cfgValuePath = setIt->second.m_elementPath + "[@" + setIt->second.m_attributeName + "]";
             std::vector<std::shared_ptr<SchemaValue>> cfgValues;
             fetchSchemaValues(cfgValuePath, cfgValues);
             if (!cfgValues.empty())
@@ -478,12 +516,12 @@ void SchemaItem::processDefinedUniqueAttributeValueSets(std::map<std::string, st
             }
             else
             {
-                throw(ParseException("Attribute " + setIt->second.m_attributeName + " not found for key " + setIt->second.m_setName));
+                throw(ParseException("Element: " + getProperty("name", "unknown") + ", attribute " + setIt->second.m_attributeName + " not found for key " + setIt->second.m_setName));
             }
         }
         else
         {
-            throw(ParseException("Duplicate key (" + setIt->second.m_setName + ") found for element " + m_properties["name"]));
+            throw(ParseException("Element: " + getProperty("name", "unknown") + ", duplicate key (" + setIt->second.m_setName + ") found for element " + m_properties["name"]));
         }
     }
 
@@ -508,7 +546,7 @@ void SchemaItem::postProcessConfig(const std::map<std::string, std::vector<std::
             auto rc = itemTypes.insert(it->second->getItemType());
             if (!rc.second)
             {
-                throw(ParseException("Duplicate itemType(" + it->second->getItemType() + ") found for element " + m_properties["name"]));
+                throw(ParseException("Element: " + getProperty("name") + ", duplicate itemType(" + it->second->getItemType() + ") found"));
             }
         }
     }
@@ -534,12 +572,12 @@ void SchemaItem::postProcessConfig(const std::map<std::string, std::vector<std::
                 }
                 else
                 {
-                    throw(ParseException("Multiple sources found for mirror from path for attribute " + it->second->getName() + " (path=" + it->second->getMirrorFromPath()));
+                    throw(ParseException("Element: " + getProperty("name") + ", multiple sources found for mirror from path for attribute " + it->second->getName() + " (path=" + it->second->getMirrorFromPath()));
                 }
             }
             else
             {
-                throw(ParseException("Mirrored from source not found for attribute " + it->second->getName() + " path=" + it->second->getMirrorFromPath()));
+                throw(ParseException("Element: " + getProperty("name") + ", mirrored from source not found for attribute '" + it->second->getName() + "' path=" + it->second->getMirrorFromPath()));
             }
         }
     }

configuration/config2/SchemaItem.hpp

@@ -27,6 +27,7 @@
 #include "SchemaValue.hpp"
 #include "platform.h"
 #include "EnvironmentEventHandlers.hpp"
+#include "ConfigPath.hpp"
 
 class EnvironmentNode;
 
@@ -50,13 +51,14 @@ class DECL_EXPORT SchemaItem : public std::enable_shared_from_this<SchemaItem>
         void insertSchemaType(const std::shared_ptr<SchemaItem> pTypeItem);
         void addChild(const std::shared_ptr<SchemaItem> &pItem) { m_children.insert({ pItem->getProperty("name"), pItem }); }
         void addChild(const std::shared_ptr<SchemaItem> &pItem, const std::string &name) { m_children.insert({ name, pItem }); }
-        void getChildren(std::vector<std::shared_ptr<SchemaItem>> &children) const;
+        void getChildren(std::vector<std::shared_ptr<SchemaItem>> &children, const std::string &name = std::string("")) const;
         std::shared_ptr<SchemaItem> getChild(const std::string &name);
         std::shared_ptr<SchemaItem> getChildByComponent(const std::string &name, std::string &componentName);
         void setItemSchemaValue(const std::shared_ptr<SchemaValue> &pValue) { m_pItemValue = pValue; }
         std::shared_ptr<SchemaValue> getItemSchemaValue() const { return m_pItemValue; }
         bool isItemValueDefined() { return m_pItemValue != nullptr; }
-        void fetchSchemaValues(const std::string &path, std::vector<std::shared_ptr<SchemaValue>> &schemaValues);
+        void fetchSchemaValues(const std::string &path, std::vector<std::shared_ptr<SchemaValue>> &schemaValues) const;
+        void doFetchSchemaValues(ConfigPath &configPath, std::vector<std::shared_ptr<SchemaValue>> &schemaValues) const;
         void addAttribute(const std::shared_ptr<SchemaValue> &pCfgValue);
         void addAttribute(const std::vector<std::shared_ptr<SchemaValue>> &attributes);
         void addAttribute(const std::map<std::string, std::shared_ptr<SchemaValue>> &attributes);

configuration/config2/SchemaTypeLimits.hpp

@@ -49,6 +49,7 @@ struct DECL_EXPORT AllowedValue
     std::string m_userMessageType;
     std::string m_userMessage;
     std::vector<DependentValue> m_dependencies;
+    std::vector<std::string> m_optionalAttributes, m_requiredAttributes;
 };
 
 

configuration/config2/SchemaValue.cpp

@@ -35,6 +35,8 @@ SchemaValue::SchemaValue(const std::string &name, bool isDefined) :
 
 SchemaValue::SchemaValue(const SchemaValue &value)
 {
+    // Primary purpose of the copy constructor is for use when a complexType is referenced. A copy is made which includes a copy
+    // of each SchemaValue in the complexType SchemaItem.
     m_pType = value.m_pType;
     m_name = value.m_name;
     m_displayName = value.m_displayName;
@@ -47,7 +49,7 @@ SchemaValue::SchemaValue(const SchemaValue &value)
     m_modifiers = value.m_modifiers;
     m_valueLimitRuleType = value.m_valueLimitRuleType;
     m_valueLimitRuleData = value.m_valueLimitRuleData;
-    m_requiredIfSet = value.m_requiredIfSet;
+    m_requiredIf = value.m_requiredIf;
     m_group = value.m_group;
 
     // special processing? Maybe after inserting?
@@ -106,19 +108,21 @@ void SchemaValue::validate(Status &status, const std::string &id, const Environm
     {
         //
         // See if there is a dependency on another value being set.
-        if (!m_requiredIfSet.empty() && isValid)
+        if (!m_requiredIf.empty() && isValid)
         {
             //
-            // Required if set string format is path[@attribute[=value]]. Search this environment value's owning node
+            // Required if string format is an xpath. Search this environment value's owning node
             // for a match.
             std::vector<std::shared_ptr<EnvironmentNode>> nodes;
-            pEnvValue->getEnvironmentNode()->fetchNodes(m_requiredIfSet, nodes);
+            pEnvValue->getEnvironmentNode()->fetchNodes(m_requiredIf, nodes);
             if (!nodes.empty())
             {
+                //
+                // Since here is a match for a requiredIf, this value MUST be set
                 if (pEnvValue->getValue().empty())
                 {
                     isValid = false;
-                    std::string msg = "Environment value required based on requiredIf rule " + m_requiredIfSet + " being set.";
+                    std::string msg = "Environment value required based on requiredIf rule " + m_requiredIf + " being set.";
                     status.addMsg(statusMsg::error, pEnvValue->getNodeId(), pEnvValue->getName(), msg);
                 }
             }
@@ -133,7 +137,7 @@ void SchemaValue::validate(Status &status, const std::string &id, const Environm
                 msg = "Value was forced to an invalid value (" + curValue + ").";
             else
                 msg = "Value is invalid (" + curValue + ").";
-            msg += "Valid value (" + m_pType->getLimitString() + ")";
+            msg += " Valid value (" + m_pType->getLimitString() + ")";
 
             status.addMsg(pEnvValue->wasForced() ? statusMsg::warning : statusMsg::error, pEnvValue->getNodeId(), pEnvValue->getName(), msg);
         }

configuration/config2/SchemaValue.hpp

@@ -83,8 +83,8 @@ class DECL_EXPORT SchemaValue
         const std::string &getValueLimitRuleType() { return m_valueLimitRuleType; }
         void setValueLimitRuleData(const std::string &data) { m_valueLimitRuleData = data; }
         const std::string &getValueLimitRuleData() { return m_valueLimitRuleData; }
-        void setRequiredIfSet(const std::string &reqIf) { m_requiredIfSet = reqIf; }
-        const std::string &getRequiredIfSet() const { return m_requiredIfSet; }
+        void setRequiredIf(const std::string &reqIf) { m_requiredIf = reqIf; }
+        const std::string &getRequiredIf() const { return m_requiredIf; }
         void setGroup(const std::string &group) { m_group = group; }
         const std::string &getGroup() const { return m_group; }
 
@@ -102,7 +102,7 @@ class DECL_EXPORT SchemaValue
         std::string m_autoGenerateType;
         std::string m_valueLimitRuleType;
         std::string m_valueLimitRuleData;
-        std::string m_requiredIfSet;
+        std::string m_requiredIf;
         std::string m_group;
         // DON'T FORGET IF DATA ADDED, IT MAY MAY TO BE COPIED IN THE COPY CONSTRUCTOR!!
 

configuration/config2/Utils.cpp

@@ -17,8 +17,11 @@ limitations under the License.
 
 #include <vector>
 #include <string>
+#include "Exceptions.hpp"
 
-std::vector<std::string> splitString(const std::string  &input, const std::string delim)
+bool getEnclosedString(const std::string source, std::string &result, std::size_t startPos, char endDelim, bool throwIfError);
+
+std::vector<std::string> splitString(const std::string &input, const std::string &delim)
 {
     size_t  start = 0, end = 0, delimLen = delim.length();
     std::vector<std::string> list;
@@ -33,3 +36,50 @@ std::vector<std::string> splitString(const std::string  &input, const std::strin
     }
     return list;
 }
+
+
+bool extractEnclosedString(const std::string source, std::string &result, char startDelim, char endDelim, bool optional)
+{
+    bool rc = false;
+    std::size_t startPos = source.find_first_of(startDelim);
+    if (startPos != std::string::npos)
+    {
+        rc = getEnclosedString(source, result, startPos, endDelim, true);
+    }
+    else if (!optional)
+    {
+        std::string msg = "Bad string parse, expected string enclosed in '" + std::string(1, startDelim) + std::string(1, endDelim) + "' at or around: " + source;
+        throw(ParseException(msg));
+    }
+    else
+    {
+        result = source;
+    }
+    return rc;
+}
+
+
+// throwIfError allows the caller to ignore an error finding the enclosed string and simply return the input string as the result string.
+// If true, the caller expects there to be an ending delimiter and wants an exception since this is an error condition as determined by the caller.
+// The return value will always reflect whether the result string was enclosed in the delimiter or not.
+bool getEnclosedString(const std::string source, std::string &result, std::size_t startPos, char endDelim, bool throwIfError)
+{
+    bool rc = false;
+    std::size_t endPos = source.find_first_of(endDelim, startPos+1);
+    if (endPos != std::string::npos)
+    {
+        result = source.substr(startPos+1, endPos-startPos-1);
+        rc = true;
+    }
+    else if (throwIfError)
+    {
+        std::string delim(1, endDelim);
+        std::string msg = "Bad string parse, expectd '" + delim + "' at or around: " + source;
+        throw(ParseException(msg));
+    }
+    else
+    {
+        result = source;
+    }
+    return rc;
+}

configuration/config2/Utils.hpp

@@ -20,7 +20,10 @@ limitations under the License.
 #define _CONFIG2_CONFIGUTILS_HPP_
 
 #include <vector>
+#include <string>
 
-std::vector<std::string> splitString(const std::string  &input, const std::string delim);
+std::vector<std::string> splitString(const std::string  &input, const std::string  &delim);
+bool extractEnclosedString(const std::string source, std::string &result, char startDelim, char endDelim, bool optional = true);
+bool getEnclosedString(const std::string source, std::string &result, std::size_t startPos, char endDelim, bool throwIfError = false);
 
 #endif

configuration/config2/XSDSchemaParser.cpp

@@ -650,15 +650,29 @@ void XSDSchemaParser::parseAllowedValue(const pt::ptree &allowedValueTree, Schem
     //
     // Parse the value for the enumeration, the add to the allowed values for the limits for this type. Note that enumerations
     // are enhanced with additional information for the UI.
-    allowedValue.m_value = allowedValueTree.get("<xmlattr>.value", "");
+    allowedValue.m_value = allowedValueTree.get("<xmlattr>.value", "XXXmissingYYY");
     allowedValue.m_displayName = allowedValueTree.get("<xmlattr>.hpcc:displayName", allowedValue.m_value);
     allowedValue.m_description = allowedValueTree.get("<xmlattr>.hpcc:description", "");
     allowedValue.m_userMessage = allowedValueTree.get("<xmlattr>.hpcc:userMessage", "");
     allowedValue.m_userMessageType = allowedValueTree.get("<xmlattr>.hpcc:userMessageType", allowedValue.m_userMessage.empty() ? "" : "info");
 
     //
+    // Parse any attribute lists
+    std::string attrList = allowedValueTree.get("<xmlattr>.hpcc:optionalAttributes", "");
+    if (attrList.length())
+    {
+        allowedValue.m_optionalAttributes = splitString(attrList, ",");
+    }
+
+    attrList = allowedValueTree.get("<xmlattr>.hpcc:requiredAttributes", "");
+    if (attrList.length())
+    {
+        allowedValue.m_requiredAttributes = splitString(attrList, ",");
+    }
+
+    //
     // Value is required. Throw an exception if not found
-    if (allowedValue.m_value.empty())
+    if (allowedValue.m_value == "XXXmissingYYY")
     {
         std::string msg = "Missing value attribute for enumeration";
         throw(ParseException(msg));
@@ -685,7 +699,7 @@ std::shared_ptr<SchemaValue> XSDSchemaParser::getSchemaValue(const pt::ptree &at
     pCfgValue->setCodeDefault(attr.get("<xmlattr>.hpcc:codeDefault", ""));
     pCfgValue->setValueLimitRuleType(attr.get("<xmlattr>.hpcc:valueLimitRuleType", ""));
     pCfgValue->setValueLimitRuleData(attr.get("<xmlattr>.hpcc:valueLimitRuleData", ""));
-    pCfgValue->setRequiredIfSet(attr.get("<xmlattr>.hpcc:requiredIfSet", ""));
+    pCfgValue->setRequiredIf(attr.get("<xmlattr>.hpcc:requiredIf", ""));
 
     std::string modList = attr.get("<xmlattr>.hpcc:modifiers", "");
     if (modList.length())

esp/scm/ws_config2.ecm

@@ -123,6 +123,8 @@ ESPstruct ChoiceType
     string MsgType("");
     string Msg("");
     ESParray<ESPstruct DependentValueType, dependentValue> Dependencies;
+    ESParray<string, Attribute> OptionalAttributes;    // Attributes to be enabled, but are optional if this choice is selected (UI)
+    ESParray<string, Attribute> RequiredAttributes;    // Attributes to be enabled and must be filled in (UI)
 };
 
 
@@ -149,6 +151,7 @@ ESPstruct NodeInfoType
 {
     string    Name("");
     string    NodeType("");
+    string    NodeName;
     string    Class("");
     string    Category("");
     bool      Hidden(false);
@@ -189,7 +192,7 @@ ESPstruct AttributeType
     string     DisplayName;
     string     Name;
     string     Group("");
-    string     tooltip("");
+    string     Tooltip("");
     ESPstruct  TypeInfo Type;
     string     CurrentValue("");
     string     DefaultValue("");

esp/services/ws_config2/ws_config2Error.hpp

@@ -33,6 +33,6 @@
 #define CFGMGR_ERROR_ENV_EXTERNAL_CHANGE     CONFIG_MGR_ERROR_START+10   // Environment was externally modified
 #define CFGMGR_ERROR_ENVIRONMENT_LOCKING     CONFIG_MGR_ERROR_START+11   // Error locking the enironment
 #define CFGMGR_ERROR_NODE_INVALID            CONFIG_MGR_ERROR_START+12   // Enironment node not valid
-#define CDGMGR_ERROR_PATH_INVALID            CONFIG_MGR_ERROR_START+13   // The path specified is not valid
+#define CFGMGR_ERROR_PATH_INVALID            CONFIG_MGR_ERROR_START+13   // The path specified is not valid
 
 #endif

esp/services/ws_config2/ws_config2Service.cpp

@@ -21,6 +21,7 @@
 #include "InsertableItem.hpp"
 #include "jexcept.hpp"
 #include "ws_config2Error.hpp"
+#include "Exceptions.hpp"
 
 static const std::string CFG2_MASTER_CONFIG_FILE = "environment.xsd";
 static const std::string CFG2_CONFIG_DIR = COMPONENTFILES_DIR  PATHSEPSTR "configschema" PATHSEPSTR "xsd" PATHSEPSTR;
@@ -46,7 +47,7 @@ bool Cws_config2Ex::onOpenSession(IEspContext &context, IEspOpenSessionRequest &
     std::string inputMasterFile = req.getMasterSchemaFile();
     std::string inputSchemaPath = req.getSchemaPath();
     std::string inputSourcePath = req.getSourcePath();
-    std::string inputActivePath = req.getSourcePath();
+    std::string inputActivePath = req.getActivePath();
     pNewSession->masterConfigFile = (inputMasterFile != "") ? inputMasterFile : CFG2_MASTER_CONFIG_FILE;
     pNewSession->username = req.getUsername();
     pNewSession->schemaPath = !inputSchemaPath.empty() ? inputSchemaPath : CFG2_CONFIG_DIR;
@@ -54,6 +55,14 @@ bool Cws_config2Ex::onOpenSession(IEspContext &context, IEspOpenSessionRequest &
     pNewSession->activePath = !inputActivePath.empty() ? inputActivePath : ACTIVE_ENVIRONMENT_FILE;
 
     //
+    // Make sure paths end with a separator
+    if (std::string(1, pNewSession->schemaPath.back()) != PATHSEPSTR)
+        pNewSession->schemaPath += PATHSEPSTR;
+
+    if (std::string(1, pNewSession->sourcePath.back()) != PATHSEPSTR)
+        pNewSession->sourcePath += PATHSEPSTR;
+
+    //
     // Only XML supported at this time
     pNewSession->configType = XML;
 
@@ -125,7 +134,7 @@ bool Cws_config2Ex::onGetEnvironmentFileList(IEspContext &context, IEspCommonSes
     StringBuffer activeConfig_md5sum;
     md5_filesum(pSession->activePath.c_str(), activeConfig_md5sum);
     IArrayOf<IEspEnvironmentFileType> environmentFiles;
-    Owned<IFile> pDir = createIFile(CFG2_SOURCE_DIR.c_str());
+    Owned<IFile> pDir = createIFile(pSession->sourcePath.c_str());
     if (pDir->exists())
     {
         Owned<IDirectoryIterator> it = pDir->directoryFiles(NULL, false, true);
@@ -269,17 +278,12 @@ bool Cws_config2Ex::onSaveEnvironmentFile(IEspContext &context, IEspSaveEnvironm
 bool Cws_config2Ex::onLockSession(IEspContext &context, IEspCommonSessionRequest &req, IEspLockSessionResponse &resp)
 {
     std::string sessionId = req.getSessionId();
-    ConfigMgrSession *pSession = getConfigSession(sessionId);
+    ConfigMgrSession *pSession = getConfigSession(sessionId, true);
     if (pSession->locked)
     {
         throw MakeStringException(CFGMGR_ERROR_ENVIRONMENT_LOCKED, "Current enironment already locked");
     }
 
-    if (pSession->curEnvironmentFile.empty())
-    {
-        throw MakeStringException(CFGMGR_ERROR_NO_ENVIRONMENT, "No environment loaded");
-    }
-
     if (pSession->externallyModified)
     {
         throw MakeStringException(CFGMGR_ERROR_ENV_EXTERNAL_CHANGE, "No environment loaded");
@@ -325,10 +329,9 @@ bool Cws_config2Ex::onGetNode(IEspContext &context, IEspNodeRequest &req, IEspGe
 {
     std::string sessionId = req.getSessionId();
     std::string id = req.getNodeId();
-    ConfigMgrSession *pSession = getConfigSession(sessionId);
+    ConfigMgrSession *pSession = getConfigSession(sessionId, true);
     Status status;
 
-
     EnvironmentMgr *pEnvMgr = pSession->m_pEnvMgr;
     std::shared_ptr<EnvironmentNode> pNode = pEnvMgr->getEnvironmentNode(id);
     if (pNode == nullptr)
@@ -400,7 +403,8 @@ bool Cws_config2Ex::onValidateEnvironment(IEspContext &context, IEspValidateEnvi
 {
     Status status;
     std::string sessionId = req.getSessionId();
-    ConfigMgrSession *pSession = getConfigSession(sessionId);
+    ConfigMgrSession *pSession = getConfigSession(sessionId, true);
+
     pSession->m_pEnvMgr->validate(status, req.getIncludeHiddenNodes());
     addStatusToResponse(status, pSession, resp);
     return true;
@@ -447,7 +451,7 @@ bool Cws_config2Ex::onGetParents(IEspContext &context, IEspNodeRequest &req, IEs
     std::string nodeId = req.getNodeId();
     std::string sessionId = req.getSessionId();
 
-    ConfigMgrSession *pSession = getConfigSession(sessionId);
+    ConfigMgrSession *pSession = getConfigSession(sessionId, true);
 
     StringArray ids;
     std::shared_ptr<EnvironmentNode> pNode = pSession->m_pEnvMgr->getEnvironmentNode(nodeId);
@@ -475,7 +479,10 @@ bool Cws_config2Ex::onGetNodeTree(IEspContext &context, IEspGetTreeRequest &req,
     std::string nodeId = req.getNodeId();
     std::string sessionId = req.getSessionId();
 
-    ConfigMgrSession *pSession = getConfigSession(sessionId);
+    ConfigMgrSession *pSession = getConfigSession(sessionId, true);
+
+    if (nodeId == "")
+        nodeId = pSession->m_pEnvMgr->getRootNodeId();
 
     std::shared_ptr<EnvironmentNode> pNode = pSession->m_pEnvMgr->getEnvironmentNode(nodeId);
     if (pNode)
@@ -493,13 +500,13 @@ bool Cws_config2Ex::onFetchNodes(IEspContext &context, IEspFetchNodesRequest &re
     std::string path = req.getPath();
     std::string startingNodeId = req.getStartingNodeId();
     std::shared_ptr<EnvironmentNode> pStartingNode;
-    ConfigMgrSession *pSession = getConfigSession(sessionId);
+    ConfigMgrSession *pSession = getConfigSession(sessionId, true);
 
     if (startingNodeId != "")
     {
         if (path[0] == '/')
         {
-            throw MakeStringException(CDGMGR_ERROR_PATH_INVALID, "Path may not begin at root if starting node specified");
+            throw MakeStringException(CFGMGR_ERROR_PATH_INVALID, "Path may not begin at root if starting node specified");
         }
 
         pStartingNode = pSession->m_pEnvMgr->getEnvironmentNode(startingNodeId);
@@ -510,20 +517,27 @@ bool Cws_config2Ex::onFetchNodes(IEspContext &context, IEspFetchNodesRequest &re
     }
     else if (path[0] != '/')
     {
-        throw MakeStringException(CDGMGR_ERROR_PATH_INVALID, "Path must begin at root (/) if no starting node is specified");
+        throw MakeStringException(CFGMGR_ERROR_PATH_INVALID, "Path must begin at root (/) if no starting node is specified");
     }
 
-    std::vector<std::shared_ptr<EnvironmentNode>> nodes;
-    pSession->m_pEnvMgr->fetchNodes(path, nodes, pStartingNode);
-    StringArray ids;
-    for ( auto &&pNode : nodes)
+    try
     {
-        if (!pNode->getSchemaItem()->isHidden())
+        std::vector<std::shared_ptr<EnvironmentNode>> nodes;
+        pSession->m_pEnvMgr->fetchNodes(path, nodes, pStartingNode);
+        StringArray ids;
+        for ( auto &&pNode : nodes)
         {
-            ids.append(pNode->getId().c_str());
+            if (!pNode->getSchemaItem()->isHidden())
+            {
+                ids.append(pNode->getId().c_str());
+            }
         }
+        resp.setNodeIds(ids);
+    }
+    catch (ParseException &pe)
+    {
+        throw MakeStringException(CFGMGR_ERROR_PATH_INVALID, "%s", pe.what());
     }
-    resp.setNodeIds(ids);
 
     return true;
 }
@@ -558,7 +572,7 @@ void Cws_config2Ex::addStatusToResponse(const Status &status, ConfigMgrSession *
 }
 
 
-ConfigMgrSession *Cws_config2Ex::getConfigSession(const std::string &sessionId)
+ConfigMgrSession *Cws_config2Ex::getConfigSession(const std::string &sessionId, bool environmentRequired)
 {
     ConfigMgrSession *pSession = nullptr;
 
@@ -574,13 +588,16 @@ ConfigMgrSession *Cws_config2Ex::getConfigSession(const std::string &sessionId)
     if (pSession == nullptr)
         throw MakeStringException(CFGMGR_ERROR_INVALID_SESSION_ID, "Session ID not valid");
 
+    if (environmentRequired && pSession->curEnvironmentFile.empty())
+        throw MakeStringException(CFGMGR_ERROR_NO_ENVIRONMENT, "No environment loaded");
+
     return pSession;
 }
 
 
 ConfigMgrSession *Cws_config2Ex::getConfigSessionForUpdate(const std::string &sessionId, const std::string &lockKey)
 {
-    ConfigMgrSession *pSession = getConfigSession(sessionId);
+    ConfigMgrSession *pSession = getConfigSession(sessionId, true);
     if (!pSession->doesKeyFit(lockKey))
     {
         throw MakeStringException(CFGMGR_ERROR_LOCK_KEY_INVALID, "Session lock key is missing or invalid");
@@ -742,6 +759,7 @@ void Cws_config2Ex::getNodeInfo(const std::shared_ptr<EnvironmentNode> &pNode, I
     getNodeInfo(pNodeSchemaItem, nodeInfo);      // fill it in based on schema
     getNodeDisplayName(pNode, nodeDisplayName);  // possibly override the displayname
     nodeInfo.setName(nodeDisplayName.c_str());
+    nodeInfo.setNodeName(pNode->getName().c_str());
 }
 
 
@@ -788,7 +806,8 @@ void Cws_config2Ex::getAttributes(const std::vector<std::shared_ptr<EnvironmentV
         pAttribute->setReadOnly(pSchemaValue->isReadOnly());
         pAttribute->setHidden(pSchemaValue->isHidden());
         pAttribute->setDeprecated(pSchemaValue->isDeprecated());
-        pAttribute->setGroup(pSchemaValue->getGroup().c_str());
+        std::string groupName = pSchemaValue->getGroup();
+        pAttribute->setGroup(groupName.empty() ? "Attributes" : groupName.c_str());
 
         std::vector<AllowedValue> allowedValues;
         pSchemaValue->getAllowedValues(allowedValues, pAttr->getEnvironmentNode());
@@ -819,6 +838,30 @@ void Cws_config2Ex::getAttributes(const std::vector<std::shared_ptr<EnvironmentV
                     pChoice->setDependencies(dependencies);
                 }
 
+                //
+                // Add optional/required attributes.
+                if (!(*valueIt).m_optionalAttributes.empty())
+                {
+                    StringArray attributeNames;
+                    for (auto &attr: (*valueIt).m_optionalAttributes)
+                    {
+                        StringBuffer atrrname(attr.c_str());
+                        attributeNames.append(atrrname);
+                    }
+                    pChoice->setOptionalAttributes(attributeNames);
+                }
+
+                if (!(*valueIt).m_requiredAttributes.empty())
+                {
+                    StringArray attributeNames;
+                    for (auto &attr: (*valueIt).m_requiredAttributes)
+                    {
+                        StringBuffer atrrname(attr.c_str());
+                        attributeNames.append(atrrname);
+                    }
+                    pChoice->setRequiredAttributes(attributeNames);
+                }
+
                 choices.append(*pChoice.getLink());
             }
             pAttribute->updateType().updateLimits().setChoiceList(choices);

esp/services/ws_config2/ws_config2Service.hpp

@@ -60,7 +60,7 @@ public:
 private:
 
     void addStatusToResponse(const Status &status, ConfigMgrSession *pSession, IEspStatusResponse &resp) const;
-    ConfigMgrSession *getConfigSession(const std::string &sessionId);
+    ConfigMgrSession *getConfigSession(const std::string &sessionId, bool environmentRequired = false);
     ConfigMgrSession *getConfigSessionForUpdate(const std::string &sessionId, const std::string &lockKey);
     bool deleteConfigSession(const std::string &sessionId);
     void getNodeResponse(const std::shared_ptr<EnvironmentNode> &pNode, IEspGetNodeResponse &resp) const;

esp/services/ws_config2/ws_config2Session.hpp

@@ -80,7 +80,7 @@ struct ConfigMgrSession {
 
     void getEnvironmentFullyQualifiedPath(const std::string &envFile, std::string &fullPath)
     {
-        fullPath = sourcePath + PATHSEPSTR + envFile;
+        fullPath = sourcePath + envFile;
     }
 
 

initfiles/componentfiles/configschema/xsd/environment.xsd

@@ -29,9 +29,9 @@
     <xs:element name="Environment" hpcc:category="root" hpcc:class="category">
         <xs:complexType>
             <xs:sequence>
-                <!--xs:element name="EnvSettings" hpcc:class="category" hpcc:displayName="System Settings" hpcc:hidden="true"></xs:element-->
+                <xs:element name="EnvSettings" hpcc:class="category" hpcc:displayName="System Settings" hpcc:hidden="true"></xs:element>
                 <xs:element type="hardware" hpcc:class="category" hpcc:displayName="Hardware" hpcc:required="true"></xs:element>
-                <!--xs:element name="Programs" hpcc:class="category" hpcc:displayName="Programs" hpcc:hidden="true"></xs:element-->
+                <xs:element name="Programs" hpcc:class="category" hpcc:displayName="Programs" hpcc:hidden="true"></xs:element>
                 <xs:element name="Software" hpcc:class="category" hpcc:displayName="Software">
                     <xs:complexType>
                         <xs:sequence>

initfiles/componentfiles/configschema/xsd/esp.xsd

@@ -20,10 +20,10 @@
                                             <xs:attribute name="description" type="xs:string" use="optional" hpcc:displayName="Description"/>
                                             <xs:attribute name="path" type="xs:string" use="required" default="/" hpcc:displayName="Path" hpcc:tooltip="The logical path of a resource used for authentication"/>
                                             <xs:attribute name="resource" type="xs:string" use="required" hpcc:displayName="Resource" hpcc:tooltip="The physical resource for which access is checked"/>
-                                            <xs:attribute name="access" default="Read">
+                                            <xs:attribute name="access" hpcc:codeDefault="Read">
                                                 <xs:simpleType>
                                                     <xs:restriction base="xs:string">
-                                                        <xs:enumeration value="" hpcc:description=""/>
+                                                        <xs:enumeration value="" hpcc:displayName="Use default" hpcc:description=""/>
                                                         <xs:enumeration value="Access" hpcc:description=""/>
                                                         <xs:enumeration value="Read" hpcc:description=""/>
                                                         <xs:enumeration value="Write" hpcc:description=""/>
@@ -75,11 +75,11 @@
                                     hpcc:valueLimitRuleType="uniqueItemType_1" hpcc:valueLimitRuleData="EspBinding@service,/Environment/Software/EspService@name"/>
                                 <xs:attribute name="type" type="xs:string" use="optional" hpcc:displayName="Security Manager Plugin" hpcc:tooltip="The Security Manager to be used by the Esp Service"/>
                                 <xs:attribute name="workunitsBasedn" type="xs:string" use="optional" default="ou=workunits,ou=ecl" hpcc:displayName="WorkUnits BaseDn" hpcc:tooltip="Base location for workunit resources (used with ldap security)" hpcc:requiredIf="xpath,xpath..."/>
-                                <xs:attribute name="wsdlServiceAddress" type="xs:string" use="optional"hpcc:displayName="wsdlServiceAddress" hpcc:tooltip="Overrides the address used by client applications to connect to the service"/>
+                                <xs:attribute name="wsdlServiceAddress" type="xs:string" use="optional" hpcc:displayName="wsdlServiceAddress" hpcc:tooltip="Overrides the address used by client applications to connect to the service"/>
                             </xs:complexType>
 
                             <xs:key name="esp_custombindingparameter_key">
-                                <xs:selector xpath="CustomBindingParameter" />
+                                <xs:selector xpath="./CustomBindingParameter" />
                                 <xs:field xpath="@key" />
                             </xs:key>
 
@@ -87,19 +87,22 @@
 
                         <xs:element name="Authentication" hpcc:docid="ESP.t4" hpcc:class="valueSet" hpcc:displayName="Authentication">
                             <xs:complexType>
-                                <xs:attribute name="method" use="required" default="none" hpcc:displayName="Method" hpcc:tooltip="The protocol to use for authenticating the service">
+                                <xs:attribute name="method" use="required" default="none" hpcc:displayName="Method" hpcc:modifiers="variableAttributes" hpcc:tooltip="The protocol to use for authenticating the service">
                                     <xs:simpleType>
                                         <xs:restriction base="xs:string">
                                             <xs:enumeration value="none" hpcc:description=""/>
                                             <xs:enumeration value="local" hpcc:description=""/>
-                                            <xs:enumeration value="ldap" hpcc:description=""/>
-                                            <xs:enumeration value="ldaps" hpcc:description=""/>
+                                            <xs:enumeration value="ldap" hpcc:requiredAttributes="ldapServer,ldapAuthMethod,ldapConnections,passwordExpirationWarningDays,checkViewPermissions" hpcc:description=""/>
+                                            <xs:enumeration value="ldaps" hpcc:requiredAttributes="ldapServer,ldapAuthMethod,ldapConnections,passwordExpirationWarningDays,checkViewPermissions" hpcc:description=""/>
+                                            <xs:enumeration value="userNameOnly" hpcc:requiredAttributes="getUserNameURL,getUserNameUnrestrictedResources" hpcc:description=""/>
                                             <xs:enumeration value="secmgrPlugin" hpcc:description=""/>
                                         </xs:restriction>
                                     </xs:simpleType>
                                 </xs:attribute>
-                                <xs:attribute name="ldapServer" type="xs:string" use="optional" hpcc:displayName="LDAP Server" hpcc:tooltip="The ldap server to be used for authentication" hpcc:requiredIf="ldap chosen"/>
-                                <xs:attribute name="ldapAuthMethod" type="xs:string" use="required" default="kerberos" hpcc:displayName="LDAP Auth Method" hpcc:tooltip="The protocol to use for LDAP authentication">
+                                <xs:attribute name="ldapServer" type="xs:string" use="optional" hpcc:displayName="LDAP Server" hpcc:requiredIf=".[@method=('ldap','ldaps')]"
+                                    hpcc:tooltip="The ldap server to be used for authentication"/>
+                                <xs:attribute name="ldapAuthMethod" type="xs:string" use="optional" default="kerberos" hpcc:displayName="LDAP Auth Method"
+                                    hpcc:requiredIf=".[@method=('ldap','ldaps')]" hpcc:tooltip="The protocol to use for LDAP authentication">
                                     <xs:simpleType>
                                         <xs:restriction base="xs:string">
                                             <xs:enumeration value="kerberos" hpcc:description=""/>
@@ -107,12 +110,16 @@
                                         </xs:restriction>
                                     </xs:simpleType>
                                 </xs:attribute>
-                                <xs:attribute name="ldapConnections" type="xs:nonNegativeInteger" use="optional" default="10" hpcc:displayName=""
-                                              hpcc:tooltip="Maximum number of connections to the LDAP server" hpcc:requiredIf=""/>
-                                <xs:attribute name="passwordExpirationWarningDays" type="xs:nonNegativeInteger" use="optional" default="10"
-                                              hpcc:displayName="Passowrd Expiration Warning Days" hpcc:tooltip="In this time period, ESP displays a warning about password expiration"/>
+                                <xs:attribute name="ldapConnections" type="xs:nonNegativeInteger" use="optional" default="10" hpcc:displayName="LDAP Connections"
+                                    hpcc:tooltip="Maximum number of connections to the LDAP server" hpcc:requiredIf=".[@method=('ldap','ldaps')]"/>
+                                <xs:attribute name="passwordExpirationWarningDays" type="xs:nonNegativeInteger" use="optional" default="10" hpcc:requiredIf=".[@method=('ldap','ldaps')]"
+                                    hpcc:displayName="Passowrd Expiration Warning Days" hpcc:tooltip="In this time period, ESP displays a warning about password expiration"/>
                                 <xs:attribute name="checkViewPermissions" type="xs:boolean" use="optional" default="false" hpcc:displayName="Check View Permissions"
-                                              hpcc:tooltip="Enable file and column access permission checking for all view enabled queries"/>
+                                    hpcc:requiredIf=".[@method=('ldap','ldaps')]" hpcc:tooltip="Enable file and column access permission checking for all view enabled queries"/>
+                                <xs:attribute name="getUserNameURL" type="xs:string" hpcc:displayName="Username URL" use="optional" default="/esp/files/GetUserName.html"
+                                    hpcc:requiredIf=".[@method='userNameOnly']" hpcc:tooltip="URL to getUserName"/>
+                                <xs:attribute name="getUserNameUnrestrictedResources" type="xs:string" hpcc:displayName="Unrestricted Resources Name" use="optional" default="/favicon.ico,/esp/files/*,/esp/xslt/*"
+                                    hpcc:requiredIf=".[@method='userNameOnly']" hpcc:tooltip="unrestricted resources for getUserNameURL"/>
                             </xs:complexType>
                         </xs:element>
 
@@ -187,28 +194,28 @@
                 </xs:complexType>
 
                 <xs:key name="espprocess_name_key">
-                    <xs:selector xpath="." />
+                    <xs:selector xpath="./" />
                     <xs:field xpath="@name" />
                 </xs:key>
 
                 <xs:key name="espprocess_controlport_key">
-                    <xs:selector xpath="." />
+                    <xs:selector xpath="./" />
                     <xs:field xpath="@controlPort" />
                 </xs:key>
 
                 <xs:keyref name="esp_servicename_keyref" refer="espservice_name_key">
-                    <xs:selector xpath="EspBinding" />
+                    <xs:selector xpath="./EspBinding" />
                     <xs:field xpath="@service" />
                 </xs:keyref>
 
                 <xs:keyref name="espprocess_Instance_keyref" refer="computerNameKey">
-                    <xs:selector xpath="Instance"/>
+                    <xs:selector xpath="./Instance"/>
                     <xs:field xpath="@computer"/>
                 </xs:keyref>
 
                 <xs:keyref name="espprocess_Instance_ipref" refer="computerIPAddressKey">
-                    <xs:selector xpath="Instance"/>
-                    <xs:field xpath="netAddress"/>
+                    <xs:selector xpath="./Instance"/>
+                    <xs:field xpath="@netAddress"/>
                 </xs:keyref>
 
             </xs:element>

initfiles/componentfiles/configschema/xsd/esp_service_smc.xsd

@@ -146,7 +146,7 @@
                 </xs:complexType>
 
                 <xs:key name="espservice_name_key" hpcc:allowDuplicate="true">
-                    <xs:selector xpath="." />
+                    <xs:selector xpath="./" />
                     <xs:field xpath="@name" />
                 </xs:key>
 

initfiles/componentfiles/configschema/xsd/hardware.xsd

@@ -8,8 +8,7 @@
       <xs:element name="Hardware" hpcc:docid="????" hpcc:doc="Section to describe the hardware that composes the environment">
         <xs:complexType>
           <xs:sequence>
-            <xs:element name="ComputerType" maxOccurs="unbounded" hpcc:displayName="Defined computer types"  hpcc:class="valueSet"
-                        hpcc:doc="Defines a generalized type of computer that is used in the environment">
+            <xs:element name="ComputerType" maxOccurs="unbounded" hpcc:displayName="Defined computer types"  hpcc:class="valueSet">
               <xs:complexType>
                 <xs:attribute name="name" type="xs:string" use="required"/>
                 <xs:attribute name="nicSpeed" type="xs:nonNegativeInteger"/>
@@ -53,37 +52,37 @@
         </xs:complexType>
 
         <xs:key name="domainKey">
-          <xs:selector xpath="Domain" />
+          <xs:selector xpath="./Domain" />
           <xs:field xpath="@name" />
         </xs:key>
 
         <xs:key name="computerTypeKey">
-          <xs:selector xpath="ComputerType" />
+          <xs:selector xpath="./ComputerType" />
           <xs:field xpath="@name" />
         </xs:key>
 
         <xs:key name="computerIPAddressKey">
-          <xs:selector xpath="Computer" />
-          <xs:field xpath="netAddress" />
+          <xs:selector xpath="./Computer" />
+          <xs:field xpath="@netAddress" />
         </xs:key>
 
         <xs:key name="computerNameKey">
-          <xs:selector xpath="Computer" />
+          <xs:selector xpath="./Computer" />
           <xs:field xpath="@name" />
         </xs:key>
 
         <xs:keyref name="Computer_computerTypeRef" refer="computerTypeKey">
-          <xs:selector xpath="Computer"/>
+          <xs:selector xpath="./Computer"/>
           <xs:field xpath="@computerType" />
         </xs:keyref>
 
         <xs:keyref name="Computer_domainRef" refer="domainKey">
-          <xs:selector xpath="Computer"/>
+          <xs:selector xpath="./Computer"/>
           <xs:field xpath="@domain" />
         </xs:keyref>
 
         <xs:key name="switchNameKey">
-          <xs:selector xpath="Switch" />
+          <xs:selector xpath="./Switch" />
           <xs:field xpath="@name" />
         </xs:key>
 

initfiles/componentfiles/configschema/xsd/roxie.xsd

@@ -51,7 +51,7 @@
                         </xs:element>
                         <xs:element name="RoxieServerProcess" hpcc:displayName="Servers" maxOccurs="unbounded"> <!--special view type needed to collect together all of these somehow -->
                             <xs:complexType>
-                                <xs:attribute name="name" type="nodeName" use="required" hpcc:readOnly="true" hpcc:mirrorFrom="/Environment/Hardware/Computer@name"/>
+                                <xs:attribute name="name" type="nodeName" use="required" hpcc:readOnly="true" hpcc:mirrorFrom="/Environment/Hardware/Computer[@name]"/>
                                 <xs:attributeGroup ref="computerNodeReference"/>
                             </xs:complexType>
                         </xs:element>

initfiles/componentfiles/configschema/xsd/types.xsd

@@ -26,13 +26,6 @@
         </xs:restriction>
     </xs:simpleType>
 
-    <xs:simpleType name="yesno">
-        <xs:restriction base="xs:string">
-            <xs:minInclusive value="yes" hpcc:displayName="Yes"/>
-            <xs:maxInclusive value="no" hpcc:displayName="No"/>
-        </xs:restriction>
-    </xs:simpleType>
-
     <xs:simpleType name="nodeName">
         <xs:restriction base="xs:string">
             <xs:pattern value="[a-zA-z0-9_\-]+"/>
@@ -83,8 +76,8 @@
     </xs:attributeGroup>
 
     <xs:attributeGroup name="computerNodeReference">
-        <xs:attribute name="computer" type="nodeName" use="required" hpcc:readOnly="true" hpcc:mirrorFrom="/Environment/Hardware/Computer@name"/>
-        <xs:attribute name="netAddress" type="ipV4Address" use="required" hpcc:readOnly="true" hpcc:mirrorFrom="/Environment/Hardware/Computer@netAddress"/>
+        <xs:attribute name="computer" type="nodeName" use="required" hpcc:readOnly="true" hpcc:mirrorFrom="/Environment/Hardware/Computer[@name]"/>
+        <xs:attribute name="netAddress" type="ipV4Address" use="required" hpcc:readOnly="true" hpcc:mirrorFrom="/Environment/Hardware/Computer[@netAddress]"/>
     </xs:attributeGroup>
 
     <xs:complexType name="hwinstance">
@@ -180,7 +173,7 @@
                     <xs:attribute name="defaultResourcesBasedn" type="xs:string"/>
                     <xs:attribute name="defaultSecurePort" type="xs:string"/>
                     <xs:attribute name="type" type="xs:string"/>
-                </complexType>
+                </xs:complexType>
             </xs:element>
         </xs:sequence>
     </xs:complexType>