HPCC-17851 New config manager core library

Adds the core library for the new configuration manager.

Signed-off-by: Ken Rowland <kenneth.rowland@lexisnexisrisk.com>

.gitignore

@@ -6,3 +6,4 @@ ln/
 node_modules
 esp/src/lib
 esp/src/build
+.vscode/

CMakeLists.txt

@@ -176,6 +176,7 @@ elseif ( NOT MAKE_DOCS_ONLY )
     HPCC_ADD_SUBDIRECTORY (services "PLATFORM")
     HPCC_ADD_SUBDIRECTORY (thorlcr "PLATFORM")
     HPCC_ADD_SUBDIRECTORY (testing)
+    HPCC_ADD_SUBDIRECTORY (configuration)
 
     if ( WIN32 )
         HPCC_ADD_SUBDIRECTORY (clienttools/IDEPlugins "CLIENTTOOLS_ONLY")

configuration/CMakeLists.txt

@@ -14,4 +14,4 @@
 #    limitations under the License.
 ################################################################################
 
-HPCC_ADD_SUBDIRECTORY (configurator)
+HPCC_ADD_SUBDIRECTORY (config2)

configuration/config2/CMakeLists.txt

@@ -0,0 +1,51 @@
+##############################################################################
+#
+#    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems®.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+##############################################################################
+
+project ( config2 )
+
+SET(  SRCS
+      EnvironmentMgr.cpp
+      XMLEnvironmentMgr.cpp
+      SchemaParser.cpp
+      SchemaItem.cpp
+      SchemaValue.cpp
+      SchemaTypeLimits.cpp
+      SchemaTypeIntegerLimits.cpp
+      SchemaTypeStringLimits.cpp
+      EnvironmentNode.cpp
+      EnvironmentValue.cpp
+      XSDComponentParser.cpp
+      XSDSchemaParser.cpp
+      XSDValueSetParser.cpp
+      Status.cpp
+)
+
+INCLUDE_DIRECTORIES(
+      ${HPCC_SOURCE_DIR}/system/include
+      ${HPCC_SOURCE_DIR}/system/jlib
+      ${HPCC_SOURCE_DIR}/deployment/deploy
+      ${CMAKE_PREFIX_PATH}/mkspecs/linux-g++
+      ${CMAKE_CURRENT_SOURCE_DIR}
+      ${CMAKE_BINARY_DIR}
+      ${CMAKE_BINARY_DIR}/oss
+)
+
+ADD_DEFINITIONS( -D_USRDLL -DCONFIG2_LIB)
+HPCC_ADD_LIBRARY( config2 SHARED ${SRCS} )
+TARGET_LINK_LIBRARIES( config2 jlib )
+
+INSTALL( TARGETS config2 LIBRARY DESTINATION ${LIB_DIR} )

configuration/config2/EnvironmentMgr.cpp

@@ -0,0 +1,240 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2017 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#include "EnvironmentMgr.hpp"
+#include "Exceptions.hpp"
+#include "XMLEnvironmentMgr.hpp"
+
+
+EnvironmentMgr *getEnvironmentMgrInstance(const EnvironmentType envType)
+{
+    EnvironmentMgr *pEnvMgr = nullptr;
+    if (envType == XML)
+    {
+        pEnvMgr = new XMLEnvironmentMgr();
+    }
+    return pEnvMgr;
+}
+
+
+EnvironmentMgr::EnvironmentMgr() :
+    m_key(0)
+{
+    m_pSchema = std::make_shared<SchemaItem>("root");  // make the root
+}
+
+
+bool EnvironmentMgr::loadSchema(const std::string &configPath, const std::string &masterConfigFile, const std::vector<std::string> &cfgParms)
+{
+    bool rc = false;
+    if (createParser())
+    {
+        rc = m_pSchemaParser->parse(configPath, masterConfigFile, cfgParms);
+        if (rc)
+        {
+            // unique attribure value sets are global across a schema. Allocate one here and pass it in
+            // for use in building the necessary references and dependencies across the schema, then pass
+            // it to the post processing for finalization. Once referencs and dependencies are built, the
+            // attribute value sets are no longer needed.
+            std::map<std::string, std::vector<std::shared_ptr<SchemaValue>>> uniqueAttributeValueSets;
+            m_pSchema->processDefinedUniqueAttributeValueSets(uniqueAttributeValueSets);  // This must be done first
+            m_pSchema->postProcessConfig(uniqueAttributeValueSets);
+        }
+    }
+    return rc;
+}
+
+
+std::string EnvironmentMgr::getLastSchemaMessage() const
+{
+    if (m_pSchemaParser)
+        return m_pSchemaParser->getLastMessage();
+    return "";
+}
+
+
+bool EnvironmentMgr::loadEnvironment(const std::string &qualifiedFilename)
+{
+    bool rc = false;
+
+    if (m_pSchema)
+    {
+        std::ifstream in;
+
+        in.open(qualifiedFilename);
+        if (in.is_open())
+        {
+            rc = doLoadEnvironment(in);
+        }
+        else
+        {
+            m_message = "Unable to open environment file '" + qualifiedFilename + "'";
+        }
+    }
+    else
+    {
+        m_message = "No schema loaded";
+    }
+    return rc;
+}
+
+
+bool EnvironmentMgr::saveEnvironment(const std::string &qualifiedFilename)
+{
+    bool rc = false;
+    if (m_pRootNode)
+    {
+        std::ofstream out;
+
+        out.open(qualifiedFilename);
+        if (out.is_open())
+        {
+            rc = save(out);
+        }
+    }
+    else
+    {
+        m_message = "No environment loaded";
+    }
+    return rc;
+}
+
+
+void EnvironmentMgr::addPath(const std::shared_ptr<EnvironmentNode> pNode)
+{
+    auto retVal = m_nodeIds.insert({pNode->getId(), pNode });
+    if (!retVal.second)
+    {
+        throw (ParseException("Attempted to insert duplicate path name " + pNode->getId() + " for node "));
+    }
+}
+
+
+std::shared_ptr<EnvironmentNode> EnvironmentMgr::getEnvironmentNode(const std::string &nodeId)
+{
+    std::shared_ptr<EnvironmentNode> pNode;
+    auto pathIt = m_nodeIds.find(nodeId);
+    if (pathIt != m_nodeIds.end())
+        pNode = pathIt->second;
+    return pNode;
+}
+
+
+
+std::shared_ptr<EnvironmentNode> EnvironmentMgr::addNewEnvironmentNode(const std::string &parentNodeId, const std::string &configType, Status &status)
+{
+    std::shared_ptr<EnvironmentNode> pNewNode;
+    std::shared_ptr<EnvironmentNode> pParentNode = getEnvironmentNode(parentNodeId);
+    if (pParentNode)
+    {
+        std::shared_ptr<SchemaItem> pNewCfgItem;
+        std::vector<std::shared_ptr<SchemaItem>> insertableItems;
+        pParentNode->getInsertableItems(insertableItems);
+        for (auto it = insertableItems.begin(); it != insertableItems.end(); ++it)
+        {
+            if ((*it)->getItemType() == configType)
+            {
+                pNewNode = addNewEnvironmentNode(pParentNode, *it, status);
+                break;
+            }
+            if (pNewNode == nullptr)
+            {
+                status.addMsg(statusMsg::error, "Configuration type (" + configType + ") not found");
+            }
+        }
+    }
+    else
+    {
+        status.addMsg(statusMsg::error, parentNodeId, "", "", "Unable to find indicated parent node");
+    }
+    return pNewNode;
+}
+
+
+std::shared_ptr<EnvironmentNode> EnvironmentMgr::addNewEnvironmentNode(const std::shared_ptr<EnvironmentNode> &pParentNode, const std::shared_ptr<SchemaItem> &pNewCfgItem, Status &status)
+{
+    std::shared_ptr<EnvironmentNode> pNewNode;
+
+    //
+    // Create the new node and add it to the parent
+    pNewNode = std::make_shared<EnvironmentNode>(pNewCfgItem, pNewCfgItem->getProperty("name"), pParentNode);
+    pNewNode->setId(getUniqueKey());
+    pParentNode->addChild(pNewNode);
+    addPath(pNewNode);
+    pNewNode->initialize();
+
+
+    //
+    // Look through the children and add any that are necessary
+    std::vector<std::shared_ptr<SchemaItem>> cfgChildren;
+    pNewCfgItem->getChildren(cfgChildren);
+    for (auto childIt = cfgChildren.begin(); childIt != cfgChildren.end(); ++childIt)
+    {
+        int numReq = (*childIt)->getMinInstances();
+        for (int i=0; i<numReq; ++i)
+        {
+            addNewEnvironmentNode(pNewNode, *childIt, status);
+        }
+    }
+
+    return pNewNode;
+}
+
+
+bool EnvironmentMgr::removeEnvironmentNode(const std::string &nodeId, Status &status)
+{
+    bool rc = false;
+    std::shared_ptr<EnvironmentNode> pNode = getEnvironmentNode(nodeId);
+
+    if (pNode)
+    {
+        std::shared_ptr<EnvironmentNode> pParentNode = pNode->getParent();
+        if (pParentNode->removeChild(pNode))
+        {
+            m_nodeIds.erase(nodeId);
+        }
+        else
+        {
+            status.addMsg(statusMsg::error, nodeId, "", "", "Unable to remove the node");
+        }
+    }
+    else
+    {
+        status.addMsg(statusMsg::error, nodeId, "", "", "Unable to find indicated node");
+    }
+
+    return rc;
+}
+
+
+std::string EnvironmentMgr::getUniqueKey()
+{
+    return std::to_string(m_key++);
+}
+
+
+void EnvironmentMgr::validate(Status &status) const
+{
+    if (m_pRootNode)
+    {
+        m_pRootNode->validate(status, true);
+    }
+    else
+    {
+        status.addMsg(statusMsg::error, "No environment loaded");
+    }
+}

configuration/config2/EnvironmentMgr.hpp

@@ -0,0 +1,86 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2017 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+
+#ifndef _CONFIG2_ENVIRONMENTMGR_HPP_
+#define _CONFIG2_ENVIRONMENTMGR_HPP_
+
+#include <string>
+#include <fstream>
+#include <vector>
+#include <atomic>
+#include "SchemaItem.hpp"
+#include "SchemaParser.hpp"
+#include "EnvironmentNode.hpp"
+#include "Status.hpp"
+#include "NameValue.hpp"
+#include "platform.h"
+
+class EnvironmentMgr;
+
+enum EnvironmentType
+{
+    UNDEFINED,
+    XML
+};
+
+
+DECL_EXPORT EnvironmentMgr *getEnvironmentMgrInstance(const EnvironmentType envType);
+
+class DECL_EXPORT EnvironmentMgr
+{
+    public:
+
+        EnvironmentMgr();
+        virtual ~EnvironmentMgr() { }
+        bool loadSchema(const std::string &configPath, const std::string &masterConfigFile, const std::vector<std::string> &cfgParms = std::vector<std::string>());
+        std::string getLastSchemaMessage() const;
+        std::string getLastEnvironmentMessage() const { return m_message;  }
+        bool loadEnvironment(const std::string &qualifiedFilename);
+        std::shared_ptr<EnvironmentNode> getEnvironmentNode(const std::string &nodeId);
+        std::shared_ptr<EnvironmentNode> addNewEnvironmentNode(const std::string &parentNodeId, const std::string &configType, Status &status);
+        bool removeEnvironmentNode(const std::string &nodeId, Status &status);
+        bool saveEnvironment(const std::string &qualifiedFilename);
+        void discardEnvironment() { m_pRootNode = nullptr; }
+        void validate(Status &status) const;
+
+
+    protected:
+
+        std::string getUniqueKey();
+        void addPath(const std::shared_ptr<EnvironmentNode> pNode);
+        virtual bool createParser() = 0;
+        std::shared_ptr<EnvironmentNode> addNewEnvironmentNode(const std::shared_ptr<EnvironmentNode> &pParentNode, const std::shared_ptr<SchemaItem> &pNewCfgItem, Status &status);
+        virtual bool doLoadEnvironment(std::istream &in) = 0;
+        virtual bool save(std::ostream &out) = 0;
+
+
+    protected:
+
+        std::shared_ptr<SchemaItem> m_pSchema;
+        std::shared_ptr<SchemaParser> m_pSchemaParser;
+        std::shared_ptr<EnvironmentNode> m_pRootNode;
+        std::map<std::string, std::shared_ptr<EnvironmentNode>> m_nodeIds;
+        std::string m_message;
+
+
+    private:
+
+        std::atomic_int m_key;
+};
+
+#endif

configuration/config2/EnvironmentNode.cpp

@@ -0,0 +1,350 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2017 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#include "EnvironmentNode.hpp"
+
+void EnvironmentNode::addChild(std::shared_ptr<EnvironmentNode> pNode)
+{
+    m_children.insert(std::make_pair(pNode->getName(), pNode));
+}
+
+
+bool EnvironmentNode::removeChild(const std::shared_ptr<EnvironmentNode> pNode)
+{
+    bool removed = false;
+    for (auto it=m_children.begin(); it!= m_children.end() && !removed; ++it)
+    {
+        if (pNode == it->second)
+        {
+            m_children.erase(it);
+            removed = true;
+        }
+    }
+    return removed;
+}
+
+
+bool EnvironmentNode::addAttribute(const std::string &name, std::shared_ptr<EnvironmentValue> pValue)
+{
+    auto retValue = m_attributes.insert(std::make_pair(name, pValue));
+    return retValue.second;
+}
+
+
+void EnvironmentNode::getChildren(std::vector<std::shared_ptr<EnvironmentNode>> &childNodes, const std::string &name) const
+{
+    if (name.empty())
+    {
+        for (auto nodeIt = m_children.begin(); nodeIt != m_children.end(); ++nodeIt)
+        {
+            childNodes.push_back(nodeIt->second);
+        }
+    }
+    else
+    {
+        auto rangeIt = m_children.equal_range(name);
+        for (auto it = rangeIt.first; it != rangeIt.second; ++it)
+        {
+            childNodes.push_back(it->second);
+        }
+    }
+}
+
+
+std::shared_ptr<EnvironmentNode> EnvironmentNode::getParent() const
+{
+    std::shared_ptr<EnvironmentNode> pParent;
+    if (!m_pParent.expired())
+    {
+        pParent = m_pParent.lock();
+    }
+    return pParent;
+}
+
+
+void EnvironmentNode::getAttributes(std::vector<std::shared_ptr<EnvironmentValue>> &attrs) const
+{
+    for (auto attrIt = m_attributes.begin(); attrIt != m_attributes.end(); ++attrIt)
+    {
+        attrs.push_back(attrIt->second);
+    }
+}
+
+
+void EnvironmentNode::addMissingAttributesFromConfig()
+{
+    std::vector<std::shared_ptr<SchemaValue>> configuredAttributes;
+    m_pSchemaItem->getAttributes(configuredAttributes);
+
+    //
+    // go through all the configured attrubutes and for each that is not present in our list, add it
+    for (auto it = configuredAttributes.begin(); it != configuredAttributes.end(); ++it)
+    {
+        auto attrIt = m_attributes.find((*it)->getName());
+        if (attrIt == m_attributes.end())
+        {
+            std::shared_ptr<SchemaValue> pCfgValue = *it;
+            std::shared_ptr<EnvironmentValue> pEnvValue = std::make_shared<EnvironmentValue>(shared_from_this(), pCfgValue, pCfgValue->getName());
+            pCfgValue->addEnvironmentValue(pEnvValue);
+            addAttribute(pCfgValue->getName(), pEnvValue);
+        }
+    }
+
+}
+
+
+void EnvironmentNode::setAttributeValues(const std::vector<NameValue> &values, Status &status, bool allowInvalid, bool forceCreate)
+{
+    for (auto it = values.begin(); it != values.end(); ++it)
+    {
+        setAttributeValue((*it).name, (*it).value, status, allowInvalid, forceCreate);
+    }
+}
+
+
+void EnvironmentNode::setAttributeValue(const std::string &attrName, const std::string &value, Status &status, bool allowInvalid, bool forceCreate)
+{
+    std::shared_ptr<EnvironmentValue> pEnvValue;
+
+    auto it = m_attributes.find(attrName);
+    if (it != m_attributes.end())
+    {
+        pEnvValue = it->second;
+    }
+
+    //
+    // Not found on this node. See if the configuration defines the attribute. If so, set the value and move on.
+    // If not and the forceCreate flag is set, create it.
+    else if (forceCreate)
+    {
+        std::shared_ptr<SchemaValue> pCfgValue = m_pSchemaItem->getAttribute(attrName);
+        pEnvValue = std::make_shared<EnvironmentValue>(shared_from_this(), pCfgValue, attrName);
+        addAttribute(attrName, pEnvValue);
+        if (!pCfgValue->isDefined())
+        {
+            status.addMsg(statusMsg::warning, getId(), attrName, "", "Undefined attribute did not exist in configuration, was created");
+        }
+    }
+
+    if (pEnvValue)
+    {
+        pEnvValue->setValue(value, &status, allowInvalid);
+    }
+    else
+    {
+        status.addMsg(statusMsg::error, getId(), attrName, "", "The attribute does not exist and was not created");
+    }
+
+}
+
+
+std::string EnvironmentNode::getAttributeValue(const std::string &name) const
+{
+    std::string value;
+    std::shared_ptr<EnvironmentValue> pAttribute = getAttribute(name);
+    if (pAttribute)
+        value = pAttribute->getValue();
+    return value;
+}
+
+
+bool EnvironmentNode::setLocalValue(const std::string &newValue, Status &status, bool force)
+{
+    bool rc = false;
+
+    //
+    // If no environment value is present, create one first
+    if (!m_pLocalValue)
+    {
+        std::shared_ptr<SchemaValue> pCfgValue = m_pSchemaItem->getItemSchemaValue();
+        m_pLocalValue = std::make_shared<EnvironmentValue>(shared_from_this(), pCfgValue, "");  // node's value has no name
+    }
+
+    rc = m_pLocalValue->setValue(newValue, &status, force);
+    return rc;
+}
+
+
+std::string EnvironmentNode::getLocalValue() const
+{
+    std::string value;
+    if (m_pLocalValue)
+    {
+        value = m_pLocalValue->getValue();
+    }
+    return value;
+}
+
+
+void EnvironmentNode::validate(Status &status, bool includeChildren) const
+{
+    //
+    // Check node value
+    if (m_pLocalValue)
+    {
+        m_pLocalValue->validate(status, "");
+    }
+
+    //
+    // Check any attributes
+    for (auto attrIt = m_attributes.begin(); attrIt != m_attributes.end(); ++attrIt)
+    {
+        attrIt->second->validate(status, m_id);
+
+        //
+        // If this value must be unique, make sure it is
+        if (attrIt->second->getSchemaValue()->isUniqueValue())
+        {
+            bool found = false;
+            std::vector<std::string> allValues;
+            attrIt->second->getAllValuesForSiblings(allValues);
+            std::set<std::string> unquieValues;
+            for (auto it = allValues.begin(); it != allValues.end() && !found; ++it)
+            {
+                auto ret = unquieValues.insert(*it);
+                found = ret.second;
+            }
+
+            if (found)
+            {
+                status.addUniqueMsg(statusMsg::error, m_id, attrIt->second->getName(), "", "Attribute value must be unique");
+            }
+        }
+
+        //
+        // Does this value need to be from another set of values?
+        if (attrIt->second->getSchemaValue()->isFromUniqueValueSet())
+        {
+            bool found = false;
+            std::vector<std::string> allValues;
+            attrIt->second->getSchemaValue()->getAllKeyRefValues(allValues);
+            for (auto it = allValues.begin(); it != allValues.end() && !found; ++it)
+                found = *it == attrIt->second->getValue();
+            if (!found)
+            {
+                status.addMsg(statusMsg::error, m_id, attrIt->second->getName(), "", "Attribute value must be from a unique set");
+            }
+        }
+    }
+
+    //
+    // Now check all children
+    if (includeChildren)
+    {
+        for (auto childIt = m_children.begin(); childIt != m_children.end(); ++childIt)
+        {
+            childIt->second->validate(status);
+        }
+    }
+}
+
+
+void EnvironmentNode::getAttributeValueForAllSiblings(const std::string &attrName, std::vector<std::string> &result) const
+{
+    std::shared_ptr<EnvironmentNode> pParentNode = m_pParent.lock();
+    if (pParentNode)
+    {
+        std::vector<std::shared_ptr<EnvironmentNode>> nodes;
+        pParentNode->getChildren(nodes, m_name);
+        for (auto it = nodes.begin(); it != nodes.end(); ++it)
+        {
+            result.push_back((*it)->getAttributeValue(attrName));
+        }
+    }
+}
+
+
+const std::shared_ptr<EnvironmentValue> EnvironmentNode::getAttribute(const std::string &name) const
+{
+    std::shared_ptr<EnvironmentValue> pValue;
+    auto it = m_attributes.find(name);
+    if (it != m_attributes.end())
+    {
+        pValue = it->second;
+    }
+    return pValue;
+}
+
+
+void EnvironmentNode::getInsertableItems(std::vector<std::shared_ptr<SchemaItem>> &insertableItems) const
+{
+    std::map<std::string, unsigned> childCounts;
+
+    //
+    // Iterate over the children and for each, create a childCount entry based on the
+    // child node's configuration type
+    for (auto childIt = m_children.begin(); childIt != m_children.end(); ++childIt)
+    {
+        std::string itemType = childIt->second->getSchemaItem()->getItemType();
+        auto findIt = childCounts.find(itemType);
+        if (findIt != childCounts.end())
+        {
+            ++findIt->second;  // increment the number of instances of this item type.
+        }
+        else
+        {
+            childCounts.insert({ itemType, 1 });
+        }
+    }
+
+    //
+    // Now get the full list of configurable items, then resolve it against the child counts from
+    // above to build a vector of insertable items
+    std::vector<std::shared_ptr<SchemaItem>> configChildren;
+    m_pSchemaItem->getChildren(configChildren);
+    for (auto cfgIt = configChildren.begin(); cfgIt != configChildren.end(); ++cfgIt)
+    {
+        auto findIt = childCounts.find((*cfgIt)->getItemType());
+        if (findIt != childCounts.end())
+        {
+            if (findIt->second < (*cfgIt)->getMaxInstances())
+            {
+                insertableItems.push_back(*cfgIt);
+            }
+        }
+        else
+        {
+            insertableItems.push_back(*cfgIt);
+        }
+    }
+}
+
+
+//
+// Called to initialize a newly added node to the environment (not just read from the environment)
+void EnvironmentNode::initialize()
+{
+    //
+    // Add any attributes that are requried
+    addMissingAttributesFromConfig();
+
+    //
+    // If we are a comonent and there is a buildSet attribute, set the value to the configItem's type
+    if (!(m_pSchemaItem->getProperty("componentName").empty())  && hasAttribute("buildSet"))
+    {
+        Status status;
+        setAttributeValue("buildSet", m_pSchemaItem->getProperty("category"), status);
+    }
+
+
+    //
+    // Initilize each attribute
+    for (auto attrIt = m_attributes.begin(); attrIt != m_attributes.end(); ++attrIt)
+    {
+        attrIt->second->initialize();
+    }
+}

configuration/config2/EnvironmentNode.hpp

@@ -0,0 +1,80 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2017 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#ifndef _CONFIG2_ENVIRONMENTNODE_HPP_
+#define _CONFIG2_ENVIRONMENTNODE_HPP_
+
+#include <memory>
+#include <map>
+#include "SchemaItem.hpp"
+#include "EnvironmentValue.hpp"
+#include "SchemaValue.hpp"
+#include "Status.hpp"
+#include "NameValue.hpp"
+#include "platform.h"
+
+
+class DECL_EXPORT EnvironmentNode : public std::enable_shared_from_this<EnvironmentNode>
+{
+    public:
+
+        EnvironmentNode(const std::shared_ptr<SchemaItem> &pCfgItem, const std::string &elemName, const std::shared_ptr<EnvironmentNode> &pParent = nullptr) :
+            m_pSchemaItem(pCfgItem), m_name(elemName), m_pParent(pParent) { }
+        ~EnvironmentNode() { }
+        const std::string &getName() const { return m_name;  }
+        void addChild(std::shared_ptr<EnvironmentNode> pNode);
+        bool removeChild(std::shared_ptr<EnvironmentNode> pNode);
+        void getChildren(std::vector<std::shared_ptr<EnvironmentNode>> &children, const std::string &name="") const;
+        bool hasChildren() const { return m_children.size() != 0; }
+        int getNumChildren() const { return m_children.size(); }
+        std::shared_ptr<EnvironmentNode> getParent() const;
+        bool addAttribute(const std::string &name, std::shared_ptr<EnvironmentValue> pValue);
+        void setAttributeValues(const std::vector<NameValue> &values, Status &status, bool allowInvalid, bool forceCreate);
+        void setAttributeValue(const std::string &name, const std::string &value, Status &status, bool allowInvalid=false, bool forceCreate=false);   // candidate for a variant?
+        std::string getAttributeValue(const std::string &name) const;                                  // candidate for a variant?
+        void addMissingAttributesFromConfig();
+        bool setLocalValue(const std::string &newValue, Status &status, bool force = false);
+        std::string getLocalValue() const;
+        void setLocalEnvValue(const std::shared_ptr<EnvironmentValue> &pEnvValue) { m_pLocalValue = pEnvValue;  }
+        const std::shared_ptr<EnvironmentValue> &getLocalEnvValue() const { return m_pLocalValue;  }
+        bool isLocalValueSet() const { return m_pLocalValue != nullptr; }
+        void getAttributes(std::vector<std::shared_ptr<EnvironmentValue>> &attrs) const;
+        const std::shared_ptr<EnvironmentValue> getAttribute(const std::string &name) const;
+        bool hasAttribute(const std::string &name) const { return m_attributes.find(name) != m_attributes.end();  }
+        bool hasAttributes() const { return m_attributes.size() != 0; }
+        void setId(const std::string &id) { m_id = id; }
+        const std::string &getId() const { return m_id;  }
+        void validate(Status &status, bool includeChildren=false) const;
+        void getAttributeValueForAllSiblings(const std::string &attrName, std::vector<std::string> &result) const;
+        const std::shared_ptr<SchemaItem> &getSchemaItem() const { return m_pSchemaItem; }
+        void getInsertableItems(std::vector<std::shared_ptr<SchemaItem>> &items) const;
+        void initialize();
+
+
+    protected:
+
+        std::string m_name;
+        std::shared_ptr<SchemaItem> m_pSchemaItem;
+        std::weak_ptr<EnvironmentNode> m_pParent;
+        std::multimap<std::string, std::shared_ptr<EnvironmentNode>> m_children;
+        std::shared_ptr<EnvironmentValue> m_pLocalValue;   // not normal because values usually in attributes
+        std::map<std::string, std::shared_ptr<EnvironmentValue>> m_attributes;
+        std::string m_id;
+};
+
+
+#endif

configuration/config2/EnvironmentValue.cpp

@@ -0,0 +1,148 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2017 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#include "EnvironmentValue.hpp"
+#include "EnvironmentNode.hpp"
+
+bool EnvironmentValue::setValue(const std::string &value, Status *pStatus, bool forceSet)
+{
+    bool rc = true;
+    std::string oldValue = m_value;
+    if (m_pSchemaValue)
+    {
+        m_forcedSet = false;
+        if (m_pSchemaValue->isValueValid(value))
+        {
+            m_value = value;
+        }
+        else if (forceSet)
+        {
+            m_value = value;
+            m_forcedSet = true;
+            if (pStatus != nullptr)
+            {
+                std::string msg = "Attribute forced to invalid value (" + m_pSchemaValue->getType()->getLimitString() + ")";
+                pStatus->addMsg(statusMsg::info, m_pMyEnvNode.lock()->getId(), m_name, "", msg);
+            }
+        }
+        else
+        {
+            rc = false;
+            if (pStatus != nullptr)
+            {
+                std::string msg = "Value not set. New value(" + value + ") not valid (" + m_pSchemaValue->getType()->getLimitString() + ")";
+                pStatus->addMsg(statusMsg::error, m_pMyEnvNode.lock()->getId(), m_name, "", msg);
+            }
+        }
+
+        if (rc)
+            m_pSchemaValue->mirrorValueToEnvironment(oldValue, value);
+
+    }
+    return rc;
+}
+
+
+void EnvironmentValue::getAllValuesForSiblings(std::vector<std::string> &result) const
+{
+    std::shared_ptr<EnvironmentNode> pEnvNode = m_pMyEnvNode.lock();
+    return pEnvNode->getAttributeValueForAllSiblings(m_pSchemaValue->getName(), result);
+}
+
+
+bool EnvironmentValue::isValueValid(const std::string &value) const
+{
+    bool rc = m_pSchemaValue->isValueValid(value, this);
+    return rc;
+}
+
+
+void EnvironmentValue::validate(Status &status, const std::string &myId) const
+{
+    //
+    // Only validate if value is set, otherwise, it's valid
+    if (isValueSet())
+    {
+        if (!m_pSchemaValue->isDefined())
+            status.addMsg(statusMsg::warning, myId, m_name, "", "No type information exists");
+
+        if (m_forcedSet)
+            status.addMsg(statusMsg::warning, myId, m_name, "", "Current value was force set to an invalid value");
+
+        // Will generate status based on current value and type
+        m_pSchemaValue->validate(status, myId, this);
+    }
+}
+
+
+// Called when a new value has been created, not read from existing environment, but created and added
+void EnvironmentValue::initialize()
+{
+    //
+    // Is there an auto generated value we should process:
+    const std::string &type = m_pSchemaValue->getAutoGenerateType();
+    if (!type.empty())
+    {
+        //
+        // type "prefix" means to use the auto generate value as a name prefix and to append numbers until a new unique name is
+        // found
+        if (type == "prefix")
+        {
+            std::string newName;
+            const std::string &prefix = m_pSchemaValue->getAutoGenerateValue();
+            std::vector<std::string> curValues;
+            m_pMyEnvNode.lock()->getAttributeValueForAllSiblings(m_name, curValues);
+            size_t count = curValues.size();
+            newName = prefix;
+            size_t n = 0;
+            while (n <= count + 1)
+            {
+                bool found = false;
+                for (auto it = curValues.begin(); it != curValues.end() && !found; ++it)
+                {
+                    if ((*it) == newName)
+                        found = true;
+                }
+
+                if (!found)
+                {
+                    setValue(newName, nullptr);
+                    break;
+                }
+                ++n;
+                newName = prefix + std::to_string(n);
+            }
+        }
+
+        //
+        // If type is configProperty, then the autogenerate value is the name of the value's node's schema item
+        // property. Set the value to this
+        else if (type == "configProperty")
+        {
+            const std::string &propertyName = m_pSchemaValue->getAutoGenerateValue();
+            std::string value;
+            value = m_pMyEnvNode.lock()->getSchemaItem()->getProperty(propertyName);
+            setValue(value, nullptr);
+        }
+    }
+}
+
+
+std::string EnvironmentValue::getNodeId() const
+{
+    return m_pMyEnvNode.lock()->getId();
+}

configuration/config2/EnvironmentValue.hpp

@@ -0,0 +1,61 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2017 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#ifndef _CONFIG2_ENVVALUE_HPP_
+#define _CONFIG2_ENVVALUE_HPP_
+
+#include <string>
+#include "SchemaValue.hpp"
+#include "Status.hpp"
+#include "platform.h"
+
+class EnvironmentNode;
+
+class DECL_EXPORT EnvironmentValue
+{
+    public:
+
+        EnvironmentValue(const std::shared_ptr<EnvironmentNode> &pMyNode, const std::shared_ptr<SchemaValue> &pCfgValue, const std::string &name="") :
+            m_pMyEnvNode(pMyNode), m_pSchemaValue(pCfgValue), m_name(name), m_forcedSet(false) { }
+        EnvironmentValue(const std::shared_ptr<EnvironmentNode> &pMyNode, const std::shared_ptr<SchemaValue> &pCfgValue, const std::string &name, const std::string initValue) :
+            EnvironmentValue(pMyNode, pCfgValue, name) { m_value = initValue; }
+
+        ~EnvironmentValue() { }
+        bool setValue(const std::string &value, Status *pStatus, bool forceSet=false);
+        bool isValueSet() const { return !m_value.empty(); }
+        const std::string &getValue() const { return m_value;  }
+        const std::string &getDefaultValue() const { return m_pSchemaValue->getDefaultValue(); }
+        bool hasDefaultValue() const { return m_pSchemaValue->hasDefaultValue(); }
+        const std::shared_ptr<SchemaValue> &getSchemaValue() const { return m_pSchemaValue;  }
+        const std::string &getName() const { return m_name;  }
+        bool wasForced() const { return m_forcedSet; }
+        bool isValueValid(const std::string &value) const;
+        void validate(Status &status, const std::string &myId) const;
+        void getAllValuesForSiblings(std::vector<std::string> &result) const;
+        std::string getNodeId() const;
+        void initialize();
+
+    private:
+
+        bool m_forcedSet;     // true when last set value was a forced set
+        std::string m_name;
+        std::string m_value;
+        std::shared_ptr<SchemaValue> m_pSchemaValue;
+        std::weak_ptr<EnvironmentNode> m_pMyEnvNode;
+};
+
+#endif

configuration/config2/Exceptions.hpp

@@ -0,0 +1,66 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2017 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#ifndef _CONFIG2_CONFIGEXCEPTIONS_HPP_
+#define _CONFIG2_CONFIGEXCEPTIONS_HPP_
+
+#include <exception>
+#include <string>
+#include <vector>
+
+class ParseException : public std::exception
+{
+    public:
+
+        ParseException(const std::string &reason) : m_reason(reason) { };
+        ParseException(const char *reason) : m_reason(reason) { };
+        ParseException() { };
+
+        void addFilename(const std::string &filename) { m_filenames.push_back(filename); }
+
+        virtual void setMessage(const char *msg) { m_reason = msg; }
+        virtual void setMessage(const std::string &msg) { m_reason = msg; }
+
+        virtual const char *what() const noexcept
+        {
+            std::string &msg = const_cast<std::string &>(m_reason);
+
+            if (m_filenames.size())
+            {
+                bool first = true;
+                msg += " While parsing: ";
+                for (auto it = m_filenames.begin(); it != m_filenames.end(); ++it)
+                {
+                    if (!first)
+                        msg += "-->";
+                    msg += (*it);
+                    first = false;
+                }
+            }
+
+            return m_reason.c_str();
+        }
+
+
+    private:
+
+        std::string m_reason;
+        std::vector<std::string> m_filenames;
+
+};
+
+#endif

configuration/config2/NameValue.hpp

@@ -0,0 +1,29 @@
+/*/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2017 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#ifndef _CONFIG2_NAMEVALUE_HPP_
+#define _CONFIG2_NAMEVALUE_HPP_
+
+#include <string>
+#include "platform.h"
+
+struct NameValue {
+    std::string name;
+    std::string value;
+};
+
+#endif

configuration/config2/SchemaItem.cpp

@@ -0,0 +1,532 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2017 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#include "SchemaItem.hpp"
+#include "Exceptions.hpp"
+#include "SchemaTypeStringLimits.hpp"
+#include "SchemaTypeIntegerLimits.hpp"
+#include <algorithm>
+
+
+// static class variables
+//std::map<std::string, std::vector<std::shared_ptr<SchemaValue>>> SchemaItem::m_uniqueAttributeValueSets;
+
+SchemaItem::SchemaItem(const std::string &name, const std::string &className, const std::shared_ptr<SchemaItem> &pParent) :
+    m_pParent(pParent),
+    m_minInstances(1),
+    m_maxInstances(1)
+{
+    //
+    // Set property defaults
+    m_properties["name"] = name;
+    m_properties["displayName"] = name;
+    m_properties["className"] = className;
+
+    //
+    // If this is a root node (no parent), then do some additional init
+    if (m_pParent.expired())
+    {
+        //
+        // Create a default type so that all values have a type
+        std::shared_ptr<SchemaType> pDefaultType = std::make_shared<SchemaType>("default");
+        std::shared_ptr<SchemaTypeLimits> pDefaultLimits = std::make_shared<SchemaTypeLimits>();
+        pDefaultType->setLimits(pDefaultLimits);
+        addSchemaValueType(pDefaultType);
+    }
+}
+
+
+void SchemaItem::addSchemaValueType(const std::shared_ptr<SchemaType> &pType)
+{
+    m_types[pType->getName()] = pType;
+}
+
+
+std::shared_ptr<SchemaType> SchemaItem::getSchemaValueType(const std::string &typeName, bool throwIfNotPresent) const
+{
+    std::shared_ptr<SchemaType> pType;
+    auto it = m_types.find(typeName);
+    if (it != m_types.end())
+    {
+        return it->second;
+    }
+    else
+    {
+        std::shared_ptr<SchemaItem> pParent = m_pParent.lock();
+        if (pParent)
+        {
+            return pParent->getSchemaValueType(typeName, throwIfNotPresent);
+        }
+    }
+
+    //
+    // Did not find the type
+    if (throwIfNotPresent)
+    {
+        std::string msg = "Unable to find type: " + typeName;
+        throw(ParseException(msg));
+    }
+    return pType;
+}
+
+
+bool SchemaItem::addUniqueName(const std::string keyName)
+{
+    auto result = m_keys.insert(keyName);
+    return result.second;  // true if keyName did not already exist
+}
+
+
+void SchemaItem::addSchemaType(const std::shared_ptr<SchemaItem> &pItem, const std::string &typeName)
+{
+    auto it = m_schemaTypes.find(typeName);
+    if (it == m_schemaTypes.end())
+    {
+        m_schemaTypes[typeName] = pItem;
+    }
+    else
+    {
+        throw(ParseException("Duplicate config type found: " + m_properties["name"]));
+    }
+}
+
+
+std::shared_ptr<SchemaItem> SchemaItem::getSchemaType(const std::string &name, bool throwIfNotPresent) const
+{
+    std::shared_ptr<SchemaItem> pItem;
+    auto it = m_schemaTypes.find(name);
+    if (it != m_schemaTypes.end())
+    {
+        return it->second;
+    }
+    else
+    {
+        std::shared_ptr<SchemaItem> pParent = m_pParent.lock();
+        if (pParent)
+        {
+            return pParent->getSchemaType(name, throwIfNotPresent);
+        }
+    }
+
+    //
+    // Did not find the type
+    if (throwIfNotPresent)
+    {
+        std::string msg = "Unable to find config type: " + name;
+        throw(ParseException(msg));
+    }
+    return pItem;
+}
+
+
+//
+// This method is used to insert a named type into the current schema item. This is done by making copies
+// of the relevant members and inserting them into this instance
+void SchemaItem::insertSchemaType(const std::shared_ptr<SchemaItem> pTypeItem)
+{
+    //
+    // To insert a schema type (for example a previously defined complexType name="" XSD definition)
+    // loop through each set of configurable pieces of the input type, make a copy of each, and add it to
+    // this element.
+
+    //
+    // Children
+    std::vector<std::shared_ptr<SchemaItem>> typeChildren;
+    pTypeItem->getChildren(typeChildren);
+    for (auto childIt = typeChildren.begin(); childIt != typeChildren.end(); ++childIt)
+    {
+        std::shared_ptr<SchemaItem> pNewItem = std::make_shared<SchemaItem>(*(*childIt));
+        addChild(pNewItem);
+    }
+
+    //
+    // Attributes
+    std::vector< std::shared_ptr<SchemaValue>> typeAttributes;
+    pTypeItem->getAttributes(typeAttributes);
+    for (auto attrIt = typeAttributes.begin(); attrIt != typeAttributes.end(); ++attrIt)
+    {
+        std::shared_ptr<SchemaValue> pNewAttr = std::make_shared<SchemaValue>(*(*attrIt));
+        addAttribute(pNewAttr);
+    }
+
+    //
+    // Type main value
+    if (pTypeItem->getItemSchemaValue() != nullptr)
+    {
+        std::shared_ptr<SchemaValue> pNewItemCfgValue = std::make_shared<SchemaValue>(*(pTypeItem->getItemSchemaValue()));
+        setItemSchemaValue(pNewItemCfgValue);
+    }
+
+    //
+    // Unique attribute sets
+    for (auto setIt = pTypeItem->m_uniqueAttributeValueSetDefs.begin(); setIt != pTypeItem->m_uniqueAttributeValueSetDefs.end(); ++setIt)
+    {
+        m_uniqueAttributeValueSetDefs.insert({ setIt->first, setIt->second });
+    }
+
+    //
+    // Unique attribute reference sets
+    for (auto it = pTypeItem->m_uniqueAttributeValueSetReferences.begin(); it != pTypeItem->m_uniqueAttributeValueSetReferences.end(); ++it)
+    {
+        m_uniqueAttributeValueSetReferences.insert({ it->first, it->second });
+    }
+}
+
+
+void SchemaItem::addAttribute(const std::shared_ptr<SchemaValue> &pCfgValue)
+{
+    auto retVal = m_attributes.insert({ pCfgValue->getName(), pCfgValue });
+    if (!retVal.second)
+    {
+        throw(ParseException("Duplicate attribute (" + pCfgValue->getName() + ") found for element " + m_properties["name"]));
+    }
+}
+
+
+void SchemaItem::addAttribute(const std::map<std::string, std::shared_ptr<SchemaValue>> &attributes)
+{
+    for (auto it = attributes.begin(); it != attributes.end(); ++it)
+        addAttribute(it->second);
+}
+
+
+void SchemaItem::addAttribute(const std::vector<std::shared_ptr<SchemaValue>> &attributes)
+{
+    for (auto it=attributes.begin(); it!=attributes.end(); ++it)
+        addAttribute(*it);
+}
+
+
+void SchemaItem::getAttributes(std::vector<std::shared_ptr<SchemaValue>> &attributes) const
+{
+    for (auto it = m_attributes.begin(); it != m_attributes.end(); ++it)
+        attributes.push_back(it->second);
+}
+
+
+std::shared_ptr<SchemaValue> SchemaItem::getAttribute(const std::string &name, bool createIfDoesNotExist) const
+{
+    std::shared_ptr<SchemaValue> pCfgValue;
+    auto it = m_attributes.find(name);
+    if (it != m_attributes.end())
+    {
+        pCfgValue = it->second;
+    }
+    else if (createIfDoesNotExist)
+    {
+        // not found, build a default cfg value for the undefined attribute
+        pCfgValue = std::make_shared<SchemaValue>(name, false);
+        pCfgValue->setType(getSchemaValueType("default"));
+    }
+    return pCfgValue;
+}
+
+
+void SchemaItem::addUniqueAttributeValueSetDefinition(const std::string &setName, const std::string &elementPath, const std::string &attributeName, bool duplicateOk)
+{
+    m_uniqueAttributeValueSetDefs.insert({ setName, SetInfo(setName, elementPath, attributeName, duplicateOk) });   // these are processed later
+}
+
+
+void SchemaItem::addReferenceToUniqueAttributeValueSet(const std::string &setName, const std::string &elementPath, const std::string &attributeName)
+{
+    m_uniqueAttributeValueSetReferences.insert({ setName, SetInfo(setName, elementPath, attributeName) });   // these are processed later
+}
+
+
+void SchemaItem::processUniqueAttributeValueSetReferences(const std::map<std::string, std::vector<std::shared_ptr<SchemaValue>>> &uniqueAttributeValueSets)
+{
+    for (auto setRefIt = m_uniqueAttributeValueSetReferences.begin(); setRefIt != m_uniqueAttributeValueSetReferences.end(); ++setRefIt)
+    {
+        auto keyIt = uniqueAttributeValueSets.find(setRefIt->second.m_setName);
+        if (keyIt != uniqueAttributeValueSets.end())
+        {
+            for (auto cfgIt = keyIt->second.begin(); cfgIt != keyIt->second.end(); ++cfgIt)
+            {
+                std::shared_ptr<SchemaValue> pKeyRefAttribute = *cfgIt;     // this is the reference attribute from which attributeName must be a member
+                std::string cfgValuePath = ((setRefIt->second.m_elementPath != ".") ? setRefIt->second.m_elementPath : "") + "@" + setRefIt->second.m_attributeName;
+                std::vector<std::shared_ptr<SchemaValue>> cfgValues;
+                findSchemaValues(cfgValuePath, cfgValues);
+                if (!cfgValues.empty())
+                {
+                    for (auto attrIt = cfgValues.begin(); attrIt != cfgValues.end(); ++attrIt)
+                    {
+                        (*attrIt)->setUniqueValueSetRef(pKeyRefAttribute);
+                    }
+                }
+                else
+                {
+                    throw(ParseException("Attribute " + (setRefIt->second.m_attributeName + " not found when adding keyRef for key " + (setRefIt->second.m_setName))));
+                }
+            }
+        }
+        else
+        {
+            throw(ParseException("Keyref to key '" + (setRefIt->second.m_setName + "' was not found")));
+        }
+    }
+}
+
+
+void SchemaItem::getChildren(std::vector<std::shared_ptr<SchemaItem>> &children)
+{
+    for (auto it = m_children.begin(); it != m_children.end(); ++it)
+    {
+        children.push_back(it->second);
+    }
+}
+
+
+std::shared_ptr<SchemaItem> SchemaItem::getChild(const std::string &name)
+{
+    std::shared_ptr<SchemaItem> pItem = std::make_shared<SchemaItem>(name, "default", shared_from_this());
+    auto it = m_children.find(name);  // only return the first one
+    if (it != m_children.end())
+    {
+        pItem = it->second;
+    }
+    return pItem;
+}
+
+
+std::shared_ptr<SchemaItem> SchemaItem::getChildByComponent(const std::string &name, std::string &componentName)
+{
+    std::shared_ptr<SchemaItem> pItem = std::make_shared<SchemaItem>(name, "default", shared_from_this());
+    auto childItRange = m_children.equal_range(name);
+    for (auto childIt = childItRange.first; childIt != childItRange.second; ++childIt)
+    {
+        if (childIt->second->getProperty("componentName") == componentName)
+        {
+            pItem = childIt->second;
+            break;
+        }
+    }
+
+    return pItem;
+}
+
+
+void SchemaItem::resetEnvironment()
+{
+    for (auto it = m_attributes.begin(); it != m_attributes.end(); ++it)
+    {
+        it->second->resetEnvironment();
+    }
+}
+
+
+
+void SchemaItem::findSchemaValues(const std::string &path, std::vector<std::shared_ptr<SchemaValue>> &schemaValues)
+{
+    bool rootPath = path[0] == '/';
+
+    //
+    // If path is from the root, and we aren't the root, pass the request to our parent
+    if (rootPath && !m_pParent.expired())
+    {
+        std::shared_ptr<SchemaItem> pParent = m_pParent.lock();
+        if (pParent)
+        {
+            return pParent->findSchemaValues(path, schemaValues);
+        }
+    }
+
+    //
+    // Break the path down and process it
+    size_t start = rootPath ? 1 : 0;    // skip leading slash if we are at the root
+    size_t end = path.find_first_of("/@", start);
+    if (end != std::string::npos)
+    {
+        std::string elem = path.substr(start, end - start);
+
+        if (rootPath)
+        {
+            if (m_properties["name"] == elem)
+            {
+                return findSchemaValues(path.substr(end + 1), schemaValues);
+            }
+            else
+            {
+                throw(ParseException("Unable to find root element '" + elem + "' when searching path '" + path + "'"));
+            }
+        }
+
+        if (path[0] == '@')
+        {
+            std::string attrName = path.substr(1);
+            auto rangeIt = m_attributes.equal_range(attrName);
+            for (auto it = rangeIt.first; it != rangeIt.second; ++it)
+            {
+                schemaValues.push_back(it->second);
+            }
+        }
+
+        else
+        {
+            auto rangeIt = m_children.equal_range(elem);
+            for (auto it = rangeIt.first; it != rangeIt.second; ++it)
+            {
+                return it->second->findSchemaValues(path.substr(end + ((path[end] == '/') ? 1 : 0)), schemaValues);
+            }
+        }
+    }
+
+    return;
+}
+
+
+void SchemaItem::processDefinedUniqueAttributeValueSets(std::map<std::string, std::vector<std::shared_ptr<SchemaValue>>> &uniqueAttributeValueSets)
+{
+    for (auto setIt = m_uniqueAttributeValueSetDefs.begin(); setIt != m_uniqueAttributeValueSetDefs.end(); ++setIt)
+    {
+        auto it = uniqueAttributeValueSets.find(setIt->first);
+        bool keyDefExists = it != uniqueAttributeValueSets.end();
+        if (!keyDefExists || setIt->second.m_duplicateOk)
+        {
+            std::string cfgValuePath = ((setIt->second.m_elementPath != ".") ? setIt->second.m_elementPath : "") + "@" + setIt->second.m_attributeName;
+            std::vector<std::shared_ptr<SchemaValue>> cfgValues;
+            findSchemaValues(cfgValuePath, cfgValues);
+            if (!cfgValues.empty())
+            {
+                //
+                // For each attribute, if it does not already exist in the list of attributes making up this
+                // key value, add it.
+                for (auto attrIt = cfgValues.begin(); attrIt != cfgValues.end(); ++attrIt)
+                {
+                    (*attrIt)->setUniqueValue(true);
+
+                    if (!keyDefExists)
+                    {
+                        std::vector<std::shared_ptr<SchemaValue>> values;
+                        values.push_back(*attrIt);
+                        it = uniqueAttributeValueSets.insert({ setIt->second.m_setName, values }).first;  // so the else condition will work
+                        keyDefExists = true;  // Now, it does exist
+                    }
+                    else
+                    {
+                        std::vector<std::shared_ptr<SchemaValue>> &values = it->second;
+                        bool found = false;
+                        for (auto cfgIt = values.begin(); cfgIt != values.end() && !found; ++cfgIt)
+                        {
+                            found = (*cfgIt) == (*attrIt);
+                        }
+                        if (!found)
+                            values.push_back(*attrIt);
+                    }
+                }
+            }
+            else
+            {
+                throw(ParseException("Attribute " + setIt->second.m_attributeName + " not found for key " + setIt->second.m_setName));
+            }
+        }
+        else
+        {
+            throw(ParseException("Duplicate key (" + setIt->second.m_setName + ") found for element " + m_properties["name"]));
+        }
+    }
+
+    //
+    // Post process all of our children now
+    for (auto it = m_children.begin(); it != m_children.end(); ++it)
+    {
+        it->second->processDefinedUniqueAttributeValueSets(uniqueAttributeValueSets);
+    }
+}
+
+
+
+void SchemaItem::postProcessConfig(const std::map<std::string, std::vector<std::shared_ptr<SchemaValue>>> &uniqueAttributeValueSets)
+{
+    //
+    // Make sure that the item type value for all children that are insertable (minRequired = 0 or maxAllowed > minRequired)
+    std::set<std::string> itemTypes;
+    for (auto it = m_children.begin(); it != m_children.end(); ++it)
+    {
+        if (it->second->isInsertable())
+        {
+            auto rc = itemTypes.insert(it->second->getItemType());
+            if (!rc.second)
+            {
+                throw(ParseException("Duplicate itemType(" + it->second->getItemType() + ") found for element " + m_properties["name"]));
+            }
+        }
+    }
+
+    processUniqueAttributeValueSetReferences(uniqueAttributeValueSets);
+
+    //
+    // Post process the attributes
+    for (auto it = m_attributes.begin(); it != m_attributes.end(); ++it)
+    {
+        //
+        // If this is a mirroed value, go find the source and attach ourselves so that if that value changes,
+        // it is replicated to us.
+        if (it->second->isMirroredValue())
+        {
+            std::vector<std::shared_ptr<SchemaValue>> cfgValues;
+            findSchemaValues(it->second->getMirrorFromPath(), cfgValues);
+            if (!cfgValues.empty() && cfgValues.size() == 1)
+            {
+                if (cfgValues.size() == 1)
+                {
+                    it->second->addMirroredSchemaValue(cfgValues[0]);
+                }
+                else
+                {
+                    throw(ParseException("Multiple sources found for mirror from path for attribute " + it->second->getName() + " (path=" + it->second->getMirrorFromPath()));
+                }
+            }
+            else
+            {
+                throw(ParseException("Mirrored from source not found for attribute " + it->second->getName() + " path=" + it->second->getMirrorFromPath()));
+            }
+        }
+    }
+
+    //
+    // Post process all of our children now
+    for (auto it = m_children.begin(); it!= m_children.end(); ++it)
+    {
+        it->second->postProcessConfig(uniqueAttributeValueSets);
+    }
+}
+
+
+std::string SchemaItem::getItemType() const
+{
+    //
+    // Return itemType based on this set of rules
+    if (!getProperty("itemType").empty())
+        return getProperty("itemType");
+    else if (!getProperty("componentName").empty())
+        return getProperty("componentName");
+
+    return getProperty("name");
+}
+
+
+std::string SchemaItem::getProperty(const std::string &name, const std::string &dflt) const
+{
+    std::string retVal = dflt;
+    auto it = m_properties.find(name);
+    if (it != m_properties.end())
+    {
+        retVal = it->second;
+    }
+    return retVal;
+}

configuration/config2/SchemaItem.hpp

@@ -0,0 +1,112 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2017 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+
+#ifndef _CONFIG2_CONFIGITEM_HPP_
+#define _CONFIG2_CONFIGITEM_HPP_
+
+#include <string>
+#include <memory>
+#include <set>
+#include <map>
+#include "SchemaType.hpp"
+#include "SchemaValue.hpp"
+#include "platform.h"
+
+
+class DECL_EXPORT SchemaItem : public std::enable_shared_from_this<SchemaItem>
+{
+    public:
+
+        SchemaItem(const std::string &name, const std::string &className = "category", const std::shared_ptr<SchemaItem> &pParent = nullptr);
+        ~SchemaItem() { }
+        std::string getItemType() const;
+        void setMinInstances(unsigned num) { m_minInstances = num; }
+        unsigned getMinInstances() const { return m_minInstances; }
+        void setMaxInstances(unsigned num) { m_maxInstances = num; }
+        unsigned getMaxInstances() const { return m_maxInstances; }
+        void addSchemaValueType(const std::shared_ptr<SchemaType> &pType);
+        std::shared_ptr<SchemaType> getSchemaValueType(const std::string &typeName, bool throwIfNotPresent = true) const;
+        void addSchemaType(const std::shared_ptr<SchemaItem> &pItem, const std::string &typeName);
+        std::shared_ptr<SchemaItem> getSchemaType(const std::string &name, bool throwIfNotPresent=true) const;
+        void insertSchemaType(const std::shared_ptr<SchemaItem> pTypeItem);
+        void addChild(const std::shared_ptr<SchemaItem> &pItem) { m_children.insert({ pItem->getProperty("name"), pItem }); }
+        void addChild(const std::shared_ptr<SchemaItem> &pItem, const std::string &name) { m_children.insert({ name, pItem }); }
+        void getChildren(std::vector<std::shared_ptr<SchemaItem>> &children);
+        std::shared_ptr<SchemaItem> getChild(const std::string &name);
+        std::shared_ptr<SchemaItem> getChildByComponent(const std::string &name, std::string &componentName);
+        void setItemSchemaValue(const std::shared_ptr<SchemaValue> &pValue) { m_pItemValue = pValue; }
+        std::shared_ptr<SchemaValue> getItemSchemaValue() const { return m_pItemValue; }
+        bool isItemValueDefined() { return m_pItemValue != nullptr; }
+        void findSchemaValues(const std::string &path, std::vector<std::shared_ptr<SchemaValue>> &schemaValues);
+        void addAttribute(const std::shared_ptr<SchemaValue> &pCfgValue);
+        void addAttribute(const std::vector<std::shared_ptr<SchemaValue>> &attributes);
+        void addAttribute(const std::map<std::string, std::shared_ptr<SchemaValue>> &attributes);
+        std::shared_ptr<SchemaValue> getAttribute(const std::string &name, bool createIfDoesNotExist=true) const;
+        void getAttributes(std::vector<std::shared_ptr<SchemaValue>> &attributes) const;
+        bool addUniqueName(const std::string keyName);
+        void addUniqueAttributeValueSetDefinition(const std::string &setName, const std::string &elementPath, const std::string &attributeName, bool duplicateOk = false);
+        void addReferenceToUniqueAttributeValueSet(const std::string &setName, const std::string &elementPath, const std::string &attributeName);
+        void processDefinedUniqueAttributeValueSets(std::map<std::string, std::vector<std::shared_ptr<SchemaValue>>> &uniqueAttributeValueSets);
+        void processUniqueAttributeValueSetReferences(const std::map<std::string, std::vector<std::shared_ptr<SchemaValue>>> &uniqueAttributeValueSets);
+        void resetEnvironment();
+        void postProcessConfig(const std::map<std::string, std::vector<std::shared_ptr<SchemaValue>>> &uniqueAttributeValueSets);
+        bool isInsertable() const { return (m_minInstances == 0) || (m_maxInstances > m_minInstances); }
+        bool isRequired() const { return m_minInstances > 0; }
+
+        std::string getProperty(const std::string &name, const std::string &dfault = std::string("")) const;
+        void setProperty(const std::string &name, const std::string &value) { m_properties[name] = value; }
+
+
+    protected:
+
+        SchemaItem() { };
+
+    protected:
+
+        std::map<std::string, std::string> m_properties;
+        unsigned m_minInstances;
+        unsigned m_maxInstances;
+        std::multimap<std::string, std::shared_ptr<SchemaItem>> m_children;
+        std::shared_ptr<SchemaValue> m_pItemValue;   // value for this item (think of it as the VALUE for an element <xx attr= att1>VALUE</xx>)
+        std::map<std::string, std::shared_ptr<SchemaValue>> m_attributes;   // attributes for this item (think in xml terms <m_name attr1="val" attr2="val" .../> where attrN is in this vector
+        std::set<std::string> m_keys;   // generic set of key values for use by any component to prevent duplicate operations
+        std::weak_ptr<SchemaItem> m_pParent;
+        std::map<std::string, std::shared_ptr<SchemaType>> m_types;
+        std::map<std::string, std::shared_ptr<SchemaItem>> m_schemaTypes;                // reusable types
+
+        // This struct handles both unique attribute sets and references to same
+        struct SetInfo {
+            SetInfo(const std::string &setName, const std::string &elementPath, const std::string &attributeName) :
+                m_setName(setName), m_elementPath(elementPath), m_attributeName(attributeName), m_duplicateOk(false) { }
+            SetInfo(const std::string &setName, const std::string &elementPath, const std::string &attributeName, bool duplicateOk) :
+                m_setName(setName), m_elementPath(elementPath), m_attributeName(attributeName), m_duplicateOk(duplicateOk) { }
+            std::string m_setName;
+            std::string m_elementPath;
+            std::string m_attributeName;
+            bool m_duplicateOk;
+        };
+
+        // Attribute unique sets and references to unique sets are stored during parsing and post processed
+        std::map<std::string, SetInfo> m_uniqueAttributeValueSetReferences;
+        std::map<std::string, SetInfo> m_uniqueAttributeValueSetDefs;
+
+        // These are the attribute value sets whose members must be unique
+        //static std::map<std::string, std::vector<std::shared_ptr<SchemaValue>>> m_uniqueAttributeValueSets;
+};
+
+#endif // _CONFIG2_CONFIGITEM_HPP_

configuration/config2/SchemaParser.cpp

@@ -0,0 +1,50 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2017 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#include "SchemaParser.hpp"
+#include "Exceptions.hpp"
+
+
+bool SchemaParser::parse(const std::string &configPath, const std::string &masterConfigFile,  const std::vector<std::string> &cfgParms)
+{
+    bool rc = true;
+    try
+    {
+        doParse(configPath, masterConfigFile, cfgParms);
+    }
+    catch (const ParseException &pe)
+    {
+        m_message = "The following error was detected while parsing the configuration: " + static_cast<std::string>(pe.what());
+        rc = false;
+    }
+    return rc;
+}
+
+
+std::vector<std::string> SchemaParser::split(const std::string  &input, const std::string  &delim)
+{
+    size_t  start = 0, end = 0, delimLen = delim.length();
+    std::vector<std::string> list;
+
+    while (end != std::string::npos)
+    {
+        end = input.find(delim, start);
+        list.push_back(input.substr(start, (end == std::string::npos) ? std::string::npos : end - start));
+        start = ((end > (std::string::npos - delimLen)) ? std::string::npos : end + delimLen);
+    }
+    return list;
+}

configuration/config2/SchemaParser.hpp

@@ -0,0 +1,56 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2017 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+
+#ifndef _CONFIG2_CONFIGPARSER_HPP_
+#define _CONFIG2_CONFIGPARSER_HPP_
+
+#include <string>
+#include <memory>
+#include <vector>
+
+
+#include "SchemaItem.hpp"
+#include "Status.hpp"
+#include "platform.h"
+
+
+class DECL_EXPORT SchemaParser
+{
+    public:
+
+        SchemaParser(std::shared_ptr<SchemaItem> &pSchema) : m_pSchemaItem(pSchema) { };
+        virtual ~SchemaParser() { };
+        virtual bool parse(const std::string &configPath, const std::string &masterConfigFile,  const std::vector<std::string> &cfgParms);
+        const std::string getLastMessage() const { return m_message;  }
+
+
+    protected:
+
+        virtual bool doParse(const std::string &configPath, const std::string &masterConfigFile,  const std::vector<std::string> &cfgParms) = 0;
+        SchemaParser() { };
+        std::vector<std::string> split(const std::string  &input, const std::string  &delim);
+
+
+    protected:
+
+        std::shared_ptr<SchemaItem> m_pSchemaItem;
+        std::string m_message;                       // a place where a message can be stored and retrieved, such as for a parse error
+};
+
+
+#endif // _CONFIG2_CONFIGPARSER_HPP_

configuration/config2/SchemaType.hpp

@@ -0,0 +1,57 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2017 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#ifndef _CONFIG2_CFGTYPE_HPP_
+#define _CONFIG2_CFGTYPE_HPP_
+
+#include <string>
+#include <memory>
+#include <map>
+#include <vector>
+#include "SchemaTypeLimits.hpp"
+#include "platform.h"
+
+
+
+class DECL_EXPORT SchemaType
+{
+    public:
+
+        SchemaType(const std::string &name) : m_name(name), m_pLimits(std::make_shared<SchemaTypeLimits>()) { }
+        SchemaType() : SchemaType("") { }
+        virtual ~SchemaType() { }
+
+        std::shared_ptr<SchemaTypeLimits> &getLimits() { return m_pLimits; }
+        void setLimits(const std::shared_ptr<SchemaTypeLimits> &pLimits) { m_pLimits = pLimits; }
+        bool isValid() const { return m_pLimits!=nullptr; }
+        const std::string &getName() const { return m_name; }
+        void setName(const std::string &name) { m_name = name;  }
+        bool isValueValid(const std::string &testValue) const { return m_pLimits->isValueValid(testValue); }
+        bool isEnumerated() const { return m_pLimits->isEnumerated(); }
+        const std::vector<AllowedValue> getEnumeratedValues() const { return m_pLimits->getEnumeratedValues(); }
+        const std::string getLimitString() const { return m_pLimits->getLimitString();  }
+
+
+    private:
+
+        std::string m_name;
+        std::shared_ptr<SchemaTypeLimits> m_pLimits;
+
+};
+
+
+#endif // _CONFIG2_CFGTYPE_HPP_

configuration/config2/SchemaTypeIntegerLimits.cpp

@@ -0,0 +1,47 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2017 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#include "SchemaTypeIntegerLimits.hpp"
+
+bool SchemaTypeIntegerLimits::doValueTest(const std::string &value) const
+{
+    bool isValid = true;
+    int testValue;
+    try
+    {
+        testValue = std::stoi(value);
+    }
+    catch (...)
+    {
+        isValid = false;  // not even an integer string
+    }
+
+    if (isValid)
+    {
+        //
+        // min/max tests
+        isValid = (m_minExclusiveTest) ? (testValue > m_min) : (testValue >= m_min);
+        isValid = isValid && ((m_maxExclusiveTest) ? (testValue < m_max) : (testValue <= m_max));
+    }
+    return isValid;
+}
+
+
+std::string SchemaTypeIntegerLimits::getLimitString() const
+{
+    return "integer limit string";
+}

configuration/config2/SchemaTypeIntegerLimits.hpp

@@ -0,0 +1,56 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2017 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#ifndef _CONFIG2_CFGINTEGERLIMITS_HPP_
+#define _CONFIG2_CFGINTEGERLIMITS_HPP_
+
+#include "SchemaTypeLimits.hpp"
+#include <limits.h>
+
+class SchemaTypeIntegerLimits : public SchemaTypeLimits
+{
+    public:
+
+        SchemaTypeIntegerLimits() : m_min(INT_MIN), m_max(INT_MAX), m_minSet(false), m_maxSet(false), m_minExclusiveTest(false), m_maxExclusiveTest(false) { }
+        virtual ~SchemaTypeIntegerLimits() { };
+        void setMinInclusive(int v) { m_minSet = true; m_min = v; }
+        void setMinExclusive(int v) { m_minSet = true; m_min = v;  m_minExclusiveTest = true; }
+        void setMaxInclusive(int v) { m_maxSet = true; m_max = v; }
+        void setMaxExclusive(int v) { m_maxSet = true; m_max = v;  m_maxExclusiveTest = true; }
+        std::string getLimitString() const;
+        virtual bool isMaxSet() const { return m_maxSet; }
+        virtual bool isMinSet() const { return m_minSet; }
+        virtual int getMax() const { return (m_maxExclusiveTest ? (m_max-1) : m_max); }
+        virtual int getMin() const { return (m_minExclusiveTest ? (m_min+1) : m_min); }
+
+
+    protected:
+
+        virtual bool doValueTest(const std::string &testValue) const;
+
+
+    protected:
+
+        bool m_maxExclusiveTest;
+        bool m_minExclusiveTest;
+        bool m_minSet;
+        bool m_maxSet;
+        int m_min;
+        int m_max;
+};
+
+#endif // _CONFIG2_CFGINTEGERLIMITS_HPP_

configuration/config2/SchemaTypeLimits.cpp

@@ -0,0 +1,49 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2017 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#include "SchemaTypeLimits.hpp"
+
+std::vector<AllowedValue> SchemaTypeLimits::getEnumeratedValues() const
+{
+    return m_enumeratedValues;
+}
+
+
+bool SchemaTypeLimits::isValueValid(const std::string &testValue) const
+{
+    bool rc = isValidEnumeratedValue(testValue);
+    if (rc)
+    {
+        rc = doValueTest(testValue);
+    }
+    return rc;
+}
+
+
+bool SchemaTypeLimits::isValidEnumeratedValue(const std::string &testValue) const
+{
+    bool rc = true;
+    if (isEnumerated())  // extra check just in case called by accident
+    {
+        rc = false;
+        for (auto it = m_enumeratedValues.begin(); it != m_enumeratedValues.end() && !rc; ++it)
+        {
+            rc = (testValue == (*it).m_value);
+        }
+    }
+    return rc;
+}

configuration/config2/SchemaTypeLimits.hpp

@@ -0,0 +1,66 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2017 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#ifndef _CONFIG2_CFGLIMITS_HPP_
+#define _CONFIG2_CFGLIMITS_HPP_
+
+#include <memory>
+#include <vector>
+#include <string>
+#include "platform.h"
+
+
+class EnvironmentValue;
+
+struct DECL_EXPORT AllowedValue
+{
+    AllowedValue(const std::string &value, const std::string &desc="") : m_value(value), m_description(desc) { }
+    std::string m_value;
+    std::string m_description;
+};
+
+class DECL_EXPORT SchemaTypeLimits
+{
+    public:
+
+        SchemaTypeLimits() { }
+        virtual ~SchemaTypeLimits() { }
+        void addAllowedValue(const std::string &value, const std::string &desc="") { m_enumeratedValues.push_back(AllowedValue(value, desc)); }
+        std::vector<AllowedValue> getEnumeratedValues() const;
+        bool isEnumerated() const { return !m_enumeratedValues.empty(); }
+        bool isValueValid(const std::string &testValue) const;
+        virtual std::string getLimitString() const { return "No value limits"; }
+        virtual bool isMaxSet() const { return false; }
+        virtual bool isMinSet() const { return false; }
+        virtual int getMax() const { return 0; }
+        virtual int getMin() const { return 0; }
+
+
+    protected:
+
+        virtual bool isValidEnumeratedValue(const std::string &testValue) const;
+        virtual bool doValueTest(const std::string &testValue) const { return true;  }
+
+
+    protected:
+
+        std::vector<AllowedValue> m_enumeratedValues;
+};
+
+
+
+#endif // _CONFIG2_CFGLIMITS_HPP_

configuration/config2/SchemaTypeStringLimits.cpp

@@ -0,0 +1,42 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2017 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#include "SchemaTypeStringLimits.hpp"
+#include "EnvironmentValue.hpp"
+#include <regex>
+
+std::string SchemaTypeStringLimits::getLimitString() const
+{
+
+    return "String limit info";
+}
+
+
+bool SchemaTypeStringLimits::doValueTest(const std::string &testValue) const
+{
+    bool isValid;
+    int len = testValue.length();
+    isValid = len >= m_minLength && len <= m_maxLength;
+
+    // test patterns
+    for (auto it = m_patterns.begin(); it != m_patterns.end() && isValid; ++it)
+    {
+        std::regex expr (*it);
+        isValid = std::regex_match(testValue, expr);
+    }
+    return isValid;
+}

configuration/config2/SchemaTypeStringLimits.hpp

@@ -0,0 +1,56 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2017 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#ifndef _CONFIG2_CFGSTRINGLIMITS_HPP_
+#define _CONFIG2_CFGSTRINGLIMITS_HPP_
+
+#include "SchemaTypeLimits.hpp"
+#include <limits.h>
+
+
+class SchemaTypeStringLimits : public SchemaTypeLimits
+{
+    public:
+
+        SchemaTypeStringLimits() : m_minLength(0), m_maxLength(INT_MAX), m_minSet(false), m_maxSet(false) { }
+        virtual ~SchemaTypeStringLimits() { };
+        void setLength(unsigned v) { m_minLength = v;  m_maxLength = v; m_minSet = m_maxSet = true; }
+        void setMinLength(int v) { m_minSet = true; m_minLength = v; }
+        void setMaxLength(int v) { m_maxSet = true; m_maxLength = v; }
+        void addPattern(const std::string &pattern) { m_patterns.push_back(pattern); }
+        std::string getLimitString() const;
+        virtual bool isMaxSet() const { return m_maxSet; }
+        virtual bool isMinSet() const { return m_minSet; }
+        virtual int getMax() const { return m_maxLength; }
+        virtual int getMin() const { return m_minLength; }
+
+
+    protected:
+
+        virtual bool doValueTest(const std::string &testValue) const;
+
+
+    protected:
+
+        int m_minLength;
+        int m_maxLength;
+        bool m_minSet;
+        bool m_maxSet;
+        std::vector<std::string> m_patterns;
+};
+
+#endif // _CONFIG2_CFGSTRINGLIMITS_HPP_

configuration/config2/SchemaValue.cpp

@@ -0,0 +1,171 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2017 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#include "SchemaValue.hpp"
+#include "EnvironmentValue.hpp"
+
+SchemaValue::SchemaValue(const std::string &name, bool isDefined) :
+    m_name(name), m_displayName(name)
+{
+    bitMask.m_required = 0;
+    bitMask.m_readOnly = 0;
+    bitMask.m_hidden = 0;
+    bitMask.m_deprecated = 0;
+    bitMask.m_isUnique = 0;
+    bitMask.m_isDefined = isDefined;
+}
+
+
+bool SchemaValue::isValueValid(const std::string &value, const EnvironmentValue *pEnvValue) const
+{
+    bool isValid = true;   // assume valid
+
+    //
+    // Check the type
+    isValid = m_pType->isValueValid(value);
+
+    //
+    // Keyed ?, then value must NOT be in the current list.
+    if (isValid && isUniqueValue() && pEnvValue != nullptr)
+    {
+        bool found = false;
+        std::vector<std::string> allValues;
+        pEnvValue->getAllValuesForSiblings(allValues);
+        for (auto it = allValues.begin(); it != allValues.end() && !found; ++it)
+            found = *it == value;
+
+        if (found)
+        {
+            return false;
+        }
+    }
+
+    //
+    // Keyref ?, then the value must be from another set
+    if (isValid && isFromUniqueValueSet() && pEnvValue != nullptr)
+    {
+        bool found = false;
+        std::vector<std::string> allValues;
+        getAllKeyRefValues(allValues);
+        for (auto it = allValues.begin(); it != allValues.end() && !found; ++it)
+            found = *it == value;
+        isValid = found;
+    }
+    return isValid;
+}
+
+
+void SchemaValue::validate(Status &status, const std::string &id, const EnvironmentValue *pEnvValue) const
+{
+    std::string curValue = pEnvValue->getValue();
+    bool isValid;
+
+    if (!m_pType->isValueValid(curValue))
+    {
+        if (pEnvValue)
+        {
+            std::string msg;
+            if (pEnvValue->wasForced())
+                msg = "Value was forced to an invalid value (" + curValue + ").";
+            else
+                msg = "Value is invalid (" + curValue + ").";
+            msg += "Valid value (" + m_pType->getLimitString() + ")";
+
+            status.addMsg(pEnvValue->wasForced() ? statusMsg::warning : statusMsg::error, pEnvValue->getNodeId(), pEnvValue->getName(), "", msg);
+        }
+        isValid = false;
+    }
+
+    // get currentvalue from pEnvValue
+    // for keyed, make sure all values are unique
+    // call pType with value to see if good
+    // call pType->limits->toString(value) if bad to get message about whats bad
+    // add to status
+
+}
+
+
+void SchemaValue::resetEnvironment()
+{
+    m_envValues.clear();
+}
+
+
+// replicates the new value throughout the environment
+void SchemaValue::mirrorValueToEnvironment(const std::string &oldValue, const std::string &newValue)
+{
+    for (auto mirrorCfgIt = m_mirrorToSchemaValues.begin(); mirrorCfgIt != m_mirrorToSchemaValues.end(); ++mirrorCfgIt)
+    {
+        (*mirrorCfgIt)->setMirroredEnvironmentValues(oldValue, newValue);
+    }
+}
+
+
+// Worker method for replicating a mirrored value to the environment values for this config value
+void SchemaValue::setMirroredEnvironmentValues(const std::string &oldValue, const std::string &newValue)
+{
+    for (auto envIt = m_envValues.begin(); envIt != m_envValues.end(); ++envIt)
+    {
+        std::shared_ptr<EnvironmentValue> pEnvValue = (*envIt).lock();
+        if (pEnvValue && pEnvValue->getValue() == oldValue)
+        {
+            pEnvValue->setValue(newValue, nullptr, true);
+        }
+    }
+}
+
+
+void SchemaValue::getAllEnvironmentValues(std::vector<std::string> &values) const
+{
+    for (auto it = m_envValues.begin(); it != m_envValues.end(); ++it)
+    {
+        values.push_back((*it).lock()->getValue());
+    }
+}
+
+
+void SchemaValue::getAllowedValues(std::vector<AllowedValue> &allowedValues, const EnvironmentValue *pEnvValue) const
+{
+    //
+    // Either the type is enumerated, or there is a keyref.
+    if (m_pType->isEnumerated())
+    {
+        allowedValues = m_pType->getEnumeratedValues();
+    }
+    else if (isFromUniqueValueSet() && pEnvValue != nullptr)
+    {
+        std::vector<std::string> refValues;
+        getAllKeyRefValues(refValues);
+        for (auto it = refValues.begin(); it != refValues.end(); ++it)
+        {
+            allowedValues.push_back({ *it, "" });
+        }
+    }
+}
+
+
+void SchemaValue::getAllKeyRefValues(std::vector<std::string> &keyRefValues) const
+{
+    std::vector<std::weak_ptr<SchemaValue>> refCfgValues = getUniqueValueSetRefs();
+    for (auto refCfgValueIt = refCfgValues.begin(); refCfgValueIt != refCfgValues.end(); ++refCfgValueIt)
+    {
+        std::shared_ptr<SchemaValue> pRefCfgValue = (*refCfgValueIt).lock();
+        std::vector<std::string> allValues;
+        pRefCfgValue->getAllEnvironmentValues(allValues);
+        keyRefValues.insert(keyRefValues.end(), allValues.begin(), allValues.end());
+    }
+}

configuration/config2/SchemaValue.hpp

@@ -0,0 +1,106 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2017 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#ifndef _CONFIG2_VALUE_HPP_
+#define _CONFIG2_VALUE_HPP_
+
+#include <memory>
+#include "SchemaType.hpp"
+#include "Status.hpp"
+#include "platform.h"
+
+
+class DECL_EXPORT SchemaValue
+{
+    public:
+
+        SchemaValue(const std::string &name, bool isDefined = true);
+        virtual ~SchemaValue() { }
+        void setType(const std::shared_ptr<SchemaType> pType) { m_pType = pType; }
+        const std::shared_ptr<SchemaType> &getType() const { return m_pType; }
+        const std::string &getName() const { return m_name; }
+        bool isValueValid(const std::string &newValue, const EnvironmentValue *pEnvValue = nullptr) const;
+        void setDisplayName(const std::string &displayName) { m_displayName = displayName; }
+        const std::string &getDisplayName() const { return m_displayName; }
+        void setRequired(bool reqd) { bitMask.m_required = reqd; }
+        bool isRequired() const { return bitMask.m_required; }
+        void setDefaultValue(const std::string &dflt) { m_default = dflt; }
+        const std::string &getDefaultValue() const { return m_default; }
+        bool hasDefaultValue() const { return !m_default.empty(); }
+        void setReadOnly(bool readOnly) { bitMask.m_readOnly = readOnly; }
+        bool isReadOnly() const { return bitMask.m_readOnly; }
+        void setHidden(bool hidden) { bitMask.m_hidden = hidden; }
+        bool isHidden() const { return bitMask.m_hidden; }
+        void setDeprecated(bool deprecated) { bitMask.m_deprecated = deprecated; }
+        bool isDeprecated() const { return bitMask.m_deprecated; }
+        void setTooltip(const std::string &tooltip) { m_tooltip = tooltip; }
+        const std::string &getTooltip() const { return m_tooltip; }
+        void addModifer(const std::string &mod) { m_modifiers.push_back(mod); }
+        void setModifiers(const std::vector<std::string> &list) { m_modifiers = list;  }
+        const std::vector<std::string> &getModifiers() const { return m_modifiers; }
+        bool hasModifiers() const { return m_modifiers.size() != 0; }
+        void setUniqueValue(bool isUnique) { bitMask.m_isUnique = isUnique; }
+        bool isUniqueValue() const { return bitMask.m_isUnique;  }
+        void setUniqueValueSetRef(const std::shared_ptr<SchemaValue> &pValue) { m_pUniqueValueSetRefs.push_back(pValue);  }
+        bool isFromUniqueValueSet() const { return !m_pUniqueValueSetRefs.empty(); }
+        const std::vector<std::weak_ptr<SchemaValue>> &getUniqueValueSetRefs() const { return m_pUniqueValueSetRefs;  }
+        bool isDefined() const { return bitMask.m_isDefined;  }
+        void resetEnvironment();
+        void setMirrorFromPath(const std::string &path) { m_mirrorFromPath = path;  }
+        const std::string &getMirrorFromPath() const { return m_mirrorFromPath;  }
+        bool isMirroredValue() const { return m_mirrorFromPath.length() != 0; }
+        void addMirroredSchemaValue(const std::shared_ptr<SchemaValue> &pVal) { m_mirrorToSchemaValues.push_back(pVal); }
+        void mirrorValueToEnvironment(const std::string &oldValue, const std::string &newValue);
+        void addEnvironmentValue(const std::shared_ptr<EnvironmentValue> &pEnvValue) { m_envValues.push_back(pEnvValue); }
+        void getAllEnvironmentValues(std::vector<std::string> &values) const;
+        void setMirroredEnvironmentValues(const std::string &oldValue, const std::string &newValue);
+        void validate(Status &status, const std::string &id, const EnvironmentValue *pEnvValue = nullptr) const;
+        void getAllowedValues(std::vector<AllowedValue> &allowedValues, const EnvironmentValue *pEnvValue = nullptr) const;
+        void setAutoGenerateType(const std::string &type) { m_autoGenerateType = type; }
+        const std::string &getAutoGenerateType() const { return m_autoGenerateType; }
+        void setAutoGenerateValue(const std::string &value) { m_autoGenerateValue = value; }
+        const std::string &getAutoGenerateValue() const { return m_autoGenerateValue; }
+        void getAllKeyRefValues(std::vector<std::string> &keyRefValues) const;
+
+
+    protected:
+
+        std::shared_ptr<SchemaType> m_pType;
+        std::vector<std::weak_ptr<EnvironmentValue>> m_envValues;
+        std::vector<std::shared_ptr<SchemaValue>> m_mirrorToSchemaValues;
+        std::string m_name;
+        std::string m_displayName;
+        std::string m_mirrorFromPath;
+        std::string m_autoGenerateValue;
+        std::string m_autoGenerateType;
+
+        struct {
+            unsigned m_required  : 1;
+            unsigned m_readOnly  : 1;
+            unsigned m_hidden    : 1;
+            unsigned m_deprecated: 1;
+            unsigned m_isUnique  : 1;
+            unsigned m_isDefined : 1;
+        } bitMask;
+
+        std::string m_default;
+        std::string m_tooltip;
+        std::vector<std::string> m_modifiers;
+        std::vector<std::weak_ptr<SchemaValue>> m_pUniqueValueSetRefs;    // this value serves as the key from which values are valid
+};
+
+#endif // _CONFIG2_VALUE_HPP_

configuration/config2/Status.cpp

@@ -0,0 +1,73 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2017 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#include "Status.hpp"
+
+
+void Status::addMsg(enum statusMsg::msgLevel level, const std::string &nodeId, const std::string &name, const std::string &referNodeId, const std::string &msg)
+{
+    statusMsg statusMsg(level, nodeId, name, referNodeId, msg);
+    m_messages.insert({level, statusMsg });
+    if (level > m_highestMsgLevel)
+        m_highestMsgLevel = level;
+}
+
+
+void Status::addUniqueMsg(enum statusMsg::msgLevel level, const std::string &nodeId, const std::string &name, const std::string &referNodeId, const std::string &msg)
+{
+    bool duplicateFound = false;
+    auto msgRange = m_messages.equal_range(level);
+    for (auto msgIt = msgRange.first; msgIt != msgRange.second && !duplicateFound; ++msgIt)
+    {
+        duplicateFound = (msgIt->second.nodeId == nodeId) && (msgIt->second.attribute == name) && (msgIt->second.msg == msg);
+    }
+
+    if (!duplicateFound)
+        addMsg(level, nodeId, name, referNodeId, msg);
+}
+
+std::vector<statusMsg> Status::getMessages() const
+{
+    std::vector<statusMsg> msgs;
+    for (auto it = m_messages.begin(); it != m_messages.end(); ++it)
+    {
+        msgs.push_back(it->second);
+    }
+    return msgs;
+}
+
+
+std::string Status::getStatusTypeString(enum statusMsg::msgLevel status) const
+{
+    std::string result = "Not found";
+    switch (status)
+    {
+        case statusMsg::info:    result = "Info";     break;
+        case statusMsg::warning: result = "Warning";  break;
+        case statusMsg::error:   result = "Error";    break;
+        case statusMsg::fatal:   result = "Fatal";    break;
+    }
+    return result;
+}
+
+void Status::add(const std::vector<statusMsg> msgs)
+{
+    for (auto msgIt = msgs.begin(); msgIt != msgs.end(); ++msgIt)
+    {
+        m_messages.insert({ (*msgIt).msgLevel, *msgIt });
+    }
+}

configuration/config2/Status.hpp

@@ -0,0 +1,70 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2017 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#ifndef _CONFIG2_NODESTATUS_HPP_
+#define _CONFIG2_NODESTATUS_HPP_
+
+#include <map>
+#include <vector>
+#include <string>
+#include "platform.h"
+
+
+
+struct DECL_EXPORT statusMsg {
+
+    enum msgLevel
+    {
+        info = 0,     // informational messages mainly
+        warning,
+        error,
+        fatal
+    };
+
+    statusMsg(enum msgLevel _msgLevel, const std::string &_nodeId, const std::string &_name, const std::string &_referNodeId, const std::string &_msg) :
+        msgLevel(_msgLevel), nodeId(_nodeId), attribute(_name), referNodeId(_referNodeId), msg(_msg) { }
+    msgLevel msgLevel;           // Message level
+    std::string nodeId;          // if not '', the node ID to which this status applies
+    std::string attribute;       // possible name of attribute in nodeId
+    std::string referNodeId;     // refernece node to which this status may also apply
+    std::string msg;             // message for user
+};
+
+class DECL_EXPORT Status
+{
+    public:
+
+        Status() : m_highestMsgLevel(statusMsg::info) { }
+        ~Status() {}
+        void addMsg(enum statusMsg::msgLevel status, const std::string &msg) { addMsg(status, "", "", "", msg); }
+        void addMsg(enum statusMsg::msgLevel status, const std::string &nodeId, const std::string &name, const std::string &referNodeId, const std::string &msg);
+        void addUniqueMsg(enum statusMsg::msgLevel status, const std::string &nodeId, const std::string &name, const std::string &referNodeId, const std::string &msg);
+        enum statusMsg::msgLevel getHighestMsgLevel() const { return m_highestMsgLevel; }
+        bool isOk() const { return m_highestMsgLevel <= statusMsg::warning; }
+        bool isError() const { return m_highestMsgLevel >= statusMsg::error; }
+        std::string getStatusTypeString(enum statusMsg::msgLevel status) const;
+        std::vector<statusMsg> getMessages() const;
+        void add(const std::vector<statusMsg> msgs);
+
+
+    private:
+
+        enum statusMsg::msgLevel m_highestMsgLevel;
+        std::multimap<enum statusMsg::msgLevel, statusMsg> m_messages;
+};
+
+#endif

configuration/config2/ValueDef.hpp

@@ -0,0 +1,29 @@
+/*/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2017 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#ifndef _CONFIG2_VALUEDEF_HPP_
+#define _CONFIG2_VALUEDEF_HPP_
+
+#include <string>
+#include "platform.h"
+
+struct ValueDef {
+    std::string name;
+    std::string value;
+};
+
+#endif

configuration/config2/XMLEnvironmentMgr.cpp

@@ -0,0 +1,164 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2017 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#include "XMLEnvironmentMgr.hpp"
+#include "XSDSchemaParser.hpp"
+
+
+bool XMLEnvironmentMgr::createParser()
+{
+    m_pSchemaParser = std::make_shared<XSDSchemaParser>(m_pSchema);
+    return true;
+}
+
+
+bool XMLEnvironmentMgr::doLoadEnvironment(std::istream &in)
+{
+
+    bool rc = true;
+    pt::ptree envTree;
+    try
+    {
+        pt::read_xml(in, envTree, pt::xml_parser::trim_whitespace | pt::xml_parser::no_comments);
+        auto rootIt = envTree.begin();
+
+        //
+        // Start at root, these better match!
+        std::string rootName = rootIt->first;
+        if (rootName == m_pSchema->getProperty("name"))
+        {
+            m_pRootNode = std::make_shared<EnvironmentNode>(m_pSchema, rootName);
+            m_pRootNode->setId(".");
+            addPath(m_pRootNode);
+            parse(rootIt->second, m_pSchema, m_pRootNode);
+        }
+    }
+    catch (const std::exception &e)
+    {
+        std::string xmlError = e.what();
+        m_message = "Unable to read/parse Environment file. Error = " + xmlError;
+        rc = false;
+    }
+    return rc;
+}
+
+
+bool XMLEnvironmentMgr::save(std::ostream &out)
+{
+    bool rc = true;
+    try
+    {
+        pt::ptree envTree, topTree;
+        serialize(envTree, m_pRootNode);
+        topTree.add_child("Environment", envTree);
+        pt::write_xml(out, topTree);
+    }
+    catch (const std::exception &e)
+    {
+        std::string xmlError = e.what();
+        m_message = "Unable to save Environment file. Error = " + xmlError;
+        rc = false;
+    }
+    return rc;
+}
+
+
+void XMLEnvironmentMgr::parse(const pt::ptree &envTree, const std::shared_ptr<SchemaItem> &pConfigItem, std::shared_ptr<EnvironmentNode> &pEnvNode)
+{
+
+    //
+    // First see if the node has a value
+    std::string value;
+    try
+    {
+        value = envTree.get<std::string>("");
+        if (!value.empty())
+        {
+            std::shared_ptr<SchemaValue> pCfgValue = pConfigItem->getItemSchemaValue();
+            std::shared_ptr<EnvironmentValue> pEnvValue = std::make_shared<EnvironmentValue>(pEnvNode, pCfgValue, "");  // node's value has no name
+            pEnvValue->setValue(value, nullptr);
+            pEnvNode->setLocalEnvValue(pEnvValue);
+        }
+    }
+    catch (...)
+    {
+        // do nothing
+    }
+
+    //
+    // Find elements in environment tree cooresponding to this config item, then parse each
+    for (auto it = envTree.begin(); it != envTree.end(); ++it)
+    {
+        std::string elemName = it->first;
+
+        //
+        // First see if there are attributes for this element (<xmlattr> === <element attr1="xx" attr2="yy" ...></element>  The attr1 and attr2 are in this)
+        if (elemName == "<xmlattr>")
+        {
+            for (auto attrIt = it->second.begin(); attrIt != it->second.end(); ++attrIt)
+            {
+                std::shared_ptr<SchemaValue> pSchemaValue = pConfigItem->getAttribute(attrIt->first);  // note, undefined attributes in schema will return a generic schema value
+                std::string curValue = attrIt->second.get_value<std::string>();
+                std::shared_ptr<EnvironmentValue> pEnvValue = std::make_shared<EnvironmentValue>(pEnvNode, pSchemaValue, attrIt->first, curValue);   // this is where we would use a variant
+                pSchemaValue->addEnvironmentValue(pEnvValue);
+                pEnvNode->addAttribute(attrIt->first, pEnvValue);
+            }
+        }
+        else
+        {
+            std::string typeName = it->second.get("<xmlattr>.buildSet", "");
+            std::shared_ptr<SchemaItem> pSchemaItem;
+            if (!typeName.empty())
+            {
+                pSchemaItem = pConfigItem->getChildByComponent(elemName, typeName);
+            }
+            else
+            {
+                pSchemaItem = pConfigItem->getChild(elemName);
+            }
+            std::shared_ptr<EnvironmentNode> pElementNode = std::make_shared<EnvironmentNode>(pSchemaItem, elemName, pEnvNode);
+            pElementNode->setId(getUniqueKey());
+            addPath(pElementNode);
+            parse(it->second, pSchemaItem, pElementNode);
+            pEnvNode->addChild(pElementNode);
+        }
+    }
+}
+
+
+void XMLEnvironmentMgr::serialize(pt::ptree &envTree, std::shared_ptr<EnvironmentNode> &pEnvNode) const
+{
+    std::vector<std::shared_ptr<EnvironmentValue>> attributes;
+    pEnvNode->getAttributes(attributes);
+    for (auto attrIt = attributes.begin(); attrIt != attributes.end(); ++attrIt)
+    {
+        envTree.put("<xmlattr>." + (*attrIt)->getName(), (*attrIt)->getValue());
+    }
+    std::shared_ptr<EnvironmentValue> pNodeValue = pEnvNode->getLocalEnvValue();
+    if (pNodeValue)
+    {
+        envTree.put_value(pNodeValue->getValue());
+    }
+    std::vector<std::shared_ptr<EnvironmentNode>> children;
+    pEnvNode->getChildren(children);
+    for (auto childIt = children.begin(); childIt != children.end(); ++childIt)
+    {
+        pt::ptree nodeTree;
+        serialize(nodeTree, *childIt);
+        envTree.add_child((*childIt)->getName(), nodeTree);
+    }
+}

configuration/config2/XMLEnvironmentMgr.hpp

@@ -0,0 +1,46 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2017 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#ifndef _CONFIG2_XMLENVIRONMENTMGR_HPP_
+#define _CONFIG2_XMLENVIRONMENTMGR_HPP_
+
+#include "EnvironmentMgr.hpp"
+
+#include <boost/property_tree/ptree.hpp>
+#include <boost/property_tree/xml_parser.hpp>
+
+namespace pt = boost::property_tree;
+
+class XMLEnvironmentMgr :   public EnvironmentMgr
+{
+    public:
+
+        XMLEnvironmentMgr() : EnvironmentMgr() { }
+        ~XMLEnvironmentMgr() { }
+
+
+    protected:
+
+        bool createParser() override;
+        bool doLoadEnvironment(std::istream &in) override;
+        bool save(std::ostream &out) override;
+        void parse(const pt::ptree &envTree, const std::shared_ptr<SchemaItem> &pConfig, std::shared_ptr<EnvironmentNode> &pEnvNode);
+        void serialize(pt::ptree &envTree, std::shared_ptr<EnvironmentNode> &pEnvNode) const;
+
+};
+
+#endif

configuration/config2/XSDComponentParser.cpp

@@ -0,0 +1,114 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2017 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+
+#include "XSDComponentParser.hpp"
+#include "XSDValueSetParser.hpp"
+#include "Exceptions.hpp"
+
+namespace pt = boost::property_tree;
+
+
+void XSDComponentParser::parseXSD(const pt::ptree &compTree)
+{
+    bool foundComponentDef = false;
+
+    pt::ptree tree = compTree.get_child("", pt::ptree());
+
+    //
+    // First time through look for attributeGroups that can be defined and for existence of a sequence element that actually defines the component
+    for (auto it = tree.begin(); it != tree.end(); ++it)
+    {
+        //
+        // Element parent (a type in realilty) and the element name help figure out how to process the XSD schema element
+        std::string elemType = it->first;
+        if (elemType == "xs:attributeGroup")
+        {
+            parseAttributeGroup(it->second);
+        }
+        else if (elemType == "xs:sequence")
+        {
+            foundComponentDef = true;
+        }
+    }
+
+
+    if (foundComponentDef)
+    {
+        pt::ptree elemTree = tree.get_child("xs:sequence.xs:element", pt::ptree());
+        if (!elemTree.empty())
+        {
+            std::string elementName = getXSDAttributeValue(elemTree, "<xmlattr>.name");
+            m_pSchemaItem->setProperty("name", elementName);
+            int minOccurs = elemTree.get("<xmlattr>.minOccurs", 1);
+            std::string maxOccursStr = elemTree.get("<xmlattr>.maxOccurs", "1");
+            int maxOccurs = (maxOccursStr != "unbounded") ? stoi(maxOccursStr) : -1;
+            m_pSchemaItem->setMinInstances(minOccurs);
+            m_pSchemaItem->setMaxInstances(maxOccurs);
+
+            //
+            // See if the element has a type. If so, then the element can have a value (other than attributes). Note does happen, but is rare
+            std::string elementDataType = elemTree.get("<xmlattr>.type", "");
+            if (!elementDataType.empty())
+            {
+                std::shared_ptr<SchemaValue> pItemCfgValue = std::make_shared<SchemaValue>("elementData");
+                pItemCfgValue->setType(m_pSchemaItem->getSchemaValueType(elementDataType));
+                pItemCfgValue->setDefaultValue(elemTree.get("<xmlattr>.default", ""));
+            }
+
+            //
+            // Parse any attributes, these are located in the xs:complexType section
+            pt::ptree attributeTree = elemTree.get_child("xs:complexType", pt::ptree());
+            for (auto attrIt = attributeTree.begin(); attrIt != attributeTree.end(); ++attrIt)
+            {
+                //
+                // Element parent (a type in realilty) and the element name help figure out how to process the XSD schema element
+                std::string elemType = attrIt->first;
+                if (elemType == "xs:attributeGroup")
+                {
+                    parseAttributeGroup(attrIt->second);
+                }
+                else if (elemType == "xs:attribute")
+                {
+                    parseAttribute(attrIt->second);
+                }
+            }
+
+            //
+            // Now parse the sequence section (these are sub keys for the component)
+            XSDSchemaParser::parseXSD(elemTree.get_child("xs:complexType.xs:sequence", pt::ptree()));
+
+            //
+            // See if any post config stuff like unique, key, or keyref
+            for (auto it = elemTree.begin(); it != elemTree.end(); ++it)
+            {
+                //
+                // Element parent (a type in realilty) and the element name help figure out how to process the XSD schema element
+                std::string elemType = it->first;
+                if (elemType == "xs:key")
+                {
+                    parseKey(it->second);
+                }
+                else if (elemType == "xs:keyref")
+                {
+                    parseKeyRef(it->second);
+                }
+            }
+
+        }
+    }
+}

configuration/config2/XSDComponentParser.hpp

@@ -0,0 +1,46 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2017 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+
+#ifndef _CONFIG2_XSDCOMPONENTPARSER_HPP_
+#define _CONFIG2_XSDCOMPONENTPARSER_HPP_
+
+#include <string>
+#include <memory>
+#include <map>
+#include <boost/property_tree/ptree.hpp>
+
+#include "XSDSchemaParser.hpp"
+
+namespace pt = boost::property_tree;
+
+class XSDComponentParser : public XSDSchemaParser
+{
+    public:
+
+        XSDComponentParser(std::shared_ptr<SchemaItem> pConfig) : XSDSchemaParser(pConfig) { }
+        virtual ~XSDComponentParser() { }
+        virtual void parseXSD(const pt::ptree &tree);
+
+
+    protected:
+
+        XSDComponentParser() { };
+};
+
+
+#endif // _CONFIG2_XSDCOMPONENTPARSER_HPP_

configuration/config2/XSDSchemaParser.cpp

@@ -0,0 +1,568 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2017 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#include <exception>
+#include "XSDSchemaParser.hpp"
+#include "Exceptions.hpp"
+#include "SchemaValue.hpp"
+
+#include "XSDComponentParser.hpp"
+#include "XSDValueSetParser.hpp"
+#include "SchemaTypeStringLimits.hpp"
+#include "SchemaTypeIntegerLimits.hpp"
+
+namespace pt = boost::property_tree;
+
+bool XSDSchemaParser::doParse(const std::string &configPath, const std::string &masterConfigFile,  const std::vector<std::string> &cfgParms)
+{
+    bool rc = true;
+
+    //
+    // Add some default types to the config. Note changing values for limits
+    std::shared_ptr<SchemaTypeStringLimits> pStringLimits;
+    std::shared_ptr<SchemaTypeIntegerLimits> pIntLimits;
+
+    std::shared_ptr<SchemaType> pType = std::make_shared<SchemaType>("xs:string");
+    pStringLimits = std::make_shared<SchemaTypeStringLimits>();
+    pType->setLimits(pStringLimits);
+    m_pSchemaItem->addSchemaValueType(pType);
+
+    pType = std::make_shared<SchemaType>("xs:token");
+    pStringLimits = std::make_shared<SchemaTypeStringLimits>();
+    pType->setLimits(pStringLimits);
+    m_pSchemaItem->addSchemaValueType(pType);
+
+    pType = std::make_shared<SchemaType>("xs:boolean");
+    std::shared_ptr<SchemaTypeLimits> pBoolLimits = std::make_shared<SchemaTypeStringLimits>();
+    pBoolLimits->addAllowedValue("true");
+    pBoolLimits->addAllowedValue("false");
+    pType->setLimits(pBoolLimits);
+    m_pSchemaItem->addSchemaValueType(pType);
+
+    pType = std::make_shared<SchemaType>("xs:integer");
+    pIntLimits = std::make_shared<SchemaTypeIntegerLimits>();
+    pType->setLimits(pIntLimits);
+    m_pSchemaItem->addSchemaValueType(pType);
+
+    pType = std::make_shared<SchemaType>("xs:int");
+    pIntLimits = std::make_shared<SchemaTypeIntegerLimits>();
+    pType->setLimits(pIntLimits);
+    m_pSchemaItem->addSchemaValueType(pType);
+
+    pType = std::make_shared<SchemaType>("xs:nonNegativeInteger");
+    pIntLimits = std::make_shared<SchemaTypeIntegerLimits>();
+    pIntLimits->setMinInclusive(0);
+    pType->setLimits(pIntLimits);
+    m_pSchemaItem->addSchemaValueType(pType);
+
+    pType = std::make_shared<SchemaType>("xs:positiveInteger");
+    pIntLimits = std::make_shared<SchemaTypeIntegerLimits>();
+    pIntLimits->setMinInclusive(1);
+    pType->setLimits(pIntLimits);
+    m_pSchemaItem->addSchemaValueType(pType);
+
+    pType = std::make_shared<SchemaType>("xs:unsignedInt");
+    pIntLimits = std::make_shared<SchemaTypeIntegerLimits>();
+    pIntLimits->setMinInclusive(0);
+    pType->setLimits(pIntLimits);
+    m_pSchemaItem->addSchemaValueType(pType);
+
+    //
+    // Get our specific XSD parameters from the input
+    m_basePath = configPath;
+    m_masterXSDFilename = masterConfigFile;
+    //m_buildsetFilename = cfgParms[2];
+    parseXSD(m_masterXSDFilename);
+
+    return rc;
+}
+
+
+void XSDSchemaParser::parseXSD(const std::string &filename)
+{
+    pt::ptree xsdTree;
+    std::string fpath = m_basePath + filename;
+    try
+    {
+        pt::read_xml(fpath, xsdTree, pt::xml_parser::trim_whitespace | pt::xml_parser::no_comments);
+    }
+    catch (const std::exception &e)
+    {
+        std::string xmlError = e.what();
+        ParseException pe("Unable to read/parse file. Check that file is formatted correctly. Error = " + xmlError);
+        pe.addFilename(filename);
+        throw(pe);
+    }
+
+    try
+    {
+        auto schemaIt = xsdTree.find("xs:schema");
+        const pt::ptree &keys = schemaIt->second.get_child("", pt::ptree());
+        parseXSD(keys);
+    }
+    catch (ParseException &pe)
+    {
+        pe.addFilename(filename);
+        throw(pe);
+    }
+}
+
+
+void XSDSchemaParser::parseXSD(const pt::ptree &keys)
+{
+    for (auto it = keys.begin(); it != keys.end(); ++it)
+    {
+        //
+        // Element parent (a type in realilty) and the element name help figure out how to process the XSD schema element
+        std::string elemType = it->first;
+        if (elemType == "xs:include")
+        {
+            std::string schemaFile = getXSDAttributeValue(it->second, "<xmlattr>.schemaLocation");
+            if (m_pSchemaItem->addUniqueName(schemaFile))
+            {
+                parseXSD(schemaFile);
+            }
+        }
+        else if (elemType == "xs:simpleType")
+        {
+            parseSimpleType(it->second);
+        }
+        else if (elemType == "xs:complexType")
+        {
+            parseComplexType(it->second);
+        }
+        else if (elemType == "xs:attributeGroup")
+        {
+            parseAttributeGroup(it->second);
+        }
+        else if (elemType == "xs:attribute")
+        {
+            parseAttribute(it->second);
+        }
+        else if (elemType == "xs:sequence")
+        {
+            parseXSD(it->second.get_child("", pt::ptree()));
+        }
+        else if (elemType == "xs:element")
+        {
+            parseElement(it->second);
+        }
+        else if (elemType == "xs:key")
+        {
+            parseKey(it->second);
+        }
+        else if (elemType == "xs:keyref")
+        {
+            parseKeyRef(it->second);
+        }
+    }
+}
+
+
+std::string XSDSchemaParser::getXSDAttributeValue(const pt::ptree &tree, const std::string &attrName, bool throwIfNotPresent, const std::string &defaultVal) const
+{
+    std::string value = defaultVal;
+    try
+    {
+        value = tree.get<std::string>(attrName);
+    }
+    catch (std::exception &e)
+    {
+        if (throwIfNotPresent)
+            throw(ParseException("Missing attribute " + attrName + "."));
+    }
+    return value;
+}
+
+
+void XSDSchemaParser::parseSimpleType(const pt::ptree &typeTree)
+{
+    std::shared_ptr<SchemaType> pCfgType = getSchemaType(typeTree, true);
+    m_pSchemaItem->addSchemaValueType(pCfgType);
+}
+
+
+void XSDSchemaParser::parseAttribute(const pt::ptree &attr)
+{
+    std::shared_ptr<SchemaValue> pCfgValue = getSchemaValue(attr);
+    m_pSchemaItem->addAttribute(pCfgValue);
+}
+
+
+void XSDSchemaParser::parseAttributeGroup(const pt::ptree &attributeTree)
+{
+    std::string groupName = getXSDAttributeValue(attributeTree, "<xmlattr>.name", false, "");  // only a named attributeGroup is supported
+
+    //
+    // If there is a name (for the attribute group) then a group of attributes is being defined. Create the group, parese it, and add it as a
+    // schema type that can be reused (usually with a ref= reference in another attribute group schema item)
+    if (!groupName.empty())
+    {
+        std::shared_ptr<SchemaItem> pValueSet = std::make_shared<SchemaItem>(groupName, "valueset", m_pSchemaItem);
+        std::shared_ptr<XSDValueSetParser> pXSDValueSetParaser = std::make_shared<XSDValueSetParser>(pValueSet);
+        pXSDValueSetParaser->parseXSD(attributeTree.get_child("", pt::ptree()));
+        m_pSchemaItem->addSchemaType(pValueSet, groupName);
+    }
+
+    //
+    // Is it a reference to a named attribute group previously saved? If so, grab the defined attributes and add them.
+    else
+    {
+        std::string refName = getXSDAttributeValue(attributeTree, "<xmlattr>.ref", false, "");  // only a named attributeGroup is supported
+        if (!refName.empty())
+        {
+            std::shared_ptr<SchemaItem> pValueSet = m_pSchemaItem->getSchemaType(refName, true);
+            if (pValueSet)
+            {
+                std::vector<std::shared_ptr<SchemaValue>> attributes;
+                pValueSet->getAttributes(attributes);
+                m_pSchemaItem->addAttribute(attributes);
+            }
+        }
+    }
+}
+
+
+void XSDSchemaParser::parseComplexType(const pt::ptree &typeTree)
+{
+    std::string complexTypeName = getXSDAttributeValue(typeTree, "<xmlattr>.name", false, "");
+    std::string className = typeTree.get("<xmlattr>.hpcc:class", "");
+    std::string catName = typeTree.get("<xmlattr>.hpcc:category", "");
+    std::string componentName = typeTree.get("<xmlattr>.hpcc:componentName", "");
+    std::string displayName = typeTree.get("<xmlattr>.hpcc:displayName", "");
+
+    if (!complexTypeName.empty())
+    {
+        if (!className.empty())
+        {
+            if (className == "component")
+            {
+                std::shared_ptr<SchemaItem> pComponent = std::make_shared<SchemaItem>(complexTypeName, "component", m_pSchemaItem);
+                pComponent->setProperty("category", catName);
+                pComponent->setProperty("componentName", componentName);
+                pComponent->setProperty("displayName", displayName);
+                pt::ptree componentTree = typeTree.get_child("", pt::ptree());
+                if (!componentTree.empty())
+                {
+                    std::shared_ptr<XSDComponentParser> pComponentXSDParaser = std::make_shared<XSDComponentParser>(std::dynamic_pointer_cast<SchemaItem>(pComponent));
+                    pComponentXSDParaser->parseXSD(typeTree);
+                    m_pSchemaItem->addSchemaType(pComponent, complexTypeName);
+                }
+                else
+                {
+                    throw(ParseException("Component definition empty: " + displayName));
+                }
+            }
+            else
+            {
+                throw(ParseException("Unrecognized class name for complex type: " + className));
+            }
+        }
+
+        //
+        // This is a complex type definition of just regular XSD statements, no special format. Create a parser and parse it
+        // and add it to the
+        else
+        {
+            std::shared_ptr<SchemaItem> pTypeItem = std::make_shared<SchemaItem>(complexTypeName, "", m_pSchemaItem);
+            pt::ptree childTree = typeTree.get_child("", pt::ptree());
+            if (!childTree.empty())
+            {
+                std::shared_ptr<XSDSchemaParser> pXSDParaser = std::make_shared<XSDSchemaParser>(pTypeItem);
+                pXSDParaser->parseXSD(childTree);
+                m_pSchemaItem->addSchemaType(pTypeItem, complexTypeName);
+            }
+            else
+            {
+                throw(ParseException("Complex type definition empty: " + displayName));
+            }
+        }
+    }
+
+    //
+    // Just a complexType delimiter, ignore and parse the children
+    else
+    {
+        parseXSD(typeTree.get_child("", pt::ptree()));
+    }
+}
+
+
+void XSDSchemaParser::parseElement(const pt::ptree &elemTree)
+{
+    std::string elementName = elemTree.get("<xmlattr>.name", "");
+    std::string className = elemTree.get("<xmlattr>.hpcc:class", "");
+    std::string category = elemTree.get("<xmlattr>.hpcc:category", "");
+    std::string displayName = elemTree.get("<xmlattr>.hpcc:displayName", "");
+    std::string typeName = elemTree.get("<xmlattr>.type", "");
+    unsigned minOccurs = elemTree.get("<xmlattr>.minOccurs", 1);
+    std::string maxOccursStr = elemTree.get("<xmlattr>.maxOccurs", "1");
+    unsigned maxOccurs = (maxOccursStr != "unbounded") ? stoi(maxOccursStr) : UINTMAX_MAX;
+
+    std::shared_ptr<SchemaItem> pConfigElement = std::make_shared<SchemaItem>(elementName, className, m_pSchemaItem);
+    pConfigElement->setProperty("displayName", displayName);
+    pConfigElement->setMinInstances(minOccurs);
+    pConfigElement->setMaxInstances(maxOccurs);
+    pConfigElement->setProperty("category", category);
+
+    pt::ptree childTree = elemTree.get_child("", pt::ptree());
+
+    // special case to set the root since the top level schema can't specify it
+    if (category == "root")  // special case to set the root since the top level schema can't specify it
+    {
+        m_pSchemaItem->setProperty("name", elementName);
+        parseXSD(childTree);
+    }
+    else
+    {
+        //
+        // If a type is specified, then either it's a simple value type (which could be previously defined) for this element, or a named complex type.
+        if (!typeName.empty())
+        {
+            const std::shared_ptr<SchemaType> pSimpleType = m_pSchemaItem->getSchemaValueType(typeName, false);
+            if (pSimpleType != nullptr)
+            {
+                std::shared_ptr<SchemaValue> pCfgValue = std::make_shared<SchemaValue>("");  // no name value since it's the element's value
+                pCfgValue->setType(pSimpleType);                      // will throw if type is not defined
+                pConfigElement->setItemSchemaValue(pCfgValue);
+            }
+            else
+            {
+                std::shared_ptr<SchemaItem> pConfigType = m_pSchemaItem->getSchemaType(typeName, false);
+                if (pConfigType != nullptr)
+                {
+                    //
+                    // Insert into this config element the component defined data (attributes, references, etc.)
+                    pConfigElement->insertSchemaType(pConfigType);
+
+                    //
+                    // Set element min/max instances to that defined by the component type def (ignore values parsed above)
+                    pConfigElement->setMinInstances(pConfigType->getMinInstances());
+                    pConfigElement->setMaxInstances(pConfigType->getMaxInstances());
+
+                    //
+                    // If a component, then set element data (allow overriding with locally parsed values)
+                    if (pConfigType->getProperty("className") == "component")
+                    {
+                        pConfigElement->setProperty("name", (!elementName.empty()) ? elementName : pConfigType->getProperty("name"));
+                        pConfigElement->setProperty("className", (!className.empty()) ? className : pConfigType->getProperty("className"));
+                        pConfigElement->setProperty("category", (!category.empty()) ? category : pConfigType->getProperty("category"));
+                        pConfigElement->setProperty("displayName", (!displayName.empty()) ? displayName : pConfigType->getProperty("displayName"));
+                        pConfigElement->setProperty("componentName", pConfigType->getProperty("componentName"));
+                    }
+                }
+                else
+                {
+                    std::string msg = "Unable to find type " + typeName + " when parsing element " + elementName;
+                    throw(ParseException(msg));
+                }
+            }
+        }
+
+        //
+        // Now, if there are children, create a parser and have at it
+        if (!childTree.empty())
+        {
+            std::shared_ptr<XSDSchemaParser> pXSDParaser = std::make_shared<XSDSchemaParser>(pConfigElement);
+            pXSDParaser->parseXSD(childTree);
+        }
+
+        //
+        // Add the element
+        m_pSchemaItem->addChild(pConfigElement);
+
+    }
+}
+
+
+std::shared_ptr<SchemaType> XSDSchemaParser::getSchemaType(const pt::ptree &typeTree, bool nameRequired)
+{
+    std::string typeName = getXSDAttributeValue(typeTree, "<xmlattr>.name", nameRequired, "");
+
+    std::shared_ptr<SchemaType> pCfgType = std::make_shared<SchemaType>(typeName);
+    std::shared_ptr<SchemaTypeLimits> pLimits;
+    auto restriction = typeTree.find("xs:restriction");
+    if (restriction != typeTree.not_found())
+    {
+        std::string baseType = getXSDAttributeValue(restriction->second, "<xmlattr>.base");
+        std::shared_ptr<SchemaType> pType = std::make_shared<SchemaType>(*(m_pSchemaItem->getSchemaValueType(baseType)));
+
+        pLimits = pType->getLimits();
+
+        if (!restriction->second.empty())
+        {
+            pt::ptree restrictTree = restriction->second.get_child("", pt::ptree());
+            if (std::dynamic_pointer_cast<SchemaTypeIntegerLimits>(pLimits) != nullptr)
+            {
+                std::shared_ptr<SchemaTypeIntegerLimits> pBaseIntLimits = std::dynamic_pointer_cast<SchemaTypeIntegerLimits>(pLimits);
+                std::shared_ptr<SchemaTypeIntegerLimits> pIntLimits = std::make_shared<SchemaTypeIntegerLimits>(*pBaseIntLimits);
+                parseIntegerTypeLimits(restrictTree, pIntLimits);
+                pLimits = pIntLimits;
+            }
+            else if (std::dynamic_pointer_cast<SchemaTypeStringLimits>(pLimits) != nullptr)
+            {
+                std::shared_ptr<SchemaTypeStringLimits> pBaseStringimits = std::dynamic_pointer_cast<SchemaTypeStringLimits>(pLimits);
+                std::shared_ptr<SchemaTypeStringLimits> pStringimits = std::make_shared<SchemaTypeStringLimits>(*pBaseStringimits);
+                parseStringTypeLimits(restrictTree, pStringimits);
+                pLimits = pStringimits;
+            }
+            else
+            {
+                std::string msg = "Unsupported base type(" + baseType + ")";
+                throw(ParseException(msg));
+            }
+        }
+    }
+
+    pCfgType->setLimits(pLimits);
+    return pCfgType;
+}
+
+
+
+void XSDSchemaParser::parseIntegerTypeLimits(const pt::ptree &restrictTree, std::shared_ptr<SchemaTypeIntegerLimits> &pIntegerLimits)
+{
+    for (auto it = restrictTree.begin(); it != restrictTree.end(); ++it)
+    {
+        std::string restrictionType = it->first;
+
+        if (restrictionType == "xs:minInclusive")
+            pIntegerLimits->setMinInclusive(it->second.get<int>("<xmlattr>.value"));
+        else if (restrictionType == "xs:maxInclusive")
+            pIntegerLimits->setMaxInclusive(it->second.get<int>("<xmlattr>.value"));
+        else if (restrictionType == "xs:minExclusive")
+            pIntegerLimits->setMinExclusive(it->second.get<int>("<xmlattr>.value"));
+        else if (restrictionType == "xs:maxExclusive")
+            pIntegerLimits->setMaxExclusive(it->second.get<int>("<xmlattr>.value"));
+        else if (restrictionType == "xs:enumeration")
+        {
+            pIntegerLimits->addAllowedValue(it->second.get("<xmlattr>.value", "badbadbad"), it->second.get("<xmlattr>.hpcc:description", ""));
+        }
+        else if (restrictionType != "<xmlattr>")
+        {
+            std::string msg = "Invalid restriction(" + it->first + ") found while parsing type";
+            throw(ParseException(msg));
+        }
+    }
+}
+
+
+void XSDSchemaParser::parseStringTypeLimits(const pt::ptree &restrictTree, std::shared_ptr<SchemaTypeStringLimits> &pStringLimits)
+{
+    for (auto it = restrictTree.begin(); it != restrictTree.end(); ++it)
+    {
+        std::string restrictionType = it->first;
+
+        if (restrictionType == "xs:minLength")
+            pStringLimits->setMinLength(it->second.get<int>("<xmlattr>.value"));
+        else if (restrictionType == "xs:maxLength")
+            pStringLimits->setMaxLength(it->second.get<int>("<xmlattr>.value"));
+        else if (restrictionType == "xs:length")
+            pStringLimits->setLength(it->second.get<int>("<xmlattr>.value"));
+        else if (restrictionType == "xs:pattern")
+            pStringLimits->addPattern(it->second.get("<xmlattr>.value", "0"));
+        else if (restrictionType == "xs:enumeration")
+        {
+            pStringLimits->addAllowedValue(it->second.get("<xmlattr>.value", "badbadbad"), it->second.get("<xmlattr>.hpcc:description", ""));
+        }
+        else if (restrictionType != "<xmlattr>")
+        {
+            std::string msg = "Invalid restriction(" + it->first + ") found while parsing type";
+            throw(ParseException(msg));
+        }
+    }
+}
+
+
+std::shared_ptr<SchemaValue> XSDSchemaParser::getSchemaValue(const pt::ptree &attr)
+{
+    std::string attrName = getXSDAttributeValue(attr, "<xmlattr>.name");
+    std::shared_ptr<SchemaValue> pCfgValue = std::make_shared<SchemaValue>(attrName);
+    pCfgValue->setDisplayName(attr.get("<xmlattr>.hpcc:displayName", attrName));
+    pCfgValue->setRequired(attr.get("<xmlattr>.use", "optional") == "required");
+    pCfgValue->setTooltip(attr.get("<xmlattr>.hpcc:tooltip", ""));
+    pCfgValue->setReadOnly(attr.get("<xmlattr>.hpcc:readOnly", "false") == "true");
+    pCfgValue->setHidden(attr.get("<xmlattr>.hpcc:hidden", "false") == "true");
+    pCfgValue->setDeprecated(attr.get("<xmlattr>.hpcc:deprecated", "false") == "true");
+    pCfgValue->setMirrorFromPath(attr.get("<xmlattr>.hpcc:mirrorFrom", ""));
+    pCfgValue->setAutoGenerateType(attr.get("<xmlattr>.hpcc:autoGenerateType", ""));
+    pCfgValue->setAutoGenerateValue(attr.get("<xmlattr>.hpcc:autoGenerateValue", ""));
+    pCfgValue->setDefaultValue(attr.get("<xmlattr>.default", ""));
+
+    std::string modList = attr.get("<xmlattr>.hpcc:modifiers", "");
+    if (modList.length())
+    {
+        pCfgValue->setModifiers(split(modList, ","));
+    }
+
+    std::string typeName = attr.get("<xmlattr>.type", "");
+    if (!typeName.empty())
+    {
+        pCfgValue->setType(m_pSchemaItem->getSchemaValueType(typeName));
+    }
+    else
+    {
+        std::shared_ptr<SchemaType> pType = getSchemaType(attr.get_child("xs:simpleType", pt::ptree()), false);
+        if (!pType->isValid())
+        {
+            throw(ParseException("Attribute " + attrName + " does not have a valid type"));
+        }
+        pCfgValue->setType(pType);
+    }
+    return pCfgValue;
+}
+
+
+void XSDSchemaParser::parseKey(const pt::ptree &keyTree)
+{
+    std::string keyName = getXSDAttributeValue(keyTree, "<xmlattr>.name");
+    bool duplicateOk = keyTree.get("<xmlattr>.hpcc:allowDuplicate", "false") == "true";
+    std::string elementName = getXSDAttributeValue(keyTree, "xs:selector.<xmlattr>.xpath", false, "");
+    std::string attrName = getXSDAttributeValue(keyTree, "xs:field.<xmlattr>.xpath", false, "");
+    std::string attributeName;
+
+    if (attrName.find_first_of('@') != std::string::npos)
+    {
+        attributeName = attrName.substr(attrName.find_first_of('@') + 1);
+    }
+    else
+    {
+        attributeName = attrName;
+    }
+
+    m_pSchemaItem->addUniqueAttributeValueSetDefinition(keyName, elementName, attributeName, duplicateOk);
+}
+
+
+void XSDSchemaParser::parseKeyRef(const pt::ptree &keyTree)
+{
+    std::string keyName = getXSDAttributeValue(keyTree, "<xmlattr>.refer");
+    std::string elementName = getXSDAttributeValue(keyTree, "xs:selector.<xmlattr>.xpath", false, "");
+    std::string attrName = getXSDAttributeValue(keyTree, "xs:field.<xmlattr>.xpath", false, "");
+    std::string attributeName;
+
+    if (attrName.find_first_of('@') != std::string::npos)
+    {
+        attributeName = attrName.substr(attrName.find_first_of('@') + 1);
+    }
+    else
+    {
+        attributeName = attrName;
+    }
+
+    m_pSchemaItem->addReferenceToUniqueAttributeValueSet(keyName, elementName, attributeName);
+}

configuration/config2/XSDSchemaParser.hpp

@@ -0,0 +1,74 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2017 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+
+#ifndef _CONFIG2_XSDCONFIGPARSER_HPP_
+#define _CONFIG2_XSDCONFIGPARSER_HPP_
+
+#include <string>
+#include <memory>
+#include <map>
+#include <boost/property_tree/ptree.hpp>
+#include <boost/property_tree/xml_parser.hpp>
+
+#include "SchemaParser.hpp"
+#include "SchemaTypeIntegerLimits.hpp"
+#include "SchemaTypeStringLimits.hpp"
+
+namespace pt = boost::property_tree;
+
+class XSDSchemaParser : public SchemaParser
+{
+    public:
+
+        XSDSchemaParser(std::shared_ptr<SchemaItem> &pConfig) :
+            SchemaParser(pConfig) { }
+        virtual ~XSDSchemaParser() { };
+
+
+    protected:
+
+        XSDSchemaParser() { };
+        virtual bool doParse(const std::string &configPath, const std::string &masterConfigFile,  const std::vector<std::string> &cfgParms) override;
+        virtual void parseXSD(const pt::ptree &tree);
+        virtual void parseXSD(const std::string &filename);
+        virtual std::string getXSDAttributeValue(const pt::ptree &tree, const std::string &attriName, bool throwIfNotPresent=true, const std::string &defaultVal = (std::string(""))) const;
+        virtual void parseAttributeGroup(const pt::ptree &attributeTree);
+        virtual void parseAttribute(const pt::ptree &attr);
+
+        virtual void parseSimpleType(const pt::ptree &typeTree);
+        virtual void parseComplexType(const pt::ptree &typeTree);
+        virtual void parseElement(const pt::ptree &elemTree);
+
+        virtual std::shared_ptr<SchemaType> getSchemaType(const pt::ptree &typeTree, bool nameRequired=true);
+        virtual std::shared_ptr<SchemaValue> getSchemaValue(const pt::ptree &attr);
+
+        virtual void parseKey(const pt::ptree &keyTree);
+        virtual void parseKeyRef(const pt::ptree &keyTree);
+        virtual void parseIntegerTypeLimits(const pt::ptree &restrictTree, std::shared_ptr<SchemaTypeIntegerLimits> &pIntegerLimits);
+        virtual void parseStringTypeLimits(const pt::ptree &restrictTree, std::shared_ptr<SchemaTypeStringLimits> &pStringLimits);
+
+    protected:
+
+        std::string m_buildsetFilename;
+        std::string m_basePath;
+        std::string m_masterXSDFilename;
+
+};
+
+
+#endif // _CONFIG2_XSDCONFIGPARSER_HPP_

configuration/config2/XSDValueSetParser.cpp

@@ -0,0 +1,52 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2017 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#include "XSDValueSetParser.hpp"
+#include "Exceptions.hpp"
+
+namespace pt = boost::property_tree;
+
+void XSDValueSetParser::parseXSD(const pt::ptree &valueSetTree)
+{
+    for (auto it = valueSetTree.begin(); it != valueSetTree.end(); ++it)
+    {
+        //
+        // Element parent (a type in realilty) and the element name help figure out how to process the XSD schema element
+        std::string elemType = it->first;
+        if (it->first == "xs:attributeGroup")
+        {
+            parseAttributeGroup(it->second);
+        }
+        else if (it->first == "xs:attribute")
+        {
+            parseAttribute(it->second);
+        }
+    }
+}
+
+
+void XSDValueSetParser::parseAttributeGroup(const pt::ptree &attributeTree)
+{
+    std::string groupRefName = getXSDAttributeValue(attributeTree, "<xmlattr>.ref");
+    std::shared_ptr<SchemaItem> pValueSet = m_pSchemaItem->getSchemaType(groupRefName, true);
+    if (pValueSet)
+    {
+        std::vector<std::shared_ptr<SchemaValue>> attributes;
+        pValueSet->getAttributes(attributes);
+        m_pSchemaItem->addAttribute(attributes);
+    }
+}

configuration/config2/XSDValueSetParser.hpp

@@ -0,0 +1,48 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2017 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+
+#ifndef _CONFIG2_XSDVALUESETPARSER_HPP_
+#define _CONFIG2_XSDVALUESETPARSER_HPP_
+
+#include <string>
+#include <memory>
+#include <map>
+#include <boost/property_tree/ptree.hpp>
+
+#include "XSDSchemaParser.hpp"
+
+namespace pt = boost::property_tree;
+
+class XSDValueSetParser : public XSDSchemaParser
+{
+    public:
+
+        XSDValueSetParser(std::shared_ptr<SchemaItem> pValueSet) :
+            XSDSchemaParser(pValueSet)  { }
+        virtual ~XSDValueSetParser() { }
+        virtual void parseXSD(const pt::ptree &valueSetTree);
+        void parseAttributeGroup(const pt::ptree &attributeTree);
+
+
+    protected:
+
+        XSDValueSetParser() { }
+};
+
+
+#endif // _CONFIG2_XSDVALUESETPARSER_HPP_

configuration/config2/configfiles/buildset.xml

@@ -0,0 +1,136 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!--
+################################################################################
+#    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems®.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+################################################################################
+-->
+<Environment>
+  <Software>
+    <Directories name="HPCCSystems">
+      <Category dir="/var/log/[NAME]/[INST]" name="log"/>
+      <Category dir="/var/lib/[NAME]/[INST]" name="run"/>
+      <Category dir="/etc/[NAME]/[INST]" name="conf"/>
+      <Category dir="/var/lib/[NAME]/[INST]/temp" name="temp"/>
+      <Category dir="/var/lib/[NAME]/hpcc-data/[COMPONENT]" name="data"/>
+      <Category dir="/var/lib/[NAME]/hpcc-data2/[COMPONENT]" name="data2"/>
+      <Category dir="/var/lib/[NAME]/hpcc-data3/[COMPONENT]" name="data3"/>
+      <Category dir="/var/lib/[NAME]/hpcc-mirror/[COMPONENT]" name="mirror"/>
+      <Category dir="/var/lib/[NAME]/queries/[INST]" name="query"/>
+      <Category dir="/var/lock/[NAME]/[INST]" name="lock"/>
+    </Directories>
+  </Software>
+  <Programs>
+    <Build name="_" url="/opt/HPCCSystems">
+      <BuildSet installSet="deploy_map.xml" name="dafilesrv" path="componentfiles/dafilesrv" processName="DafilesrvProcess" schema="dafilesrv.xsd"/>
+      <BuildSet installSet="deploy_map.xml" name="dali" path="componentfiles/dali" processName="DaliServerProcess" schema="dali.xsd"/>
+      <BuildSet installSet="deploy_map.xml" name="backupnode" path="componentfiles/backupnode" processName="BackupNodeProcess" schema="backupnode.xsd"/>
+      <BuildSet installSet="deploy_map.xml" name="daliplugin" path="componentfiles/daliplugin" processName="DaliServerPlugin" schema="daliplugin.xsd"/>
+      <BuildSet installSet="deploy_map.xml" name="dfuplus" path="componentfiles/dfuplus" processName="DfuplusProcess" schema="dfuplus.xsd" overide="no"/>
+      <BuildSet installSet="deploy_map.xml" name="dfuserver" path="componentfiles/dfuserver" processName="DfuServerProcess" schema="dfuserver.xsd"/>
+      <BuildSet deployable="no" installSet="deploy_map.xml" name="DropZone" path="componentfiles/DropZone" processName="DropZone" schema="dropzone.xsd"/>
+      <BuildSet installSet="deploy_map.xml" name="eclagent" path="componentfiles/eclagent" processName="EclAgentProcess" schema="eclagent_config.xsd"/>
+      <BuildSet installSet="deploy_map.xml" name="eclminus" path="componentfiles/eclminus" overide="no"/>
+      <BuildSet installSet="deploy_map.xml" name="eclplus" path="componentfiles/eclplus" processName="EclPlusProcess" schema="eclplus.xsd" overide="no"/>
+      <BuildSet installSet="eclccserver_deploy_map.xml" name="eclccserver" path="componentfiles/configxml" processName="EclCCServerProcess" schema="eclccserver.xsd"/>
+      <BuildSet installSet="eclscheduler_deploy_map.xml" name="eclscheduler" path="componentfiles/configxml" processName="EclSchedulerProcess" schema="eclscheduler.xsd"/>
+      <BuildSet installSet="deploy_map.xml" name="esp" path="componentfiles/esp" processName="EspProcess" schema="esp.xsd"/>
+      <BuildSet deployable="no" installSet="deploy_map.xml" name="espsmc" path="componentfiles/espsmc" processName="EspService" schema="espsmcservice.xsd">
+        <Properties defaultPort="8010" defaultResourcesBasedn="ou=SMC,ou=EspServices,ou=ecl" defaultSecurePort="18010" type="WsSMC">
+          <Authenticate access="Read" description="Root access to SMC service" path="/" required="Read" resource="SmcAccess"/>
+          <AuthenticateFeature description="Access to SMC service" path="SmcAccess" resource="SmcAccess" service="ws_smc"/>
+          <AuthenticateFeature description="Access to thor queues" path="ThorQueueAccess" resource="ThorQueueAccess" service="ws_smc"/>
+          <AuthenticateFeature description="Access to roxie control commands" path="RoxieControlAccess" resource="RoxieControlAccess" service="ws_smc"/>
+          <AuthenticateFeature description="Access to DFU" path="DfuAccess" resource="DfuAccess" service="ws_dfu"/>
+          <AuthenticateFeature description="Access to DFU XRef" path="DfuXrefAccess" resource="DfuXrefAccess" service="ws_dfuxref"/>
+          <AuthenticateFeature description="Access to machine information" path="MachineInfoAccess" resource="MachineInfoAccess" service="ws_machine"/>
+          <AuthenticateFeature description="Access to SNMP metrics information" path="MetricsAccess" resource="MetricsAccess" service="ws_machine"/>
+          <AuthenticateFeature description="Access to DFU workunits" path="DfuWorkunitsAccess" resource="DfuWorkunitsAccess" service="ws_fs"/>
+          <AuthenticateFeature description="Access to DFU exceptions" path="DfuExceptionsAccess" resource="DfuExceptions" service="ws_fs"/>
+          <AuthenticateFeature description="Access to spraying files" path="FileSprayAccess" resource="FileSprayAccess" service="ws_fs"/>
+          <AuthenticateFeature description="Access to despraying of files" path="FileDesprayAccess" resource="FileDesprayAccess" service="ws_fs"/>
+          <AuthenticateFeature description="Access to upload files to dropzone" path="FileUploadAccess" resource="FileUploadAccess" service="ws_fs" />
+          <AuthenticateFeature description="Access to files in dropzone" path="FileIOAccess" resource="FileIOAccess" service="ws_fileio"/>
+          <AuthenticateFeature description="Access to permissions for file scopes" path="FileScopeAccess" resource="FileScopeAccess" service="ws_access"/>
+          <AuthenticateFeature description="Access to WS ECL service" path="WsEclAccess" resource="WsEclAccess" service="ws_ecl"/>
+          <AuthenticateFeature description="Access to cluster topology" path="ClusterTopologyAccess" resource="ClusterTopologyAccess" service="ws_topology"/>
+          <AuthenticateFeature description="Access to own workunits" path="OwnWorkunitsAccess" resource="OwnWorkunitsAccess" service="ws_workunits"/>
+          <AuthenticateFeature description="Access to others&apos; workunits" path="OthersWorkunitsAccess" resource="OthersWorkunitsAccess" service="ws_workunits"/>
+          <AuthenticateFeature description="Access to ECL direct service" path="EclDirectAccess" resource="EclDirectAccess" service="ecldirect"/>
+          <AuthenticateFeature description="Access to ESDL configuration service" path="ESDLConfigAccess" resource="ESDLConfigAccess" service="ws_esdlconfig"/>
+          <ProcessFilters>
+            <Platform name="Windows">
+              <ProcessFilter name="any">
+                <Process name="dafilesrv"/>
+              </ProcessFilter>
+              <ProcessFilter multipleInstances="true" name="DfuServerProcess"/>
+              <ProcessFilter multipleInstances="true" name="EclCCServerProcess"/>
+              <ProcessFilter multipleInstances="true" name="EspProcess">
+                <Process name="dafilesrv" remove="true"/>
+              </ProcessFilter>
+            </Platform>
+            <Platform name="Linux">
+              <ProcessFilter name="any">
+                <Process name="dafilesrv"/>
+              </ProcessFilter>
+              <ProcessFilter multipleInstances="true" name="DfuServerProcess"/>
+              <ProcessFilter multipleInstances="true" name="EclCCServerProcess"/>
+              <ProcessFilter multipleInstances="true" name="EspProcess">
+                <Process name="dafilesrv" remove="true"/>
+              </ProcessFilter>
+              <ProcessFilter name="GenesisServerProcess">
+                <Process name="httpd"/>
+                <Process name="atftpd"/>
+                <Process name="dhcpd"/>
+              </ProcessFilter>
+            </Platform>
+          </ProcessFilters>
+        </Properties>
+      </BuildSet>
+      <BuildSet installSet="deploy_map.xml" name="ftslave" path="componentfiles/ftslave" processName="FTSlaveProcess" schema="ftslave_linux.xsd"/>
+      <BuildSet installSet="deploy_map.xml" name="hqltest" path="componentfiles/hqltest" processName="HqlTestProcess" overide="no"/>
+      <BuildSet deployable="no" installSet="deploy_map.xml" name="ldapServer" path="componentfiles/ldapServer" processName="LDAPServerProcess" schema="ldapserver.xsd"/>
+      <BuildSet installSet="roxie_deploy_map.xml" name="roxie" path="componentfiles/configxml" processName="RoxieCluster" schema="roxie.xsd"/>
+      <BuildSet installSet="deploy_map.xml" name="sasha" path="componentfiles/sasha" processName="SashaServerProcess" schema="sasha.xsd"/>
+      <BuildSet deployable="no" installSet="deploy_map.xml" name="SiteCertificate" path="componentfiles/SiteCertificate" processName="SiteCertificate" schema="SiteCertificate.xsd" overide="no"/>
+      <BuildSet installSet="deploy_map.xml" name="soapplus" path="componentfiles/soapplus" processName="SoapPlusProcess" schema="soapplus.xsd" overide="no"/>
+      <BuildSet installSet="deploy_map.xml" name="thor" path="componentfiles/thor" processName="ThorCluster" schema="thor.xsd"/>
+      <BuildSet deployable="no" installSet="deploy_map.xml" name="topology" path="componentfiles/topology" processName="Topology" schema="topology.xsd"/>
+      <BuildSet deployable="no" installSet="deploy_map.xml" name="ws_ecl" path="componentfiles/ws_ecl" processName="EspService" schema="esp_service_wsecl2.xsd">
+        <Properties bindingType="ws_eclSoapBinding" defaultPort="8002" defaultResourcesBasedn="ou=WsEcl,ou=EspServices,ou=ecl" defaultSecurePort="18002" plugin="ws_ecl" type="ws_ecl">
+          <Authenticate access="Read" description="Root access to WS ECL service" path="/" required="Read" resource="WsEclAccess"/>
+          <AuthenticateFeature description="Access to WS ECL service" path="WsEclAccess" resource="WsEclAccess" service="ws_ecl"/>
+        </Properties>
+      </BuildSet>
+      <BuildSet deployable="no" installSet="deploy_map.xml" name="ecldirect" path="componentfiles/ecldirect" processName="EspService" schema="esp_service_ecldirect.xsd">
+        <Properties bindingType="EclDirectSoapBinding" defaultPort="8008" defaultResourcesBasedn="ou=EclDirectAccess,ou=EspServices,ou=ecl" defaultSecurePort="18008" plugin="ecldirect" type="ecldirect">
+          <Authenticate access="Read" description="Root access to ECL Direct service" path="/" required="Read" resource="EclDirectAccess"/>
+          <AuthenticateFeature description="Access to ECL Direct service" path="EclDirectAccess" resource="EclDirectAccess" service="ecldirect"/>
+        </Properties>
+      </BuildSet>
+      <BuildSet deployable="no" installSet="deploy_map.xml" name="DynamicESDL" path="componentfiles/esdlsvcengine" processName="EspService" schema="esdlsvcengine.xsd">
+        <Properties bindingType="EsdlBinding" defaultPort="8043" defaultResourcesBasedn="ou=EspServices,ou=ecl" defaultSecurePort="18043" plugin="esdl_svc_engine" type="DynamicESDL"/>
+      </BuildSet>
+      <BuildSet deployable="no" installSet="deploy_map.xml" name="cassandraloggingagent" path="componentfiles/cassandraloggingagent" processName="CassandraLoggingAgent" schema="cassandraloggingagent.xsd"/>
+      <BuildSet deployable="no" installSet="deploy_map.xml" name="esploggingagent" path="componentfiles/esploggingagent" processName="ESPLoggingAgent" schema="esploggingagent.xsd"/>
+      <BuildSet deployable="no" installSet="deploy_map.xml" name="loggingmanager" path="componentfiles/loggingmanager" processName="LoggingManager" schema="loggingmanager.xsd"/>
+      <BuildSet deployable="no" installSet="deploy_map.xml" name="wslogging" path="componentfiles/wslogging" processName="EspService" schema="wslogging.xsd">
+        <Properties bindingType="loggingservice_binding" defaultPort="8146" defaultResourcesBasedn="ou=EspServices,ou=ecl" defaultSecurePort="18146" plugin="wslogging" type="wslogging">
+          <Authenticate access="Read" description="Root access to WS Logging service" path="/" required="Read" resource="WsLoggingAccess"/>
+          <AuthenticateFeature description="Access to WS Logging service" path="WsLoggingAccess" resource="WsLoggingAccess" service="wslogging"/>
+        </Properties>
+      </BuildSet>
+    </Build>
+  </Programs>
+</Environment>

configuration/config2/configfiles/dafilesrv.xsd

@@ -0,0 +1,46 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<xs:schema
+    xmlns:xs="http://www.w3.org/2001/XMLSchema" elementFormDefault="qualified" attributeFormDefault="unqualified"
+    xmlns:hpcc="someuri">
+    <xs:include schemaLocation="types.xsd"/>
+    <xs:complexType name="dafilesrv" hpcc:class="component" hpcc:category="espporocess" hpcc:displayName="Da File serve Process">
+        <xs:attributeGroup name="executionSettings">
+            <xs:attribute name="parallelRequestLimit" type="percent" use="optional" default="20" hpcc:tooltip="Defines the maximum number of concurrent dafilesrv requests allowed. Requests that exceed the limit will be delayed. A value of 0 disables throttling. Overrides global settting."/>
+            <xs:attribute name="throttleDelayMs" type="xs:nonNegativeInteger" use="optional" default="5000" hpcc:tooltip="Defines how many milliseconds delayed requests will be delayed by. Overrides global settting."/>
+            <xs:attribute name="throttleCPULimit" type="xs:nonNegativeInteger" use="optional" default="75" hpcc:tooltip="if after the initial delay, the CPU % falls below this setting, the transaction will be allowed to continue, i.e. the limit can be exceeded this way. Overrides global settting."/>
+        </xs:attributeGroup>
+        <xs:sequence>
+            <xs:element name="DafilesrvProcess" hpcc:class="valueSet" hpcc:docid="daFs.t">
+                <xs:annotation>
+                    <xs:documentation>Describes an DFUFileSrv installation</xs:documentation>
+                </xs:annotation>
+                <xs:complexType>
+                    <xs:sequence>
+                        <xs:element name="Instance" maxOccurs="unbounded" hpcc:class="valueSet" hpcc:displayName="Bound Computers">
+                            <xs:complexType>
+                                <xs:attributeGroup ref="computerNodeReference"/>
+                                <xs:attribute name="directory" type="xs:string" use="optional" default="c$\dafilesrv" hpcc:hidden="true"/>
+                                <xs:attributeGroup ref="executionSettings"/>
+                            </xs:complexType>
+                        </xs:element>
+                    </xs:sequence>
+                    <xs:attributeGroup ref="buildInfo"/>
+                    <xs:attribute name="name" type="xs:string" use="required" hpcc:tooltip="Name for this process"/>
+                    <xs:attribute name="description" type="xs:string" use="optional" default="DaFileSrv process" hpcc:tooltip="Description for this process"/>
+                    <xs:attribute name="version" type="xs:string" use="optional" default="1" hpcc:tooltip="Version identifier used to select which process will be started"/>
+                    <xs:attributeGroup ref="executionSettings"/>
+                </xs:complexType>
+            </xs:element>
+        </xs:sequence>
+
+        <xs:keyref name="dafilesrv_Instance_keyref" refer="computerNameKey">
+            <xs:selector xpath="Instance"/>
+            <xs:field xpath="@computer"/>
+        </xs:keyref>
+
+        <xs:keyref name="dafilesrv_Instance_ipref" refer="computerIPAddressKey">
+            <xs:selector xpath="Instance"/>
+            <xs:field xpath="netAddress"/>
+        </xs:keyref>
+    </xs:complexType>
+</xs:schema>

configuration/config2/configfiles/dali.xsd

@@ -0,0 +1,426 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+################################################################################
+#    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems®.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+################################################################################
+-->
+<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema" elementFormDefault="qualified" attributeFormDefault="unqualified">
+
+    <xs:element name="DaliServerProcess">
+        <xs:annotation>
+            <xs:documentation>Describes a Dali server installation</xs:documentation>
+        </xs:annotation>
+        <xs:complexType>
+            <!--DOC-Autobuild-code-->
+            <xs:annotation>
+                <xs:appinfo>
+                    <doc>
+                        <docid>da.t1</docid>
+                    </doc>
+                </xs:appinfo>
+            </xs:annotation>
+            <xs:sequence>
+                <xs:element name="Instance" maxOccurs="unbounded">
+                    <!-- Define the information for this thing -->
+                    <xs:annotation>
+                        <xs:appinfo>
+                            <class>instanceSet</class>
+                            <dislpayName>Instances</dislpayName>
+                            <tooltip>Adds instances of bindings for this stuff to the rest of the stuff</tooltip>
+                        </xs:appinfo>
+                    </xs:annotation>
+                    <xs:complexType>
+                        <xs:attribute name="computer" type="computerType" use="required">
+                            <xs:annotation>
+                                <xs:appinfo>
+                                    <colIndex>1</colIndex>
+                                </xs:appinfo>
+                            </xs:annotation>
+                        </xs:attribute>
+                        <xs:attribute name="netAddress" type="xs:string" use="optional">
+                            <xs:annotation>
+                                <xs:appinfo>
+                                    <viewType>readonly</viewType>
+                                    <colIndex>2</colIndex>
+                                </xs:appinfo>
+                            </xs:annotation>
+                        </xs:attribute>
+                        <xs:attribute name="directory" type="absolutePath" use="optional">
+                            <xs:annotation>
+                                <xs:appinfo>
+                                    <viewType>hidden</viewType>
+                                    <required>true</required>
+                                    <colIndex>3</colIndex>
+                                </xs:appinfo>
+                            </xs:annotation>
+                        </xs:attribute>
+                        <xs:attribute name="port" type="xs:string" use="optional" default="7070">
+                            <xs:annotation>
+                                <xs:appinfo>
+                                    <required>true</required>
+                                    <colIndex>4</colIndex>
+                                </xs:appinfo>
+                            </xs:annotation>
+                        </xs:attribute>
+                    </xs:complexType>
+
+                    <xs:unique name="uniqueComputerName">
+                        <xs:selector xpath="/Hardware/Computer"/>
+                        <xs:field xpath="@name"/>
+                    </xs:unique>
+                    <xs:keyref name="keyrefComputerName" refer="uniqueComputerName">
+
+                    </xskeyref>
+
+
+<Hardware><Computer computerType="linuxmachine" domain="localdomain" name="node084202" netAddress="10.173.84.202"/>
+
+                </xs:element>
+                <xs:element name="Notes" maxOccurs="unbounded">
+                    <xs:annotation>
+                        <xs:appinfo>
+                            <viewChildNodes>true</viewChildNodes>
+                        </xs:appinfo>
+                    </xs:annotation>
+                    <xs:complexType>
+                        <xs:sequence>
+                            <xs:element name="Note" type="xs:string" minOccurs="0" maxOccurs="1"/>
+                        </xs:sequence>
+                        <xs:attribute name="severity" use="optional" default="Minor">
+                            <xs:annotation>
+                                <xs:appinfo>
+                                    <tooltip>Significance of this note.</tooltip>
+                                    <title>Severity</title>
+                                    <colIndex>1</colIndex>
+                                </xs:appinfo>
+                            </xs:annotation>
+                            <xs:simpleType>
+                                <xs:restriction base="xs:string">
+                                    <xs:enumeration value="Minor"/>
+                                    <xs:enumeration value="Normal"/>
+                                    <xs:enumeration value="Critical"/>
+                                </xs:restriction>
+                            </xs:simpleType>
+                        </xs:attribute>
+                        <xs:attribute name="date" type="AutoTimeStampType" use="optional">
+                            <xs:annotation>
+                                <xs:appinfo>
+                                    <title>Date / Time</title>
+                                    <tooltip>Date and time this note was entered</tooltip>
+                                    <viewType>readonly</viewType>
+                                    <width>120</width>
+                                    <colIndex>2</colIndex>
+                                </xs:appinfo>
+                            </xs:annotation>
+                        </xs:attribute>
+                        <xs:attribute name="computer" type="AutoComputerType" use="optional">
+                            <xs:annotation>
+                                <xs:appinfo>
+                                    <title>Computer</title>
+                                    <tooltip>Computer from which this note was entered</tooltip>
+                                    <viewType>readonly</viewType>
+                                    <colIndex>3</colIndex>
+                                </xs:appinfo>
+                            </xs:annotation>
+                        </xs:attribute>
+                        <xs:attribute name="user" type="AutoUseridType" use="optional">
+                            <xs:annotation>
+                                <xs:appinfo>
+                                    <title>User</title>
+                                    <tooltip>User account from which this note was entered</tooltip>
+                                    <viewType>readonly</viewType>
+                                    <colIndex>4</colIndex>
+                                </xs:appinfo>
+                            </xs:annotation>
+                        </xs:attribute>
+                    </xs:complexType>
+                </xs:element>
+            </xs:sequence>
+            <xs:attributeGroup ref="Store"/>
+            <xs:attributeGroup ref="Backup"/>
+            <xs:attributeGroup ref="LDAP"/>
+            <xs:attributeGroup ref="DFS"/>
+            <xs:attribute name="build" type="buildType" use="required">
+                <xs:annotation>
+                    <xs:appinfo>
+                        <tooltip>The build name to be deployed</tooltip>
+                        <viewType>hidden</viewType>
+                    </xs:appinfo>
+                </xs:annotation>
+            </xs:attribute>
+            <xs:attribute name="buildSet" type="buildSetType" use="required">
+                <xs:annotation>
+                    <xs:appinfo>
+                        <viewType>hidden</viewType>
+                    </xs:appinfo>
+                </xs:annotation>
+            </xs:attribute>
+            <xs:attribute name="name" type="xs:string" use="optional">
+                <xs:annotation>
+                    <xs:appinfo>
+                        <tooltip>Name for this process</tooltip>
+                        <required>true</required>
+                    </xs:appinfo>
+                </xs:annotation>
+            </xs:attribute>
+            <xs:attribute name="description" type="xs:string" use="optional" default="Dali Server process">
+                <xs:annotation>
+                    <xs:appinfo>
+                        <tooltip>Description for this process</tooltip>
+                    </xs:appinfo>
+                </xs:annotation>
+            </xs:attribute>
+            <xs:attribute name="msgLevel" type="xs:nonNegativeInteger" use="optional" default="100">
+                <xs:annotation>
+                    <xs:appinfo>
+                        <tooltip>Severity threshold for reporting errors in log file</tooltip>
+                    </xs:appinfo>
+                </xs:annotation>
+            </xs:attribute>
+            <xs:attribute name="dataPath" type="absolutePath" use="optional">
+                <xs:annotation>
+                    <xs:appinfo>
+                        <tooltip>Directory in which dali's data files will be written to</tooltip>
+                        <onchange>
+                            <message>                Warn:This is the path for critical data for the Dali server and must be changed with care! Please refer to the documentation for more details.              </message>
+                        </onchange>
+                    </xs:appinfo>
+                </xs:annotation>
+            </xs:attribute>
+            <xs:attribute name="LogDir" type="absolutePath" use="optional">
+                <xs:annotation>
+                    <xs:appinfo>
+                        <tooltip>Directory in which to store server log files </tooltip>
+                    </xs:appinfo>
+                </xs:annotation>
+            </xs:attribute>
+            <xs:attribute name="AuditLogDir" type="absolutePath" use="optional">
+                <xs:annotation>
+                    <xs:appinfo>
+                        <tooltip>Directory in which to store audit log files</tooltip>
+                    </xs:appinfo>
+                </xs:annotation>
+            </xs:attribute>
+            <xs:attribute name="environment" type="absolutePath">
+                <xs:annotation>
+                    <xs:appinfo>
+                        <autogenforwizard>1</autogenforwizard>
+                        <autogendefaultvalue>$defaultenvfile</autogendefaultvalue>
+                        <tooltip>Path to an xml file containing an Environment to use</tooltip>
+                    </xs:appinfo>
+                </xs:annotation>
+            </xs:attribute>
+        </xs:complexType>
+    </xs:element>
+    <xs:attributeGroup name="Store">
+        <!--DOC-Autobuild-code-->
+        <xs:annotation>
+            <xs:appinfo>
+                <docid>da.t2</docid>
+            </xs:appinfo>
+        </xs:annotation>
+        <xs:attribute name="lightweightCoalesce" type="xs:boolean" use="optional" default="true">
+            <xs:annotation>
+                <xs:appinfo>
+                    <tooltip>Enable non memory loaded consolidation of store</tooltip>
+                </xs:appinfo>
+            </xs:annotation>
+        </xs:attribute>
+        <xs:attribute name="IdlePeriod" type="xs:nonNegativeInteger" use="optional" default="600">
+            <xs:annotation>
+                <xs:appinfo>
+                    <tooltip>Period of client to server quiet time to trigger store save</tooltip>
+                </xs:appinfo>
+            </xs:annotation>
+        </xs:attribute>
+        <xs:attribute name="IdleRate" type="xs:nonNegativeInteger" use="optional" default="10">
+            <xs:annotation>
+                <xs:appinfo>
+                    <tooltip>Number of transaction per minute to be considered quiet time</tooltip>
+                </xs:appinfo>
+            </xs:annotation>
+        </xs:attribute>
+        <xs:attribute name="MinTime" type="xs:nonNegativeInteger" use="optional" default="86400">
+            <xs:annotation>
+                <xs:appinfo>
+                    <tooltip>Minimum amount of time between lightweight store saves</tooltip>
+                </xs:appinfo>
+            </xs:annotation>
+        </xs:attribute>
+        <xs:attribute name="StartTime" type="xs:string" use="optional">
+            <xs:annotation>
+                <xs:appinfo>
+                    <tooltip>Start time of lightweight coalesce checking</tooltip>
+                </xs:appinfo>
+            </xs:annotation>
+        </xs:attribute>
+        <xs:attribute name="EndTime" type="xs:string" use="optional">
+            <xs:annotation>
+                <xs:appinfo>
+                    <tooltip>End time of lightweight coalesce checking</tooltip>
+                </xs:appinfo>
+            </xs:annotation>
+        </xs:attribute>
+        <xs:attribute name="keepStores" type="xs:nonNegativeInteger" use="optional" default="10">
+            <xs:annotation>
+                <xs:appinfo>
+                    <tooltip>Number of old saved stores to keep</tooltip>
+                </xs:appinfo>
+            </xs:annotation>
+        </xs:attribute>
+        <xs:attribute name="recoverFromIncErrors" type="xs:boolean" default="true">
+            <xs:annotation>
+                <xs:appinfo>
+                    <tooltip>Switch on to auto recover from corruption to delta files on load</tooltip>
+                </xs:appinfo>
+            </xs:annotation>
+        </xs:attribute>
+    </xs:attributeGroup>
+    <xs:attributeGroup name="Backup">
+        <!--DOC-Autobuild-code-->
+        <xs:annotation>
+            <xs:appinfo>
+                <docid>da.t3</docid>
+            </xs:appinfo>
+        </xs:annotation>
+        <xs:attribute name="backupComputer" type="computerType" use="optional">
+            <xs:annotation>
+                <xs:appinfo>
+                    <tooltip>Backup computer</tooltip>
+                </xs:appinfo>
+            </xs:annotation>
+        </xs:attribute>
+        <xs:attribute name="asyncBackup" type="xs:boolean" use="optional" default="true">
+            <xs:annotation>
+                <xs:appinfo>
+                    <tooltip>Asynchronous backup of transactions</tooltip>
+                </xs:appinfo>
+            </xs:annotation>
+        </xs:attribute>
+        <xs:attribute name="useNFSBackupMount" type="xs:boolean" use="optional" default="false">
+            <xs:annotation>
+                <xs:appinfo>
+                    <tooltip>Create and use a NFS mount point for backups</tooltip>
+                </xs:appinfo>
+            </xs:annotation>
+        </xs:attribute>
+        <xs:attribute name="backupLargeWarningThreshold" type="xs:nonNegativeInteger" use="optional" default="50">
+            <xs:annotation>
+                <xs:appinfo>
+                    <tooltip>Granularity of pending backups to begin issue warnings about backlog</tooltip>
+                </xs:appinfo>
+            </xs:annotation>
+        </xs:attribute>
+        <xs:attribute name="backupSoftQueueLimit" type="xs:nonNegativeInteger" use="optional" default="200">
+            <xs:annotation>
+                <xs:appinfo>
+                    <tooltip>Limit above which the backup queue will start introducing delays until the backup queue decreases below this limit</tooltip>
+                </xs:appinfo>
+            </xs:annotation>
+        </xs:attribute>
+        <xs:attribute name="backupSoftQueueLimitDelay" type="xs:nonNegativeInteger" use="optional" default="200">
+            <xs:annotation>
+                <xs:appinfo>
+                    <tooltip>The maximum delay to introduce when the backupSoftQueueLimit has been introduced</tooltip>
+                </xs:appinfo>
+            </xs:annotation>
+        </xs:attribute>
+    </xs:attributeGroup>
+    <xs:attributeGroup name="DFS">
+        <!--DOC-Autobuild-code-->
+        <xs:annotation>
+            <xs:appinfo>
+                <docid>da.t5</docid>
+            </xs:appinfo>
+        </xs:annotation>
+        <xs:attribute name="forceGroupUpdate" type="xs:boolean" use="optional" default="false">
+            <xs:annotation>
+                <xs:appinfo>
+                    <tooltip>Force group updates on startup, if environment mismatch</tooltip>
+                </xs:appinfo>
+            </xs:annotation>
+        </xs:attribute>
+        <xs:attribute name="numThreads" type="xs:nonNegativeInteger" use="optional" default="30">
+            <xs:annotation>
+                <xs:appinfo>
+                    <tooltip>Number of threads to use to process DFS requests</tooltip>
+                </xs:appinfo>
+            </xs:annotation>
+        </xs:attribute>
+    </xs:attributeGroup>
+    <xs:attributeGroup name="LDAP">
+        <!--DOC-Autobuild-code-->
+        <xs:annotation>
+            <xs:appinfo>
+                <docid>da.t4</docid>
+            </xs:appinfo>
+        </xs:annotation>
+        <xs:attribute name="ldapServer" type="ldapServerType" use="optional">
+            <xs:annotation>
+                <xs:appinfo>
+                    <tooltip>The ldap server to be used for authentication.</tooltip>
+                </xs:appinfo>
+            </xs:annotation>
+        </xs:attribute>
+        <xs:attribute name="ldapProtocol" use="optional" default="ldap">
+            <xs:annotation>
+                <xs:appinfo>
+                    <tooltip>The protocol to use - standard ldap or ldap over SSL.</tooltip>
+                </xs:appinfo>
+            </xs:annotation>
+            <xs:simpleType>
+                <xs:restriction base="xs:string">
+                    <xs:enumeration value="ldap"/>
+                    <xs:enumeration value="ldaps"/>
+                </xs:restriction>
+            </xs:simpleType>
+        </xs:attribute>
+        <xs:attribute name="authMethod" use="optional" default="kerberos">
+            <xs:annotation>
+                <xs:appinfo>
+                    <tooltip>The protocol to use for LDAP authentication.</tooltip>
+                </xs:appinfo>
+            </xs:annotation>
+            <xs:simpleType>
+                <xs:restriction base="xs:string">
+                    <xs:enumeration value="kerberos"/>
+                    <xs:enumeration value="simple"/>
+                </xs:restriction>
+            </xs:simpleType>
+        </xs:attribute>
+        <xs:attribute name="filesDefaultUser" use="optional" type="xs:string">
+            <xs:annotation>
+                <xs:appinfo>
+                    <tooltip>The default username for Files access (ActiveDirectory).</tooltip>
+                </xs:appinfo>
+            </xs:annotation>
+        </xs:attribute>
+        <xs:attribute name="filesDefaultPassword" use="optional" type="xs:string">
+            <xs:annotation>
+                <xs:appinfo>
+                    <viewType>password</viewType>
+                    <tooltip>The default password for filesDefaultUser.</tooltip>
+                </xs:appinfo>
+            </xs:annotation>
+        </xs:attribute>
+        <xs:attribute name="checkScopeScans" type="xs:boolean" use="optional" default="true">
+            <xs:annotation>
+                <xs:appinfo>
+                    <tooltip>Enable LDAP checking for all logical file listings</tooltip>
+                </xs:appinfo>
+            </xs:annotation>
+        </xs:attribute>
+    </xs:attributeGroup>
+</xs:schema>

configuration/config2/configfiles/eclagent.xsd

@@ -0,0 +1,60 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<xs:schema
+    xmlns:xs="http://www.w3.org/2001/XMLSchema" elementFormDefault="qualified" attributeFormDefault="unqualified"
+    xmlns:hpcc="someuri">
+    <xs:include schemaLocation="types.xsd"/>
+    <xs:attributeGroup name="Options" hpcc:displayName="Options" hpcc:tooltip="Options" hpcc:docid="EA.t2">
+        <xs:attribute name="allowedPipePrograms" type="xs:string" use="optional" default="*" hpcc:tab="Eclagent" hpcc:tooltip="Comma separated list of allowed PIPE programs (* for allow all)"/>
+        <xs:attribute name="daliServers" type="xs:string" use="required" hpcc:autogenforwizard="true" hpcc:tooltip="Specifies the dali server to which this eclagent is attached"/>
+        <xs:attribute name="defaultMemoryLimitMB" type="xs:nonNegativeInteger" use="optional" default="300" hpcc:tooltip="Default memory limit in MB for eclagent"/>
+        <xs:attribute name="heapUseHugePages" type="xs:boolean" default="false" hpcc:tooltip="Use memory from huge pages if they have been configured"/>
+        <xs:attribute name="heapUseTransparentHugePages" type="xs:boolean" default="true" hpcc:tooltip="Use memory from transparent huge pages"/>
+        <xs:attribute name="heapRetainMemory" type="xs:boolean" default="false" hpcc:tooltip="Retain and do not return unused memory to the operating system"/>
+        <xs:attribute name="pluginDirectory" type="absolutePath" use="optional" default="/opt/HPCCSystems/plugins/" hpcc:tooltip="Directory where plugins are located"/>
+        <xs:attribute name="traceLevel" type="xs:nonNegativeInteger" use="optional" default="0" hpcc:tooltip="Trace level"/>
+        <xs:attribute name="thorConnectTimeout" type="xs:nonNegativeInteger" use="optional" default="600" hpcc:tooltip="Default connection timeout when sending query to Thor"/>
+        <xs:attribute name="wuQueueName" type="xs:string" use="optional" default="" hpcc:tab="Eclagent" hpcc:tooltip="eclAgent Workunit Execution Queue Name" hpcc:autogenforwizard="true" hpcc:autogensuffix="_queue"/>
+    </xs:attributeGroup>
+    <xs:complexType name="eclAgent" hpcc:class="component" hpcc:category="agentprocess" hpcc:displayName="ECL Agent Process">
+        <xs:sequence>
+            <xs:element name="EclAgentProcess" hpcc:docid="EA.t1">
+                <xs:complexType>
+                    <xs:sequence>
+                        <xs:element name="Instance" maxOccurs="unbounded" hpcc:displayName="Bound Computers">
+                            <xs:complexType>
+                                <xs:attribute name="computer" type="xs:string" use="required"/>
+                                <xs:attribute name="netAddress" type="xs:string" use="optional" hpcc:readonly="true"/>
+                                <xs:attribute name="directory" type="xs:string" use="optional" default="c$\eclagent" hpcc:hidden="true"/>
+                            </xs:complexType>
+                        </xs:element>
+
+                        <xs:element name="Notes" minOccurs="0" maxOccurs="unbounded" hpcc:viewChildNodes="??true">
+                            <xs:complexType>
+                                <xs:sequence>
+                                    <xs:element name="Note" type="xs:string" minOccurs="0" maxOccurs="1"/>
+                                </xs:sequence>
+                                <xs:attribute name="severity" use="optional" default="Minor" hpcc:displayName="Severity" hpcc:tooltip="Significance of this note.">
+                                    <xs:simpleType>
+                                        <xs:restriction base="xs:string">
+                                            <xs:enumeration value="Minor"/>
+                                            <xs:enumeration value="Normal"/>
+                                            <xs:enumeration value="Critical"/>
+                                        </xs:restriction>
+                                    </xs:simpleType>
+                                </xs:attribute>
+                                <xs:attribute name="date" type="AutoTimeStampType" use="optional" hpcc:readOnly="true" hpcc:displayName="Date / Time" hpcc:tooltip="Date and time this note was entered"/>
+                                <xs:attribute name="computer" type="AutoComputerType" use="optional" hpcc:readOnly="true" hpcc:displayName="computer" hpcc:tooltip="Computer from which this note was entered"/>
+                                <xs:attribute name="user" type="AutoUseridType" use="optional" hpcc:readOnly="true" hpcc:displayName="User" hpcc:tooltip="User account from which this note was entered"/>
+                            </xs:complexType>
+                        </xs:element>
+
+                    </xs:sequence>
+                    <xs:attributeGroup ref="buildInfo"/>
+                    <xs:attribute name="name" type="xs:string" use="required" hpcc:tooltip="Name for this process"/>
+                    <xs:attribute name="description" type="xs:string" use="optional" default="EclAgent process" hpcc:tooltip="Description for this process"/>
+                    <xs:attributeGroup ref="Options"/>
+                </xs:complexType>
+            </xs:element>
+        </xs:sequence>
+    </xs:complexType>
+</xs:schema>

configuration/config2/configfiles/esp.xsd

@@ -0,0 +1,215 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<xs:schema
+    xmlns:xs="http://www.w3.org/2001/XMLSchema" elementFormDefault="qualified" attributeFormDefault="unqualified"
+    xmlns:hpcc="someuri">
+    <xs:include schemaLocation="types.xsd"/>
+    <xs:complexType name="espProcess" hpcc:class="component" hpcc:category="espporocess" hpcc:componentName="esp" hpcc:displayName="ESP Process">
+        <xs:sequence>
+            <xs:element name="EspProcess" hpcc:docid="ESP.t6" maxOccurs="unbounded">
+                <xs:annotation>
+                    <xs:documentation>ESP Service Bindings</xs:documentation>
+                </xs:annotation>
+                <xs:complexType>
+                    <xs:sequence>
+
+                        <xs:element name="EspBinding" maxOccurs="unbounded" hpcc:itemType="espbinding" hpcc:displayName="ESP Service Bindings" hpcc:class="valueSet">
+                            <xs:complexType>
+                                <xs:element name="Authenticate" minOccurs="0" maxOccurs="unbounded" hpcc:displayName="URL Authentication" hpcc:docid="ESP.t1">
+                                    <xs:complexType>
+                                        <xs:attribute name="description" type="xs:string" use="optional" hpcc:displayName="Description"/>
+                                        <xs:attribute name="path" type="xs:string" use="required" default="/" hpcc:displayName="Path" hpcc:tooltip="The logical path of a resource used for authentication"/>
+                                        <xs:attribute name="resource" type="xs:string" use="required" hpcc:displayName="Resource" hpcc:tooltip="The physical resource for which access is checked"/>
+                                        <xs:attribute name="access" use="optional" default="Read">
+                                            <xs:simpleType>
+                                                <xs:restriction base="xs:string">
+                                                    <xs:enumeration value="" hpcc:description=""/>
+                                                    <xs:enumeration value="Access" hpcc:description=""/>
+                                                    <xs:enumeration value="Read" hpcc:description=""/>
+                                                    <xs:enumeration value="Write" hpcc:description=""/>
+                                                    <xs:enumeration value="Full" hpcc:description=""/>
+                                                    <xs:enumeration value="None" hpcc:description=""/>
+                                                </xs:restriction>
+                                            </xs:simpleType>
+                                        </xs:attribute>
+                                    </xs:complexType>
+                                </xs:element>
+                                <xs:element name="AuthenticateFeature" minOccurs="0" maxOccurs="unbounded" hpcc:displayName="Feature Authentication" hpcc:docid="ESP.t2">
+                                    <xs:complexType>
+                                        <xs:attribute name="authenticate" use="optional" default="Yes" hpcc:displayName="Authenticate" hpcc:tooltip="Validate access rights for this capability?">
+                                            <xs:simpleType>
+                                                <xs:restriction base="xs:string">
+                                                    <xs:enumeration value="Yes" hpcc:description="Validate access rights for this capability"/>
+                                                    <xs:enumeration value="No" hpcc:description="No validation"/>
+                                                </xs:restriction>
+                                            </xs:simpleType>
+                                        </xs:attribute>
+                                        <xs:attribute name="description" type="xs:string" use="optional" hpcc:displayName="Description"/>
+                                        <xs:attribute name="resource" type="xs:string" use="required" hpcc:displayName="Resource"/>
+                                    </xs:complexType>
+                                </xs:element>
+
+                                <xs:element name="CustomBindingParameter" minOccurs="0" maxOccurs="unbounded" hpcc:displayName="Custom Binding Parameters">
+                                    <xs:complexType>
+                                        <xs:attribute name="key" type="xs:string" use="optional" hpcc:displayName="Key" />
+                                        <xs:attribute name="value" type="xs:string" use="optional" hpcc:displayName="Value"/>
+                                    </xs:complexType>
+                                </xs:element>
+
+
+                            </xs:complexType>
+
+                            <xs:attribute name="name" type="xs:string" use="required" hpcc:displayName="Binding Name" hpcc:autoName="" />
+                            <xs:attribute name="defaultServiceVersion" type="version" use="optional" hpcc:displayName="Default Service Version" hpcc:tooltip="The default version for WSDL, XSD and the ESP form"/>
+                            <xs:attribute name="defaultForPort" type="xs:boolean" use="required" default="true" hpcc:displayName="Default for port" hpcc:tooltip="This binding is determines root access"/>
+                            <xs:attribute name="port" type="xs:nonNegativeInteger" use="required" hpcc:displayName="Port" hpcc:tooltip="Port to which the service is bound"/>
+                            <xs:attribute name="protocol" use="required" default="http" hpcc:displayName="Protocol" hpcc:tooltip="The protocol to use">
+                                <xs:simpleType>
+                                    <xs:restriction base="xs:string">
+                                        <xs:enumeration value="http" hpcc:descritpion=""/>
+                                        <xs:enumeration value="https" hpcc:descritpion=""/>
+                                    </xs:restriction>
+                                </xs:simpleType>
+                            </xs:attribute>
+                            <xs:attribute name="resourcesBasedn" type="xs:string" use="optional" hpcc:displayName="" hpcc:tooltip="Base location for resources (used with ldap security)"/>
+                            <xs:attribute name="service" type="xs:string" use="required" hpcc:displayName="Service"/>
+                            <xs:attribute name="type" type="xs:string" use="optional" hpcc:displayName="Security Manager Plugin" hpcc:tooltip="The Security Manager to be used by the Esp Service"/>
+                            <xs:attribute name="workunitsBasedn" type="xs:string" use="optional" default="ou=workunits,ou=ecl" hpcc:displayName="WorkUnits BaseDn" hpcc:tooltip="Base location for workunit resources (used with ldap security)" hpcc:requiredIf="xpath,xpath..."/>
+                            <xs:attribute name="wsdlServiceAddress" type="xs:string" use="optional"hpcc:displayName="wsdlServiceAddress" hpcc:tooltip="Overrides the address used by client applications to connect to the service"/>
+
+                            <xs:key name="esp_custombindingparameter_key">
+                                <xs:selector xpath="CustomBindingParameter" />
+                                <xs:field xpath="@key" />
+                            </xs:key>
+
+                        </xs:element>
+
+                        <xs:element name="Authentication" hpcc:docid="ESP.t4" hpcc:class="valueSet" hpcc:displayName="Authentication">
+                            <xs:complexType>
+                                <xs:attribute name="method" use="required" default="none" hpcc:displayName="Method" hpcc:tooltip="The protocol to use for authenticating the service">
+                                    <xs:simpleType>
+                                        <xs:restriction base="xs:string">
+                                            <xs:enumeration value="none" hpcc:description=""/>
+                                            <xs:enumeration value="local" hpcc:description=""/>
+                                            <xs:enumeration value="ldap" hpcc:description=""/>
+                                            <xs:enumeration value="ldaps" hpcc:description=""/>
+                                            <xs:enumeration value="secmgrPlugin" hpcc:description=""/>
+                                        </xs:restriction>
+                                    </xs:simpleType>
+                                </xs:attribute>
+                                <xs:attribute name="ldapServer" type="xs:string" use="optional" hpcc:displayName="LDAP Server" hpcc:tooltip="The ldap server to be used for authentication" hpcc:requiredIf="ldap chosen"/>
+                                <xs:attribute name="ldapAuthMethod" type="xs:string" use="required" default="kerberos" hpcc:displayName="LDAP Auth Method" hpcc:tooltip="The protocol to use for LDAP authentication">
+                                    <xs:simpleType>
+                                        <xs:restriction base="xs:string">
+                                            <xs:enumeration value="kerberos" hpcc:description=""/>
+                                            <xs:enumeration value="simple" hpcc:description=""/>
+                                        </xs:restriction>
+                                    </xs:simpleType>
+                                </xs:attribute>
+                                <xs:attribute name="ldapConnections" type="xs:nonNegativeInteger" use="optional" default="10" hpcc:displayName=""
+                                              hpcc:tooltip="Maximum number of connections to the LDAP server" hpcc:requiredIf=""/>
+                                <xs:attribute name="passwordExpirationWarningDays" type="xs:nonNegativeInteger" use="optional" default="10"
+                                              hpcc:displayName="Passowrd Expiration Warning Days" hpcc:tooltip="In this time period, ESP displays a warning about password expiration"/>
+                                <xs:attribute name="checkViewPermissions" type="xs:boolean" use="optional" default="false" hpcc:displayName="Check View Permissions"
+                                              hpcc:tooltip="Enable file and column access permission checking for all view enabled queries"/>
+                            </xs:complexType>
+                        </xs:element>
+
+                        <xs:element name="AuthDomain" minOccurs="0" maxOccurs="unbounded" hpcc:displayName="AuthDomain" hpcc:class="valueSet" hpcc:docid="ESP.t7">
+                            <xs:complexType>
+                                <xs:attribute name="authType" use="required" default="AuthTypeMixed" hpcc:displayName="" hpcc:tooltip="User session Authentication type">
+                                    <xs:simpleType>
+                                        <xs:restriction base="xs:string">
+                                            <xs:enumeration value="AuthTypeMixed" hpcc:description=""/>
+                                            <xs:enumeration value="AuthPerSessionOnly" hpcc:description=""/>
+                                            <xs:enumeration value="AuthPerRequestOnly" hpcc:description=""/>
+                                        </xs:restriction>
+                                    </xs:simpleType>
+                                </xs:attribute>
+                                <xs:attribute name="domainName" type="xs:string" use="optional" hpcc:displayName="Domain Name" hpcc:tooltip="Unique string to identify an auth domain in case > 1 domains in an esp"/>
+                                <xs:attribute name="logonURL" type="xs:string" use="optional" default="/esp/files/eclwatch/templates/Login.html" hpcc:displayName="Logon URL" hpcc:tooltip="Logon URL"/>
+                                <xs:attribute name="logoutURL" type="xs:string" use="optional" default="" hpcc:displayName="Logout URL" hpcc:tooltip="Logout URL"/>
+                                <xs:attribute name="sessionTimeoutMinutes" type="xs:integer" use="optional" default="60" hpcc:displayName="Session Timeout Minutes" hpcc:tooltip="Inactive session duration, in minutes. Specify 0 for default timeout, -1 for never timeout"/>
+                                <xs:attribute name="resourceURL" type="xs:string" use="optional" default="/favicon.ico,/esp/files/img/favicon.ico,/esp/files/eclwatch/img/Loginlogo.png,/esp/files/dojo/*,/esp/files/eclwatch/nls/*"
+                                              hpcc:displayName="Resurce URL" hpcc:tooltip="??"/>
+                            </xs:complexType>
+                        </xs:element>
+
+                        <xs:element name="HTTPS" minOccurs="0" hpcc:class="valueSet" hpcc:requiredIf="">
+                            <xs:complexType>
+                                <xs:attribute name="acceptSelfSigned" type="xs:boolean" use="optional" default="true" hpcc:displayName="Accept Self Signed" hpcc:tooltip="whether to accept self-signed certificates"/>
+                                <xs:attribute name="CA_Certificates_Path" type="xs:string" use="optional" default="ca.pem" hpcc:displayName="Certificates Path" hpcc:tooltip="path to the file that contains CA certificates"/>
+                                <xs:attribute name="certificateFileName" type="xs:string" use="optional" default="certificate.cer" hpcc:displayName="Certificate Filename" hpcc:tooltip="Name of destination file in which the certificate will be written"/>
+                                <xs:attribute name="cipherList" type="xs:string" use="optional" default="ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+3DES:!aNULL:!MD5"
+                                              hpcc:displayName="Cipher List" hpcc:tooltip="Sets the ordered list of available ciphers for use by openssl.  See openssl documentation on ciphers for information on use and formatting."/>
+                                <xs:attribute name="passphrase" type="xs:string" use="optional" hpcc:modifiers="mask,verify" hpcc:displayName="Passphrase" hpcc:tooltip="The passphrase used to generate the private key" />
+                                <xs:attribute name="privateKeyFileName" type="xs:string" use="optional" default="privatekey.cer" hpcc:displayName="Private Key Filename" hpcc:tooltip="Name of destination file in which the private key will be written" />
+                            </xs:complexType>
+                        </xs:element>
+
+                        <xs:element name="Instance" maxOccurs="unbounded" hpcc:class="valueSet">
+                            <xs:complexType>
+                                <xs:sequence>
+                                    <xs:element name="Certificate" type="xs:string" minOccurs="0" hpcc:displayName="Certificate" hpcc:tooltip="A certificate can be pasted here from the clipboard.  Otherwise, the generated certificate is displayed here after deployment" />
+                                    <xs:element name="PrivateKey" type="xs:string" minOccurs="0" hpcc:displayName="Private Key" hpcc:tooltip="A private key can be pasted here from the clipboard.  Otherwise, the generated private key is displayed here after deployment" />
+                                    <xs:element name="CSR" type="xs:string" minOccurs="0" hpcc:displayName="Certificate Signing Request" hpcc:tooltip="The generated Certificate Signing Request (CSR) is displayed here after first deployment" />
+                                </xs:sequence>
+                                <xs:attributeGroup ref="computerNodeReference"/>
+                                <xs:attribute name="directory" type="absolutePath" use="optional" hpcc:hidden="true" />
+                                <xs:attribute name="FQDN" type="xs:string" use="optional" hpcc:displayName="Fully Qualified Domain Name" hpcc:tooltip="Enter the fully qualified domain name" />
+                            </xs:complexType>
+                        </xs:element>
+                        <xs:element name="Notes" type="usernotes"  hpcc:displayName="Notes"/>
+                    </xs:sequence>
+
+                    <xs:attributeGroup ref="buildInfo"/>
+                    <xs:attribute name="name" type="xs:string" use="required" hpcc:displayName="Name" hpcc:autoName="autoincrement=pathto where to count" hpcc:tooltip="Name for this process"/>
+                    <xs:attribute name="description" type="xs:string" use="optional" default="ESP server" hpcc:displayName="Description" hpcc:tooltip="Description for this process"/>
+                    <xs:attribute name="daliServers" type="xs:string" use="required" hpcc:displayName="Dali Servers" hpcc:tooltip="Specifies the dali server to which this ESP is attached"/>
+                    <xs:attribute name="enableSEHMapping" type="xs:boolean" use="optional" default="true" hpcc:displayName="Enable SEH Mapping" hpcc:tooltip="Enables SEH to exception mapping" />
+                    <xs:attribute name="httpConfigAccess" type="xs:boolean" use="optional" default="true" hpcc:displayName="HTTP Config Access" hpcc:tooltip="Allows esp config file to be viewed via a web browser"/>
+                    <xs:attribute name="formOptionsAccess" type="xs:boolean" use="optional" default="false" hpcc:displayName="Form Options Access" hpcc:tooltip="Allows show Options in test form page" />
+                    <xs:attribute name="maxRequestEntityLength" type="xs:nonNegativeInteger" use="optional" default="8000000" hpcc:displayName="Max Request Entity Length" hpcc:tooltip="The maximum length of request entity allowed" />
+                    <xs:attribute name="maxConcurrentThreads" type="xs:nonNegativeInteger" use="optional" default="0" hpcc:displayName="Max Concurrent Threads" hpcc:tooltip="The maximum number of concurrent threads. 0 means unlimited" />
+                    <xs:attribute name="maxBacklogQueueSize" type="xs:nonNegativeInteger" use="optional" default="200" hpcc:displayName="Max Backlog Queue Size" hpcc:tooltip="Sets the sockets parameter for the maximum number of backlogged requests" />
+                    <xs:attribute name="perfReportDelay" type="xs:nonNegativeInteger" use="optional" default="60" hpcc:displayName="Resource Usage Stats Logging Frequency" hpcc:tooltip="Sets the frequency for logging resource usage stats" />
+                    <xs:attribute name="portalurl" type="xs:string" default="http://hpccsystems.com/download" hpcc:hidden="true" hpcc:tooltip="portal to HPCC Systems® web site" />
+                    <xs:attribute name="controlPort" type="xs:nonNegativeInteger" use="optional" default="8010" hpcc:displayName="Control Port" hpcc:tooltip="Sets the network port for ESP control" />
+                    <xs:attribute name="logLevel" type="xs:nonNegativeInteger" use="optional" default="1" hpcc:displayName="Log Level" hpcc:tooltip="ets the log level [0: none, 1: min, 5: noraml, 10: max]" />
+                    <xs:attribute name="componentfilesDir" type="xs:string" use="optional" default="${COMPONENTFILES_PATH}" hpcc:displayName="Component Files Dir" hpcc:tooltip="Sets the componentfiles directory" />
+                    <xs:attribute name="logRequests" type="xs:boolean" use="optional" default="true" hpcc:displayName="Log Requests"/>
+                    <xs:attribute name="logResponses" type="xs:boolean" use="optional" default="false" hpcc:displayName="Log Responses"/>
+                    <xs:attribute name="txSummaryLevel" type="xs:nonNegativeInteger" use="optional" default="1" hpcc:displayName="Tx Summary Level" hpcc:tooltip="Sets the TxSummary level [0: none, 1: min, 5: noraml, 10: max]" />
+                    <xs:attribute name="txSummaryResourceReq" type="xs:boolean" use="optional" default="false" hpcc:displayName="Tx Summary Resource Req" hpcc:tooltip="Log TxSummary for Resource Requests" />
+                    <xs:attribute name="memCachedOptionString" type="xs:string" use="optional" hpcc:displayName="memcached Option String" hpcc:tooltip="Option string used by ESP memcached client" />
+
+                </xs:complexType>
+
+                <xs:key name="espprocess_name_key">
+                    <xs:selector xpath="." />
+                    <xs:field xpath="@name" />
+                </xs:key>
+
+                <xs:key name="espprocess_controlport_key">
+                    <xs:selector xpath="." />
+                    <xs:field xpath="@controlPort" />
+                </xs:key>
+
+                <xs:keyref name="esp_servicename_keyref" refer="espservice_name_key">
+                    <xs:selector xpath="EspBinding" />
+                    <xs:field xpath="@service" />
+                </xs:keyref>
+
+                <xs:keyref name="espprocess_Instance_keyref" refer="computerNameKey">
+                    <xs:selector xpath="Instance"/>
+                    <xs:field xpath="@computer"/>
+                </xs:keyref>
+
+                <xs:keyref name="espprocess_Instance_ipref" refer="computerIPAddressKey">
+                    <xs:selector xpath="Instance"/>
+                    <xs:field xpath="netAddress"/>
+                </xs:keyref>
+
+            </xs:element>
+        </xs:sequence>
+    </xs:complexType>
+</xs:schema>

configuration/config2/configfiles/esp_service_smc.xsd

@@ -0,0 +1,46 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<xs:schema
+    xmlns:xs="http://www.w3.org/2001/XMLSchema" elementFormDefault="qualified" attributeFormDefault="unqualified"
+    xmlns:hpcc="someuri">
+    <xs:include schemaLocation="types.xsd"/>
+    <xs:complexType name="espsmc" hpcc:class="component" hpcc:category="espservice" hpcc:componentName="espsmc" hpcc:displayName="ESP SMC" hpcc:namePrefix="myesmsmc">
+
+        <xs:attributeGroup name="Monitoring" hpcc:docid="SMC-T02">
+            <xs:attribute name="monitorDaliFileServer" type="xs:boolean" use="true" default="false" hpcc:displayName="Monior Dali File Server" hpcc:tooltip="Warn if dafilesrv process is not running on computers"/>
+            <xs:attribute name="excludePartitions" type="xs:string" use="optional" default="/dev*,/sys,/proc/*" hpcc:displayName="Exclude Partitions" hpcc:tooltip="Comma, space or semicolon delimited list of partitions not to be monitored for free space"/>
+            <xs:attribute name="warnIfCpuLoadOver" type="xs:nonNegativeInteger" use="optional" default="95" hpcc:displayName="Warn CPU Load" hpcc:tooltip="CPU load over this value is flagged as warning in monitoring output"/>
+            <xs:attribute name="warnIfFreeStorageUnder" type="xs:nonNegativeInteger" use="optional" default="5" hpcc:displayName="Warn Free Storage" hpcc:tooltip="Available disk storage space under this value is flagged as warning in monitoring output"/>
+            <xs:attribute name="warnIfFreeMemoryUnder" type="xs:nonNegativeInteger" use="optional" default="5" hpcc:displayName="Warn Free Memory" hpcc:tooltip="Available memory under this value is flagged as warning in monitoring output"/>
+        </xs:attributeGroup>
+
+        <xs:sequence>
+            <xs:element name="EspService" maxOccurs="unbounded" hpcc:docid="SMC-T01">
+                <xs:complexType>
+                    <xs:attributeGroup ref="buildInfo"/>
+                    <xs:attribute name="name" type="xs:string" use="required" hpcc:displayName="Name" hpcc:autoGenerateType="prefix" hpcc:autoGenerateValue="EclWatch" hpcc:tooltip="Name for this ESP service" />
+                    <xs:attribute name="description" type="xs:string" use="optional" hpcc:displayName="Description" hpcc:tooltip="Description for this process" />
+                    <xs:attribute name="syntaxCheckQueue" type="xs:string" use="optional" hpcc:displayName="Syntax Check Queue" hpcc:tooltip="Queue Name of ECL server which is used for ECL Syntax Check" />
+                    <xs:attribute name="pluginsPath" type="xs:string" use="optional" default="${PLUGINS_PATH}" hpcc:displayName="Plugins Path" hpcc:tooltip="Path where plugin files are deployed" />
+                    <xs:attribute name="viewTimeout" type="xs:nonNegativeInteger" use="optional" default="1000" hpcc:displayName="View Timeout" hpcc:tooltip="timeout for XXXX (in seconds)" />
+                    <xs:attribute name="clusterQueryStateThreadPoolSize" type="xs:nonNegativeInteger" use="optional"  default="25" hpcc:displayName="" hpcc:tooltip="Default thread pool size for checking query state on clusters" />
+                    <xs:attribute name="AWUsCacheTimeout" type="xs:nonNegativeInteger" use="optional" default="15" hpcc:displayName="AWUs Cache Timeout" hpcc:tooltip="timeout for archived WU search cache (in minutes)" />
+                    <xs:attribute name="NodeGroupCacheMinutes" type="xs:nonNegativeInteger" use="optional" default="30" hpcc:displayName="Node Group Cache Minutes" hpcc:tooltip="timeout for node group cache (in minutes)" />
+                    <xs:attribute name="ActivityInfoCacheSeconds" type="xs:nonNegativeInteger" use="optional" default="10" hpcc:displayName="Activity Info Cache Sectonds" hpcc:tooltip="timeout for activity info cache (in seconds)" />
+                    <xs:attribute name="serverForArchivedECLWU" type="xs:string" use="optional" hpcc:displayName="Server For Archived ECLWU" hpcc:tooltip="Specify Sasha server for archiving ECL workunits" />
+                    <xs:attribute name="enableSystemUseRewrite" type="xs:boolean" use="optional" default="false" hpcc:displayName="Enable system Use Rewrite" hpcc:tooltip="To disable ESP Service links for System Servers that use rewrite rules" />
+                    <xs:attribute name="defaultTargetCluster" type="xs:string" use="optional" hpcc:displayName="Default Target Cluster" hpcc:tooltip="Default target for published queries" />
+                    <xs:attribute name="disableUppercaseTranslation" type="xs:boolean" use="optional" default="false" hpcc:displayName="Disable Uppercase Translation" hpcc:tooltip="To disable upper case translation for filter values in ViewKeyFile function" />
+                    <xs:attribute name="enableLogDaliConnection" type="xs:boolean" use="optional" default="false" hpcc:displayName="Enable Log Dali Connection" hpcc:tooltip="Enable ESP/Dali Connection ID to be logged into esp.xml" />
+                    <xs:attribute name="allowNewRoxieOnDemandQuery" type="xs:boolean" use="optional" default="false" hpcc:displayName="Allow New Roxie On Demand Query" hpcc:tooltip="allow new queries to be used by roxie on demand and roxie browser" />
+                    <xs:attributeGroup ref="Monitoring"/>
+                </xs:complexType>
+
+                <xs:key name="espservice_name_key" hpcc:allowDuplicate="true">
+                    <xs:selector xpath="." />
+                    <xs:field xpath="@name" />
+                </xs:key>
+
+            </xs:element>
+        </xs:sequence>
+    </xs:complexType>
+</xs:schema>

configuration/config2/configfiles/esp_service_wsecl2.xsd

@@ -0,0 +1,51 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<xs:schema
+    xmlns:xs="http://www.w3.org/2001/XMLSchema" elementFormDefault="qualified" attributeFormDefault="unqualified"
+    xmlns:hpcc="someuri">
+    <xs:include schemaLocation="types.xsd"/>
+    <xs:complexType name="ws_ecl" hpcc:class="component" hpcc:category="espservice" hpcc:componentName="ws_ecl" hpcc:displayName="WS ECL??" hpcc:namePrefix="myeclWatch">
+        <xs:sequence>
+            <xs:element name="EspService" maxOccurs="unbounded" hpcc:docid="MyWS2-T03">
+                <xs:complexType>
+                    <xs:sequence>
+
+                        <xs:element name="ProcessCluster" minOccurs="0" maxOccurs="unbounded" hpcc:displayName="VIPS" hpcc:class="valueSet" hpcc:docid="MyWS2-T01">
+                            <xs:complexType>
+                                <xs:attribute name="name" type="xs:string" use="required" hpcc:hidden="true" hpcc:autoGenerateType="prefix" hpcc:autoGenerateValue="proc"/>
+                                <xs:attribute name="roxie" type="xs:string" use="required" hpcc:displayName="Roxie" hpcc:tooltip=""
+                                              hpcc:category="select" hpcc:keyref="keyref to roxie names"/> <!-- type=xpathType /Environment/Software/RoxieCluster[@name!=$./*ProcessCluster/@roxie] -->
+                                <xs:attribute name="vip" type="xs:string" use="required" hpcc:displayName="VIP" hpcc:tooltip=""/>
+                                <xs:attribute name="dnsInterval" type="xs:integer" use="optional" hpcc:displayName="DNS Cache timeout interval"
+                                              hpcc:tooltip="DNS lookup cache timeout in seconds. Set to 0 to resolve DNS for every transaction.  Set to -1 (default) to keep DNS lookup cached indefinitely."/>
+                                <xs:attribute name="sendTargetToRoxie" type="xs:boolean" use="optional" hpcc:displayName="Send Target To Roxie" default="true"
+                                              hpcc:tooltip="Send roxie the target from which to run query (disable for backward compatibility issues)"/>
+                            </xs:complexType>
+                        </xs:element>
+
+                        <xs:element name="Target" minOccurs="0" maxOccurs="unbounded" hpcc:displayName="Restrict To Target(s)" hpcc:docid="MyWS2-T02">
+                            <xs:complexType>
+                                <xs:attribute name="name" type="xs:string" use="required" hpcc:category="select" hpcc:keyref="keyref to thor names (hthor, thor, roxie, see configmgr)"
+                                              hpcc:tooltip="WsEcl will only display specified targets, if none specified WsEcl will display all targets."/> <!-- type="topologyClusterType" -->
+                            </xs:complexType>
+                        </xs:element>
+                    </xs:sequence>
+
+                    <xs:attributeGroup ref="buildInfo"/>
+                    <xs:attribute name="name" type="xs:string" use="required" hpcc:autogenerateType="Prefix" hpcc:autoGenerateValue="wsecl" hpcc:displayName="Service Name" hpcc:tooltip="Name for this ESP service" />
+                    <xs:attribute name="description" type="xs:string" use="optional" default="WS ECL Service" hpcc:displayName="Description" hpcc:tooltip="Allows creation of web services using ECL language" />
+                    <xs:attribute name="roxieTimeout" type="xs:unsignedInt" use="optional" default="300" hpcc:displayName="Roxie Timeout"
+                                  hpcc:tooltip="Timeout (in seconds) for WsEcl connections to roxie (0 == wait forever)" />
+                    <xs:attribute name="workunitTimeout" type="xs:unsignedInt" use="optional" default="600" hpcc:displayName="Workunit Timeout"
+                                  hpcc:tooltip="Timeout (in seconds), for WsEcl to wait for workunit to complete (0 == wait forever)" />
+
+                </xs:complexType>
+
+                <xs:key name="espservice_name_key" hpcc:allowDuplicate="true">
+                    <xs:selector xpath="." />
+                    <xs:field xpath="@name" />
+                </xs:key>
+
+            </xs:element>
+        </xs:sequence>
+    </xs:complexType>
+</xs:schema>

configuration/config2/configfiles/hardware.xsd

@@ -0,0 +1,93 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<xs:schema
+    xmlns:xs="http://www.w3.org/2001/XMLSchema" elementFormDefault="qualified" attributeFormDefault="unqualified"
+    xmlns:hpcc="someuri">
+  <xs:include schemaLocation="types.xsd"/>
+  <xs:complexType name="hardware" hpcc:class="component" hpcc:category="" hpcc:displayName="Hardware Configuration">
+    <xs:sequence>
+      <xs:element name="Hardware" hpcc:docid="????" hpcc:doc="Section to describe the hardware that composes the environment">
+        <xs:complexType>
+          <xs:sequence>
+            <xs:element name="ComputerType" maxOccurs="unbounded" hpcc:displayName="Defined computer types"  hpcc:class="valueSet"
+                        hpcc:doc="Defines a generalized type of computer that is used in the environment">
+              <xs:complexType>
+                <xs:attribute name="name" type="xs:string" use="required"/>
+                <xs:attribute name="nicSpeed" type="xs:nonNegativeInteger"/>
+                <xs:attribute name="manufacturer" type="xs:string"/>
+                <xs:attribute name="memory" type="xs:nonNegativeInteger"/>
+                <xs:attribute name="computerType" type="xs:string"/>
+                <xs:attribute name="opSys">
+                  <xs:simpleType>
+                    <xs:restriction base="xs:string">
+                      <xs:enumeration value="linux"/>
+                      <xs:enumeration value="solaris"/>
+                      <xs:enumeration value="W2K"/>
+                    </xs:restriction>
+                  </xs:simpleType>
+                </xs:attribute>
+              </xs:complexType>
+            </xs:element>
+
+            <xs:element name="Domain" hpcc:class="valueSet" hpcc:doc="Describes a domain that is contained in the environment">
+              <xs:complexType>
+                <xs:attribute name="name" type="xs:string" use="required"/>
+                <xs:attribute name="username" type="xs:string"/>
+                <xs:attribute name="password" type="xs:string" hpcc:modifers="mask,verify" />
+              </xs:complexType>
+            </xs:element>
+
+            <xs:element name="Switch" maxOccurs="unbounded" hpcc:class="valueSet" hpcc:doc="Describes a switch that is contained in the environment">
+              <xs:complexType>
+                <xs:attribute name="name" type="xs:string"/>
+              </xs:complexType>
+            </xs:element>
+
+            <xs:element name="Computer" maxOccurs="unbounded" hpcc:class="valueSet" hpcc:doc="Describes a computer that is contained in the environment">
+              <xs:complexType>
+                <xs:attributeGroup ref="computerNode"/>
+                <xs:attribute name="domain" type="xs:string"/>
+                <xs:attribute name="computerType" type="xs:string"/>
+              </xs:complexType>
+            </xs:element>
+          </xs:sequence>
+        </xs:complexType>
+
+        <xs:key name="domainKey">
+          <xs:selector xpath="Domain" />
+          <xs:field xpath="@name" />
+        </xs:key>
+
+        <xs:key name="computerTypeKey">
+          <xs:selector xpath="ComputerType" />
+          <xs:field xpath="@name" />
+        </xs:key>
+
+        <xs:key name="computerIPAddressKey">
+          <xs:selector xpath="Computer" />
+          <xs:field xpath="netAddress" />
+        </xs:key>
+
+        <xs:key name="computerNameKey">
+          <xs:selector xpath="Computer" />
+          <xs:field xpath="@name" />
+        </xs:key>
+
+        <xs:keyref name="Computer_computerTypeRef" refer="computerTypeKey">
+          <xs:selector xpath="Computer"/>
+          <xs:field xpath="@computerType" />
+        </xs:keyref>
+
+        <xs:keyref name="Computer_domainRef" refer="domainKey">
+          <xs:selector xpath="Computer"/>
+          <xs:field xpath="@domain" />
+        </xs:keyref>
+
+        <xs:key name="switchNameKey">
+          <xs:selector xpath="Switch" />
+          <xs:field xpath="@name" />
+        </xs:key>
+
+      </xs:element>
+    </xs:sequence>
+  </xs:complexType>
+</xs:schema>

configuration/config2/configfiles/hpcc.xsd

@@ -0,0 +1,26 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+################################################################################
+#    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems®.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+################################################################################
+-->
+<xs:schema
+    xmlns:xs="http://www.w3.org/2001/XMLSchema" elementFormDefault="qualified" attributeFormDefault="unqualified">
+
+    <xs:attribute name="readOnly" type="xs:boolean" default="false"/>
+    <xs:attribute name="docid" type="xs:string" default=""/>
+    <xs:attribute name="hidden" type="xs:boolean" default="false"/>
+    <xs:attribute name="tooltip" type="xs:string" default=""/>
+</xs:schema>

configuration/config2/configfiles/newenv.xsd

@@ -0,0 +1,50 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+################################################################################
+#    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems®.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+################################################################################
+-->
+<xs:schema
+    xmlns:xs="http://www.w3.org/2001/XMLSchema" elementFormDefault="qualified" attributeFormDefault="unqualified"
+    xmlns:hpcc="something hpcc.xsd">
+  <xs:include schemaLocation="types.xsd"/>
+  <xs:include schemaLocation="hardware.xsd"/>
+  <xs:include schemaLocation="esp.xsd"/>
+  <xs:include schemaLocation="esp_service_wsecl2.xsd"/>
+  <xs:include schemaLocation="esp_service_smc.xsd"/>
+  <xs:include schemaLocation="dafilesrv.xsd"/>
+  <xs:include schemaLocation="eclagent.xsd"/>
+  <xs:element name="Environment" hpcc:category="root" hpcc:class="category">
+    <xs:complexType>
+      <xs:sequence>
+        <xs:element name="EnvSettings" hpcc:class="category" hpcc:displayName="System Settings" hpcc:hidden="true"></xs:element>
+        <xs:element type="hardware" hpcc:class="category" hpcc:displayName="Hardware" hpcc:required="true"></xs:element>
+        <xs:element name="Programs" hpcc:class="category" hpcc:displayName="Programs" hpcc:hidden="true"></xs:element>
+        <xs:element name="Software" hpcc:class="category" hpcc:displayName="Software">
+          <xs:complexType>
+            <xs:sequence>
+              <xs:element type="dafilesrv"/>  <!-- this is like an include right now -->
+              <xs:element type="eclAgent"/>
+              <xs:element type="ws_ecl"/>
+              <xs:element type="espsmc"/>
+              <xs:element type="espProcess"/>
+            </xs:sequence>
+          </xs:complexType>
+
+        </xs:element>
+      </xs:sequence>
+    </xs:complexType>
+  </xs:element>
+</xs:schema>

configuration/config2/configfiles/types.xsd

@@ -0,0 +1,88 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+################################################################################
+#    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems®.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+################################################################################
+ -->
+<xs:schema
+    xmlns:xs="http://www.w3.org/2001/XMLSchema" elementFormDefault="qualified" attributeFormDefault="unqualified"
+    xmlns:hpcc="someurl">
+    <xs:simpleType name="percent">
+        <xs:restriction base="xs:integer">
+            <xs:minInclusive value="0"/>
+            <xs:maxInclusive value="100"/>
+        </xs:restriction>
+    </xs:simpleType>
+    <xs:simpleType name="nodeName">
+        <xs:restriction base="xs:string">
+            <xs:pattern value="[a-zA-z0-9_\-]+"/>
+            <!-- need regex to eliminate special characters -->
+        </xs:restriction>
+    </xs:simpleType>
+    <xs:simpleType name="version">
+        <xs:restriction base="xs:string">
+            <xs:pattern value=""/>
+            <!-- regex for d[.d[.d[.d]]] -->
+        </xs:restriction>
+    </xs:simpleType>
+    <xs:simpleType name="ipV4Address">
+        <xs:restriction base="xs:token">
+            <xs:pattern value="/^0*([1-9]?\\d|1\\d\\d|2[0-4]\\d|25[0-5])\\.0*([1-9]?\\d|1\\d\\d|2[0-4]\\d|25[0-5])\\.0*([1-9]?\\d|1\\d\\d|2[0-4]\\d|25[0-5])\\.0*([1-9]?\\d|1\\d\\d|2[0-4]\\d|25[0-5])$/"/>
+        </xs:restriction>
+    </xs:simpleType>
+    <xs:simpleType name="absolutePath">
+        <xs:restriction base="xs:string">
+            <xs:pattern value="/^\\/.*/"/>
+        </xs:restriction>
+    </xs:simpleType>
+    <xs:simpleType name="AutoTimeStampType">
+        <xs:restriction base="xs:string" hpcc:autoType="timestamp"/>
+    </xs:simpleType>
+    <xs:simpleType name="AutoComputerType">
+        <xs:restriction base="xs:string" hpcc:autoType="requestIP"/>
+    </xs:simpleType>
+    <xs:simpleType name="AutoUseridType">
+        <xs:restriction base="xs:string" hpcc:autoType="requestUser"/>
+    </xs:simpleType>
+    <xs:attributeGroup name="computerNode">
+        <xs:attribute name="name" type="nodeName" use="required"/>
+        <xs:attribute name="netAddress" type="ipV4Address" use="required"/>
+    </xs:attributeGroup>
+    <xs:attributeGroup name="computerNodeReference">
+        <xs:attribute name="computer" type="nodeName" use="required" hpcc:readOnly="true" hpcc:mirrorFrom="/Environment/Hardware/Computer@name"/>
+        <xs:attribute name="netAddress" type="ipV4Address" use="required" hpcc:readOnly="true" hpcc:mirrorFrom="/Environment/Hardware/Computer@netAddress"/>
+    </xs:attributeGroup>
+    <xs:complexType name="usernotes">
+        <xs:sequence>
+            <xs:element name="Note" type="xs:string" minOccurs="0"/>
+        </xs:sequence>
+        <xs:attribute name="severity" use="optional" default="Minor" hpcc:displayName="Severity" hpcc:tooltip="Significance of this note">
+            <xs:simpleType>
+                <xs:restriction base="xs:string">
+                    <xs:enumeration value="Minor"/>
+                    <xs:enumeration value="Normal"/>
+                    <xs:enumeration value="Critical"/>
+                </xs:restriction>
+            </xs:simpleType>
+        </xs:attribute>
+        <xs:attribute name="date" type="AutoTimeStampType" use="optional" hpcc:displayName="Date/Time" hpcc:readOnly="true" hpcc:tooltip="Date and time this note was entered"/>
+        <xs:attribute name="computer" type="AutoComputerType" use="optional" hpcc:displayName="Computer" hpcc:readOnly="true" hpcc:tooltip="Computer from which this note was entered"/>
+        <xs:attribute name="user" type="AutoUseridType" use="optional" hpcc:displayName="User" hpcc:readOnly="true" hpcc:tooltip="User account from which this note was entered"/>
+    </xs:complexType>
+    <xs:attributeGroup name="buildInfo">
+        <xs:attribute name="build" type="xs:string" use="required" hpcc:hidden="true" hpcc:tooltip="The build name to be deployed"/>
+        <xs:attribute name="buildSet" type="xs:string" use="required" hpcc:hidden="true" hpcc:autoGenerateType="configProperty" hpcc:autoGenerateValue="componentName"/>
+    </xs:attributeGroup>
+</xs:schema>

configuration/config2/test.cpp

@@ -0,0 +1,246 @@
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <memory.h>
+#include <string>
+#include <exception>
+#include <iostream>
+
+#include "Exceptions.hpp"
+
+//#include "libxml/parser.h"
+//#include "libxml/tree.h"
+
+
+// namespace pt = boost::property_tree;
+
+const std::string c_path = "configfiles/"; ///opt/HPCCSystems/componentfiles/config2xml/";
+
+// void parseComponentXSD(pt::ptree &xsdTree);
+// void parseIncludes(pt::ptree &xsdTree);
+
+#include "SchemaItem.hpp"
+#include "XSDSchemaParser.hpp"
+#include "XMLEnvironmentMgr.hpp"
+
+
+int main()
+{
+
+    try
+    {
+        //std::shared_ptr<ConfigItem> pConfig = std::make_shared<ConfigItem>("root");
+
+        //ConfigParser *pCfgParser = new XSDConfigParser("", pConfig);
+        //std::string fpath = c_path + "types.xsd";  //dafilesrv.xsd";
+        // std::shared_ptr<ConfigParser> pCfgParser = std::make_shared<XSDConfigParser>("");
+        // std::shared_ptr<ConfigItem> pNull;
+        // std::shared_ptr<ConfigItem> pConfig = std::make_shared<ConfigItem>("myconfig", pNull);
+
+        //pCfgParser->parseEnvironmentConfig("newenv.xsd", "");
+
+        EnvironmentMgr *pEnvMgr = getEnvironmentMgrInstance("XML");
+        std::vector<std::string> cfgParms;
+        cfgParms.push_back("buildset.xml");  // not used right now
+        pEnvMgr->loadSchema(c_path, "newenv.xsd", cfgParms);
+        pEnvMgr->loadEnvironment(c_path + "/environment.xml");
+
+        // 158
+        //auto pNode = envMgr.getNodeFromPath("158");
+        // auto pNode = pEnvMgr->getEnvironmentNode("74");     // 29 is Hardware/Computer
+        auto pNode = pEnvMgr->getEnvironmentNode("35");
+
+        //auto x = pNode->getAllFieldValues("name");
+
+        auto list = pNode->getInsertableItems();
+
+        Status status;
+        auto pNewNode = pEnvMgr->addNewEnvironmentNode("35", "ws_ecl", status);
+        auto newList = pNewNode->getInsertableItems();
+        pEnvMgr->addNewEnvironmentNode("35", "ws_ecl", status);
+        pEnvMgr->addNewEnvironmentNode("35", "ws_ecl", status);
+
+
+        /*auto attributes = pNode->getAttributes();
+        for (auto it = attributes.begin(); it != attributes.end(); ++it)
+        {
+            if ((*it)->getName() == "service")
+            {
+                std::shared_ptr<EnvironmentValue> pEnvValue = *it;
+                const std::shared_ptr<ConfigValue> &pCfgValue = pEnvValue->getCfgValue();
+                auto values = pCfgValue->getAllowedValues(pEnvValue.get());
+                int i = 3;
+            }
+        }*/
+
+
+        // keyref needs to look local first, then search the config tree, but field is ALWAYS local
+        // Used during validation.
+
+        //
+        // Validate the environment
+
+
+
+
+        //
+        // Value set test
+        //std::vector<EnvironmentMgr::NameValue> newValues;
+        //newValues.push_back({ "name", "namehasbeenchanged" });
+        //pEnvMgr->setAttributeValues("158", newValues, "", false);
+
+        //pEnvMgr->saveEnvironment("testout.xml", status);
+
+        auto results = pEnvMgr->getEnvironmentNode(".");
+
+    }
+    catch (ParseException &e)
+    {
+        std::cout << "Error: " << e.what() << "\n";
+    }
+
+    //pt::ptree xsdTree;
+
+    // try
+    // {
+    //     std::string fpath = c_path + "types.xsd";  //dafilesrv.xsd";
+    //     pt::read_xml(fpath, xsdTree);
+
+    //     // throw(ParseException("exception throw test"));
+
+    //     parseComponentXSD(xsdTree);
+
+
+        //
+        // Let's try iterating
+        // for (pt::ptree::const_iterator it=xsdTree.begin(); it!=xsdTree.end(); ++it)
+        // {
+        //     std::cout << "here: " << it->first << std::endl;
+        // }
+
+        // auto it = xsdTree.find("xs:schema");
+        // bool isEnd = (it == xsdTree.not_found());
+        // std::cout << "here " << it->first << "  " << it->second.get_value<std::string>() << std::endl;
+
+        // parseIncludes(it->second);
+
+        // const pt::ptree &attributes = it->second.get_child("<xmlattr>", pt::ptree());
+        // for (auto attrIt=attributes.begin(); attrIt!=attributes.end(); ++attrIt)
+        // {
+        //     std::cout << "attr = " << attrIt->first.data() << " second = " << attrIt->second.get_value<std::string>() << std::endl;
+        // }
+
+
+        // std::cout << "First level keys: " << std::endl;
+        // const pt::ptree &keys = it->second.get_child("", pt::ptree());
+
+        // std::string val = it->second.get_value<std::string>("xs:include.<xmlattr>.schemaLocation");
+        // std::cout << "value: " << val << std::endl;
+
+        // for (auto keyIt=keys.begin(); keyIt!=keys.end(); ++keyIt)
+        // {
+        //     std::cout << "key = >>" << keyIt->first << "<< " << std::endl;
+        //     if (keyIt->first == "xs:include")
+        //     {
+        //         int i = 5;
+        //         std::string val = keyIt->second.get("<xmlattr>.schemaLocation", "not found");
+        //         std::cout << "   schema Location = " << val << std::endl;
+        //     }
+        //     //if (keyIt->first.get_value())
+        // }
+
+
+
+        std::cout << "Success\n";
+
+        // std::cout << std::endl << "Using Boost "
+        //   << BOOST_VERSION / 100000     << "."  // major version
+        //   << BOOST_VERSION / 100 % 1000 << "."  // minor version
+        //   << BOOST_VERSION % 100                // patch level
+        //   << std::endl;
+    // }
+    // catch (const ParseException &e)
+    // {
+    //     std::cout << "Error: " << e.what() << "\n";
+    // }
+    // catch (std::exception &e)
+    // {
+    //     std::cout << "Error: " << e.what() << "\n";
+    // }
+    // return 0;
+}
+
+
+
+// void parseComponentXSD(pt::ptree &xsdTree)
+// {
+//     //
+//     // Get to the schema
+//     auto schemaIt = xsdTree.find("xs:schema");
+
+//     //
+//     // Since we only support includes for a component schema at the top level, look for includes first and process them.
+//     // Note that these includes may NOT define elements or anything like that. Only types, attribute groups, keyrefs
+//     const pt::ptree &keys = schemaIt->second.get_child("", pt::ptree());
+//     for (auto it=keys.begin(); it!=keys.end(); ++it)
+//     {
+//         if (it->first == "xs:include")
+//         {
+//             std::string schema = it->second.get("<xmlattr>.schemaLocation", "not found");
+//             std::cout << "Parsing found include: " << schema << std::endl;
+//         }
+//     }
+
+//     std::vector<std::string> vec;
+//     vec.push_back("Hello");
+//     vec.push_back("There");
+
+
+    //
+    // Now look for any special sections we support
+    // for (auto it=keys.begin(); it!=keys.end(); ++it)
+    // {
+        // if (it->first == "xs:attributeGroup")
+        // {
+        //     std::string groupName = it->second.get("<xmlattr>.name", "not found");
+        //     std::cout << "Attribute group name: " << groupName << std::endl;
+
+        //     //
+        //     // Lets parse the attributes now
+        //     const pt::ptree &attrs = it->second.get_child("", pt::ptree());
+        //     for (auto attributeIt = attrs.begin(); attributeIt != attrs.end(); ++attributeIt)
+        //     {
+        //         auto ss = attributeIt->first;
+        //         if (attributeIt->first == "xs:attribute")
+        //         {
+        //             std::cout << "found attribute" << groupName << std::endl;
+        //             XSDAttribute attr;
+        //             attr.parse(attributeIt->second);
+        //         }
+        //     }
+//         // }
+//         if (it->first == "xs:simpleType")
+//         {
+//             //std::string typeName = it->second.get("<xmlattr>.name", "not found");
+//             //int i = 4;
+//             std::shared_ptr<CfgType> pType = XSDTypeParser::parseSimpleType(it->second);
+//         }
+//     }
+
+
+// }
+
+
+
+// void parseIncludes(pt::ptree &xsdTree)
+// {
+//     const pt::ptree &keys = xsdTree.get_child("", pt::ptree());
+//     for (auto it=keys.begin(); it!=keys.end(); ++it)
+//     {
+//         if (it->first == "xs:include")
+//         {
+//             std::string schema = it->second.get("<xmlattr>.schemaLocation", "not found");
+//             std::cout << "Parsing found include: " << schema << std::endl;
+//         }
+//     }
+// }