HPCC-10699 Configuration: Add support for esp service plugins

  - add esp service plugin support to
     ./opt/HPCCSystems/componentfiles/plugins/<service_name>/<config files>

  - service plugins may require configuration files including:

      - buildset.xml : explicit services configured through ConfigMgr
         Installed into
         ./opt/HPCCSystems/componentfiles/configxml/plugins/<service_name>/

      - genenvrules.conf : explicit services that should not be
         generated by default. Installed into
         ./opt/HPCCSystems/componentfiles/configxml/plugins/<service_name>/

      - cgencomplist.xml : implicit services share an explicit service
         component name with a custom xsl, explict services define their
         own component name. Installed into
         ./opt/HPCCSystems/componentfiles/configxml/plugins/<service_name>/

      - esp_service_<service_name>.xsd : explicit services configured
         through ConfigMgr. Installed into
         ./opt/HPCCSystems/componentfiles/configxml/

      - esp_service_<service_name>.xsl : explicit and implicit services
         requires at a minimum binding info.  Refer to WS-SMC section in
         esp_service_WsSMC.xsl as an example of what is required.
         Installed into
         ./opt/HPCCSystems/componentfiles/configxml/atmark_temp/

Signed-off-by: Gleb Aronsky <gleb.aronsky@lexisnexis.com>

deployment/deploy/CMakeLists.txt

@@ -35,11 +35,11 @@ set (    SRCS
          DeployTask.cpp 
          EspDeploymentEngine.cpp 
          espconfiggenengine.cpp 
-#        HoleDeploymentEngine.cpp 
          RoxieDeploymentEngine.cpp 
          thorconfiggenengine.cpp 
          ThorDeploymentEngine.cpp 
-         XslFunctions.cpp 
+         XslFunctions.cpp
+         ../deployutils/confighelper.cpp
     )
 
 include_directories ( 
@@ -50,6 +50,9 @@ include_directories (
          ./../../common/environment
          ./../../dali/base 
          ./../../system/security/securesocket 
+         ${CMAKE_BINARY_DIR}
+         ${CMAKE_BINARY_DIR}/oss
+         ./../deployutils
     )
 
 ADD_DEFINITIONS ( -D_USRDLL -DDEPLOY_EXPORTS )

deployment/deploy/XMLTags.h

@@ -36,6 +36,7 @@
 #define XML_TAG_BUILDSET               "BuildSet"
 #define XML_TAG_CLUSTER                "Cluster"
 #define XML_TAG_COMPONENT              "Component"
+#define XML_TAG_COMPONENTS             "Components"
 #define XML_TAG_COMPUTER               "Computer"
 #define XML_TAG_COMPUTERTYPE           "ComputerType"
 #define XML_TAG_DALISERVERINSTANCE     "DaliServerInstance"
@@ -272,7 +273,7 @@
 #define TAG_PATH                        "path"
 #define TAG_RUNTIME                     "runtime"
 #define TAG_BUILD                       "build"
-#define TAG_BUILDSET                    "buildSet"
+#define TAG_BUILDSET                    "buildset"
 #define TAG_DIRECTORY                   "directory"
 #define TAG_NODENAME                    "nodeName"
 #define TAG_PREFIX                      "prefix"

deployment/deploy/configgenengine.cpp

@@ -21,6 +21,7 @@
 #include "xslprocessor.hpp"
 #include "configgenengine.hpp"
 #include "XMLTags.h"
+#include "confighelper.hpp"
 
 #ifdef _WINDOWS 
 #define CONFIGGEN_COMP_LIST "cgencomplist_win.xml"
@@ -56,6 +57,12 @@ int CConfigGenEngine::determineInstallFiles(IPropertyTree& processNode, CInstall
             compListPath.clear().append(m_inDir).append(PATHSEPCHAR).append(CONFIGGEN_COMP_LIST);
 
         Owned<IPropertyTree> deployNode = createPTreeFromXMLFile(compListPath.str(), ipt_caseInsensitive);
+
+        CConfigHelper *pConfigHelper = CConfigHelper::getInstance(NULL, NULL, m_pCallback.get());
+
+        compListPath.clear().set(m_inDir).append(PATHSEPCHAR);
+        pConfigHelper->addPluginsToConfigGenCompList(deployNode.get(), compListPath.str());
+
         StringBuffer srcFilePath;
         srcFilePath.ensureCapacity(_MAX_PATH);
         const bool bFindStartable = &m_process == &processNode && m_startable == unknown;

deployment/deploy/espconfiggenengine.cpp

@@ -257,13 +257,6 @@ void CEspConfigGenEngine::processServiceModules(const char* moduleType,
          {
             if (m_deployFlags & DEFLAGS_CONFIGFILES)
             {
-               //each esp_service_module for a given service name is expected to produce necessary
-               //and sufficient information for itself (EspService node) and all its EspBindings 
-               //(we may have multiple bindings for the same service i.e. EspService[@name='xyz']).
-               //Therefore, we only need to process one instance of EspService exactly once. 
-               if (serviceNamesProcessed.find(serviceName) != serviceNamesProcessed.end())
-                  continue;
-
                serviceNamesProcessed.insert(serviceName);
 
                serviceXsltOutputFiles.append(dest.c_str());

deployment/deployutils/CMakeLists.txt

@@ -28,8 +28,8 @@ set (    SRCS
          buildset.cpp 
          computerpicker.cpp 
          configenvhelper.cpp
-         wizardInputs.cpp 
-         deployutils.cpp 
+         wizardInputs.cpp
+         deployutils.cpp
          confighelper.cpp
     )
 
@@ -42,7 +42,7 @@ include_directories (
          ./../../dali/base 
          ./../../esp/esplib
          ../configgen
-         ../deploy 
+         ../deploy
          ${CMAKE_BINARY_DIR}
          ${CMAKE_BINARY_DIR}/oss
     )

deployment/deployutils/confighelper.cpp

@@ -1,64 +1,119 @@
 #include "confighelper.hpp"
-#include "XMLTags.h"
-#include "jlib.hpp"
+#include "jexcept.hpp"
+#include "jfile.hpp"
+#include "jmutex.hpp"
 #include "jprop.hpp"
-#include "jptree.hpp"
+#include "jfile.hpp"
+#include "../deploy/XMLTags.h"
 #include "build-config.h"
 
 #define STANDARD_CONFIG_BUILDSETFILE "buildset.xml"
 #define STANDARD_CONFIG_CONFIGXML_DIR "/componentfiles/configxml/"
+#define STANDARD_CONFIG_PLUGIN_DIR_NAME "/plugins/"
+#define STANDARD_CONFIG_PLUGINS_DIR STANDARD_CONFIG_CONFIGXML_DIR STANDARD_CONFIG_PLUGIN_DIR_NAME
+#define PLUGIN_CGEN_COMP_LIST  "cgencomplist.xml"
+#define ENV_GEN_RULES_DO_NOT_GENERATE_PROP "do_not_generate"
 
-CConfigHelper::CConfigHelper() : m_pDefBldSet(NULL)
+CConfigHelper::CConfigHelper(IDeploymentCallback *pCallBack): m_pDefBldSet(NULL)
 {
+    if (pCallBack != NULL)
+    {
+        m_cbDeployment.set(pCallBack);
+    }
+    m_strConfigXMLDir.clear();
 }
 
 CConfigHelper::~CConfigHelper()
 {
 }
 
-CConfigHelper* CConfigHelper::getInstance(const IPropertyTree *cfg, const char* esp_name)
+CConfigHelper* CConfigHelper::getInstance(const IPropertyTree *cfg, const char* esp_name, IDeploymentCallback *pCallBack)
 {
-  static CConfigHelper *pConfigHelper = NULL;
-
-  if (pConfigHelper != NULL)
-  {
-      return pConfigHelper;
-  }
-
-  if (cfg == NULL || esp_name == NULL)
-  {
-      return NULL;
-  }
-
-  pConfigHelper = new CConfigHelper();
-
-  StringBuffer xpath;
-
-  xpath.setf("%s/%s/%s[%s='%s']/%s",XML_TAG_SOFTWARE, XML_TAG_ESPPROCESS, XML_TAG_ESPSERVICE, XML_ATTR_NAME, esp_name, XML_TAG_LOCALCONFFILE);
-  pConfigHelper->m_strConfFile = cfg->queryProp(xpath.str());
+    static CConfigHelper *p_sConfigHelper = NULL;
+    static CSingletonLock slock;
+    StringBuffer xpath1, xpath2;
 
-  xpath.clear().appendf("%s/%s/%s[%s='%s']/%s",XML_TAG_SOFTWARE, XML_TAG_ESPPROCESS, XML_TAG_ESPSERVICE, XML_ATTR_NAME, esp_name, XML_TAG_LOCALENVCONFFILE);
-  pConfigHelper->m_strEnvConfFile = cfg->queryProp(xpath.str());
-
-  if (pConfigHelper->m_strConfFile.length() > 0 && pConfigHelper->m_strEnvConfFile.length() > 0)
-  {
-    Owned<IProperties> pParams = createProperties(pConfigHelper->m_strConfFile);
-    Owned<IProperties> pEnvParams = createProperties(pConfigHelper->m_strEnvConfFile);
-
-    pConfigHelper->m_strConfigXMLDir = pEnvParams->queryProp(TAG_PATH);
-
-    if ( pConfigHelper->m_strConfigXMLDir.length() == 0)
+    if (slock.lock())
     {
-      pConfigHelper->m_strConfigXMLDir = INSTALL_DIR;
-    }
-
-    pConfigHelper->m_strBuildSetFileName = pParams->queryProp(TAG_BUILDSET);
+        if (p_sConfigHelper != NULL)
+        {
+            slock.unlock();
 
-    pConfigHelper->m_strBuildSetFilePath.append(pConfigHelper->m_strConfigXMLDir).append(STANDARD_CONFIG_CONFIGXML_DIR).append( pConfigHelper->m_strBuildSetFileName.length() > 0 ? pConfigHelper->m_strBuildSetFileName : STANDARD_CONFIG_BUILDSETFILE);
-    pConfigHelper->m_pDefBldSet.set(createPTreeFromXMLFile(pConfigHelper->m_strBuildSetFilePath.str()));
-  }
+            return p_sConfigHelper;
+        }
+        if (cfg == NULL || esp_name == NULL)
+        {
+            slock.unlock();
 
-  return pConfigHelper;
+            return p_sConfigHelper = new CConfigHelper(pCallBack);
+        }
+        else
+        {
+            p_sConfigHelper = new CConfigHelper(pCallBack);
+
+            xpath1.clear().appendf("%s/%s/%s[%s='%s']/%s",XML_TAG_SOFTWARE, XML_TAG_ESPPROCESS, XML_TAG_ESPSERVICE, XML_ATTR_NAME, esp_name, XML_TAG_LOCALCONFFILE);
+            p_sConfigHelper->m_strConfFile = cfg->queryProp(xpath1.str());
+
+            xpath2.clear().appendf("%s/%s/%s[%s='%s']/%s",XML_TAG_SOFTWARE, XML_TAG_ESPPROCESS, XML_TAG_ESPSERVICE, XML_ATTR_NAME, esp_name, XML_TAG_LOCALENVCONFFILE);
+            p_sConfigHelper->m_strEnvConfFile = cfg->queryProp(xpath2.str());
+
+            if (p_sConfigHelper->m_strConfFile.length() > 0 && p_sConfigHelper->m_strEnvConfFile.length() > 0 && checkFileExists(p_sConfigHelper->m_strConfFile.str()) && checkFileExists(p_sConfigHelper->m_strEnvConfFile.str()))
+            {
+                Owned<IProperties> pParams = createProperties(p_sConfigHelper->m_strConfFile.str());
+                Owned<IProperties> pEnvParams = createProperties(p_sConfigHelper->m_strEnvConfFile.str());
+
+                p_sConfigHelper->m_strConfigXMLDir = pEnvParams->queryProp(TAG_PATH);
+
+                if (p_sConfigHelper->m_strConfigXMLDir.length() == 0)
+                {
+                  p_sConfigHelper->m_strConfigXMLDir = INSTALL_DIR;
+                }
+
+                p_sConfigHelper->m_strBuildSetFileName = pParams->queryProp(TAG_BUILDSET);
+
+                p_sConfigHelper->m_strBuildSetFilePath.append(p_sConfigHelper->m_strConfigXMLDir).append(STANDARD_CONFIG_CONFIGXML_DIR).append(
+                            p_sConfigHelper->m_strBuildSetFileName.length() > 0 ? p_sConfigHelper->m_strBuildSetFileName : STANDARD_CONFIG_BUILDSETFILE);
+
+                if (p_sConfigHelper->m_cbDeployment.get() != NULL)
+                {
+                    p_sConfigHelper->m_cbDeployment->printStatus(STATUS_NORMAL, NULL, NULL, NULL,
+                                              "Adding plugin buildset %s", p_sConfigHelper->m_strBuildSetFilePath.str());
+                }
+
+                try
+                {
+                    p_sConfigHelper->m_pDefBldSet.set(createPTreeFromXMLFile(p_sConfigHelper->m_strBuildSetFilePath.str()));
+                    p_sConfigHelper->appendBuildSetFromPlugins();
+                }
+                catch (IException *e)
+                {
+                    if (p_sConfigHelper->m_cbDeployment.get() != NULL)
+                    {
+                        StringBuffer msg;
+                        e->errorMessage(msg);
+
+                        p_sConfigHelper->m_cbDeployment->printStatus(STATUS_ERROR, NULL, NULL, NULL,
+                                                    "Unable to add pluging buildset %s with error %s", p_sConfigHelper->m_strBuildSetFilePath.str(), msg.str());
+                    }
+                    delete e;
+                }
+
+                slock.unlock();
+                return p_sConfigHelper;
+            }
+            else
+            {
+                delete p_sConfigHelper;
+                p_sConfigHelper = NULL;
+
+                slock.unlock();
+                throw MakeStringException(-1, "Config file does not define values for %s and %s", xpath1.str(), xpath2.str());
+
+                return NULL;
+            }
+        }
+    }
+    return p_sConfigHelper;
 }
 
 bool CConfigHelper::isInBuildSet(const char* comp_process_name, const char* comp_name) const
@@ -77,6 +132,76 @@ bool CConfigHelper::isInBuildSet(const char* comp_process_name, const char* comp
   }
 }
 
+IPropertyTree* CConfigHelper::getBuildSetTree()
+{
+    this->m_pDefBldSet->Link();
+
+    return this->m_pDefBldSet;
+}
+
+void CConfigHelper::appendBuildSetFromPlugins()
+{
+    const char *pMask = "*";
+
+    StringBuffer strPath(this->m_strConfigXMLDir);
+    strPath.append(STANDARD_CONFIG_CONFIGXML_DIR).append(STANDARD_CONFIG_PLUGIN_DIR_NAME);
+
+    Owned<IFile> pluginRootDir = createIFile(strPath.str());
+
+    if (checkFileExists(strPath.str()) == false)
+    {
+        if (m_cbDeployment.get() != NULL)
+        {
+            m_cbDeployment->printStatus(STATUS_WARN, NULL, NULL, NULL,
+                                      "Could not find plugin directory at %s", strPath.str());
+        }
+        return;
+    }
+
+    Owned<IDirectoryIterator> pluginFiles = pluginRootDir->directoryFiles(pMask, false, true);
+
+    ForEach(*pluginFiles)
+    {
+        if (pluginFiles->query().isDirectory() == true)
+        {
+            StringBuffer strPluginBuildSetPath;
+
+            strPluginBuildSetPath.append(pluginFiles->query().queryFilename()).append("/").append(STANDARD_CONFIG_BUILDSETFILE);
+
+            if (checkFileExists(strPluginBuildSetPath.str()) == true)
+            {
+                StringBuffer strXPath;
+
+                strXPath.appendf("./%s/%s/%s", XML_TAG_PROGRAMS, XML_TAG_BUILD, XML_TAG_BUILDSET);
+
+                Owned<IPropertyTree> pPluginBuildSet = createPTreeFromXMLFile(strPluginBuildSetPath.str());
+
+                if (m_cbDeployment.get() != NULL)
+                {
+                    m_cbDeployment->printStatus(STATUS_NORMAL, NULL, NULL, NULL,
+                                            "Loading plugin BuildSet from  %s", strPluginBuildSetPath.str());
+                }
+
+                Owned<IPropertyTreeIterator> pBuildSetIterator = pPluginBuildSet->getElements(strXPath);
+
+                ForEach(*pBuildSetIterator)
+                {
+                    m_pDefBldSet->addPropTree(strXPath.str(), LINK(&(pBuildSetIterator->query())));
+                }
+            }
+            else
+            {
+                // Log message that buildset plugin file is missing
+                if (m_cbDeployment != NULL)
+                {
+                    m_cbDeployment->printStatus(STATUS_WARN, NULL, NULL, NULL,
+                                              "buildset.xml file is missing.  Looked in %s", strPluginBuildSetPath.str());
+                }
+            }
+        }
+    }
+}
+
 void CConfigHelper::getNewComponentListFromBuildSet(const IPropertyTree *pEnvTree, StringArray &sCompArray) const
 {
     if (pEnvTree == NULL || m_pDefBldSet == NULL)
@@ -102,7 +227,7 @@ void CConfigHelper::getNewComponentListFromBuildSet(const IPropertyTree *pEnvTre
     }
 }
 
-void CConfigHelper::addNewComponentsFromBuildSetToEnv(IPropertyTree *pEnvTree)
+void CConfigHelper::addNewComponentsFromBuildSetToEnv(IPropertyTree *pEnvTree) const
 {
     if (pEnvTree == NULL)
         return;
@@ -125,3 +250,191 @@ void CConfigHelper::addNewComponentsFromBuildSetToEnv(IPropertyTree *pEnvTree)
         pEnvTree->queryPropTree(XML_TAG_PROGRAMS"/"XML_TAG_BUILD)->addPropTree(XML_TAG_BUILDSET, createPTreeFromIPT(m_pDefBldSet->queryPropTree(xpath.str())));
     }
 }
+
+void CConfigHelper::addPluginsToConfigGenCompList(IPropertyTree *pCGenComplist,  const char *pPath) const
+{
+    if (pCGenComplist == NULL)
+    {
+        return;
+    }
+
+    const char *pMask = "*";
+
+
+    StringBuffer strPath;
+
+    if (m_strConfigXMLDir.length() > 0)
+    {
+        strPath.set(m_strConfigXMLDir.str());
+        strPath.append(STANDARD_CONFIG_CONFIGXML_DIR).append(STANDARD_CONFIG_PLUGIN_DIR_NAME);
+    }
+    else if(pPath != NULL)
+    {
+        strPath.set(pPath).append(STANDARD_CONFIG_PLUGIN_DIR_NAME);
+    }
+    else
+    {
+        return;
+    }
+
+    Owned<IFile> pluginRootDir = createIFile(strPath.str());
+
+    if (checkFileExists(strPath.str()) == false)
+    {
+        return;
+    }
+
+    Owned<IDirectoryIterator> pluginFiles = pluginRootDir->directoryFiles(pMask, false, true);
+
+    ForEach(*pluginFiles)
+    {
+        if (pluginFiles->query().isDirectory() == true)
+        {
+            StringBuffer strPluginCGenCompListPath;
+
+            strPluginCGenCompListPath.append(pluginFiles->query().queryFilename()).append("/").append(PLUGIN_CGEN_COMP_LIST);
+
+            if (checkFileExists(strPluginCGenCompListPath.str()) == true)
+            {
+                StringBuffer strXPath;
+
+                strXPath.appendf("./%s", XML_TAG_COMPONENT);
+
+                Owned<IPropertyTree> pPluginCGenCompList;
+
+                try
+                {
+                    pPluginCGenCompList.set(createPTreeFromXMLFile(strPluginCGenCompListPath.str()));
+                }
+                catch (IException *e)
+                {
+                    if (m_cbDeployment.get() != NULL)
+                    {
+                        m_cbDeployment->printStatus(STATUS_WARN, NULL, NULL, NULL,
+                                                "Unable to load cgencomplist.xml from  %s", strPluginCGenCompListPath.str());
+                    }
+                    delete e;
+                }
+
+                Owned<IPropertyTreeIterator> pCGenCompListIterator = pPluginCGenCompList->getElements(strXPath);
+
+                ForEach(*pCGenCompListIterator)
+                {
+                    StringBuffer strXPath2(XML_TAG_COMPONENT);
+                    StringBuffer strXPath3(XML_TAG_COMPONENT"/"XML_TAG_FILE);
+                    strXPath2.appendf("[%s='%s']", XML_ATTR_NAME, pCGenCompListIterator->query().queryProp(XML_ATTR_NAME));
+
+                    if (pCGenComplist->hasProp(strXPath2.str()) == false)
+                    {
+                        pCGenComplist->addPropTree(XML_TAG_COMPONENT, LINK(&(pCGenCompListIterator->query())));
+
+                        if (m_cbDeployment.get() != NULL)
+                        {
+                            m_cbDeployment->printStatus(STATUS_NORMAL, NULL, NULL, NULL,
+                                                "Loaded %s from  %s", strXPath2.str(), strPluginCGenCompListPath.str());
+                        }
+                    }
+                    else
+                    {
+                        Owned<IPropertyTreeIterator> pFileListIter = pPluginCGenCompList->getElements(strXPath3);
+
+                        ForEach(*pFileListIter)
+                        {
+                            pCGenComplist->queryPropTree(strXPath2.str())->addPropTree(XML_TAG_FILE, LINK((&(pFileListIter->query()))));
+
+                            if (m_cbDeployment.get() != NULL)
+                            {
+                                m_cbDeployment->printStatus(STATUS_NORMAL, NULL, NULL, NULL,
+                                                "Loading %s from  %s", strXPath3.str(), strPluginCGenCompListPath.str());
+                            }
+                        }
+                    }
+                }
+            }
+            else
+            {
+                if (m_cbDeployment != NULL)
+                {
+                    m_cbDeployment->printStatus(STATUS_WARN, NULL, NULL, NULL,
+                                              "cgencomplist.xml file is missing.  Looked in %s", strPluginCGenCompListPath.str());
+                }
+            }
+        }
+    }
+}
+
+void CConfigHelper::addPluginsToGenEnvRules(IProperties *pGenEnvRulesProps) const
+{
+    if (pGenEnvRulesProps == NULL)
+    {
+        return;
+    }
+
+    const char *pMask = "*";
+
+    StringBuffer strPath(this->m_strConfigXMLDir);
+    strPath.append(STANDARD_CONFIG_CONFIGXML_DIR).append(STANDARD_CONFIG_PLUGIN_DIR_NAME);
+
+    Owned<IFile> pluginRootDir = createIFile(strPath.str());
+
+    if (checkFileExists(strPath.str()) == false)
+    {
+        return;
+    }
+
+    Owned<IDirectoryIterator> pluginFiles = pluginRootDir->directoryFiles(pMask, false, true);
+
+    ForEach(*pluginFiles)
+    {
+        if (pluginFiles->query().isDirectory() == true)
+        {
+            StringBuffer strPluginGenEnvRulesPath;
+
+            strPluginGenEnvRulesPath.append(pluginFiles->query().queryFilename()).append("/").append(STANDARD_CONFIG_ALGORITHMFILE);
+
+            if (checkFileExists(strPluginGenEnvRulesPath.str()) == true)
+            {
+
+                Owned<IProperties> pPluginGenEnvPropList = createProperties(strPluginGenEnvRulesPath.str());
+                Owned<IPropertyIterator> pPluginGenEnvPropListIterator = pPluginGenEnvPropList->getIterator();
+
+                ForEach(*pPluginGenEnvPropListIterator)
+                {
+                    const char *pKeyName = pPluginGenEnvPropListIterator->getPropKey();
+
+                    if (pKeyName != NULL && *pKeyName != 0 && strcmp(pKeyName, ENV_GEN_RULES_DO_NOT_GENERATE_PROP) == 0)
+                    {
+                        StringBuffer strProp;
+
+                        if (pGenEnvRulesProps->hasProp(pKeyName) == false)
+                        {
+                            pPluginGenEnvPropList->getProp(pKeyName, strProp);
+                            pGenEnvRulesProps->appendProp(ENV_GEN_RULES_DO_NOT_GENERATE_PROP, strProp.str());
+
+                            if (m_cbDeployment != NULL)
+                            {
+                                m_cbDeployment->printStatus(STATUS_NORMAL, NULL, NULL, NULL,
+                                                            "Adding genenvrules %s", strProp.str());
+                            }
+                        }
+                        else
+                        {
+                            pPluginGenEnvPropList->getProp(pKeyName, strProp.clear());
+                            strProp.append(",");
+                            pGenEnvRulesProps->getProp(pKeyName, strProp);
+                            pGenEnvRulesProps->setProp(pKeyName, strProp.str());
+                        }
+                    }
+                }
+            }
+            else
+            {
+                if (m_cbDeployment != NULL)
+                {
+                    m_cbDeployment->printStatus(STATUS_WARN, NULL, NULL, NULL,
+                                               "Failed to load plug-in genenvrules.conf file %s", strPluginGenEnvRulesPath.str());
+                }
+            }
+        }
+    }
+}

deployment/deployutils/confighelper.hpp

@@ -1,4 +1,10 @@
+#ifndef CONFIGHELPER_HPP_INCL
+#define CONFIGHELPER_HPP_INCL
+
 #include "deployutils.hpp"
+#include "../deploy/deploy.hpp"
+
+#define STANDARD_CONFIG_ALGORITHMFILE "genenvrules.conf"
 
 class DEPLOYUTILS_API CConfigHelper
 {
@@ -6,7 +12,7 @@ public:
 
   virtual ~CConfigHelper();
 
-  static CConfigHelper* getInstance(const IPropertyTree *cfg = NULL, const char* esp_name = NULL);
+  static CConfigHelper* getInstance(const IPropertyTree *cfg = NULL, const char* esp_name = NULL, IDeploymentCallback *pCallBack = NULL);
 
   bool isInBuildSet(const char* comp_process_name, const char* comp_name) const;
 
@@ -31,22 +37,28 @@ public:
     return m_strBuildSetFilePath.toCharArray();
   }
 
-  const IPropertyTree* getBuildSetTree() const
-  {
-      return m_pDefBldSet;
-  }
-
   void getNewComponentListFromBuildSet(const IPropertyTree *pEnvTree, StringArray &sCompArray) const;
-  void addNewComponentsFromBuildSetToEnv(IPropertyTree *pEnvTree);
+  void addNewComponentsFromBuildSetToEnv(IPropertyTree *pEnvTree) const;
+  void addPluginsToConfigGenCompList(IPropertyTree *pCGenComplist, const char *pPath = NULL) const;
+  void addPluginsToGenEnvRules(IProperties *pGenEnvRulesProps) const;
+
+  IPropertyTree* getBuildSetTree();
 
 protected:
 
-  CConfigHelper();
+  CConfigHelper(IDeploymentCallback *m_cbDeployment = NULL);
 
   Owned<IPropertyTree> m_pDefBldSet;
+  Linked<IDeploymentCallback> m_cbDeployment;
   StringBuffer  m_strConfigXMLDir;
   StringBuffer  m_strBuildSetFileName;
   StringBuffer  m_strEnvConfFile;
   StringBuffer  m_strConfFile;
   StringBuffer  m_strBuildSetFilePath;
+
+  void appendBuildSetFromPlugins();
+
 };
+
+
+#endif // CONFIGHELPER_HPP_INCL

deployment/deployutils/deployutils.cpp

@@ -1819,6 +1819,8 @@ IPropertyTree* generateTreeFromXsd(const IPropertyTree* pEnv, IPropertyTree* pSc
     if (genEnvConf.length() && checkFileExists(genEnvConf.str()))
       algProp.setown(createProperties(genEnvConf.str()));
 
+    CConfigHelper::getInstance()->addPluginsToGenEnvRules(algProp.get());
+
     enum GenOptional {GENOPTIONAL_ALL, GENOPTIONAL_NONE, GENOPTIONAL_COMPS};
     GenOptional genOpt = GENOPTIONAL_COMPS;
     algProp->getProp("do_not_gen_optional", prop);

deployment/deployutils/wizardInputs.cpp

@@ -24,12 +24,11 @@
 #include "XMLTags.h"
 #include "jencrypt.hpp"
 #include "buildset.hpp"
+#include "confighelper.hpp"
 #include "build-config.h"
 
 #define STANDARD_CONFIGXMLDIR COMPONENTFILES_DIR"/configxml/"
-#define STANDARD_CONFIG_BUILDSETFILE "buildset.xml"
 #define STANDARD_CONFIG_DIR CONFIG_DIR
-#define STANDARD_CONFIG_ALGORITHMFILE "genenvrules.conf"
 
 //---------------------------------------------------------------------------
 //  CWizardInputs
@@ -125,13 +124,21 @@ void CWizardInputs::setEnvironment()
      Owned<IProperties> pEnvParams = createProperties(pEnvConfFile);
      StringBuffer sb, fileName;
      
-     fileName.append((pEnvParams->queryProp("path")!= NULL ? (sb.clear().append(pEnvParams->queryProp("path")).append("/componentfiles/configxml/")) : STANDARD_CONFIGXMLDIR));
-     fileName.append((pParams->queryProp("buildset") != NULL ? (sb.clear().append(pParams->queryProp("buildset"))) : STANDARD_CONFIG_BUILDSETFILE));
+     CConfigHelper *pConfigHelper = CConfigHelper::getInstance(m_cfg, m_service.str());
 
-     if(fileName.length() && checkFileExists(fileName.str()))
-       m_buildSetTree.setown(createPTreeFromXMLFile(fileName.str()));
-     else
-       throw MakeStringException( -1 , "The buildSetFile %s does not exists", fileName.str());
+     if (pConfigHelper == NULL)
+     {
+         throw MakeStringException( -1 , "Error loading buildset from configuration");
+     }
+
+     IPropertyTree* pBuildSet = pConfigHelper->getBuildSetTree();
+
+     if (strlen(pConfigHelper->getBuildSetFileName()) == 0 || pBuildSet == NULL)
+     {
+         throw MakeStringException( -1 , "The buildSetFile %s/%s does not exists", pConfigHelper->getBuildSetFilePath(), pConfigHelper->getBuildSetFileName());
+     }
+
+     m_buildSetTree.setown(pBuildSet);
      
      fileName.clear().append((pEnvParams->queryProp("configs") != NULL ? (sb.clear().append(pEnvParams->queryProp("configs")).append("/")): STANDARD_CONFIG_DIR));
      fileName.append((pParams->queryProp("wizardalgorithm") != NULL ? (sb.clear().append(pParams->queryProp("wizardalgorithm"))) : STANDARD_CONFIG_ALGORITHMFILE));
@@ -154,6 +161,7 @@ void CWizardInputs::setWizardRules()
 
    if(m_algProp)
    {
+     CConfigHelper::getInstance()->addPluginsToGenEnvRules(m_algProp.get());
      Owned<IPropertyIterator> iter = m_algProp->getIterator();
      StringBuffer prop;
      ForEach(*iter)

esp/services/WsDeploy/WsDeployService.cpp

@@ -195,7 +195,6 @@ void expandRange(IPropertyTree* pComputers)
   }
 }
 
-
 CWsDeployExCE::~CWsDeployExCE()
 {
     m_pCfg.clear();
@@ -251,6 +250,8 @@ void CWsDeployExCE::init(IPropertyTree *cfg, const char *process, const char *se
   StringBuffer xpath;
   m_envFile.clear();
 
+  m_pConfigHelper = CConfigHelper::getInstance(cfg,service);
+
   xpath.clear().appendf("Software/EspProcess/EspService[@name='%s']/LocalEnvConfFile", service);
   const char* tmp = cfg->queryProp(xpath.str());
   if (tmp && *tmp)
@@ -6270,6 +6271,20 @@ void CWsDeployFileInfo::initFileInfo(bool createOrOverwrite, bool bClearEnv)
     StringBuffer s("<?xml version=\"1.0\" encoding=\"UTF-8\"?><Environment></Environment>");
     Owned<IPropertyTree> pNewTree = createPTreeFromXMLString(s);
 
+    if ( strlen(m_pService->m_pConfigHelper->getBuildSetFilePath()) > 0 )
+    {
+        try
+        {
+          Owned<IPropertyTree> pDefBldSet = m_pService->m_pConfigHelper->getBuildSetTree(); //createPTreeFromXMLFile( m_pService->m_pConfigHelper->getBuildSetFilePath() );
+          pNewTree->addPropTree(XML_TAG_PROGRAMS, createPTreeFromIPT(pDefBldSet->queryPropTree("./Programs")));
+          pNewTree->addPropTree(XML_TAG_SOFTWARE, createPTreeFromIPT(pDefBldSet->queryPropTree("./Software")));
+        }
+        catch(IException* e)
+        {
+          e->Release();
+        }
+    }
+
     if(!pNewTree->queryPropTree(XML_TAG_SOFTWARE))
     {
       pNewTree->addPropTree(XML_TAG_SOFTWARE, createPTree());

esp/services/WsDeploy/WsDeployService.hpp

@@ -49,6 +49,7 @@ class CCloudTask;
 class CCloudActionHandler;
 class CWsDeployEx;
 class CWsDeployExCE;
+class CConfigHelper;
 
 interface IConfigFileObserver : extends IObserver
 {
@@ -950,6 +951,8 @@ public:
     const char* getProcessName() { return m_process.str(); }
     const char* getSourceDir() { return m_sourceDir.str(); }
 
+    CConfigHelper *m_pConfigHelper;
+
 private:
   virtual void getWizOptions(StringBuffer& sb);