Merge pull request #8167 from garonsky/HPCC-14310_for_jon_2

HPCC-14310 Add Envgen (wizard) cmd line pararms to specifiy comp IPs

Reviewed-By: Kevin Wang <kevin.wang@lexisnexis.com>
Reviewed-By: Richard Chapman <rchapman@hpccsystems.com>

deployment/deployutils/deployutils.cpp

@@ -3406,11 +3406,12 @@ void formIPList(const char* ip, StringArray& formattedIpList)
      throw MakeStringException(-1, "List of IP Addresses cannot be empty");
 }
 
-void buildEnvFromWizard(const char * wizardXml, const char* service,IPropertyTree* cfg, StringBuffer& envXml, MapStringTo<StringBuffer>* dirMap)
+void buildEnvFromWizard(const char * wizardXml, const char* service,IPropertyTree* cfg, StringBuffer& envXml, StringArray& arrBuildSetWithAssignedIPs,
+                StringArray& arrAssignedIPs, MapStringTo<StringBuffer>* dirMap)
 {
   if(wizardXml && *wizardXml)
   {
-    CWizardInputs wizardInputs(wizardXml, service, cfg, dirMap);
+    CWizardInputs wizardInputs(wizardXml, service, cfg, dirMap, arrBuildSetWithAssignedIPs, arrAssignedIPs);
     wizardInputs.setEnvironment();
     wizardInputs.generateEnvironment(envXml);
     if(envXml.length() == 0)

deployment/deployutils/deployutils.hpp

@@ -78,7 +78,7 @@ extern DEPLOYUTILS_API void UpdateRefAttributes(IPropertyTree* pEnv, const char*
 extern DEPLOYUTILS_API bool ensureUniqueName(const IPropertyTree* pEnv, IPropertyTree* pParentNode, const char* szText);
 extern DEPLOYUTILS_API void addInstanceToCompTree(const IPropertyTree* pEnvRoot,const IPropertyTree* pInstance,StringBuffer& dups,StringBuffer& resp, IConstEnvironment* pConstEnv);
 extern DEPLOYUTILS_API void formIPList(const char* ip, StringArray& formattedIpList);
-extern DEPLOYUTILS_API void buildEnvFromWizard(const char * xml, const char* service,IPropertyTree* cfg, StringBuffer& envXml, MapStringTo<StringBuffer>* dirMap=NULL);
+extern DEPLOYUTILS_API void buildEnvFromWizard(const char * xml, const char* service,IPropertyTree* cfg, StringBuffer& envXml, StringArray& arrBuildSetWithAssignedIPs, StringArray& arrAssignedIPs, MapStringTo<StringBuffer>* dirMap=NULL);
 extern DEPLOYUTILS_API void runScript(StringBuffer& output, StringBuffer& errMsg, const char* pathToScript);
 extern DEPLOYUTILS_API bool validateIPS(const char* ipAddressList);
 extern DEPLOYUTILS_API void getSummary(const IPropertyTree* pEnvRoot, StringBuffer& respXmlStr , bool prepareLink);

deployment/deployutils/wizardInputs.cpp

@@ -30,14 +30,26 @@
 #define STANDARD_CONFIGXMLDIR COMPONENTFILES_DIR"/configxml/"
 #define STANDARD_CONFIG_DIR CONFIG_DIR
 
+CInstDetails::CInstDetails(StringBuffer compName, const StringArray &ipAssigned) : m_compName(compName)
+{
+    m_ipAssigned.clear();
+
+    for (int i = 0; i < ipAssigned.ordinality(); i++)
+    {
+        m_ipAssigned.append(ipAssigned.item(i));
+    }
+}
+
 //---------------------------------------------------------------------------
 //  CWizardInputs
 //---------------------------------------------------------------------------
 CWizardInputs::CWizardInputs(const char* xmlArg,const char *service, 
                              IPropertyTree * cfg, 
-                             MapStringTo<StringBuffer>* dirMap): m_service(service), 
+                             MapStringTo<StringBuffer>* dirMap,
+                             StringArray &arrBuildSetsWithAssignedIPs,
+                             StringArray &arrAssignedIPs): m_service(service),
                              m_cfg(cfg), m_overrideDirs(dirMap), m_roxieOnDemand(true),
-                             m_supportNodes(0)
+                             m_supportNodes(0), m_arrBuildSetsWithAssignedIPs(arrBuildSetsWithAssignedIPs), m_arrAssignedIPs(arrAssignedIPs)
 {
   m_pXml.setown(createPTreeFromXMLString(xmlArg && *xmlArg ? xmlArg : "<XmlArgs/>"));
 }
@@ -259,14 +271,17 @@ CInstDetails* CWizardInputs::getServerIPMap(const char* compName, const char* bu
 
     if(m_compOnAllNodes.find(buildSetName) != NotFound)
       return instDetails;
-    
-    if (m_ipaddress.ordinality() + m_supportNodes == 1)
+
+    if (m_arrBuildSetsWithAssignedIPs.find(buildSetName) != NotFound)
+        instDetails = new CInstDetails(compName, getIpAddrMap(buildSetName));
+
+    if (m_ipaddress.ordinality() + m_supportNodes == 1 && instDetails == NULL)
     {
       instDetails = new CInstDetails(compName, m_ipaddress.item(0));
       m_compIpMap.setValue(buildSetName,instDetails);
       return instDetails;
     }
-    else if (m_supportNodes == 1 && strcmp(buildSetName, "roxie") && strcmp(buildSetName, "thor" ))
+    else if (m_supportNodes == 1 && strcmp(buildSetName, "roxie") && strcmp(buildSetName, "thor" ) && instDetails == NULL)
     {
       instDetails = new CInstDetails(compName, m_ipaddressSupport.item(0));
       m_compIpMap.setValue(buildSetName,instDetails);
@@ -280,10 +295,10 @@ CInstDetails* CWizardInputs::getServerIPMap(const char* compName, const char* bu
       {
         StringArray* pIpAddrMap = NULL;
 
-        if (x == 0 && m_supportNodes > 0 && !strcmp(buildSetName, "thor"))
-          pIpAddrMap = &m_ipaddressSupport;
-        else
+        if (m_arrBuildSetsWithAssignedIPs.find(buildSetName) != NotFound || !(x == 0 && m_supportNodes > 0 && !strcmp(buildSetName, "thor")))
           pIpAddrMap = &getIpAddrMap(buildSetName);
+        else
+          pIpAddrMap = &m_ipaddressSupport;
 
         unsigned numOfIPSAlreadyTaken = getCntForAlreadyAssignedIPS(buildSetName);
 
@@ -534,6 +549,28 @@ IPropertyTree* CWizardInputs::createEnvironment()
     pComputer->addProp(XML_ATTR_NETADDRESS, m_ipaddress.item(i));
   }
 
+  for(unsigned i = 0; i < m_arrBuildSetsWithAssignedIPs.ordinality(); i++)
+  {
+      const StringArray &strIPs = getIpAddrMap(m_arrBuildSetsWithAssignedIPs.item(i));
+
+      for (unsigned i2 = 0; i2 < strIPs.ordinality(); i2++)
+      {
+          VStringBuffer strXPath("./%s/[%s=\"%s\"]", XML_TAG_COMPUTER, XML_ATTR_NETADDRESS, strIPs.item(i2));
+          if (pCompTree->hasProp(strXPath.str()))
+              continue;
+
+          IPropertyTree* pComputer = pCompTree->addPropTree(XML_TAG_COMPUTER,createPTree());
+          ipaddr.ipset(strIPs.item(i2));
+          ipaddr.getNetAddress(sizeof(x),&x);
+          name.setf("node%03d%03d", (x >> 16) & 0xFF, (x >> 24) & 0xFF);
+          getUniqueName(pCompTree, name, XML_TAG_COMPUTER, "");
+          pComputer->addProp(XML_ATTR_COMPUTERTYPE, "linuxmachine");
+          pComputer->addProp(XML_ATTR_DOMAIN, "localdomain");
+          pComputer->addProp(XML_ATTR_NAME, name.str());
+          pComputer->addProp(XML_ATTR_NETADDRESS, strIPs.item(i2));
+      }
+  }
+
   pNewEnvTree->addPropTree(XML_TAG_HARDWARE, createPTreeFromIPT(pCompTree));
   //Before we generate software tree check for dependencies of component for do_not_generate ,roxie, thor
   checkForDependencies();
@@ -1171,6 +1208,19 @@ void CWizardInputs::addComponentToSoftware(IPropertyTree* pNewEnvTree, IProperty
           assignedIP.clear().append(m_ipaddress.item(i));
           addInstanceToTree(pNewEnvTree, assignedIP, processName, buildSetName,sbl.str());
         }
+
+        int nCount = 1;
+        for (unsigned i = 0; i < m_arrBuildSetsWithAssignedIPs.ordinality(); i++)
+        {
+            const StringArray &strIPList = getIpAddrMap(m_arrBuildSetsWithAssignedIPs.item(i));
+
+            for (unsigned i2 = 0; i2 < strIPList.ordinality(); i2++)
+            {
+              sbl.set("s").append(m_ipaddress.ordinality() + m_ipaddressSupport.ordinality() + nCount++);
+              assignedIP.set(strIPList.item(i2));
+              addInstanceToTree(pNewEnvTree, assignedIP, processName, buildSetName,sbl.str());
+            }
+        }
       }
       else if (numOfIpNeeded > 0)
       {
@@ -1208,13 +1258,25 @@ void CWizardInputs::addComponentToSoftware(IPropertyTree* pNewEnvTree, IProperty
 
 StringArray& CWizardInputs::getIpAddrMap(const char* buildSetName)
 {
-  if (m_supportNodes == 0)
-    return m_ipaddress;
-  else
-  {
-    if (!strcmp(buildSetName, "roxie") || !strcmp(buildSetName, "thor" ))
-      return m_ipaddress;
-  }
+    if (buildSetName && *buildSetName)
+    {
+        for (int i = 0; i < m_arrBuildSetsWithAssignedIPs.ordinality(); i++)
+        {
+            if (stricmp(buildSetName, m_arrBuildSetsWithAssignedIPs.item(i)) == 0)
+            {
+                m_sipaddress.clear();
+                formIPList(m_arrAssignedIPs.item(i), m_sipaddress);
+                return m_sipaddress;
+            }
+        }
+    }
+    if (m_supportNodes == 0)
+        return m_ipaddress;
+    else
+    {
+        if (!strcmp(buildSetName, "roxie") || !strcmp(buildSetName, "thor" ))
+          return m_ipaddress;
+    }
 
-  return m_ipaddressSupport;
+    return m_ipaddressSupport;
 }

deployment/deployutils/wizardInputs.hpp

@@ -48,7 +48,9 @@ class CInstDetails : public CInterface, implements IInterface
     CInstDetails(StringBuffer compName, StringBuffer ipAssigned):m_compName(compName)
     {
       m_ipAssigned.append(ipAssigned.str());
-    };
+    }
+    CInstDetails(StringBuffer compName, const StringArray &ipAssigned);
+
     virtual ~CInstDetails(){};
 
     IMPLEMENT_IINTERFACE;
@@ -77,8 +79,7 @@ class CWizardInputs : public CInterface, implements IInterface
 {
 // Construction
 public:
-  CWizardInputs(){};
-  CWizardInputs(const char* xmlArg, const char* service, IPropertyTree* cfg, MapStringTo<StringBuffer>* dirMap);
+  CWizardInputs(const char* xmlArg, const char* service, IPropertyTree* cfg, MapStringTo<StringBuffer>* dirMap, StringArray &arrBuildSetsWithAssignedIPs, StringArray &arrAssignedIPs);
   virtual ~CWizardInputs(); 
   
   void setEnvironment();
@@ -123,6 +124,8 @@ private:
    StringArray m_compOnAllNodes;
    StringArray m_doNotGenOptOnComps;
    StringArray m_clusterForTopology;
+   StringArray &m_arrBuildSetsWithAssignedIPs;
+   StringArray &m_arrAssignedIPs;
    MapStringToStringArray m_compForTopology; 
    MapStringToStringArray m_invalidServerCombo;
    unsigned m_supportNodes;
@@ -135,6 +138,7 @@ private:
    
    StringArray m_ipaddress;
    StringArray m_ipaddressSupport;
+   StringArray m_sipaddress; //user specified ip addresses
    MapStringToMyClass<CInstDetails> m_compIpMap; 
    Owned<IPropertyTree> m_pXml;
    IPropertyTree* m_cfg;

deployment/envgen/main.cpp

@@ -104,6 +104,8 @@ void usage()
   puts("          -override espsmc,@enableSystemUseRewrite,true\"");
   puts("   -set_xpath_attrib_value <XPATH> <ATTRIBUTE> <VALUE>: sets or add the xpath with attribute and value.");
   puts("          Example: \"-set_xpath_attrib_value  Software/Topology/Cluster[@name=\"thor\"]/ThorCluster @process thor123\"");
+  puts("   -assign_ips <BuildSet> <IP or IP Range>:  assign IPs to specific buildset components");
+  puts("          Example: \" -assign_ips dali 10.0.0.1-10");
   puts("   -help: print out this usage.");
 }
 
@@ -121,6 +123,8 @@ int main(int argc, char** argv)
   StringBufferArray arrXPaths;
   StringBufferArray arrAttrib;
   StringBufferArray arrValues;
+  StringArray arrAssignIPRanges;
+  StringArray arrBuildSetWithAssignedIPs;
 
   int i = 1;
   bool writeToFiles = false;
@@ -235,6 +239,12 @@ int main(int argc, char** argv)
       i++;
       overrides.append(argv[i++]);
     }
+    else if(stricmp(argv[i], "-assign_ips") == 0)
+    {
+      i++;
+      arrBuildSetWithAssignedIPs.append(argv[i++]);
+      arrAssignIPRanges.append(argv[i++]);
+    }
     else
     {
       fprintf(stderr, "Error: unknown command line parameter: %s\n", argv[i]);
@@ -263,6 +273,12 @@ int main(int argc, char** argv)
   try
   {
     validateIPS(ipAddrs.str());
+
+    for (int nIdx = 0; nIdx < arrAssignIPRanges.length(); nIdx++)
+    {
+      validateIPS(arrAssignIPRanges.item(nIdx));
+    }
+
     StringBuffer optionsXml, envXml;
     const char* pServiceName = "WsDeploy_wsdeploy_esp";
     Owned<IPropertyTree> pCfg = createPTreeFromXMLFile(ENVGEN_PATH_TO_ESP_CONFIG);
@@ -270,7 +286,7 @@ int main(int argc, char** argv)
     optionsXml.appendf("<XmlArgs supportNodes=\"%d\" roxieNodes=\"%d\" thorNodes=\"%d\" slavesPerNode=\"%d\" roxieOnDemand=\"%s\" ipList=\"%s\"/>", supportNodes, roxieNodes,
       thorNodes, slavesPerNode, roxieOnDemand?"true":"false", ipAddrs.str());
 
-    buildEnvFromWizard(optionsXml, pServiceName, pCfg, envXml, &dirMap);
+    buildEnvFromWizard(optionsXml, pServiceName, pCfg, envXml, arrBuildSetWithAssignedIPs, arrAssignIPRanges, &dirMap);
 
     if(envXml.length())
     {
@@ -336,7 +352,6 @@ int main(int argc, char** argv)
         toXML(pEnvTree, envXml.clear());
       }
 
-
       StringBuffer env;
       StringBuffer thisip;
       queryHostIP().getIpText(thisip);

esp/services/WsDeploy/WsDeployService.cpp

@@ -6101,7 +6101,8 @@ bool CWsDeployFileInfo::buildEnvironment(IEspContext &context, IEspBuildEnvironm
   {
     if(!strcmp(sbName.str(), m_userWithLock.str()) || !strcmp(sbUserIp.str(), m_userIp.str()))
     { 
-        buildEnvFromWizard(xml, m_pService->getName(), m_pService->getCfg(), envXml);
+        StringArray empty;
+        buildEnvFromWizard(xml, m_pService->getName(), m_pService->getCfg(), envXml, empty, empty);
         setSkipNotification(true);
 
         if(envXml.length())