/*##############################################################################
Copyright (C) 2011 HPCC Systems.
All rights reserved. This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see .
############################################################################## */
#include "jliball.hpp"
#include "XMLTags.h"
#include "configengcallback.hpp"
#include "deploy.hpp"
#include "build-config.h"
#define STANDARD_INDIR COMPONENTFILES_DIR"/configxml"
#define STANDARD_OUTDIR RUNTIME_DIR
void usage()
{
const char* version = "1.1";
printf("HPCC Systems configuration generator. version %s. Usage:\n", version);
puts(" configgen -env -ip [options]");
puts("");
puts("options: ");
puts(" -env : The configuration environment to be parsed.");
puts(" -ip : The ip address that will be matched against a component ");
puts(" instance's ip address. If matched, configuration files are ");
puts(" generated for that component");
puts(" -c : Optional component name for which the ");
puts(" configuration is generated. If -t is also specified, it is ");
puts(" ignored");
puts(" -t : Optional component type for which the ");
puts(" configuration is generated. If -c is also specified, the ");
puts(" component name is used");
puts(" -id : The input directory for the supporting ");
puts(" xml environment files like xsd's, xsl's and ");
puts(" configgencomplist.xml. If not specified, the following ");
puts(" defaults are used. ");
puts(" For win32, 'c:\\trunk\\initfiles\\componentfiles\\configxml'");
puts(" For Linux, '"COMPONENTFILES_DIR"/configxml/'");
puts(" -od : The output directory for the generated files.");
puts(" If not specified, the following defaults are used. ");
puts(" For win32, '.'");
puts(" For Linux, '"CONFIG_DIR"'");
puts(" -ldapconfig : Generates a .ldaprc file and puts it in the specified");
puts(" output directory. If output directory is not specified,");
puts(" default output directory is used as mentioned in -od option");
puts(" if a LDAPServer is not defined in the environment, the .ldaprc ");
puts(" file is not generated. If an -ip is not provided, the first");
puts(" instance of the first LDAPserver is used to generate the ");
puts(" .ldaprc file");
puts(" -list: Lists out the components for a specific ip in the format");
puts(" componentType=componentName;config file directory. Does not ");
puts(" generate any output files. If masters and slaves exist for ");
puts(" a component like Roxie or thor, then only the master entry ");
puts(" is returned. ");
puts(" -listall: Lists out all the components specified in the environment");
puts(" that have an instance defined. Does not require an ip. Does ");
puts(" not generate any output files. Output is written to stdout ");
puts(" in the csv format as follows");
puts(" ProcessType,componentName,instanceip,instanceport,runtimedir,logdir");
puts(" Missing fields will be empty.");
puts(" -listdirs: Lists out any directories that need to be created during ");
puts(" init time. Currently, directories for any drop zones ");
puts(" with the same ip as the -ip option are returned. Format is ");
puts(" one directory per line.");
puts(" -listdropzones: Lists out all the dropzones defined in the environment ");
puts(" Does not require an ip. Does not generate any output files.");
puts(" Output is written to stdout. Format is as follows,");
puts(" one entry per line");
puts(" dropzone node ip,dropzone directory");
puts(" -listcommondirs: Lists out all directories that are listed under ");
puts(" Software/Directories section in the following format. ");
puts(" =");
puts(" Each directory will be listed on a new line.");
puts(" -listldaps: Lists out all LDAPServer instances defined in the ");
puts(" environment in the following format. If the same component");
puts(" has more than one instance, it will be listed as two separate.");
puts(" entries in the output");
puts(" componentName,instanceip");
puts(" -machines: Lists out all names or ips of machines specified in the environment");
puts(" Output is written to stdout, one machine per line.");
puts(" -validateonly: Validates the environment, without generating permanent ");
puts(" configurations. Returns 0 if environment is valid and non zero ");
puts(" in other cases. Validation errors are printed to stderr.");
puts(" Ignores -od flag, if supplied.");
puts(" -v : Print verbose output to stdout");
puts(" -help: print out this usage.");
}
void deleteRecursive(const char* path)
{
Owned pDir = createIFile(path);
if (pDir->exists())
{
if (pDir->isDirectory())
{
Owned it = pDir->directoryFiles(NULL, false, true);
ForEach(*it)
{
StringBuffer name;
it->getName(name);
StringBuffer childPath(path);
childPath.append(PATHSEPCHAR);
childPath.append(name);
deleteRecursive(childPath.str());
}
}
pDir->remove();
}
}
void copyDirectoryRecursive(const char *source, const char *target)
{
bool first = true;
Owned dir = createDirectoryIterator(source, "*");
ForEach (*dir)
{
IFile &sourceFile = dir->query();
if (sourceFile.isFile())
{
StringBuffer targetname(target);
targetname.append(PATHSEPCHAR);
dir->getName(targetname);
OwnedIFile destFile = createIFile(targetname.str());
if (first)
{
if (!recursiveCreateDirectory(target))
throw MakeStringException(-1,"Cannot create directory %s",target);
first = false;
}
copyFile(destFile, &sourceFile);
}
else if (sourceFile.isDirectory())
{
StringBuffer newSource(source);
StringBuffer newTarget(target);
newSource.append(PATHSEPCHAR);
newTarget.append(PATHSEPCHAR);
dir->getName(newSource);
dir->getName(newTarget);
copyDirectoryRecursive(newSource.str(), newTarget.str());
}
}
}
//returns temp path that ends with path sep
//
#ifdef _WIN32
extern DWORD getLastError() { return ::GetLastError(); }
void getTempPath(char* tempPath, unsigned int bufsize, const char* subdir/*=NULL*/)
{
::GetTempPath(bufsize, tempPath);
::GetLongPathName(tempPath, tempPath, bufsize);
if (subdir && *subdir)
{
const int len = strlen(tempPath);
char* p = tempPath + len;
strcpy(p, subdir);
p += strlen(subdir);
*p++ = '\\';
*p = '\0';
}
}
#else//Linux specifics follow
extern DWORD getLastError() { return errno; }
void getTempPath(char* tempPath, unsigned int bufsize, const char* subdir/*=NULL*/)
{
assert(bufsize > 5);
strcpy(tempPath, "/tmp/");
if (subdir && *subdir)
{
strcat(tempPath, subdir);
strcat(tempPath, "/");
}
}
#endif
void replaceDotWithHostIp(IPropertyTree* pTree, bool verbose)
{
StringBuffer ip;
queryHostIP().getIpText(ip);
const char* attrs[] = {"@netAddress", "@roxieAddress", "@daliAddress"};
StringBuffer xPath;
for (int i = 0; i < sizeof(attrs)/sizeof(char*); i++)
{
xPath.clear().appendf(".//*[%s]", attrs[i]);
Owned iter = pTree->getElements(xPath.str());
ForEach(*iter)
{
IPropertyTree* pComponent = &iter->query();
Owned iAttr = pComponent->getAttributes();
ForEach(*iAttr)
{
const char* attrName = iAttr->queryName();
if (!strcmp(attrName, attrs[i]))
{
String sAttrVal(iAttr->queryValue());
String dot(".");
if (sAttrVal.equals(dot) || sAttrVal.indexOf(".:") == 0 || sAttrVal.indexOf("http://.:") == 0)
{
StringBuffer sb(sAttrVal);
if (sAttrVal.equals(dot))
sb.replaceString(".", ip.str());
else
{
ip.append(":");
sb.replaceString(".:", ip.str());
ip.remove(ip.length() - 1, 1);
}
pComponent->setProp(attrName, sb.str());
if (verbose)
fprintf(stdout, "Replacing '.' with host ip '%s' for component/attribute '%s'[@'%s']\n", ip.str(), pComponent->queryName(), attrName);
}
}
}
}
}
}
int processRequest(const char* in_cfgname, const char* out_dirname, const char* in_dirname,
const char* compName, const char* compType, const char* in_filename,
const char* out_filename, bool generateOutput, const char* ipAddr,
bool listComps, bool verbose, bool listallComps, bool listdirs,
bool listdropzones, bool listcommondirs, bool listMachines, bool validateOnly,
bool listldaps, bool ldapconfig)
{
Owned pEnv = createPTreeFromXMLFile(in_cfgname);
short nodeIndex = 1;
short index = 1;
short compTypeIndex = 0;
short buildSetIndex = 0;
StringBuffer lastCompAdded;
StringBuffer xPath("*");
CConfigEngCallback callback(verbose);
Owned iter = pEnv->getElements(xPath.str());
Owned m_pConstEnvironment;
Owned m_pEnvironment;
replaceDotWithHostIp(pEnv, verbose);
StringBuffer envXML;
toXML(pEnv, envXML);
Owned factory = getEnvironmentFactory();
m_pEnvironment.setown(factory->loadLocalEnvironment(envXML));
m_pConstEnvironment.set(m_pEnvironment);
if (validateOnly)
{
char tempdir[_MAX_PATH];
StringBuffer sb;
while(true)
{
sb.clear().appendf("%d", msTick());
getTempPath(tempdir, sizeof(tempdir), sb.str());
if (!checkDirExists(tempdir))
{
if (recursiveCreateDirectory(tempdir))
break;
}
}
try
{
Owned m_configGenMgr;
CConfigEngCallback callback(verbose, true);
m_configGenMgr.setown(createConfigGenMgr(*m_pConstEnvironment, callback, NULL, in_dirname?in_dirname:"", tempdir, NULL, NULL, NULL));
m_configGenMgr->deploy(DEFLAGS_CONFIGFILES, DEBACKUP_NONE, false, false);
deleteRecursive(tempdir);
}
catch(IException* e)
{
deleteRecursive(tempdir);
throw e;
}
}
else if (ldapconfig)
{
char tempdir[_MAX_PATH];
StringBuffer sb;
while(true)
{
sb.clear().appendf("%d", msTick());
getTempPath(tempdir, sizeof(tempdir), sb.str());
if (!checkDirExists(tempdir))
{
if (recursiveCreateDirectory(tempdir))
break;
}
}
StringBuffer out;
xPath.clear().append(XML_TAG_SOFTWARE"/"XML_TAG_LDAPSERVERPROCESS);
Owned ldaps = pEnv->getElements(xPath.str());
Owned pSelComps(createPTree("SelectedComponents"));
bool flag = false;
ForEach(*ldaps)
{
IPropertyTree* ldap = &ldaps->query();
IPropertyTree* inst;
int count = 1;
xPath.clear().appendf(XML_TAG_INSTANCE"[%d]", count);
while ((inst = ldap->queryPropTree(xPath.str())) != NULL)
{
if (ipAddr && *ipAddr && strcmp(ipAddr, inst->queryProp(XML_ATTR_NETADDRESS)))
{
ldap->removeTree(inst);
continue;
}
if (!flag)
{
inst->addProp(XML_ATTR_DIRECTORY, ".");
sb.clear().append(tempdir).append(PATHSEPCHAR).append(ldap->queryProp(XML_ATTR_NAME));
xPath.clear().appendf(XML_TAG_INSTANCE"[%d]", ++count);
flag = true;
}
else
{
ldap->removeTree(inst);
}
}
if (flag)
{
pSelComps->addPropTree(XML_TAG_LDAPSERVERPROCESS, createPTreeFromIPT(ldap));
break;
}
}
if (flag)
{
try
{
toXML(pEnv, envXML.clear());
m_pEnvironment.setown(factory->loadLocalEnvironment(envXML));
m_pConstEnvironment.set(m_pEnvironment);
Owned m_configGenMgr;
m_configGenMgr.setown(createConfigGenMgr(*m_pConstEnvironment, callback, pSelComps, in_dirname?in_dirname:"", tempdir, compName, compType, ipAddr));
m_configGenMgr->deploy(DEFLAGS_CONFIGFILES, DEBACKUP_NONE, false, false);
copyDirectoryRecursive(sb.str(), out_dirname);
deleteRecursive(tempdir);
}
catch (IException* e)
{
deleteRecursive(tempdir);
throw e;
}
}
}
else if (!listComps && !listallComps && !listdirs && !listdropzones && !listcommondirs && !listMachines
&& !listldaps)
{
Owned m_configGenMgr;
m_configGenMgr.setown(createConfigGenMgr(*m_pConstEnvironment, callback, NULL, in_dirname?in_dirname:"", out_dirname?out_dirname:"", compName, compType, ipAddr));
m_configGenMgr->deploy(DEFLAGS_CONFIGFILES, DEBACKUP_NONE, false, false);
}
else if (listldaps)
{
StringBuffer out;
xPath.appendf("Software/%s/", XML_TAG_LDAPSERVERPROCESS);
Owned ldaps = pEnv->getElements(xPath.str());
ForEach(*ldaps)
{
IPropertyTree* ldap = &ldaps->query();
Owned insts = ldap->getElements(XML_TAG_INSTANCE);
ForEach(*insts)
{
IPropertyTree* inst = &insts->query();
StringBuffer computerName(inst->queryProp(XML_ATTR_COMPUTER));
xPath.clear().appendf("Hardware/Computer[@name=\"%s\"]", computerName.str());
IPropertyTree* pComputer = pEnv->queryPropTree(xPath.str());
if (pComputer)
{
const char* netAddr = pComputer->queryProp("@netAddress");
out.appendf("%s,%s\n", ldap->queryProp(XML_ATTR_NAME), netAddr);
}
}
}
fprintf(stdout, "%s", out.str());
}
else if (listdirs || listdropzones)
{
StringBuffer out;
xPath.clear().appendf("Software/%s", XML_TAG_DROPZONE);
Owned dropZonesInsts = pEnv->getElements(xPath.str());
ForEach(*dropZonesInsts)
{
IPropertyTree* pDropZone = &dropZonesInsts->query();
StringBuffer computerName(pDropZone->queryProp(XML_ATTR_COMPUTER));
xPath.clear().appendf("Hardware/Computer[@name=\"%s\"]", computerName.str());
IPropertyTree* pComputer = pEnv->queryPropTree(xPath.str());
if (pComputer)
{
const char* netAddr = pComputer->queryProp("@netAddress");
if (listdropzones)
out.appendf("%s,%s\n", netAddr, pDropZone->queryProp(XML_ATTR_DIRECTORY));
else if (matchDeployAddress(ipAddr, netAddr))
out.appendf("%s\n", pDropZone->queryProp(XML_ATTR_DIRECTORY));
}
}
fprintf(stdout, "%s", out.str());
}
else if (listcommondirs)
{
StringBuffer out;
StringBuffer name;
xPath.clear().appendf("Software/Directories/@name");
name.append(pEnv->queryProp(xPath.str()));
xPath.clear().appendf("Software/Directories/Category");
Owned dirInsts = pEnv->getElements(xPath.str());
ForEach(*dirInsts)
{
IPropertyTree* pDir = &dirInsts->query();
StringBuffer dirName(pDir->queryProp("@dir"));
int len = strrchr(dirName.str(), '/') - dirName.str();
dirName.setLength(len);
if (strstr(dirName.str(), "/[INST]") || strstr(dirName.str(), "/[COMPONENT]"))
continue;
dirName.replaceString("[NAME]", name.str());
out.appendf("%s=%s\n", pDir->queryProp(XML_ATTR_NAME), dirName.str());
}
fprintf(stdout, "%s", out.str());
}
else if (listMachines)
{
StringBuffer out;
Owned computers = pEnv->getElements("Hardware/Computer");
ForEach(*computers)
{
IPropertyTree* pComputer = &computers->query();
const char *netAddress = pComputer->queryProp("@netAddress");
StringBuffer xpath;
const char* name = pComputer->queryProp(XML_ATTR_NAME);
bool isHPCCNode = false, isSqlOrLdap = false;
xpath.clear().appendf(XML_TAG_SOFTWARE"/*[//"XML_ATTR_COMPUTER"='%s']", name);
Owned it = pEnv->getElements(xpath.str());
ForEach(*it)
{
IPropertyTree* pComponent = &it->query();
if (!strcmp(pComponent->queryName(), "MySQLProcess") ||
!strcmp(pComponent->queryName(), "LDAPServerProcess"))
isSqlOrLdap = true;
else
{
isHPCCNode = true;
break;
}
}
if (!isHPCCNode && isSqlOrLdap)
continue;
out.appendf("%s,", netAddress ? netAddress : "");
const char *computerType = pComputer->queryProp("@computerType");
if (computerType)
{
xpath.clear().appendf("Hardware/ComputerType[@name='%s']", computerType);
IPropertyTree *pType = pEnv->queryPropTree(xpath.str());
out.appendf("%s", pType->queryProp("@opSys"));
}
out.newline();
}
fprintf(stdout, "%s", out.str());
}
else
{
StringBuffer out;
Owned pSelectedComponents = getInstances(&m_pConstEnvironment->getPTree(), compName, compType, ipAddr, true);
Owned it = pSelectedComponents->getElements("*");
ForEach(*it)
{
IPropertyTree* pComponent = &it->query();
if (listComps)
{
if (!strcmp(pComponent->queryProp("@buildSet"), "roxie") || !strcmp(pComponent->queryProp("@buildSet"), "thor"))
{
StringBuffer sbChildren;
bool isMaster = false;
Owned itInst = pComponent->getElements("*");
ForEach(*itInst)
{
IPropertyTree* pInst = &itInst->query();
String instName(pInst->queryName());
if (!strcmp(instName.toCharArray(), "ThorMasterProcess") || instName.startsWith("RoxieServerProcess"))
{
isMaster = true;
out.appendf("%s=%s;%s%c%s;%s\n", pComponent->queryProp("@name"), pComponent->queryProp("@buildSet"), out_dirname, PATHSEPCHAR, pComponent->queryProp("@name"),"master");
}
else if (!strcmp(instName.toCharArray(), "RoxieSlaveProcess"))
sbChildren.appendf("%s=%s;%s%c%s;%s\n", pComponent->queryProp("@name"), pComponent->queryProp("@buildSet"), out_dirname, PATHSEPCHAR, pComponent->queryProp("@name"),"slave");
}
if (!isMaster)
out.append(sbChildren);
}
else
out.appendf("%s=%s;%s%c%s\n", pComponent->queryProp("@name"), pComponent->queryProp("@buildSet"), out_dirname, PATHSEPCHAR, pComponent->queryProp("@name"));
}
else if (listallComps)
{
StringBuffer netAddr;
StringBuffer port;
StringBuffer processName(pComponent->queryName());
bool multiInstances = false;
if(!strcmp(processName.str(), "ThorCluster") || !strcmp(processName.str(), "RoxieCluster"))
{
processName.clear();
multiInstances = true;
}
if (pComponent->numChildren())
{
Owned itComp = pComponent->getElements("*");
ForEach(*itComp)
{
IPropertyTree* pInst = &itComp->query();
if (!strcmp(pInst->queryName(), "ThorSlaveProcess") || !strcmp(pInst->queryName(), "ThorSpareProcess"))
continue;
netAddr.clear().append(pInst->queryProp("@netAddress"));
port.clear().append(pInst->queryProp("@port"));
if (multiInstances)
processName.clear().append(pInst->queryName());
out.appendf("%s,%s,%s,%s,%s%c%s,%s\n", processName.str(),
pComponent->queryProp("@name"), netAddr.str(), port.str(),
STANDARD_OUTDIR, PATHSEPCHAR, pComponent->queryProp("@name"), pComponent->queryProp("@logDir"));
}
}
else
{
netAddr.clear().append(pComponent->queryProp("@netAddress"));
port.clear().append(pComponent->queryProp("@port"));
out.appendf("%s,%s,%s,%s,%s%c%s,%s\n", pComponent->queryName(),
pComponent->queryProp("@name"), netAddr.str(), port.str(),
STANDARD_OUTDIR, PATHSEPCHAR, pComponent->queryProp("@name"), pComponent->queryProp("@logDir"));
}
}
}
fprintf(stdout, "%s", out.str());
}
return 0;
}
int main(int argc, char** argv)
{
InitModuleObjects();
Owned globals = createProperties(true);
const char* in_filename = NULL;
const char* in_cfgname = NULL;
const char* out_dirname = STANDARD_OUTDIR;
const char* in_dirname = STANDARD_INDIR;
const char* out_filename = NULL;
const char* compName = NULL;
const char* compType = NULL;
StringBuffer ipAddr = NULL;
bool generateOutput = true;
bool listComps = false;
bool verbose = false;
bool listallComps = false;
bool listdirs = false;
bool listdropzones = false;
bool listcommondirs = false;
bool listMachines = false;
bool validateOnly = false;
bool ldapconfig = false;
bool listldaps = false;
int i = 1;
bool writeToFiles = false;
int port = 80;
while(ierrorCode(), excpt->errorMessage(errMsg).str());
releaseAtoms();
excpt->Release();
return 1;
}
catch(...)
{
fprintf(stderr, "Unknown exception\n");
releaseAtoms();
return 1;
}
releaseAtoms();
return 0;
}