ws_packageprocessService.cpp 49 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295
  1. /*##############################################################################
  2. HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems®.
  3. Licensed under the Apache License, Version 2.0 (the "License");
  4. you may not use this file except in compliance with the License.
  5. You may obtain a copy of the License at
  6. http://www.apache.org/licenses/LICENSE-2.0
  7. Unless required by applicable law or agreed to in writing, software
  8. distributed under the License is distributed on an "AS IS" BASIS,
  9. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  10. See the License for the specific language governing permissions and
  11. limitations under the License.
  12. ############################################################################## */
  13. #pragma warning (disable : 4786)
  14. #include "ws_packageprocessService.hpp"
  15. #include "daclient.hpp"
  16. #include "dalienv.hpp"
  17. #include "dadfs.hpp"
  18. #include "dfuutil.hpp"
  19. #include "ws_fs.hpp"
  20. #include "ws_workunits.hpp"
  21. #include "packageprocess_errors.h"
  22. #include "referencedfilelist.hpp"
  23. #include "package.h"
  24. #define SDS_LOCK_TIMEOUT (5*60*1000) // 5mins, 30s a bit short
  25. void CWsPackageProcessEx::init(IPropertyTree *cfg, const char *process, const char *service)
  26. {
  27. packageMapAndSet.subscribe();
  28. }
  29. bool CWsPackageProcessEx::onEcho(IEspContext &context, IEspEchoRequest &req, IEspEchoResponse &resp)
  30. {
  31. StringBuffer respMsg;
  32. ISecUser* user = context.queryUser();
  33. if(user != NULL)
  34. {
  35. const char* name = user->getName();
  36. if (name && *name)
  37. respMsg.appendf("%s: ", name);
  38. }
  39. const char* reqMsg = req.getRequest();
  40. if (reqMsg && *reqMsg)
  41. respMsg.append(reqMsg);
  42. else
  43. respMsg.append("??");
  44. resp.setResponse(respMsg.str());
  45. return true;
  46. }
  47. inline StringBuffer &buildPkgSetId(StringBuffer &id, const char *process)
  48. {
  49. if (!process || !*process)
  50. process = "*";
  51. return id.append("default_").append(process).replace('*', '#').replace('?', '~');
  52. }
  53. IPropertyTree *getPkgSetRegistry(const char *process, bool readonly)
  54. {
  55. Owned<IRemoteConnection> globalLock = querySDS().connect("/PackageSets/", myProcessSession(), RTM_LOCK_WRITE|RTM_CREATE_QUERY, SDS_LOCK_TIMEOUT);
  56. if (!globalLock)
  57. throw MakeStringException(PKG_DALI_LOOKUP_ERROR, "Unable to connect to PackageSet information in dali /PackageSets");
  58. IPropertyTree *pkgSets = globalLock->queryRoot();
  59. if (!pkgSets)
  60. throw MakeStringException(PKG_DALI_LOOKUP_ERROR, "Unable to open PackageSet information in dali /PackageSets");
  61. if (!process || !*process)
  62. process = "*";
  63. StringBuffer id;
  64. buildPkgSetId(id, process);
  65. //Only lock the branch for the target we're interested in.
  66. VStringBuffer xpath("/PackageSets/PackageSet[@id='%s']", id.str());
  67. Owned<IRemoteConnection> conn = querySDS().connect(xpath.str(), myProcessSession(), readonly ? RTM_LOCK_READ : RTM_LOCK_WRITE, SDS_LOCK_TIMEOUT);
  68. if (!conn)
  69. {
  70. if (readonly)
  71. return NULL;
  72. Owned<IPropertyTree> pkgSet = createPTree();
  73. pkgSet->setProp("@id", id.str());
  74. pkgSet->setProp("@process", process);
  75. pkgSets->addPropTree("PackageSet", pkgSet.getClear());
  76. globalLock->commit();
  77. conn.setown(querySDS().connect(xpath.str(), myProcessSession(), RTM_LOCK_WRITE, SDS_LOCK_TIMEOUT));
  78. }
  79. return (conn) ? conn->getRoot() : NULL;
  80. }
  81. ////////////////////////////////////////////////////////////////////////////////////////
  82. const unsigned roxieQueryRoxieTimeOut = 60000;
  83. #define SDS_LOCK_TIMEOUT (5*60*1000) // 5mins, 30s a bit short
  84. bool isFileKnownOnCluster(const char *logicalname, IConstWUClusterInfo *clusterInfo, IUserDescriptor* userdesc)
  85. {
  86. Owned<IDistributedFile> dst = queryDistributedFileDirectory().lookup(logicalname, userdesc, true);
  87. if (dst)
  88. {
  89. SCMStringBuffer processName;
  90. clusterInfo->getRoxieProcess(processName);
  91. if (dst->findCluster(processName.str()) != NotFound)
  92. return true; // file already known for this cluster
  93. }
  94. return false;
  95. }
  96. bool isFileKnownOnCluster(const char *logicalname, const char *target, IUserDescriptor* userdesc)
  97. {
  98. Owned<IConstWUClusterInfo> clusterInfo = getTargetClusterInfo(target);
  99. if (!clusterInfo)
  100. throw MakeStringException(PKG_TARGET_NOT_DEFINED, "Could not find information about target cluster %s ", target);
  101. return isFileKnownOnCluster(logicalname, clusterInfo, userdesc);
  102. }
  103. void cloneFileInfoToDali(unsigned updateFlags, StringArray &notFound, IPropertyTree *packageMap, const char *lookupDaliIp, IConstWUClusterInfo *dstInfo, const char *srcCluster, const char *remotePrefix, IUserDescriptor* userdesc, bool allowForeignFiles)
  104. {
  105. StringBuffer user;
  106. StringBuffer password;
  107. if (userdesc)
  108. {
  109. userdesc->getUserName(user);
  110. userdesc->getPassword(password);
  111. }
  112. Owned<IReferencedFileList> wufiles = createReferencedFileList(user, password, allowForeignFiles, false);
  113. wufiles->addFilesFromPackageMap(packageMap);
  114. SCMStringBuffer processName;
  115. dstInfo->getRoxieProcess(processName);
  116. wufiles->resolveFiles(processName.str(), lookupDaliIp, remotePrefix, srcCluster, !(updateFlags & (DALI_UPDATEF_REPLACE_FILE | DALI_UPDATEF_CLONE_FROM)), false, false);
  117. StringBuffer defReplicateFolder;
  118. getConfigurationDirectory(NULL, "data2", "roxie", processName.str(), defReplicateFolder);
  119. Owned<IDFUhelper> helper = createIDFUhelper();
  120. wufiles->cloneAllInfo(updateFlags, helper, true, false, dstInfo->getRoxieRedundancy(), dstInfo->getChannelsPerNode(), dstInfo->getRoxieReplicateOffset(), defReplicateFolder);
  121. Owned<IReferencedFileIterator> iter = wufiles->getFiles();
  122. ForEach(*iter)
  123. {
  124. IReferencedFile &item = iter->query();
  125. if (item.getFlags() & (RefFileNotFound | RefFileNotOnSource))
  126. notFound.append(item.getLogicalName());
  127. }
  128. }
  129. void cloneFileInfoToDali(unsigned updateFlags, StringArray &notFound, IPropertyTree *packageMap, const char *lookupDaliIp, const char *dstCluster, const char *srcCluster, const char *prefix, IUserDescriptor* userdesc, bool allowForeignFiles)
  130. {
  131. Owned<IConstWUClusterInfo> clusterInfo = getTargetClusterInfo(dstCluster);
  132. if (!clusterInfo)
  133. throw MakeStringException(PKG_TARGET_NOT_DEFINED, "Could not find information about target cluster %s ", dstCluster);
  134. cloneFileInfoToDali(updateFlags, notFound, packageMap, lookupDaliIp, clusterInfo, srcCluster, prefix, userdesc, allowForeignFiles);
  135. }
  136. void makePackageActive(IPropertyTree *pkgSet, IPropertyTree *psEntryNew, const char *target, bool activate)
  137. {
  138. if (activate)
  139. {
  140. VStringBuffer xpath("PackageMap[@querySet='%s'][@active='1']", target);
  141. Owned<IPropertyTreeIterator> psEntries = pkgSet->getElements(xpath.str());
  142. ForEach(*psEntries)
  143. {
  144. IPropertyTree &entry = psEntries->query();
  145. if (psEntryNew != &entry)
  146. entry.setPropBool("@active", false);
  147. }
  148. }
  149. if (psEntryNew->getPropBool("@active") != activate)
  150. psEntryNew->setPropBool("@active", activate);
  151. }
  152. //////////////////////////////////////////////////////////
  153. #define PKGADD_DFS_OVERWRITE 0x0001
  154. #define PKGADD_ALLOW_FOREIGN 0x0002
  155. #define PKGADD_PRELOAD_ALL 0x0004
  156. #define PKGADD_MAP_ACTIVATE 0x0100
  157. #define PKGADD_MAP_CREATE 0x0200
  158. #define PKGADD_MAP_REPLACE 0x0400
  159. #define PKGADD_SEG_ADD 0x1000
  160. #define PKGADD_SEG_REPLACE 0x2000
  161. class PackageMapUpdater
  162. {
  163. public:
  164. Owned<IRemoteConnection> globalLock;
  165. Owned<IUserDescriptor> userdesc;
  166. Owned<IConstWUClusterInfo> clusterInfo;
  167. Owned<IPropertyTree> pmPart;
  168. IPropertyTree *packageMaps;
  169. IPropertyTree *pmExisting;
  170. StringBuffer daliIP;
  171. StringBuffer srcCluster;
  172. StringBuffer prefix;
  173. StringBuffer pmid;
  174. StringBuffer pkgSetId;
  175. StringAttr process;
  176. StringAttr target;
  177. unsigned flags;
  178. PackageMapUpdater() : flags(0), packageMaps(NULL), pmExisting(NULL){}
  179. inline bool checkFlag(unsigned check)
  180. {
  181. return (flags & check) != 0;
  182. }
  183. inline void setFlag(unsigned flag, bool on=true)
  184. {
  185. if (on)
  186. flags |= flag;
  187. }
  188. IConstWUClusterInfo *ensureClusterInfo()
  189. {
  190. clusterInfo.setown(getTargetClusterInfo(target));
  191. if (!clusterInfo)
  192. throw MakeStringException(PKG_TARGET_NOT_DEFINED, "Could not find information about target cluster %s ", target.str());
  193. return clusterInfo;
  194. }
  195. void setPMID(const char *_target, const char *name, bool globalScope)
  196. {
  197. if (!name || !*name)
  198. throw MakeStringExceptionDirect(PKG_MISSING_PARAM, "PackageMap name parameter required");
  199. if (!globalScope)
  200. {
  201. target.set(_target);
  202. if (target.isEmpty())
  203. throw MakeStringExceptionDirect(PKG_MISSING_PARAM, "Target cluster parameter required");
  204. ensureClusterInfo();
  205. pmid.append(target).append("::");
  206. }
  207. pmid.append(name);
  208. pmid.toLowerCase();
  209. }
  210. void setProcess(const char *name)
  211. {
  212. process.set(name);
  213. buildPkgSetId(pkgSetId, process);
  214. }
  215. void setUser(const char *user, const char *password)
  216. {
  217. if (user && *user && *password && *password)
  218. {
  219. userdesc.setown(createUserDescriptor());
  220. userdesc->set(user, password);
  221. }
  222. }
  223. void setDerivedDfsLocation(const char *dfsLocation, const char *srcProcess)
  224. {
  225. splitDerivedDfsLocation(dfsLocation, srcCluster, daliIP, prefix, srcProcess, srcProcess, NULL, NULL);
  226. if (srcCluster.length())
  227. {
  228. if (!isProcessCluster(daliIP, srcCluster))
  229. throw MakeStringException(PKG_INVALID_CLUSTER_TYPE, "Process cluster %s not found on %s DALI", srcCluster.str(), daliIP.length() ? daliIP.str() : "local");
  230. }
  231. }
  232. void convertExisting()
  233. {
  234. Linked<IPropertyTree> pmPart = pmExisting;
  235. const char *s = strstr(pmid.str(), "::");
  236. if (s)
  237. pmPart->addProp("@id", s+2);
  238. packageMaps->removeTree(pmExisting);
  239. Owned<IPropertyTree> pmTree = createPTree("PackageMap", ipt_ordered);
  240. pmTree->setProp("@id", pmid);
  241. pmTree->setPropBool("@multipart", true);
  242. pmTree->addPropTree("Part", pmPart.getClear());
  243. pmExisting = packageMaps->addPropTree("PackageMap", pmTree.getClear());
  244. }
  245. void init()
  246. {
  247. VStringBuffer xpath("PackageMap[@id='%s']", pmid.str());
  248. globalLock.setown(querySDS().connect("/PackageMaps", myProcessSession(), RTM_LOCK_WRITE|RTM_CREATE_QUERY, SDS_LOCK_TIMEOUT));
  249. packageMaps = globalLock->queryRoot();
  250. pmExisting = packageMaps->queryPropTree(xpath);
  251. if (pmExisting && !pmExisting->getPropBool("@multipart", false))
  252. convertExisting();
  253. }
  254. void createPart(const char *partname, const char *xml)
  255. {
  256. if (!partname || !*partname)
  257. throw MakeStringExceptionDirect(PKG_INFO_NOT_DEFINED, "No PackageMap Part name provided");
  258. if (!xml || !*xml)
  259. throw MakeStringExceptionDirect(PKG_INFO_NOT_DEFINED, "No PackageMap content provided");
  260. pmPart.setown(createPTreeFromXMLString(xml));
  261. if (!pmPart)
  262. throw MakeStringExceptionDirect(PKG_INFO_NOT_DEFINED, "Invalid PackageMap content");
  263. pmPart->addProp("@id", partname);
  264. StringBuffer lcPmid(pmid);
  265. pmid = lcPmid.toLowerCase().str();
  266. Owned<IPropertyTreeIterator> iter = pmPart->getElements("Package");
  267. ForEach(*iter)
  268. {
  269. IPropertyTree &item = iter->query();
  270. if (checkFlag(PKGADD_PRELOAD_ALL))
  271. item.setPropBool("@preload", true);
  272. Owned<IPropertyTreeIterator> superFiles = item.getElements("SuperFile");
  273. ForEach(*superFiles)
  274. {
  275. IPropertyTree &superFile = superFiles->query();
  276. StringBuffer lc(superFile.queryProp("@id"));
  277. const char *id = lc.toLowerCase().str();
  278. if (*id == '~')
  279. id++;
  280. superFile.setProp("@id", id);
  281. Owned<IPropertyTreeIterator> subFiles = superFile.getElements("SubFile");
  282. ForEach(*subFiles)
  283. {
  284. IPropertyTree &subFile = subFiles->query();
  285. id = subFile.queryProp("@value");
  286. if (id && *id == '~')
  287. {
  288. StringAttr value(id+1);
  289. subFile.setProp("@value", value.get());
  290. }
  291. }
  292. }
  293. }
  294. }
  295. void cloneDfsInfo(unsigned updateFlags, StringArray &filesNotFound)
  296. {
  297. cloneFileInfoToDali(updateFlags, filesNotFound, pmPart, daliIP, ensureClusterInfo(), srcCluster, prefix, userdesc, checkFlag(PKGADD_ALLOW_FOREIGN));
  298. }
  299. void doCreate(const char *partname, const char *xml, unsigned updateFlags, StringArray &filesNotFound)
  300. {
  301. createPart(partname, xml);
  302. if (pmExisting)
  303. {
  304. if (!checkFlag(PKGADD_MAP_REPLACE))
  305. throw MakeStringException(PKG_NAME_EXISTS, "PackageMap %s already exists, either delete it or specify overwrite", pmid.str());
  306. }
  307. cloneDfsInfo(updateFlags, filesNotFound);
  308. if (pmExisting)
  309. packageMaps->removeTree(pmExisting);
  310. Owned<IPropertyTree> pmTree = createPTree("PackageMap", ipt_ordered);
  311. pmTree->setProp("@id", pmid);
  312. pmTree->setPropBool("@multipart", true);
  313. pmTree->addPropTree("Part", pmPart.getClear());
  314. packageMaps->addPropTree("PackageMap", pmTree.getClear());
  315. VStringBuffer xpath("PackageMap[@id='%s'][@querySet='%s']", pmid.str(), target.get());
  316. Owned<IPropertyTree> pkgSet = getPkgSetRegistry(process, false);
  317. IPropertyTree *psEntry = pkgSet->queryPropTree(xpath);
  318. if (!psEntry)
  319. {
  320. psEntry = pkgSet->addPropTree("PackageMap", createPTree("PackageMap"));
  321. psEntry->setProp("@id", pmid);
  322. psEntry->setProp("@querySet", target);
  323. }
  324. makePackageActive(pkgSet, psEntry, target, checkFlag(PKGADD_MAP_ACTIVATE));
  325. }
  326. void create(const char *partname, const char *xml, unsigned updateFlags, StringArray &filesNotFound)
  327. {
  328. init();
  329. doCreate(partname, xml, updateFlags, filesNotFound);
  330. }
  331. void addPart(const char *partname, const char *xml, unsigned updateFlags, StringArray &filesNotFound)
  332. {
  333. init();
  334. if (!pmExisting)
  335. {
  336. doCreate(partname, xml, updateFlags, filesNotFound);
  337. return;
  338. }
  339. createPart(partname, xml);
  340. VStringBuffer xpath("Part[@id='%s']", partname);
  341. IPropertyTree *existingPart = pmExisting->queryPropTree(xpath);
  342. if (existingPart && !checkFlag(PKGADD_SEG_REPLACE))
  343. throw MakeStringException(PKG_NAME_EXISTS, "Package Part %s already exists, remove, or specify 'delete previous'", partname);
  344. cloneDfsInfo(updateFlags, filesNotFound);
  345. if (existingPart)
  346. pmExisting->removeTree(existingPart);
  347. pmExisting->addPropTree("Part", pmPart.getClear());
  348. }
  349. IPropertyTree *ensurePart(const char *partname)
  350. {
  351. if (!pmExisting)
  352. throw MakeStringException(PKG_NAME_EXISTS, "PackageMap %s not found", pmid.str());
  353. VStringBuffer xpath("Part[@id='%s']", partname);
  354. IPropertyTree *existingPart = pmExisting->queryPropTree(xpath);
  355. if (!existingPart)
  356. throw MakeStringException(PKG_NAME_EXISTS, "PackageMap %s Part %s not found", pmid.str(), partname);
  357. return existingPart;
  358. }
  359. void removePart(const char *partname)
  360. {
  361. init();
  362. pmExisting->removeTree(ensurePart(partname));
  363. }
  364. StringBuffer &getPartContent(const char *partname, StringBuffer &content)
  365. {
  366. init();
  367. return toXML(ensurePart(partname), content);
  368. }
  369. };
  370. void getPackageListInfo(IPropertyTree *mapTree, IEspPackageListMapData *pkgList)
  371. {
  372. pkgList->setId(mapTree->queryProp("@id"));
  373. pkgList->setTarget(mapTree->queryProp("@querySet"));
  374. Owned<IPropertyTreeIterator> iter = mapTree->getElements("Package");
  375. IArrayOf<IConstPackageListData> results;
  376. ForEach(*iter)
  377. {
  378. IPropertyTree &item = iter->query();
  379. Owned<IEspPackageListData> res = createPackageListData("", "");
  380. res->setId(item.queryProp("@id"));
  381. if (item.hasProp("@queries"))
  382. res->setQueries(item.queryProp("@queries"));
  383. results.append(*res.getClear());
  384. }
  385. pkgList->setPkgListData(results);
  386. }
  387. void getAllPackageListInfo(IPropertyTree *mapTree, StringBuffer &info)
  388. {
  389. info.append("<PackageMap id='").append(mapTree->queryProp("@id")).append("'");
  390. Owned<IPropertyTreeIterator> iter = mapTree->getElements("Package");
  391. ForEach(*iter)
  392. {
  393. IPropertyTree &item = iter->query();
  394. info.append("<Package id='").append(item.queryProp("@id")).append("'");
  395. if (item.hasProp("@queries"))
  396. info.append(" queries='").append(item.queryProp("@queries")).append("'");
  397. info.append("></Package>");
  398. }
  399. info.append("</PackageMap>");
  400. }
  401. void listPkgInfo(double version, const char *target, const char *process, const IPropertyTree* packageMaps, IPropertyTree* pkgSetRegistry, IArrayOf<IConstPackageListMapData>* results)
  402. {
  403. StringBuffer xpath("PackageMap");
  404. if (target && *target)
  405. xpath.appendf("[@querySet='%s']", target);
  406. Owned<IPropertyTreeIterator> iter = pkgSetRegistry->getElements(xpath.str());
  407. ForEach(*iter)
  408. {
  409. IPropertyTree &item = iter->query();
  410. const char *id = item.queryProp("@id");
  411. if (!id || !*id)
  412. continue;
  413. StringBuffer xpath;
  414. xpath.append("PackageMap[@id='").append(id).append("']");
  415. IPropertyTree *mapTree = packageMaps->queryPropTree(xpath);
  416. if (!mapTree)
  417. continue;
  418. Owned<IEspPackageListMapData> res = createPackageListMapData("", "");
  419. res->setActive(item.getPropBool("@active"));
  420. if (process && *process && (version >= 1.01))
  421. res->setProcess(process);
  422. getPackageListInfo(mapTree, res);
  423. if (target && *target)
  424. res->setTarget(target);
  425. else
  426. res->setTarget(item.queryProp("@querySet"));
  427. results->append(*res.getClear());
  428. }
  429. }
  430. void listPkgInfo(double version, const char *target, const char *process, const IPropertyTree* packageMaps, IArrayOf<IConstPackageListMapData>* results)
  431. {
  432. Owned<IPropertyTree> pkgSetRegistry = getPkgSetRegistry((process && *process) ? process : "*", true);
  433. if (pkgSetRegistry) //will be NULL if no package map
  434. {
  435. listPkgInfo(version, target, process, packageMaps, pkgSetRegistry, results);
  436. }
  437. }
  438. void getPkgInfo(const IPropertyTree *packageMaps, const char *target, const char *process, StringBuffer &info)
  439. {
  440. Owned<IPropertyTree> tree = createPTree("PackageMaps");
  441. Owned<IPropertyTree> pkgSetRegistry = getPkgSetRegistry(process, true);
  442. if (!pkgSetRegistry)
  443. {
  444. toXML(tree, info);
  445. return;
  446. }
  447. StringBuffer xpath("PackageMap[@active='1']");
  448. if (target && *target)
  449. xpath.appendf("[@querySet='%s']", target);
  450. Owned<IPropertyTreeIterator> iter = pkgSetRegistry->getElements(xpath.str());
  451. ForEach(*iter)
  452. {
  453. IPropertyTree &item = iter->query();
  454. const char *id = item.queryProp("@id");
  455. if (id)
  456. {
  457. StringBuffer xpath;
  458. xpath.append("PackageMap[@id='").append(id).append("']");
  459. IPropertyTree *mapTree = packageMaps->queryPropTree(xpath);
  460. if (mapTree)
  461. mergePTree(tree, mapTree);
  462. }
  463. }
  464. toXML(tree, info);
  465. }
  466. bool deletePkgInfo(const char *name, const char *target, const char *process, bool globalScope)
  467. {
  468. Owned<IRemoteConnection> pkgSetsConn = querySDS().connect("/PackageSets/", myProcessSession(), RTM_LOCK_WRITE, SDS_LOCK_TIMEOUT);
  469. if (!pkgSetsConn)
  470. throw MakeStringException(PKG_NONE_DEFINED, "No package sets defined");
  471. IPropertyTree* packageSets = pkgSetsConn->queryRoot();
  472. StringBuffer pkgSetId;
  473. buildPkgSetId(pkgSetId, process);
  474. VStringBuffer pkgSet_xpath("PackageSet[@id='%s']", pkgSetId.str());
  475. IPropertyTree *pkgSetRegistry = packageSets->queryPropTree(pkgSet_xpath.str());
  476. if (!pkgSetRegistry)
  477. throw MakeStringException(PKG_TARGET_NOT_DEFINED, "No package sets defined for %s", process);
  478. StringBuffer lcTarget(target);
  479. target = lcTarget.toLowerCase().str();
  480. StringBuffer lcName(name);
  481. name = lcName.toLowerCase().str();
  482. Owned<IPropertyTree> mapEntry;
  483. StringBuffer xpath;
  484. if (!globalScope)
  485. {
  486. xpath.appendf("PackageMap[@id='%s::%s'][@querySet='%s']", target, name, target);
  487. mapEntry.setown(pkgSetRegistry->getPropTree(xpath.str()));
  488. }
  489. if (!mapEntry)
  490. {
  491. xpath.clear().appendf("PackageMap[@id='%s'][@querySet='%s']", name, target);
  492. mapEntry.setown(pkgSetRegistry->getPropTree(xpath.str()));
  493. if (!mapEntry)
  494. throw MakeStringException(PKG_DELETE_NOT_FOUND, "Unable to delete %s - information not found", lcName.str());
  495. }
  496. StringAttr pmid(mapEntry->queryProp("@id"));
  497. pkgSetRegistry->removeTree(mapEntry);
  498. xpath.clear().appendf("PackageSet/PackageMap[@id='%s']", pmid.get());
  499. if (!packageSets->hasProp(xpath))
  500. {
  501. Owned<IRemoteConnection> pkgMapsConn = querySDS().connect("/PackageMaps/", myProcessSession(), RTM_LOCK_WRITE, SDS_LOCK_TIMEOUT);
  502. if (!pkgMapsConn)
  503. throw MakeStringException(PKG_DALI_LOOKUP_ERROR, "Unable to retrieve PackageMaps information from dali [/PackageMaps]");
  504. IPropertyTree *pkgMaps = pkgMapsConn->queryRoot();
  505. if (!pkgMaps)
  506. throw MakeStringException(PKG_DALI_LOOKUP_ERROR, "Unable to retrieve PackageMaps information from dali [/PackageMaps]");
  507. IPropertyTree *mapTree = pkgMaps->queryPropTree(xpath.clear().appendf("PackageMap[@id='%s']", pmid.get()).str());
  508. if (mapTree)
  509. pkgMaps->removeTree(mapTree);
  510. }
  511. return true;
  512. }
  513. void activatePackageMapInfo(const char *target, const char *name, const char *process, bool globalScope, bool activate)
  514. {
  515. if (!target || !*target)
  516. throw MakeStringExceptionDirect(PKG_TARGET_NOT_DEFINED, "No target defined");
  517. if (!name || !*name)
  518. throw MakeStringExceptionDirect(PKG_MISSING_PARAM, "No pmid specified");
  519. Owned<IRemoteConnection> globalLock = querySDS().connect("PackageSets", myProcessSession(), RTM_LOCK_WRITE|RTM_CREATE_QUERY, SDS_LOCK_TIMEOUT);
  520. if (!globalLock)
  521. throw MakeStringException(PKG_DALI_LOOKUP_ERROR, "Unable to retrieve PackageSets information from dali /PackageSets");
  522. StringBuffer lcTarget(target);
  523. target = lcTarget.toLowerCase().str();
  524. StringBuffer lcName(name);
  525. name = lcName.toLowerCase().str();
  526. IPropertyTree *root = globalLock->queryRoot();
  527. if (!root)
  528. throw MakeStringException(PKG_ACTIVATE_NOT_FOUND, "Unable to retrieve PackageSet information");
  529. StringBuffer pkgSetId;
  530. buildPkgSetId(pkgSetId, process);
  531. VStringBuffer xpath("PackageSet[@id='%s']", pkgSetId.str());
  532. IPropertyTree *pkgSetTree = root->queryPropTree(xpath);
  533. if (pkgSetTree)
  534. {
  535. IPropertyTree *mapTree = NULL;
  536. if (!globalScope)
  537. {
  538. xpath.clear().appendf("PackageMap[@querySet='%s'][@id='%s::%s']", target, target, name);
  539. mapTree = pkgSetTree->queryPropTree(xpath);
  540. }
  541. if (!mapTree)
  542. {
  543. xpath.clear().appendf("PackageMap[@querySet='%s'][@id='%s']", target, name);
  544. mapTree = pkgSetTree->queryPropTree(xpath);
  545. }
  546. if (!mapTree)
  547. throw MakeStringException(PKG_ACTIVATE_NOT_FOUND, "PackageMap %s not found on target %s", name, target);
  548. makePackageActive(pkgSetTree, mapTree, target, activate);
  549. }
  550. }
  551. void PackageMapAndSet::load(const char* path, IPropertyTree* t)
  552. {
  553. Owned<IRemoteConnection> globalLock = querySDS().connect(path, myProcessSession(), RTM_LOCK_READ, SDS_LOCK_TIMEOUT);
  554. if (!globalLock)
  555. throw MakeStringException(PKG_DALI_LOOKUP_ERROR, "Unable to retrieve %s information from dali /%s", path, path);
  556. t->removeProp(path);
  557. IPropertyTree *root = globalLock->queryRoot();
  558. if (root)
  559. t->addPropTree(path, createPTreeFromIPT(root));
  560. }
  561. void PackageMapAndSet::load(unsigned flags)
  562. {
  563. Owned<IPropertyTree> t = createPTreeFromIPT(tree);
  564. if (flags & PMAS_RELOAD_PACKAGE_SET)
  565. load("PackageSets", t);
  566. if (flags & PMAS_RELOAD_PACKAGE_MAP)
  567. load("PackageMaps", t);
  568. tree.setown(t.getClear());
  569. }
  570. bool CWsPackageProcessEx::readPackageMapString(const char *packageMapString, StringBuffer &target, StringBuffer &process, StringBuffer &packageMap)
  571. {
  572. if (!packageMapString || !*packageMapString)
  573. return false;
  574. StringArray plist;
  575. plist.appendListUniq(packageMapString, ",");
  576. if (plist.length() < 3)
  577. return false;
  578. target.set(plist.item(0));
  579. process.set(plist.item(1));
  580. packageMap.set(plist.item(2));
  581. if (!target.length() || !packageMap.length())
  582. return false;
  583. return true;
  584. }
  585. void CWsPackageProcessEx::getPkgInfoById(const char *packageMapId, IPropertyTree* tree)
  586. {
  587. if (!packageMapId || !*packageMapId)
  588. return;
  589. Owned<IPropertyTree> packageMaps = packageMapAndSet.getPackageMaps();
  590. if (!packageMaps)
  591. throw MakeStringException(PKG_DALI_LOOKUP_ERROR, "Unable to retrieve information about package maps from dali server");
  592. StringBuffer lcPMID(packageMapId);
  593. packageMapId = lcPMID.toLowerCase().str();
  594. StringBuffer xpath;
  595. xpath.append("PackageMap[@id='").append(packageMapId).append("']");
  596. IPropertyTree *mapTree = packageMaps->queryPropTree(xpath);
  597. if (mapTree)
  598. mergePTree(tree, mapTree);
  599. }
  600. bool CWsPackageProcessEx::onAddPackage(IEspContext &context, IEspAddPackageRequest &req, IEspAddPackageResponse &resp)
  601. {
  602. PackageMapUpdater updater;
  603. updater.setFlag(PKGADD_MAP_CREATE);
  604. updater.setFlag(PKGADD_MAP_ACTIVATE, req.getActivate());
  605. updater.setFlag(PKGADD_MAP_REPLACE, req.getOverWrite());
  606. updater.setFlag(PKGADD_ALLOW_FOREIGN, req.getAllowForeignFiles());
  607. updater.setFlag(PKGADD_PRELOAD_ALL, req.getPreloadAllPackages());
  608. updater.setPMID(req.getTarget(), req.getPackageMap(), req.getGlobalScope());
  609. updater.setProcess(req.getProcess());
  610. updater.setUser(context.queryUserId(), context.queryPassword());
  611. updater.setDerivedDfsLocation(req.getDaliIp(), req.getSourceProcess());
  612. unsigned updateFlags = 0;
  613. if (req.getOverWrite())
  614. updateFlags |= (DALI_UPDATEF_PACKAGEMAP | DALI_UPDATEF_REPLACE_FILE | DALI_UPDATEF_CLONE_FROM | DALI_UPDATEF_SUPERFILES);
  615. if (req.getReplacePackageMap())
  616. updateFlags |= DALI_UPDATEF_PACKAGEMAP;
  617. if (req.getUpdateCloneFrom())
  618. updateFlags |= DALI_UPDATEF_CLONE_FROM;
  619. if (req.getUpdateSuperFiles())
  620. updateFlags |= DALI_UPDATEF_SUPERFILES;
  621. if (req.getAppendCluster())
  622. updateFlags |= DALI_UPDATEF_APPEND_CLUSTER;
  623. StringArray filesNotFound;
  624. updater.create(req.getPackageMap(), req.getInfo(), updateFlags, filesNotFound);
  625. resp.setFilesNotFound(filesNotFound);
  626. resp.updateStatus().setCode(0);
  627. resp.updateStatus().setDescription(StringBuffer("Successfully loaded ").append(req.getPackageMap()));
  628. return true;
  629. }
  630. void CWsPackageProcessEx::deletePackage(const char *packageMap, const char *target, const char *process, bool globalScope, StringBuffer &returnMsg, int &returnCode)
  631. {
  632. bool ret = deletePkgInfo(packageMap, target, process, globalScope);
  633. (ret) ? returnMsg.append("Successfully ") : returnMsg.append("Unsuccessfully ");
  634. returnMsg.append("deleted ").append(packageMap).append(" from ").append(target).append(";");
  635. if (!ret)
  636. returnCode = -1;
  637. return;
  638. }
  639. bool CWsPackageProcessEx::onDeletePackage(IEspContext &context, IEspDeletePackageRequest &req, IEspDeletePackageResponse &resp)
  640. {
  641. int returnCode = 0;
  642. StringBuffer returnMsg;
  643. IArrayOf<IConstPackageMapEntry>& packageMaps = req.getPackageMaps();
  644. ForEachItemIn(p, packageMaps)
  645. {
  646. IConstPackageMapEntry& item=packageMaps.item(p);
  647. if (!item.getId() || !*item.getId())
  648. {
  649. returnMsg.appendf("PackageMap[%d]: Package map Id not specified; ", p);
  650. continue;
  651. }
  652. if (!item.getTarget() || !*item.getTarget())
  653. {
  654. returnMsg.appendf("PackageMap[%d]: Target not specified;", p);
  655. continue;
  656. }
  657. StringBuffer target, processName, packageMap;
  658. packageMap.set(item.getId());
  659. target.set(item.getTarget());
  660. if (!item.getProcess() || !*item.getProcess())
  661. processName.set("*");
  662. else
  663. processName.set(item.getProcess());
  664. deletePackage(packageMap.str(), target.str(), processName.str(), req.getGlobalScope(), returnMsg, returnCode);
  665. }
  666. if (!packageMaps.length())
  667. {
  668. StringAttr pkgMap(req.getPackageMap());
  669. StringAttr processName(req.getProcess());
  670. if (!processName.length())
  671. processName.set("*");
  672. deletePackage(pkgMap.get(), req.getTarget(), processName.get(), req.getGlobalScope(), returnMsg, returnCode);
  673. }
  674. resp.updateStatus().setDescription(returnMsg.str());
  675. resp.updateStatus().setCode(returnCode);
  676. return true;
  677. }
  678. bool CWsPackageProcessEx::onActivatePackage(IEspContext &context, IEspActivatePackageRequest &req, IEspActivatePackageResponse &resp)
  679. {
  680. resp.updateStatus().setCode(0);
  681. activatePackageMapInfo(req.getTarget(), req.getPackageMap(), req.getProcess(), req.getGlobalScope(), true);
  682. return true;
  683. }
  684. bool CWsPackageProcessEx::onDeActivatePackage(IEspContext &context, IEspDeActivatePackageRequest &req, IEspDeActivatePackageResponse &resp)
  685. {
  686. resp.updateStatus().setCode(0);
  687. activatePackageMapInfo(req.getTarget(), req.getPackageMap(), req.getProcess(), req.getGlobalScope(), false);
  688. return true;
  689. }
  690. bool CWsPackageProcessEx::onListPackage(IEspContext &context, IEspListPackageRequest &req, IEspListPackageResponse &resp)
  691. {
  692. Owned<IPropertyTree> packageMaps = packageMapAndSet.getPackageMaps();
  693. if (!packageMaps)
  694. throw MakeStringException(PKG_DALI_LOOKUP_ERROR, "Unable to retrieve information about package maps from dali server");
  695. resp.updateStatus().setCode(0);
  696. IArrayOf<IConstPackageListMapData> results;
  697. StringAttr process(req.getProcess());
  698. listPkgInfo(context.getClientVersion(), req.getTarget(), process.length() ? process.get() : "*", packageMaps, &results);
  699. resp.setPkgListMapData(results);
  700. return true;
  701. }
  702. bool CWsPackageProcessEx::onListPackages(IEspContext &context, IEspListPackagesRequest &req, IEspListPackagesResponse &resp)
  703. {
  704. Owned<IPropertyTree> packageMaps = packageMapAndSet.getPackageMaps();
  705. if (!packageMaps)
  706. throw MakeStringException(PKG_DALI_LOOKUP_ERROR, "Unable to retrieve information about package maps from dali server");
  707. double version = context.getClientVersion();
  708. const char* targetReq = req.getTarget();
  709. const char* processReq = req.getProcess();
  710. const char* processFilterReq = req.getProcessFilter();
  711. IArrayOf<IConstPackageListMapData> results;
  712. if ((!processReq || !*processReq) && (processFilterReq && *processFilterReq))
  713. listPkgInfo(version, targetReq, processFilterReq, packageMaps, &results);
  714. else
  715. {
  716. Owned<IPropertyTree> pkgSetRegistryRoot = packageMapAndSet.getPackageSets();
  717. if (!pkgSetRegistryRoot)
  718. throw MakeStringException(PKG_DALI_LOOKUP_ERROR, "Unable to retrieve information about package sets from dali server");
  719. Owned<IPropertyTreeIterator> iter = pkgSetRegistryRoot->getElements("PackageSet");
  720. ForEach(*iter)
  721. {
  722. try
  723. {
  724. Owned<IPropertyTree> pkgSetRegistry= &iter->get();
  725. StringBuffer process;
  726. pkgSetRegistry->getProp("@process", process);
  727. if (process.length() && (streq(process.str(), "*") || WildMatch(process.str(), processReq, true)))
  728. listPkgInfo(version, targetReq, process.str(), packageMaps, pkgSetRegistry, &results);
  729. }
  730. catch(IException* e)
  731. {
  732. int err = e->errorCode();
  733. //Dali throws an exception if packagemap is not available for a process.
  734. if (err == PKG_DALI_LOOKUP_ERROR)
  735. e->Release();
  736. else
  737. throw e;
  738. }
  739. }
  740. if ((version >=1.01) && processReq && *processReq)
  741. {//Show warning if multiple packages are active for the same target.
  742. ForEachItemIn(i, results)
  743. {
  744. IEspPackageListMapData& r1 = (IEspPackageListMapData&) results.item(i);
  745. if (!r1.getActive())
  746. continue;
  747. const char* target1 = r1.getTarget();
  748. for (unsigned ii = i+1; ii<results.length(); ++ii)
  749. {
  750. IEspPackageListMapData& r2 = (IEspPackageListMapData&) results.item(ii);
  751. if (!r2.getActive())
  752. continue;
  753. if (!streq(target1, r2.getTarget()))
  754. continue;
  755. StringBuffer warning;
  756. warning.appendf("Error: package %s is also active.", r1.getId());
  757. r2.setDescription(warning.str());
  758. }
  759. }
  760. }
  761. }
  762. resp.setPackageMapList(results);
  763. resp.updateStatus().setCode(0);
  764. return true;
  765. }
  766. bool CWsPackageProcessEx::onGetPackage(IEspContext &context, IEspGetPackageRequest &req, IEspGetPackageResponse &resp)
  767. {
  768. Owned<IPropertyTree> packageMaps = packageMapAndSet.getPackageMaps();
  769. if (!packageMaps)
  770. throw MakeStringException(PKG_DALI_LOOKUP_ERROR, "Unable to retrieve information about package maps from dali server");
  771. resp.updateStatus().setCode(0);
  772. StringAttr process(req.getProcess());
  773. StringBuffer info;
  774. getPkgInfo(packageMaps, req.getTarget(), process.length() ? process.get() : "*", info);
  775. resp.setInfo(info);
  776. return true;
  777. }
  778. bool CWsPackageProcessEx::onGetPackageMapById(IEspContext &context, IEspGetPackageMapByIdRequest &req, IEspGetPackageMapByIdResponse &resp)
  779. {
  780. try
  781. {
  782. const char* pkgMapId = req.getPackageMapId();
  783. if (!pkgMapId && !*pkgMapId)
  784. throw MakeStringException(PKG_MISSING_PARAM, "PackageMap Id not specified");
  785. StringBuffer info;
  786. Owned<IPropertyTree> tree = createPTree("PackageMaps");
  787. getPkgInfoById(pkgMapId, tree);
  788. toXML(tree, info);
  789. resp.setInfo(info.str());
  790. resp.updateStatus().setCode(0);
  791. }
  792. catch (IException *e)
  793. {
  794. StringBuffer retMsg;
  795. resp.updateStatus().setDescription(e->errorMessage(retMsg).str());
  796. resp.updateStatus().setCode(-1);
  797. }
  798. return true;
  799. }
  800. bool CWsPackageProcessEx::onValidatePackage(IEspContext &context, IEspValidatePackageRequest &req, IEspValidatePackageResponse &resp)
  801. {
  802. StringArray warnings;
  803. StringArray errors;
  804. StringArray unmatchedQueries;
  805. StringArray unusedPackages;
  806. StringArray unmatchedFiles;
  807. Owned<IHpccPackageSet> set;
  808. Owned<IPropertyTree> mapTree;
  809. const char *target = req.getTarget();
  810. if (!target || !*target)
  811. throw MakeStringException(PKG_TARGET_NOT_DEFINED, "Target cluster required");
  812. Owned<IConstWUClusterInfo> clusterInfo = getTargetClusterInfo(target);
  813. if (!clusterInfo)
  814. throw MakeStringException(PKG_TARGET_NOT_DEFINED, "Target cluster not found");
  815. SCMStringBuffer process;
  816. clusterInfo->getRoxieProcess(process);
  817. if (!process.length())
  818. throw MakeStringException(PKG_TARGET_NOT_DEFINED, "Roxie process not found");
  819. const char* pmID = req.getPMID();
  820. const char* info = req.getInfo();
  821. if (req.getActive()) //validate active map
  822. {
  823. mapTree.setown(resolveActivePackageMap(process.str(), target, true));
  824. if (!mapTree)
  825. throw MakeStringException(PKG_PACKAGEMAP_NOT_FOUND, "Active package map not found");
  826. }
  827. else if (pmID && *pmID)
  828. {
  829. mapTree.setown(getPackageMapById(req.getGlobalScope() ? NULL : target, pmID, true));
  830. if (!mapTree)
  831. throw MakeStringException(PKG_PACKAGEMAP_NOT_FOUND, "Package map %s not found", pmID);
  832. }
  833. else if (info && *info)
  834. {
  835. mapTree.setown(createPTreeFromXMLString(info));
  836. if (!mapTree)
  837. throw MakeStringException(PKG_LOAD_PACKAGEMAP_FAILED, "Error processing package file content");
  838. }
  839. else
  840. {
  841. throw MakeStringException(PKG_PACKAGEMAP_NOT_FOUND, "package map not specified");
  842. }
  843. if (req.getCheckDFS())
  844. {
  845. Owned<IReferencedFileList> pmfiles = createReferencedFileList(context.queryUserId(), context.queryPassword(), true, false);
  846. pmfiles->addFilesFromPackageMap(mapTree);
  847. pmfiles->resolveFiles(process.str(), NULL, NULL, NULL, true, false, false);
  848. Owned<IReferencedFileIterator> files = pmfiles->getFiles();
  849. StringArray notInDFS;
  850. ForEach(*files)
  851. {
  852. IReferencedFile &file = files->query();
  853. if (file.getFlags() & RefFileNotFound)
  854. notInDFS.append(file.getLogicalName());
  855. }
  856. resp.updateFiles().setNotInDFS(notInDFS);
  857. }
  858. const char *id = mapTree->queryProp("@id");
  859. Owned<IHpccPackageMap> map = createPackageMapFromPtree(mapTree, target, id);
  860. if (!map)
  861. throw MakeStringException(PKG_LOAD_PACKAGEMAP_FAILED, "Error loading package map %s", id);
  862. StringArray queriesToVerify;
  863. const char *queryid = req.getQueryIdToVerify();
  864. if (queryid && *queryid)
  865. queriesToVerify.append(queryid);
  866. ForEachItemIn(i, req.getQueriesToVerify())
  867. {
  868. queryid = req.getQueriesToVerify().item(i);
  869. if (queryid && *queryid)
  870. queriesToVerify.appendUniq(queryid);
  871. }
  872. map->validate(queriesToVerify, warnings, errors, unmatchedQueries, unusedPackages, unmatchedFiles);
  873. resp.setPMID(map->queryPackageId());
  874. resp.setWarnings(warnings);
  875. resp.setErrors(errors);
  876. resp.updateQueries().setUnmatched(unmatchedQueries);
  877. resp.updatePackages().setUnmatched(unusedPackages);
  878. resp.updateFiles().setUnmatched(unmatchedFiles);
  879. resp.updateStatus().setCode(0);
  880. return true;
  881. }
  882. bool CWsPackageProcessEx::onGetPackageMapSelectOptions(IEspContext &context, IEspGetPackageMapSelectOptionsRequest &req, IEspGetPackageMapSelectOptionsResponse &resp)
  883. {
  884. try
  885. {
  886. bool includeTargets = req.getIncludeTargets();
  887. bool includeProcesses = req.getIncludeProcesses();
  888. if (includeTargets || includeProcesses)
  889. {
  890. Owned<IEnvironmentFactory> factory = getEnvironmentFactory();
  891. Owned<IConstEnvironment> env = factory->openEnvironment();
  892. if (!env)
  893. throw MakeStringException(PKG_DALI_LOOKUP_ERROR,"Failed to get environment information.");
  894. Owned<IPropertyTree> root = &env->getPTree();
  895. IArrayOf<IConstTargetData> targets;
  896. CConstWUClusterInfoArray clusters;
  897. getEnvironmentClusterInfo(root, clusters);
  898. ForEachItemIn(c, clusters)
  899. {
  900. SCMStringBuffer str;
  901. IConstWUClusterInfo &cluster = clusters.item(c);
  902. Owned<IEspTargetData> target = createTargetData("", "");
  903. target->setName(cluster.getName(str).str());
  904. ClusterType clusterType = cluster.getPlatform();
  905. if (clusterType == ThorLCRCluster)
  906. target->setType(THORCLUSTER);
  907. else if (clusterType == RoxieCluster)
  908. target->setType(ROXIECLUSTER);
  909. else
  910. target->setType(HTHORCLUSTER);
  911. if (!includeProcesses)
  912. {
  913. targets.append(*target.getClear());
  914. continue;
  915. }
  916. StringArray processes;
  917. if (clusterType == ThorLCRCluster)
  918. {
  919. const StringArray &thors = cluster.getThorProcesses();
  920. ForEachItemIn(i, thors)
  921. {
  922. const char* process = thors.item(i);
  923. if (process && *process)
  924. processes.append(process);
  925. }
  926. }
  927. else if (clusterType == RoxieCluster)
  928. {
  929. SCMStringBuffer process;
  930. cluster.getRoxieProcess(process);
  931. if (process.length())
  932. processes.append(process.str());
  933. }
  934. else if (clusterType == HThorCluster)
  935. {
  936. SCMStringBuffer process;
  937. cluster.getAgentQueue(process);
  938. if (process.length())
  939. processes.append(process.str());
  940. }
  941. if (processes.length())
  942. target->setProcesses(processes);
  943. targets.append(*target.getClear());
  944. }
  945. resp.setTargets(targets);
  946. }
  947. if (req.getIncludeProcessFilters())
  948. {
  949. StringArray processFilters;
  950. processFilters.append("*");
  951. Owned<IPropertyTree> pkgSets = packageMapAndSet.getPackageSets();
  952. if (pkgSets)
  953. {
  954. Owned<IPropertyTreeIterator> iter = pkgSets->getElements("PackageSet");
  955. ForEach(*iter)
  956. {
  957. StringBuffer process;
  958. iter->query().getProp("@process", process);
  959. if (process.length() && !processFilters.contains(process.str()))
  960. processFilters.append(process.str());
  961. }
  962. }
  963. resp.setProcessFilters(processFilters);
  964. }
  965. resp.updateStatus().setCode(0);
  966. }
  967. catch (IException *e)
  968. {
  969. StringBuffer retMsg;
  970. resp.updateStatus().setDescription(e->errorMessage(retMsg).str());
  971. resp.updateStatus().setCode(-1);
  972. }
  973. return true;
  974. }
  975. bool CWsPackageProcessEx::onGetQueryFileMapping(IEspContext &context, IEspGetQueryFileMappingRequest &req, IEspGetQueryFileMappingResponse &resp)
  976. {
  977. const char *queryname = req.getQueryName();
  978. if (!queryname || !*queryname)
  979. throw MakeStringException(PKG_INVALID_QUERY_NAME, "Query name required");
  980. const char *target = req.getTarget();
  981. if (!target || !*target)
  982. throw MakeStringException(PKG_TARGET_NOT_DEFINED, "Target cluster required");
  983. Owned<IConstWUClusterInfo> clusterInfo = getTargetClusterInfo(target);
  984. if (!clusterInfo)
  985. throw MakeStringException(PKG_TARGET_NOT_DEFINED, "Unable to find target cluster");
  986. if (clusterInfo->getPlatform()!=RoxieCluster)
  987. throw MakeStringException(PKG_INVALID_CLUSTER_TYPE, "Roxie target required");
  988. Owned<IHpccPackageSet> set;
  989. Owned<IHpccPackageMap> ownedmap;
  990. const IHpccPackageMap *map = NULL;
  991. const char *pmid = req.getPMID();
  992. if (pmid && *pmid)
  993. {
  994. Owned<IPropertyTree> pm = getPackageMapById(req.getGlobalScope() ? NULL : target, pmid, true);
  995. ownedmap.setown(createPackageMapFromPtree(pm, target, pmid));
  996. if (!ownedmap)
  997. throw MakeStringException(PKG_LOAD_PACKAGEMAP_FAILED, "Error loading package map %s", req.getPMID());
  998. map = ownedmap;
  999. }
  1000. else
  1001. {
  1002. SCMStringBuffer process;
  1003. clusterInfo->getRoxieProcess(process);
  1004. set.setown(createPackageSet(process.str()));
  1005. if (!set)
  1006. throw MakeStringException(PKG_CREATE_PACKAGESET_FAILED, "Unable to create PackageSet");
  1007. map = set->queryActiveMap(target);
  1008. if (!map)
  1009. throw MakeStringException(PKG_PACKAGEMAP_NOT_FOUND, "Active package map not found");
  1010. }
  1011. Owned<IPropertyTree> fileInfo = createPTree();
  1012. map->gatherFileMappingForQuery(queryname, fileInfo);
  1013. StringArray unmappedFiles;
  1014. Owned<IPropertyTreeIterator> it = fileInfo->getElements("File");
  1015. ForEach(*it)
  1016. unmappedFiles.append(it->query().queryProp(NULL));
  1017. resp.setUnmappedFiles(unmappedFiles);
  1018. IArrayOf<IEspSuperFile> superArray;
  1019. it.setown(fileInfo->getElements("SuperFile"));
  1020. ForEach(*it)
  1021. {
  1022. IPropertyTree &superTree = it->query();
  1023. Owned<IEspSuperFile> superItem = createSuperFile();
  1024. superItem->setName(superTree.queryProp("@name"));
  1025. StringArray subArray;
  1026. Owned<IPropertyTreeIterator> subfiles = superTree.getElements("SubFile");
  1027. ForEach(*subfiles)
  1028. subArray.append(subfiles->query().queryProp(NULL));
  1029. superItem->setSubFiles(subArray);
  1030. superArray.append(*superItem.getClear());
  1031. }
  1032. resp.setSuperFiles(superArray);
  1033. return true;
  1034. }
  1035. bool CWsPackageProcessEx::onAddPartToPackageMap(IEspContext &context, IEspAddPartToPackageMapRequest &req, IEspAddPartToPackageMapResponse &resp)
  1036. {
  1037. PackageMapUpdater updater;
  1038. updater.setFlag(PKGADD_SEG_ADD);
  1039. updater.setFlag(PKGADD_SEG_REPLACE, req.getDeletePrevious());
  1040. updater.setFlag(PKGADD_ALLOW_FOREIGN, req.getAllowForeignFiles());
  1041. updater.setFlag(PKGADD_PRELOAD_ALL, req.getPreloadAllPackages());
  1042. updater.setPMID(req.getTarget(), req.getPackageMap(), req.getGlobalScope());
  1043. updater.setProcess(req.getProcess());
  1044. updater.setUser(context.queryUserId(), context.queryPassword());
  1045. updater.setDerivedDfsLocation(req.getDaliIp(), req.getSourceProcess());
  1046. unsigned updateFlags = 0;
  1047. if (req.getDeletePrevious())
  1048. updateFlags |= DALI_UPDATEF_PACKAGEMAP;
  1049. if (req.getUpdateCloneFrom())
  1050. updateFlags |= DALI_UPDATEF_CLONE_FROM;
  1051. if (req.getUpdateSuperFiles())
  1052. updateFlags |= DALI_UPDATEF_SUPERFILES;
  1053. if (req.getAppendCluster())
  1054. updateFlags |= DALI_UPDATEF_APPEND_CLUSTER;
  1055. StringArray filesNotFound;
  1056. updater.addPart(req.getPartName(), req.getContent(), updateFlags, filesNotFound);
  1057. resp.setFilesNotFound(filesNotFound);
  1058. resp.updateStatus().setCode(0);
  1059. resp.updateStatus().setDescription(VStringBuffer("Successfully loaded Part %s to PackageMap %s", req.getPartName(), updater.pmid.str()));
  1060. return true;
  1061. }
  1062. bool CWsPackageProcessEx::onGetPartFromPackageMap(IEspContext &context, IEspGetPartFromPackageMapRequest &req, IEspGetPartFromPackageMapResponse &resp)
  1063. {
  1064. PackageMapUpdater updater;
  1065. updater.setPMID(req.getTarget(), req.getPackageMap(), req.getGlobalScope());
  1066. StringBuffer content;
  1067. resp.setContent(updater.getPartContent(req.getPartName(), content));
  1068. return true;
  1069. }
  1070. bool CWsPackageProcessEx::onRemovePartFromPackageMap(IEspContext &context, IEspRemovePartFromPackageMapRequest &req, IEspRemovePartFromPackageMapResponse &resp)
  1071. {
  1072. PackageMapUpdater updater;
  1073. updater.setPMID(req.getTarget(), req.getPackageMap(), req.getGlobalScope());
  1074. updater.removePart(req.getPartName());
  1075. resp.updateStatus().setCode(0);
  1076. resp.updateStatus().setDescription(VStringBuffer("Successfully removed Part %s from PackageMap %s", req.getPartName(), updater.pmid.str()));
  1077. return true;
  1078. }
  1079. int CWsPackageProcessSoapBindingEx::onFinishUpload(IEspContext &ctx, CHttpRequest* request, CHttpResponse* response,
  1080. const char *service, const char *method, StringArray& fileNames, StringArray& files, IMultiException *meIn)
  1081. {
  1082. if (meIn && (meIn->ordinality() > 0))
  1083. {
  1084. StringBuffer msg;
  1085. WARNLOG("Exception(s) in EspHttpBinding::onFinishUpload - %s", meIn->errorMessage(msg).append('\n').str());
  1086. if ((ctx.getResponseFormat() == ESPSerializationXML) || (ctx.getResponseFormat() == ESPSerializationJSON))
  1087. {
  1088. response->handleExceptions(NULL, meIn, "FileSpray", "UploadFile", NULL);
  1089. return 0;
  1090. }
  1091. else
  1092. return EspHttpBinding::onFinishUpload(ctx, request, response, service, method, fileNames, files, meIn);
  1093. }
  1094. StringBuffer respStr;
  1095. Owned<IEspWsPackageProcess> iserv = (IEspWsPackageProcess*)getService();
  1096. if(iserv == NULL)
  1097. {
  1098. WARNLOG("Exception(s) in %s::%s - Service not available", service, method);
  1099. respStr.append("{\"Code\":-1,\"Exception\":\"Service not available\"}");
  1100. }
  1101. else
  1102. {
  1103. checkRequest(ctx);
  1104. Owned<CAddPackageRequest> esp_request = new CAddPackageRequest(&ctx, "WsPackageProcess", request->queryParameters(), request->queryAttachments());
  1105. Owned<CAddPackageResponse> esp_response = new CAddPackageResponse("WsPackageProcess");
  1106. StringBuffer source;
  1107. source.setf("WsPackageProcess::%s()", method);
  1108. Owned<IMultiException> me = MakeMultiException(source.str());
  1109. try
  1110. {
  1111. if (!files.length())
  1112. throw MakeStringExceptionDirect(PKG_INFO_NOT_DEFINED, "Package content not found");
  1113. esp_request->setInfo(files.item(0));
  1114. iserv->onAddPackage(ctx, *esp_request.get(), *esp_response.get());
  1115. }
  1116. catch (IMultiException* mex)
  1117. {
  1118. me->append(*mex);
  1119. mex->Release();
  1120. }
  1121. catch (IException* e)
  1122. {
  1123. me->append(*e);
  1124. }
  1125. catch (...)
  1126. {
  1127. me->append(*MakeStringExceptionDirect(-1, "Unknown Exception"));
  1128. }
  1129. if (!me->ordinality())
  1130. {
  1131. respStr.append("{\"Code\":0,\"Description\":\"Package Map added\"}");
  1132. }
  1133. else
  1134. {
  1135. StringBuffer msg;
  1136. WARNLOG("Exception(s) in %s::%s - %s", service, method, me->errorMessage(msg).str());
  1137. respStr.appendf("{\"Code\":-1,\"Exception\":\"%s\"}", msg.str());
  1138. }
  1139. }
  1140. response->setContent(respStr.str());
  1141. response->setContentType(HTTP_TYPE_APPLICATION_JSON_UTF8);
  1142. response->send();
  1143. return 0;
  1144. }