pkgimpl.hpp 21 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627
  1. /*##############################################################################
  2. HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems®.
  3. Licensed under the Apache License, Version 2.0 (the "License");
  4. you may not use this file except in compliance with the License.
  5. You may obtain a copy of the License at
  6. http://www.apache.org/licenses/LICENSE-2.0
  7. Unless required by applicable law or agreed to in writing, software
  8. distributed under the License is distributed on an "AS IS" BASIS,
  9. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  10. See the License for the specific language governing permissions and
  11. limitations under the License.
  12. ############################################################################## */
  13. #ifndef WUPACKAGE_IMPL_HPP
  14. #define WUPACKAGE_IMPL_HPP
  15. #include "platform.h"
  16. #include "jprop.hpp"
  17. #include "jptree.hpp"
  18. #include "jregexp.hpp"
  19. #include "package.h"
  20. #include "referencedfilelist.hpp"
  21. class CPackageSuperFileArray : implements ISimpleSuperFileEnquiry, public CInterface
  22. {
  23. IArrayOf<IPropertyTree> subFiles;
  24. public:
  25. IMPLEMENT_IINTERFACE;
  26. CPackageSuperFileArray(IPropertyTreeIterator &_subs)
  27. {
  28. ForEach(_subs)
  29. {
  30. IPropertyTree &sub = _subs.query();
  31. sub.Link();
  32. subFiles.append(sub);
  33. }
  34. }
  35. virtual unsigned numSubFiles() const
  36. {
  37. return subFiles.length();
  38. }
  39. virtual bool getSubFileName(unsigned num, StringBuffer &name) const
  40. {
  41. if (subFiles.isItem(num))
  42. {
  43. name.append(subFiles.item(num).queryProp("@value"));
  44. return true;
  45. }
  46. else
  47. return false;
  48. }
  49. virtual unsigned findSubName(const char *subname) const
  50. {
  51. ForEachItemIn(idx, subFiles)
  52. {
  53. if (stricmp(subFiles.item(idx).queryProp("@value"), subname))
  54. return idx;
  55. }
  56. return NotFound;
  57. }
  58. virtual unsigned getContents(StringArray &contents) const
  59. {
  60. ForEachItemIn(idx, subFiles)
  61. {
  62. contents.append(subFiles.item(idx).queryProp("@value"));
  63. }
  64. return subFiles.length();
  65. }
  66. };
  67. class WORKUNIT_API CPackageNode : implements IHpccPackage, public CInterface
  68. {
  69. protected:
  70. Owned<IPropertyTree> node;
  71. Owned<IProperties> mergedEnvironment;
  72. hash64_t hash;
  73. void mergeEnvironment(const CPackageNode *base);
  74. virtual IPropertyTreeIterator *lookupElements(const char *xpath1, const char *xpath2) const = 0;
  75. inline StringBuffer makeSuperFileXPath(StringBuffer &xpath, const char *superFileName) const
  76. {
  77. superFileName = skipForeign(superFileName);
  78. return xpath.append("SuperFile[@id='").appendLower(strlen(superFileName), superFileName).append("']");
  79. }
  80. ISimpleSuperFileEnquiry *resolveSuperFile(const char *superFileName) const;
  81. inline bool hasProp(const char *xpath) const
  82. {
  83. return (node) ? node->hasProp(xpath) : false;
  84. }
  85. virtual const char *queryId() const
  86. {
  87. return (node) ? node->queryProp("@id") : NULL;
  88. }
  89. virtual bool isCompulsory() const
  90. {
  91. return (node) ? node->getPropBool("@compulsory", false) : false;
  92. }
  93. virtual bool isPreload() const
  94. {
  95. return (node) ? node->getPropBool("@preload", false) : false;
  96. }
  97. virtual bool resolveLocally() const
  98. {
  99. if (isCompulsory())
  100. return false;
  101. return (node) ? node->getPropBool("@resolveLocally", false) : true; // default is false for explicit package files, but true for the default empty package
  102. }
  103. virtual RecordTranslationMode getSysFieldTranslationEnabled() const { return RecordTranslationMode::None; }
  104. virtual RecordTranslationMode getEnableFieldTranslation() const
  105. {
  106. const char *val = queryEnv("control:enableFieldTranslation");
  107. if (!val) val = queryEnv("enableFieldTranslation"); // Backward compatibility
  108. if (val)
  109. return getTranslationMode(val);
  110. else
  111. return getSysFieldTranslationEnabled();
  112. }
  113. CPackageNode()
  114. {
  115. hash = 0;
  116. }
  117. virtual const IHpccPackage *queryRootPackage()
  118. {
  119. return NULL;
  120. }
  121. public:
  122. IMPLEMENT_IINTERFACE;
  123. CPackageNode(IPropertyTree *p);
  124. ~CPackageNode()
  125. {
  126. }
  127. // Load mergedEnvironment from local XML node
  128. void loadEnvironment();
  129. virtual const char *queryEnv(const char *varname) const
  130. {
  131. return mergedEnvironment->queryProp(varname);
  132. }
  133. virtual hash64_t queryHash() const
  134. {
  135. return hash;
  136. }
  137. virtual const IPropertyTree *queryTree() const
  138. {
  139. return node;
  140. }
  141. virtual void checkPreload();
  142. virtual bool validate(StringArray &warn, StringArray &err) const;
  143. };
  144. enum baseResolutionState
  145. {
  146. basesUnresolved=0,
  147. basesResolving=1,
  148. basesResolved=2
  149. };
  150. template <class TYPE>
  151. class CResolvedPackage : public TYPE
  152. {
  153. public:
  154. typedef CResolvedPackage<TYPE> self;
  155. CIArrayOf<self> bases;
  156. baseResolutionState baseResolution;
  157. CResolvedPackage<TYPE>(IPropertyTree *p) : TYPE(p), baseResolution(basesUnresolved) {}
  158. virtual aindex_t getBaseCount() const {return bases.length();}
  159. const self *getResolvedBase(aindex_t pos) const
  160. {
  161. if (pos < getBaseCount())
  162. return &bases.item(pos);
  163. return NULL;
  164. }
  165. virtual const TYPE *getBaseNode(aindex_t pos) const {return (const TYPE *) getResolvedBase(pos);}
  166. void appendBase(const IHpccPackage *base)
  167. {
  168. if (base)
  169. {
  170. const self *p = dynamic_cast<const self *>(base);
  171. bases.append(const_cast<self &>(*LINK(p))); // should really be an arrayof<const base> but that would require some fixing in jlib
  172. TYPE::hash = pkgHash64Data(sizeof(p->hash), &p->hash, TYPE::hash);
  173. TYPE::mergeEnvironment(p);
  174. }
  175. }
  176. void resolveBases(const IHpccPackageMap *packages)
  177. {
  178. if (baseResolution==basesResolved)
  179. return;
  180. if (baseResolution==basesResolving)
  181. throw MakeStringExceptionDirect(0, "PACKAGE_ERROR: circular or invalid base package definition");
  182. TYPE::loadEnvironment();
  183. if (packages)
  184. {
  185. Owned<IPropertyTreeIterator> baseIterator = TYPE::node->getElements("Base");
  186. if (!baseIterator->first())
  187. appendBase(TYPE::queryRootPackage());
  188. else
  189. {
  190. baseResolution=basesResolving;
  191. do
  192. {
  193. IPropertyTree &baseElem = baseIterator->query();
  194. const char *baseId = baseElem.queryProp("@id");
  195. if (!baseId)
  196. throw MakeStringException(PACKAGE_MISSING_ID, "PACKAGE_ERROR: base element missing id attribute");
  197. const IHpccPackage *base = packages->queryPackage(baseId);
  198. if (!base)
  199. throw MakeStringException(PACKAGE_NOT_FOUND, "PACKAGE_ERROR: base package %s not found", baseId);
  200. CResolvedPackage<TYPE> *rebase = dynamic_cast<CResolvedPackage<TYPE> *>(const_cast<IHpccPackage *>(base));
  201. rebase->resolveBases(packages);
  202. appendBase(base);
  203. }
  204. while(baseIterator->next());
  205. }
  206. TYPE::checkPreload();
  207. baseResolution=basesResolved;
  208. }
  209. }
  210. // Search this package and any bases for an element matching xpath1, then return iterator for its children that match xpath2
  211. IPropertyTreeIterator *lookupElements(const char *xpath1, const char *xpath2) const
  212. {
  213. IPropertyTree *parentNode = TYPE::node->queryPropTree(xpath1);
  214. if (parentNode)
  215. return parentNode->getElements(xpath2);
  216. ForEachItemIn(idx, bases)
  217. {
  218. const self &basePackage = bases.item(idx);
  219. IPropertyTreeIterator *it = basePackage.lookupElements(xpath1, xpath2);
  220. if (it)
  221. return it;
  222. }
  223. return NULL;
  224. }
  225. const char *locateSuperFile(const char *superFileName) const
  226. {
  227. if (!superFileName || !*superFileName || !TYPE::node)
  228. return NULL;
  229. StringBuffer xpath;
  230. if (TYPE::hasProp(TYPE::makeSuperFileXPath(xpath, superFileName)))
  231. return TYPE::queryId();
  232. ForEachItemIn(idx, bases)
  233. {
  234. if (bases.item(idx).hasProp(xpath))
  235. return bases.item(idx).queryId();
  236. }
  237. return NULL;
  238. }
  239. bool hasSuperFile(const char *superFileName) const
  240. {
  241. if (locateSuperFile(superFileName))
  242. return true;
  243. return false;
  244. }
  245. virtual bool validate(StringArray &warn, StringArray &err) const
  246. {
  247. return TYPE::validate(warn, err);
  248. }
  249. };
  250. typedef CResolvedPackage<CPackageNode> CHpccPackage;
  251. //================================================================================================
  252. // CPackageMap - an implementation of IPackageMap using a string map
  253. //================================================================================================
  254. template <class TYPE, class IFACE>
  255. class CPackageMapOf : implements IHpccPackageMap, public CInterface
  256. {
  257. public:
  258. typedef CResolvedPackage<TYPE> packageType;
  259. MapStringToMyClassViaBase<packageType, IFACE> packages;
  260. StringAttr packageId;
  261. StringAttr querySet;
  262. bool active;
  263. bool compulsory;
  264. StringArray wildMatches, wildIds;
  265. StringArray parts;
  266. public:
  267. IMPLEMENT_IINTERFACE;
  268. CPackageMapOf(const char *_packageId, const char *_querySet, bool _active)
  269. : packageId(_packageId), querySet(_querySet), active(_active), packages(true), compulsory(false)
  270. {
  271. }
  272. // IPackageMap interface
  273. virtual bool isActive() const
  274. {
  275. return active;
  276. }
  277. virtual const packageType *queryResolvedPackage(const char *name) const
  278. {
  279. return name ? packages.getValue(name) : NULL;
  280. }
  281. virtual const IHpccPackage *queryPackage(const char *name) const
  282. {
  283. return name ? (IFACE*)packages.getValue(name) : NULL;
  284. }
  285. virtual const char *queryPackageId() const
  286. {
  287. return packageId;
  288. }
  289. virtual const packageType *matchResolvedPackage(const char *name) const
  290. {
  291. if (name && *name)
  292. {
  293. const packageType *pkg = queryResolvedPackage(name);
  294. if (pkg)
  295. return pkg;
  296. const char *tail = name + strlen(name)-1;
  297. while (tail>name && isdigit(*tail))
  298. tail--;
  299. if (*tail=='.' && tail>name)
  300. {
  301. StringAttr notail(name, tail-name);
  302. pkg = queryResolvedPackage(notail);
  303. if (pkg)
  304. return pkg;
  305. }
  306. ForEachItemIn(idx, wildMatches)
  307. {
  308. if (WildMatch(name, wildMatches.item(idx), true))
  309. return queryResolvedPackage(wildIds.item(idx));
  310. }
  311. }
  312. return NULL;
  313. }
  314. const IHpccPackage *matchPackage(const char *name) const
  315. {
  316. return (IFACE *) matchResolvedPackage(name);
  317. }
  318. void loadPackage(IPropertyTree &packageTree)
  319. {
  320. const char *id = packageTree.queryProp("@id");
  321. if (!id || !*id)
  322. throw MakeStringException(PACKAGE_MISSING_ID, "Invalid package map - Package element missing id attribute");
  323. Owned<packageType> package = new packageType(&packageTree);
  324. packages.setValue(id, package.get());
  325. const char *queries = packageTree.queryProp("@queries");
  326. if (queries && *queries)
  327. {
  328. wildMatches.append(queries);
  329. wildIds.append(id);
  330. }
  331. }
  332. void loadPart(IPropertyTree &part)
  333. {
  334. const char *id = part.queryProp("@id");
  335. if (id && *id)
  336. parts.append(id);
  337. Owned<IPropertyTreeIterator> partPackages = part.getElements("Package");
  338. ForEach(*partPackages)
  339. loadPackage(partPackages->query());
  340. }
  341. const StringArray &getPartIds() const override
  342. {
  343. return parts;
  344. }
  345. void load(IPropertyTree *xml)
  346. {
  347. if (!xml)
  348. return;
  349. compulsory = xml->getPropBool("@compulsory");
  350. Owned<IPropertyTreeIterator> allpackages = xml->getElements("Package"); //old style non-part packages first
  351. ForEach(*allpackages)
  352. loadPackage(allpackages->query());
  353. Owned<IPropertyTreeIterator> parts = xml->getElements("Part"); //new multipart packagemap
  354. ForEach(*parts)
  355. loadPart(parts->query());
  356. HashIterator it(packages); //package bases can be across parts
  357. ForEach (it)
  358. {
  359. packageType *pkg = packages.getValue((const char *)it.query().getKey());
  360. if (pkg)
  361. pkg->resolveBases(this);
  362. }
  363. }
  364. void load(const char *id)
  365. {
  366. Owned<IPropertyTree> xml = getPackageMapById(id, true);
  367. load(xml);
  368. }
  369. virtual void gatherFileMappingForQuery(const char *queryname, IPropertyTree *fileInfo) const
  370. {
  371. Owned<IPropertyTree> query = resolveQueryAlias(querySet, queryname, true);
  372. if (!query)
  373. throw MakeStringException(PACKAGE_QUERY_NOT_FOUND, "Query %s not found", queryname);
  374. Owned<IReferencedFileList> filelist = createReferencedFileList(NULL, true, false);
  375. Owned<IWorkUnitFactory> wufactory = getWorkUnitFactory(NULL, NULL);
  376. Owned<IConstWorkUnit> cw = wufactory->openWorkUnit(query->queryProp("@wuid"));
  377. const IHpccPackage *pkg = matchPackage(query->queryProp("@id"));
  378. filelist->addFilesFromQuery(cw, pkg);
  379. Owned<IReferencedFileIterator> refFiles = filelist->getFiles();
  380. ForEach(*refFiles)
  381. {
  382. IReferencedFile &rf = refFiles->query();
  383. if (!(rf.getFlags() & RefFileInPackage))
  384. fileInfo->addProp("File", rf.getLogicalName());
  385. else
  386. {
  387. Owned<ISimpleSuperFileEnquiry> ssfe = pkg->resolveSuperFile(rf.getLogicalName());
  388. if (ssfe && ssfe->numSubFiles()>0)
  389. {
  390. IPropertyTree *superInfo = fileInfo->addPropTree("SuperFile");
  391. superInfo->setProp("@name", rf.getLogicalName());
  392. unsigned count = ssfe->numSubFiles();
  393. while (count--)
  394. {
  395. StringBuffer subfile;
  396. ssfe->getSubFileName(count, subfile);
  397. superInfo->addProp("SubFile", subfile.str());
  398. }
  399. }
  400. }
  401. }
  402. }
  403. virtual bool validate(StringArray &queriesToCheck, StringArray &warn, StringArray &err,
  404. StringArray &unmatchedQueries, StringArray &unusedPackages, StringArray &unmatchedFiles) const
  405. {
  406. bool isValid = true;
  407. MapStringTo<bool> referencedPackages;
  408. Owned<IPropertyTree> qs = getQueryRegistry(querySet, true);
  409. if (!qs)
  410. throw MakeStringException(PACKAGE_TARGET_NOT_FOUND, "Target %s not found", querySet.str());
  411. HashIterator it(packages);
  412. ForEach (it)
  413. {
  414. const char *packageId = (const char *)it.query().getKey();
  415. packageType *pkg = packages.getValue(packageId);
  416. if (!pkg)
  417. continue;
  418. if (!pkg->validate(warn, err))
  419. isValid = false;
  420. Owned<IPropertyTreeIterator> baseNodes = pkg->queryTree()->getElements("Base");
  421. ForEach(*baseNodes)
  422. {
  423. const char *baseId = baseNodes->query().queryProp("@id");
  424. if (!baseId || !*baseId)
  425. {
  426. VStringBuffer msg("Package '%s' contains Base element with no id attribute", packageId);
  427. err.append(msg.str());
  428. continue;
  429. }
  430. if (!referencedPackages.getValue(baseId))
  431. referencedPackages.setValue(baseId, true);
  432. }
  433. }
  434. Owned<IPropertyTree> tempQuerySet=createPTree(ipt_fast);
  435. Owned<IPropertyTreeIterator> queries;
  436. if (queriesToCheck.length())
  437. {
  438. ForEachItemIn(i, queriesToCheck)
  439. {
  440. VStringBuffer xpath("Query[@id='%s']", queriesToCheck.item(i));
  441. Owned<IPropertyTree> queryEntry = qs->getPropTree(xpath);
  442. if (queryEntry)
  443. tempQuerySet->addPropTree("Query", queryEntry.getClear());
  444. else
  445. {
  446. VStringBuffer msg("Query %s not found in %s queryset", queriesToCheck.item(i), querySet.str());
  447. err.append(msg);
  448. }
  449. }
  450. queries.setown(tempQuerySet->getElements("Query"));
  451. }
  452. else
  453. queries.setown(qs->getElements("Query"));
  454. if (!queries->first())
  455. {
  456. warn.append("No Queries found");
  457. return isValid;
  458. }
  459. Owned<IWorkUnitFactory> wufactory = getWorkUnitFactory(NULL, NULL);
  460. ForEach(*queries)
  461. {
  462. const char *queryid = queries->query().queryProp("@id");
  463. if (queryid && *queryid)
  464. {
  465. Owned<IReferencedFileList> filelist = createReferencedFileList(NULL, true, false);
  466. Owned<IConstWorkUnit> cw = wufactory->openWorkUnit(queries->query().queryProp("@wuid"));
  467. StringArray libnames, unresolvedLibs;
  468. gatherLibraryNames(libnames, unresolvedLibs, *wufactory, *cw, qs);
  469. IPointerArrayOf<IHpccPackage> libraries;
  470. ForEachItemIn(libitem, libnames)
  471. {
  472. const char *libname = libnames.item(libitem);
  473. const IHpccPackage *libpkg = matchPackage(libname);
  474. if (libpkg)
  475. libraries.append(LINK(const_cast<IHpccPackage*>(libpkg)));
  476. else
  477. unmatchedQueries.append(libname);
  478. }
  479. bool isCompulsory = filelist->addFilesFromQuery(cw, this, queryid);
  480. unsigned unmatchedCount=0;
  481. Owned<IReferencedFileIterator> refFiles = filelist->getFiles();
  482. ForEach(*refFiles)
  483. {
  484. IReferencedFile &rf = refFiles->query();
  485. unsigned flags = rf.getFlags();
  486. if (flags & RefFileInPackage)
  487. {
  488. if (flags & RefFileSuper)
  489. {
  490. const char *pkgid = rf.queryPackageId();
  491. if (!pkgid || !*pkgid)
  492. continue;
  493. ForEachItemIn(libPkgItem, libraries)
  494. {
  495. IHpccPackage *libpkg = libraries.item(libPkgItem);
  496. const char *libpkgid = libpkg->locateSuperFile(rf.getLogicalName());
  497. if (libpkgid && !strieq(libpkgid, pkgid))
  498. {
  499. VStringBuffer msg("For query %s SuperFile %s defined in package %s redefined for library %s in package %s",
  500. queryid, rf.getLogicalName(), pkgid, libpkg->queryId(), libpkgid);
  501. warn.append(msg.str());
  502. }
  503. }
  504. }
  505. continue;
  506. }
  507. VStringBuffer fullname("%s/%s", queryid, rf.getLogicalName());
  508. if (!(flags & RefFileNotOptional))
  509. fullname.append("/Optional");
  510. else if (isCompulsory)
  511. fullname.append("/Compulsory");
  512. unmatchedFiles.append(fullname);
  513. unmatchedCount++;
  514. }
  515. const IHpccPackage *matched = matchPackage(queryid);
  516. if (matched)
  517. {
  518. const char *matchId = matched->queryTree()->queryProp("@id");
  519. if (!referencedPackages.getValue(matchId))
  520. referencedPackages.setValue(matchId, true);
  521. if (unmatchedCount && matched->isCompulsory())
  522. {
  523. VStringBuffer msg("Compulsory query %s has query files not defined in package %s", queryid, matchId);
  524. err.append(msg.str());
  525. isValid=false;
  526. }
  527. }
  528. else
  529. unmatchedQueries.append(queryid);
  530. }
  531. }
  532. ForEach (it)
  533. {
  534. const char *packageId = (const char *)it.query().getKey();
  535. if (!referencedPackages.getValue(packageId))
  536. unusedPackages.append(packageId);
  537. }
  538. return isValid;
  539. }
  540. };
  541. typedef CPackageMapOf<CPackageNode, IHpccPackage> CHpccPackageMap;
  542. //================================================================================================
  543. // CHpccPackageSet - an implementation of IHpccPackageSet
  544. //================================================================================================
  545. class WORKUNIT_API CHpccPackageSet : implements IHpccPackageSet, public CInterface
  546. {
  547. IArrayOf<CHpccPackageMap> packageMaps;
  548. StringAttr process;
  549. public:
  550. IMPLEMENT_IINTERFACE;
  551. CHpccPackageSet(const char *_process);
  552. void load(IPropertyTree *xml);
  553. virtual const IHpccPackageMap *queryActiveMap(const char *queryset) const;
  554. };
  555. #endif