thorplugin.cpp 18 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639
  1. /*##############################################################################
  2. HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems®.
  3. Licensed under the Apache License, Version 2.0 (the "License");
  4. you may not use this file except in compliance with the License.
  5. You may obtain a copy of the License at
  6. http://www.apache.org/licenses/LICENSE-2.0
  7. Unless required by applicable law or agreed to in writing, software
  8. distributed under the License is distributed on an "AS IS" BASIS,
  9. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  10. See the License for the specific language governing permissions and
  11. limitations under the License.
  12. ############################################################################## */
  13. #include "jexcept.hpp"
  14. #include "jmisc.hpp"
  15. #include "jthread.hpp"
  16. #include "jsocket.hpp"
  17. #include "jprop.hpp"
  18. #include "jdebug.hpp"
  19. #include "jlzw.hpp"
  20. #include "eclrtl.hpp"
  21. #ifdef _USE_BINUTILS
  22. #include "bfd.h"
  23. #elif defined(__APPLE__)
  24. #include <mach-o/getsect.h>
  25. #include <sys/mman.h>
  26. #include <sys/stat.h>
  27. #elif !defined(_WIN32)
  28. #include <sys/mman.h>
  29. #include <sys/stat.h>
  30. #include <elf.h>
  31. #endif
  32. #include "thorplugin.hpp"
  33. void * SimplePluginCtx::ctxMalloc(size_t size)
  34. {
  35. return rtlMalloc(size);
  36. }
  37. void * SimplePluginCtx::ctxRealloc(void * _ptr, size_t size)
  38. {
  39. return rtlRealloc(_ptr, size);
  40. }
  41. void SimplePluginCtx::ctxFree(void * _ptr)
  42. {
  43. rtlFree(_ptr);
  44. }
  45. char * SimplePluginCtx::ctxStrdup(char * _ptr)
  46. {
  47. return strdup(_ptr);
  48. }
  49. int SimplePluginCtx::ctxGetPropInt(const char *propName, int defaultValue) const
  50. {
  51. return defaultValue;
  52. }
  53. const char * SimplePluginCtx::ctxQueryProp(const char *propName) const
  54. {
  55. return NULL;
  56. }
  57. //-------------------------------------------------------------------------------------------------------------------
  58. class HelperDll : public CInterface, implements ILoadedDllEntry
  59. {
  60. SharedObject so;
  61. StringAttr name;
  62. Linked<const IFileIO> dllFile;
  63. bool logLoad;
  64. public:
  65. IMPLEMENT_IINTERFACE;
  66. HelperDll(const char *_name, const IFileIO *dllFile);
  67. ~HelperDll();
  68. //interface ILoadedDllEntry
  69. virtual HINSTANCE getInstance() const;
  70. virtual void * getEntry(const char * name) const;
  71. virtual bool IsShared();
  72. virtual const char * queryVersion() const;
  73. virtual const char * queryName() const;
  74. virtual const byte * getResource(unsigned id) const;
  75. virtual bool getResource(size32_t & len, const void * & data, const char * type, unsigned id, bool trace) const;
  76. bool load(bool isGlobal, bool raiseOnError);
  77. bool loadCurrentExecutable();
  78. virtual void logLoaded();
  79. virtual bool checkVersion(const char *expected);
  80. };
  81. class PluginDll : public HelperDll
  82. {
  83. ECLPluginDefinitionBlockEx pb;
  84. public:
  85. PluginDll(const char *_name, const IFileIO *_dllFile) : HelperDll(_name, _dllFile) {}
  86. bool init(IPluginContextEx * pluginCtx);
  87. virtual bool checkVersion(const char *expected);
  88. virtual void logLoaded();
  89. };
  90. HelperDll::HelperDll(const char *_name, const IFileIO *_dllFile)
  91. : name(_name), dllFile(_dllFile)
  92. {
  93. logLoad = false;
  94. }
  95. bool HelperDll::load(bool isGlobal, bool raiseOnError)
  96. {
  97. if (!so.load(name, isGlobal, raiseOnError))
  98. return false;
  99. return true;
  100. }
  101. bool HelperDll::loadCurrentExecutable()
  102. {
  103. if (!so.loadCurrentExecutable())
  104. return false;
  105. return true;
  106. }
  107. HelperDll::~HelperDll()
  108. {
  109. if (logLoad)
  110. DBGLOG("Unloading dll %s", name.get());
  111. }
  112. HINSTANCE HelperDll::getInstance() const
  113. {
  114. return so.getInstanceHandle();
  115. }
  116. void * HelperDll::getEntry(const char * name) const
  117. {
  118. return GetSharedProcedure(so.getInstanceHandle(), name);
  119. }
  120. bool HelperDll::IsShared()
  121. {
  122. return CInterface::IsShared();
  123. }
  124. const char * HelperDll::queryVersion() const
  125. {
  126. return "";
  127. }
  128. void HelperDll::logLoaded()
  129. {
  130. logLoad = true;
  131. DBGLOG("Loaded DLL %s", name.get());
  132. }
  133. bool HelperDll::checkVersion(const char *expected)
  134. {
  135. return true;
  136. }
  137. const char * HelperDll::queryName() const
  138. {
  139. return name.get();
  140. }
  141. const byte * HelperDll::getResource(unsigned id) const
  142. {
  143. #ifdef _WIN32
  144. HINSTANCE dllHandle = so.getInstanceHandle();
  145. HRSRC hrsrc = FindResource(dllHandle, MAKEINTRESOURCE(id), "BIGSTRING");
  146. if (hrsrc)
  147. return (const byte *) LoadResource(dllHandle, hrsrc);
  148. return NULL;
  149. #else
  150. StringBuffer resourceName;
  151. resourceName.appendf("BIGSTRING_%d_txt_start", id);
  152. return (const byte *) getEntry(resourceName.str());
  153. #endif
  154. }
  155. const byte resourceHeaderVersion=1;
  156. const size32_t resourceHeaderLength = sizeof(byte) + sizeof(byte) + sizeof(bool) + sizeof(size32_t);
  157. bool HelperDll::getResource(size32_t & len, const void * & data, const char * type, unsigned id, bool trace) const
  158. {
  159. #ifdef _WIN32
  160. HINSTANCE dllHandle = so.getInstanceHandle();
  161. HRSRC hrsrc = FindResource(dllHandle, MAKEINTRESOURCE(id), type);
  162. if (!hrsrc)
  163. return false;
  164. len = SizeofResource(dllHandle, hrsrc);
  165. data = (const byte *) LoadResource(dllHandle, hrsrc);
  166. return true;
  167. #else
  168. StringBuffer symName;
  169. symName.append(type).append("_").append(id).append("_txt_start");
  170. data = (const void *) getEntry(symName.str());
  171. if (!data)
  172. {
  173. if (trace)
  174. printf("Failed to locate symbol %s\n", symName.str());
  175. return false;
  176. }
  177. byte bom;
  178. byte version;
  179. bool compressed;
  180. MemoryBuffer mb;
  181. mb.setBuffer(resourceHeaderLength, const_cast<void *>(data));
  182. mb.read(bom);
  183. if (bom!=0x80)
  184. return false;
  185. mb.read(version);
  186. if (version>resourceHeaderVersion)
  187. return false;
  188. mb.read(compressed).read(len);
  189. len+=resourceHeaderLength;
  190. return true;
  191. #endif
  192. }
  193. #ifdef _USE_BINUTILS
  194. struct SecScanParam
  195. {
  196. MemoryBuffer &result;
  197. const char *sectionName;
  198. SecScanParam(MemoryBuffer &_result, const char *_sectionName)
  199. : result(_result), sectionName(_sectionName)
  200. {
  201. }
  202. };
  203. static void secscan (bfd *file, sec_ptr sec, void *userParam)
  204. {
  205. SecScanParam *param = (SecScanParam *) userParam;
  206. if (strcmp(param->sectionName, bfd_section_name (file, sec))==0)
  207. {
  208. bfd_size_type size = bfd_section_size (file, sec);
  209. void *data = (void *) param->result.reserve(size);
  210. bfd_get_section_contents(file, sec, data, 0, size);
  211. }
  212. }
  213. #endif
  214. static bool getResourceFromMappedFile(const char * filename, const byte * start_addr, MemoryBuffer &data, const char * type, unsigned id)
  215. {
  216. #if defined(_WIN32) || defined (_USE_BINUTILS)
  217. throwUnexpected();
  218. #elif defined(__APPLE__)
  219. VStringBuffer sectname("%s_%u", type, id);
  220. // The first bytes are the Mach-O header
  221. const struct mach_header_64 *mh = (const struct mach_header_64 *) start_addr;
  222. if (mh->magic != MH_MAGIC_64)
  223. {
  224. DBGLOG("Failed to extract resource %s: Does not appear to be a Mach-O 64-bit binary", filename);
  225. return false;
  226. }
  227. unsigned long len = 0;
  228. unsigned char *data2 = getsectiondata(mh, "__TEXT", sectname.str(), &len);
  229. data.append(len, data2);
  230. return true;
  231. #else
  232. // The first bytes are the ELF header
  233. const Elf64_Ehdr * hdr = (const Elf64_Ehdr *) start_addr;
  234. if (memcmp(hdr->e_ident, ELFMAG, SELFMAG) != 0)
  235. {
  236. DBGLOG("Failed to extract resource %s: Does not appear to be a ELF binary", filename);
  237. return false;
  238. }
  239. if (hdr->e_ident[EI_CLASS] != ELFCLASS64)
  240. {
  241. DBGLOG("Failed to extract resource %s: Does not appear to be a ELF 64-bit binary", filename);
  242. return false;
  243. }
  244. //Check that there is a symbol table for the sections.
  245. if (hdr->e_shstrndx == SHN_UNDEF)
  246. {
  247. DBGLOG("Failed to extract resource %s: Does not include a section symbol table", filename);
  248. return false;
  249. }
  250. //Now walk the sections comparing the section names
  251. Elf64_Half numSections = hdr->e_shnum;
  252. const Elf64_Shdr * sectionHeaders = reinterpret_cast<const Elf64_Shdr *>(start_addr + hdr->e_shoff);
  253. const Elf64_Shdr & symbolTableSection = sectionHeaders[hdr->e_shstrndx];
  254. const char * symbolTable = (const char *)start_addr + symbolTableSection.sh_offset;
  255. VStringBuffer sectname("%s_%u", type, id);
  256. for (unsigned iSect= 0; iSect < numSections; iSect++)
  257. {
  258. const Elf64_Shdr & section = sectionHeaders[iSect];
  259. const char * sectionName = symbolTable + section.sh_name;
  260. if (streq(sectionName, sectname))
  261. {
  262. data.append(section.sh_size, start_addr + section.sh_offset);
  263. return true;
  264. }
  265. }
  266. DBGLOG("Failed to extract resource %s: Does not include a matching entry", filename);
  267. return false;
  268. #endif
  269. }
  270. extern bool getResourceFromFile(const char *filename, MemoryBuffer &data, const char * type, unsigned id)
  271. {
  272. #ifdef _WIN32
  273. HINSTANCE dllHandle = LoadLibraryEx(filename, NULL, LOAD_LIBRARY_AS_DATAFILE|LOAD_LIBRARY_AS_IMAGE_RESOURCE);
  274. if (dllHandle == NULL)
  275. dllHandle = LoadLibraryEx(filename, NULL, LOAD_LIBRARY_AS_DATAFILE); // the LOAD_LIBRARY_AS_IMAGE_RESOURCE flag is not supported on all versions of Windows
  276. if (dllHandle == NULL)
  277. {
  278. DBGLOG("Failed to load library %s: %d", filename, GetLastError());
  279. return false;
  280. }
  281. HRSRC hrsrc = FindResource(dllHandle, MAKEINTRESOURCE(id), type);
  282. if (!hrsrc)
  283. return false;
  284. size32_t len = SizeofResource(dllHandle, hrsrc);
  285. const void *rdata = (const void *) LoadResource(dllHandle, hrsrc);
  286. data.append(len, rdata);
  287. FreeLibrary(dllHandle);
  288. return true;
  289. #elif defined (_USE_BINUTILS)
  290. bfd_init ();
  291. bfd *file = bfd_openr(filename, NULL);
  292. if (file)
  293. {
  294. StringBuffer sectionName;
  295. sectionName.append(type).append("_").append(id).append(".data");
  296. SecScanParam param(data, sectionName.str());
  297. if (bfd_check_format (file, bfd_object))
  298. bfd_map_over_sections (file, secscan, &param);
  299. bfd_close (file);
  300. }
  301. return data.length() != 0;
  302. #else
  303. struct stat stat_buf;
  304. VStringBuffer sectname("%s_%u", type, id);
  305. int fd = open(filename, O_RDONLY);
  306. if (fd == -1 || fstat(fd, &stat_buf) == -1)
  307. {
  308. DBGLOG("Failed to load library %s: %d", filename, errno);
  309. return false;
  310. }
  311. bool ok = false;
  312. __uint64 size = stat_buf.st_size;
  313. const byte *start_addr = (const byte *) mmap(0, size, PROT_READ, MAP_FILE | MAP_PRIVATE, fd, 0);
  314. if (start_addr == MAP_FAILED)
  315. {
  316. DBGLOG("Failed to load library %s: %d", filename, errno);
  317. }
  318. else
  319. {
  320. ok = getResourceFromMappedFile(filename, start_addr, data, type, id);
  321. munmap((void *)start_addr, size);
  322. }
  323. close(fd);
  324. return ok;
  325. #endif
  326. }
  327. //-------------------------------------------------------------------------------------------------------------------
  328. bool PluginDll::init(IPluginContextEx * pluginCtx)
  329. {
  330. HINSTANCE h = getInstance();
  331. assertex(h != (HINSTANCE) -1);
  332. EclPluginSetCtxEx pSetCtxEx = (EclPluginSetCtxEx) GetSharedProcedure(h,"setPluginContextEx");
  333. if (pSetCtxEx)
  334. pSetCtxEx(pluginCtx);
  335. else
  336. {
  337. // Older plugins may only support setPluginContext - fall back to that
  338. EclPluginSetCtx pSetCtx = (EclPluginSetCtx) GetSharedProcedure(h,"setPluginContext");
  339. if (pSetCtx)
  340. pSetCtx(pluginCtx);
  341. }
  342. EclPluginDefinition p= (EclPluginDefinition) GetSharedProcedure(h,"getECLPluginDefinition");
  343. if (!p)
  344. return false;
  345. pb.size = sizeof(ECLPluginDefinitionBlockEx);
  346. if (!p(&pb))
  347. {
  348. pb.compatibleVersions = NULL;
  349. pb.size = sizeof(ECLPluginDefinitionBlock);
  350. if (!p(&pb))
  351. return false;
  352. }
  353. return true;
  354. }
  355. bool PluginDll::checkVersion(const char *expected)
  356. {
  357. assertex(expected);
  358. if (stricmp(pb.version, expected) == 0)
  359. return true;
  360. if (pb.compatibleVersions)
  361. {
  362. const char **finger = pb.compatibleVersions;
  363. while (*finger)
  364. {
  365. if (stricmp(*finger, expected) == 0)
  366. return true;
  367. finger++;
  368. }
  369. }
  370. return false;
  371. }
  372. void PluginDll::logLoaded()
  373. {
  374. HelperDll::logLoaded();
  375. DBGLOG("Current reported version is %s", pb.version);
  376. if (pb.compatibleVersions)
  377. {
  378. const char **finger = pb.compatibleVersions;
  379. while (*finger)
  380. {
  381. DBGLOG("Compatible version %s", *finger);
  382. finger++;
  383. }
  384. }
  385. }
  386. extern DLLSERVER_API ILoadedDllEntry * createDllEntry(const char *path, bool isGlobal, const IFileIO *dllFile)
  387. {
  388. Owned<HelperDll> result = new HelperDll(path, dllFile);
  389. if (!result->load(isGlobal, true))
  390. throw MakeStringException(0, "Failed to create ILoadedDllEntry for dll %s", path);
  391. return result.getClear();
  392. }
  393. extern DLLSERVER_API ILoadedDllEntry * createExeDllEntry(const char *path)
  394. {
  395. Owned<HelperDll> result = new HelperDll(path, NULL);
  396. if (!result->loadCurrentExecutable())
  397. throw MakeStringException(0, "Failed to create ILoadedDllEntry for current executable");
  398. return result.getClear();
  399. }
  400. extern DLLSERVER_API bool decompressResource(size32_t len, const void *data, MemoryBuffer &result)
  401. {
  402. bool hasVersion = len && (*(const byte *)data == 0x80);
  403. MemoryBuffer src;
  404. src.setBuffer(len, const_cast<void *>(data), false);
  405. byte version = 1;
  406. if (hasVersion)
  407. {
  408. src.skip(1);
  409. src.read(version);
  410. }
  411. switch (version)
  412. {
  413. case 1:
  414. decompressToBuffer(result, src);
  415. break;
  416. default:
  417. throwUnexpected();
  418. }
  419. return true;
  420. }
  421. extern DLLSERVER_API bool decompressResource(size32_t len, const void *data, StringBuffer &result)
  422. {
  423. MemoryBuffer tgt;
  424. decompressResource(len, data, tgt);
  425. tgt.append((char)0);
  426. unsigned expandedLen = tgt.length();
  427. result.setBuffer(expandedLen, reinterpret_cast<char *>(tgt.detach()), expandedLen-1);
  428. return true;
  429. }
  430. extern DLLSERVER_API void appendResource(MemoryBuffer & mb, size32_t len, const void *data, bool compress)
  431. {
  432. mb.append((byte)0x80).append(resourceHeaderVersion);
  433. if (compress)
  434. compressToBuffer(mb, len, data);
  435. else
  436. appendToBuffer(mb, len, data);
  437. }
  438. extern DLLSERVER_API void compressResource(MemoryBuffer & compressed, size32_t len, const void *data)
  439. {
  440. appendResource(compressed, len, data, true);
  441. }
  442. extern DLLSERVER_API bool getEmbeddedWorkUnitXML(ILoadedDllEntry *dll, StringBuffer &xml)
  443. {
  444. size32_t len = 0;
  445. const void * data = NULL;
  446. if (!dll->getResource(len, data, "WORKUNIT", 1000))
  447. return false;
  448. return decompressResource(len, data, xml);
  449. }
  450. extern DLLSERVER_API bool getEmbeddedManifestXML(ILoadedDllEntry *dll, StringBuffer &xml)
  451. {
  452. size32_t len = 0;
  453. const void * data = NULL;
  454. if (!dll->getResource(len, data, "MANIFEST", 1000))
  455. return false;
  456. return decompressResource(len, data, xml);
  457. }
  458. extern DLLSERVER_API bool checkEmbeddedWorkUnitXML(ILoadedDllEntry *dll)
  459. {
  460. size32_t len = 0;
  461. const void * data = NULL;
  462. return dll->getResource(len, data, "WORKUNIT", 1000, false);
  463. }
  464. extern DLLSERVER_API bool getResourceXMLFromFile(const char *filename, const char *type, unsigned id, StringBuffer &xml)
  465. {
  466. MemoryBuffer data;
  467. if (!getResourceFromFile(filename, data, type, id))
  468. return false;
  469. return decompressResource(data.length(), data.toByteArray(), xml);
  470. }
  471. extern DLLSERVER_API bool getWorkunitXMLFromFile(const char *filename, StringBuffer &xml)
  472. {
  473. return getResourceXMLFromFile(filename, "WORKUNIT", 1000, xml);
  474. }
  475. extern DLLSERVER_API bool getManifestXMLFromFile(const char *filename, StringBuffer &xml)
  476. {
  477. return getResourceXMLFromFile(filename, "MANIFEST", 1000, xml);
  478. }
  479. //-------------------------------------------------------------------------------------------------------------------
  480. //-------------------------------------------------------------------------------------------------------------------
  481. bool SafePluginMap::addPlugin(const char *path, const char *dllname)
  482. {
  483. if (!endsWithIgnoreCase(path, SharedObjectExtension))
  484. {
  485. if (trace)
  486. DBGLOG("Ecl plugin %s ignored", path);
  487. return false;
  488. }
  489. try
  490. {
  491. CriticalBlock b(crit);
  492. ILoadedDllEntry *dll = map.getValue(dllname);
  493. if (!dll)
  494. {
  495. Owned<PluginDll> n = new PluginDll(path, NULL);
  496. if (!n->load(true, false) || !n->init(pluginCtx))
  497. throw MakeStringException(0, "Failed to load plugin %s", path);
  498. if (trace)
  499. n->logLoaded();
  500. map.setValue(dllname, n); // note: setValue links arg
  501. return true;
  502. }
  503. return false;
  504. }
  505. catch (IException * e) // MORE - not sure why we don't throw exceptions back here...
  506. {
  507. EXCLOG(e, "Loading plugin");
  508. e->Release();
  509. return false;
  510. }
  511. }
  512. ILoadedDllEntry * SafePluginMap::getPluginDll(const char *id, const char *version, bool checkVersion)
  513. {
  514. CriticalBlock b(crit);
  515. Linked<PluginDll> ret = static_cast<PluginDll *>(map.getValue(id));
  516. if (ret && checkVersion)
  517. {
  518. if (!ret->checkVersion(version))
  519. return NULL;
  520. }
  521. return ret.getLink();
  522. }
  523. void SafePluginMap::loadFromList(const char * pluginsList)
  524. {
  525. const char *pluginDir = pluginsList;
  526. for (;*pluginDir;)
  527. {
  528. StringBuffer thisPlugin;
  529. while (*pluginDir && *pluginDir != ENVSEPCHAR)
  530. thisPlugin.append(*pluginDir++);
  531. if(*pluginDir)
  532. pluginDir++;
  533. if(!thisPlugin.length())
  534. continue;
  535. Owned<IFile> file = createIFile(thisPlugin.str());
  536. if (file->isDirectory() == foundYes)
  537. loadFromDirectory(thisPlugin);
  538. else
  539. {
  540. StringBuffer tail;
  541. splitFilename(thisPlugin, NULL, NULL, &tail, &tail);
  542. addPlugin(thisPlugin, tail.str());
  543. }
  544. }
  545. }
  546. void SafePluginMap::loadFromDirectory(const char * pluginDirectory)
  547. {
  548. const char * mask = "*" SharedObjectExtension;
  549. Owned<IFile> pluginDir = createIFile(pluginDirectory);
  550. Owned<IDirectoryIterator> pluginFiles = pluginDir->directoryFiles(mask,false,false);
  551. ForEach(*pluginFiles)
  552. {
  553. const char *thisPlugin = pluginFiles->query().queryFilename();
  554. StringBuffer tail;
  555. splitFilename(thisPlugin, NULL, NULL, &tail, &tail);
  556. addPlugin(thisPlugin, tail.str());
  557. }
  558. }