pyembed.cpp 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461
  1. /*##############################################################################
  2. HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
  3. Licensed under the Apache License, Version 2.0 (the "License");
  4. you may not use this file except in compliance with the License.
  5. You may obtain a copy of the License at
  6. http://www.apache.org/licenses/LICENSE-2.0
  7. Unless required by applicable law or agreed to in writing, software
  8. distributed under the License is distributed on an "AS IS" BASIS,
  9. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  10. See the License for the specific language governing permissions and
  11. limitations under the License.
  12. ############################################################################## */
  13. #include "platform.h"
  14. #include "Python.h"
  15. #include "eclrtl.hpp"
  16. #include "jexcept.hpp"
  17. #include "jthread.hpp"
  18. #include "hqlplugins.hpp"
  19. #ifdef _WIN32
  20. #define EXPORT __declspec(dllexport)
  21. #else
  22. #define EXPORT
  23. #endif
  24. static const char * compatibleVersions[] = {
  25. "Python2.7 Embed Helper 1.0.0",
  26. NULL };
  27. static const char *version = "Python2.7 Embed Helper 1.0.0";
  28. static const char * EclDefinition =
  29. "EXPORT Language := SERVICE\n"
  30. " boolean getEmbedContext():cpp,pure,namespace='pyembed',entrypoint='getEmbedContext',prototype='IEmbedContext* getEmbedContext()';\n"
  31. " boolean syntaxCheck(const varstring src):cpp,pure,namespace='pyembed',entrypoint='syntaxCheck';\n"
  32. "END;"
  33. "EXPORT getEmbedContext := Language.getEmbedContext;"
  34. "EXPORT syntaxCheck := Language.syntaxCheck;"
  35. "EXPORT boolean supportsImport := true;"
  36. "EXPORT boolean supportsScript := true;";
  37. extern "C" EXPORT bool getECLPluginDefinition(ECLPluginDefinitionBlock *pb)
  38. {
  39. if (pb->size == sizeof(ECLPluginDefinitionBlockEx))
  40. {
  41. ECLPluginDefinitionBlockEx * pbx = (ECLPluginDefinitionBlockEx *) pb;
  42. pbx->compatibleVersions = compatibleVersions;
  43. }
  44. else if (pb->size != sizeof(ECLPluginDefinitionBlock))
  45. return false;
  46. pb->magicVersion = PLUGIN_VERSION;
  47. pb->version = version;
  48. pb->moduleName = "python";
  49. pb->ECL = EclDefinition;
  50. pb->flags = PLUGIN_DLL_MODULE | PLUGIN_MULTIPLE_VERSIONS;
  51. pb->description = "Python2.7 Embed Helper";
  52. return true;
  53. }
  54. namespace pyembed {
  55. // Use class OwnedPyObject for any objects that are not 'borrowed references'
  56. // so that the appropriate Py_DECREF call is made when the OwnedPyObject goes
  57. // out of scope, even if the function returns prematurely (such as via an exception).
  58. // In particular, checkPythonError is a lot easier to call safely if this is used.
  59. class OwnedPyObject
  60. {
  61. PyObject *ptr;
  62. public:
  63. inline OwnedPyObject() : ptr(NULL) {}
  64. inline OwnedPyObject(PyObject *_ptr) : ptr(_ptr) {}
  65. inline ~OwnedPyObject() { if (ptr) Py_DECREF(ptr); }
  66. inline PyObject * get() const { return ptr; }
  67. inline PyObject * operator -> () const { return ptr; }
  68. inline operator PyObject *() const { return ptr; }
  69. inline void clear() { if (ptr) Py_DECREF(ptr); ptr = NULL; }
  70. inline void setown(PyObject *_ptr) { clear(); ptr = _ptr; }
  71. inline void set(PyObject *_ptr) { clear(); ptr = _ptr; if (ptr) Py_INCREF(ptr);}
  72. inline PyObject *getLink() { if (ptr) Py_INCREF(ptr); return ptr;}
  73. inline PyObject **ref() { return &ptr; }
  74. };
  75. // call checkPythonError to throw an exception if Python error state is set
  76. static void checkPythonError()
  77. {
  78. PyObject* err = PyErr_Occurred();
  79. if (err)
  80. {
  81. OwnedPyObject pType, pValue, pTraceBack;
  82. PyErr_Fetch(pType.ref(), pType.ref(), pType.ref());
  83. OwnedPyObject errStr = PyObject_Str(err);
  84. OwnedPyObject valStr = PyObject_Str(pValue);
  85. PyErr_Clear();
  86. VStringBuffer errMessage("%s: %s", PyString_AsString(errStr), PyString_AsString(valStr));
  87. rtlFail(0, errMessage.str());
  88. }
  89. }
  90. // The Python Global Interpreter Lock (GIL) won't know about C++-created threads, so we need to
  91. // call PyGILState_Ensure() and PyGILState_Release at the start and end of every function.
  92. // Wrapping them in a class like this ensures that the release always happens even if
  93. // the function exists prematurely
  94. class GILstateWrapper
  95. {
  96. PyGILState_STATE gstate;
  97. public:
  98. GILstateWrapper()
  99. {
  100. gstate = PyGILState_Ensure();
  101. }
  102. ~GILstateWrapper()
  103. {
  104. PyGILState_Release(gstate);
  105. }
  106. };
  107. // Use a global object to ensure that the Python interpreter is initialized on main thread
  108. static class Python27GlobalState
  109. {
  110. public:
  111. Python27GlobalState()
  112. {
  113. // Initialize the Python Interpreter
  114. Py_Initialize();
  115. PyEval_InitThreads();
  116. tstate = PyEval_SaveThread();
  117. }
  118. ~Python27GlobalState()
  119. {
  120. PyEval_RestoreThread(tstate);
  121. // Finish the Python Interpreter
  122. Py_Finalize();
  123. }
  124. protected:
  125. PyThreadState *tstate;
  126. } globalState;
  127. // There is a singleton PythonThreadContext per thread. This allows us to
  128. // ensure that we can make repeated calls to a Python function efficiently.
  129. class PythonThreadContext
  130. {
  131. public:
  132. PyThreadState *threadState;
  133. public:
  134. PythonThreadContext()
  135. {
  136. threadState = PyEval_SaveThread();
  137. }
  138. ~PythonThreadContext()
  139. {
  140. PyEval_RestoreThread(threadState);
  141. script.clear();
  142. }
  143. inline PyObject * importFunction(const char *text)
  144. {
  145. if (!prevtext || strcmp(text, prevtext) != 0)
  146. {
  147. prevtext.clear();
  148. // Name should be in the form module.function
  149. const char *funcname = strrchr(text, '.');
  150. if (!funcname)
  151. rtlFail(0, "Expected module.function");
  152. StringBuffer modname(funcname-text, text);
  153. funcname++; // skip the '.'
  154. // If the modname is preceded by a path, add it to the python path before importing
  155. const char *pathsep = strrchr(modname, PATHSEPCHAR);
  156. if (pathsep)
  157. {
  158. StringBuffer path(pathsep-modname, modname);
  159. modname.remove(0, 1+pathsep-modname);
  160. PyObject *sys_path = PySys_GetObject("path");
  161. OwnedPyObject new_path = PyString_FromString(path);
  162. if (sys_path)
  163. {
  164. PyList_Append(sys_path, new_path);
  165. checkPythonError();
  166. }
  167. }
  168. module.setown(PyImport_ImportModule(modname));
  169. checkPythonError();
  170. PyObject *dict = PyModule_GetDict(module); // this is a borrowed reference and does not need to be released
  171. script.set(PyDict_GetItemString(dict, funcname));
  172. checkPythonError();
  173. if (!script || !PyCallable_Check(script))
  174. rtlFail(0, "Object is not callable");
  175. prevtext.set(text);
  176. }
  177. return script.getLink();
  178. }
  179. inline PyObject *compileEmbeddedScript(const char *text)
  180. {
  181. if (!prevtext || strcmp(text, prevtext) != 0)
  182. {
  183. prevtext.clear();
  184. // Try compiling as a eval first... if that fails, try as a script.
  185. script.setown(Py_CompileString(text, "", Py_eval_input));
  186. if (!script)
  187. {
  188. PyErr_Clear();
  189. StringBuffer wrapped;
  190. wrapPythonText(wrapped, text);
  191. script.setown(Py_CompileString(wrapped, "<embed>", Py_file_input));
  192. }
  193. checkPythonError();
  194. prevtext.set(text);
  195. }
  196. return script.getLink();
  197. }
  198. private:
  199. static StringBuffer &wrapPythonText(StringBuffer &out, const char *in)
  200. {
  201. out.append("def __user__():\n ");
  202. char c;
  203. while ((c = *in++) != '\0')
  204. {
  205. out.append(c);
  206. if (c=='\n')
  207. out.append(" ");
  208. }
  209. out.append("\n__result__ = __user__()\n");
  210. return out;
  211. }
  212. GILstateWrapper GILState;
  213. OwnedPyObject module;
  214. OwnedPyObject script;
  215. StringAttr prevtext;
  216. };
  217. // Each call to a Python function will use a new Python27EmbedFunctionContext object
  218. // This takes care of ensuring that the Python GIL is locked while we are executing python code,
  219. // and released when we are not
  220. class Python27EmbedContextBase : public CInterfaceOf<IEmbedFunctionContext>
  221. {
  222. public:
  223. Python27EmbedContextBase(PythonThreadContext *_sharedCtx)
  224. : sharedCtx(_sharedCtx)
  225. {
  226. PyEval_RestoreThread(sharedCtx->threadState);
  227. locals.setown(PyDict_New());
  228. globals.setown(PyDict_New());
  229. PyDict_SetItemString(locals, "__builtins__", PyEval_GetBuiltins( )); // required for import to work
  230. }
  231. ~Python27EmbedContextBase()
  232. {
  233. sharedCtx->threadState = PyEval_SaveThread();
  234. }
  235. virtual bool getBooleanResult()
  236. {
  237. assertex(result);
  238. if (!PyBool_Check(result))
  239. throw MakeStringException(MSGAUD_user, 0, "pyembed: Type mismatch on result");
  240. return result == Py_True;
  241. }
  242. virtual double getRealResult()
  243. {
  244. assertex(result);
  245. return (__int64) PyFloat_AsDouble(result);
  246. }
  247. virtual __int64 getSignedResult()
  248. {
  249. assertex(result);
  250. return (__int64) PyLong_AsLongLong(result);
  251. }
  252. virtual unsigned __int64 getUnsignedResult()
  253. {
  254. return (__int64) PyLong_AsUnsignedLongLong(result);
  255. }
  256. virtual void getStringResult(size32_t &__len, char * &__result)
  257. {
  258. assertex(result);
  259. __len = PyString_Size(result);
  260. const char * chars = PyString_AsString(result);
  261. checkPythonError();
  262. __result = (char *)rtlMalloc(__len);
  263. memcpy(__result, chars, __len);
  264. }
  265. protected:
  266. PythonThreadContext *sharedCtx;
  267. OwnedPyObject locals;
  268. OwnedPyObject globals;
  269. OwnedPyObject result;
  270. OwnedPyObject script;
  271. };
  272. class Python27EmbedScriptContext : public Python27EmbedContextBase
  273. {
  274. public:
  275. Python27EmbedScriptContext(PythonThreadContext *_sharedCtx, const char *options)
  276. : Python27EmbedContextBase(_sharedCtx)
  277. {
  278. }
  279. ~Python27EmbedScriptContext()
  280. {
  281. }
  282. virtual void bindBooleanParam(const char *name, bool val)
  283. {
  284. OwnedPyObject vval = PyBool_FromLong(val ? 1 : 0);
  285. PyDict_SetItemString(locals, name, vval);
  286. }
  287. virtual void bindRealParam(const char *name, double val)
  288. {
  289. OwnedPyObject vval = PyFloat_FromDouble(val);
  290. PyDict_SetItemString(locals, name, vval);
  291. }
  292. virtual void bindSignedParam(const char *name, __int64 val)
  293. {
  294. OwnedPyObject vval = PyLong_FromLongLong(val);
  295. PyDict_SetItemString(locals, name, vval);
  296. }
  297. virtual void bindUnsignedParam(const char *name, unsigned __int64 val)
  298. {
  299. OwnedPyObject vval = PyLong_FromUnsignedLongLong(val);
  300. PyDict_SetItemString(locals, name, vval);
  301. }
  302. virtual void bindStringParam(const char *name, size32_t len, const char *val)
  303. {
  304. OwnedPyObject vval = PyString_FromStringAndSize(val, len);
  305. PyDict_SetItemString(locals, name, vval);
  306. }
  307. virtual void bindVStringParam(const char *name, const char *val)
  308. {
  309. OwnedPyObject vval = PyString_FromString(val);
  310. PyDict_SetItemString(locals, name, vval);
  311. }
  312. virtual void importFunction(const char *text)
  313. {
  314. throwUnexpected();
  315. }
  316. virtual void compileEmbeddedScript(const char *text)
  317. {
  318. script.setown(sharedCtx->compileEmbeddedScript(text));
  319. }
  320. virtual void callFunction()
  321. {
  322. result.setown(PyEval_EvalCode((PyCodeObject *) script.get(), locals, globals));
  323. checkPythonError();
  324. if (!result || result == Py_None)
  325. result.set(PyDict_GetItemString(locals, "__result__"));
  326. if (!result || result == Py_None)
  327. result.set(PyDict_GetItemString(globals, "__result__"));
  328. }
  329. };
  330. class Python27EmbedImportContext : public Python27EmbedContextBase
  331. {
  332. public:
  333. Python27EmbedImportContext(PythonThreadContext *_sharedCtx, const char *options)
  334. : Python27EmbedContextBase(_sharedCtx)
  335. {
  336. argcount = 0;
  337. }
  338. ~Python27EmbedImportContext()
  339. {
  340. }
  341. virtual void bindBooleanParam(const char *name, bool val)
  342. {
  343. addArg(PyBool_FromLong(val ? 1 : 0));
  344. }
  345. virtual void bindRealParam(const char *name, double val)
  346. {
  347. addArg(PyFloat_FromDouble(val));
  348. }
  349. virtual void bindSignedParam(const char *name, __int64 val)
  350. {
  351. addArg(PyLong_FromLongLong(val));
  352. }
  353. virtual void bindUnsignedParam(const char *name, unsigned __int64 val)
  354. {
  355. addArg(PyLong_FromUnsignedLongLong(val));
  356. }
  357. virtual void bindStringParam(const char *name, size32_t len, const char *val)
  358. {
  359. addArg(PyString_FromStringAndSize(val, len));
  360. }
  361. virtual void bindVStringParam(const char *name, const char *val)
  362. {
  363. addArg(PyString_FromString(val));
  364. }
  365. virtual void importFunction(const char *text)
  366. {
  367. script.setown(sharedCtx->importFunction(text));
  368. }
  369. virtual void compileEmbeddedScript(const char *text)
  370. {
  371. throwUnexpected();
  372. }
  373. virtual void callFunction()
  374. {
  375. result.setown(PyObject_CallObject(script, args));
  376. checkPythonError();
  377. }
  378. private:
  379. void addArg(PyObject *arg)
  380. {
  381. if (argcount)
  382. _PyTuple_Resize(args.ref(), argcount+1);
  383. else
  384. args.setown(PyTuple_New(1));
  385. PyTuple_SET_ITEM((PyTupleObject *) args.get(), argcount++, arg); // Note - 'steals' the arg reference
  386. }
  387. int argcount;
  388. OwnedPyObject args;
  389. };
  390. static __thread PythonThreadContext* threadContext; // We reuse per thread, for speed
  391. static __thread ThreadTermFunc threadHookChain;
  392. static void releaseContext()
  393. {
  394. delete threadContext;
  395. if (threadHookChain)
  396. (*threadHookChain)();
  397. }
  398. class Python27EmbedContext : public CInterfaceOf<IEmbedContext>
  399. {
  400. public:
  401. virtual IEmbedFunctionContext *createFunctionContext(bool isImport, const char *options)
  402. {
  403. if (!threadContext)
  404. {
  405. threadContext = new PythonThreadContext;
  406. threadHookChain = addThreadTermFunc(releaseContext);
  407. }
  408. if (isImport)
  409. return new Python27EmbedImportContext(threadContext, options);
  410. else
  411. return new Python27EmbedScriptContext(threadContext, options);
  412. }
  413. };
  414. extern IEmbedContext* getEmbedContext()
  415. {
  416. return new Python27EmbedContext;
  417. }
  418. extern bool syntaxCheck(const char *script)
  419. {
  420. return true; // MORE
  421. }
  422. } // namespace