py3embed.cpp 63 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805
  1. /*##############################################################################
  2. HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems®.
  3. Licensed under the Apache License, Version 2.0 (the "License");
  4. you may not use this file except in compliance with the License.
  5. You may obtain a copy of the License at
  6. http://www.apache.org/licenses/LICENSE-2.0
  7. Unless required by applicable law or agreed to in writing, software
  8. distributed under the License is distributed on an "AS IS" BASIS,
  9. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  10. See the License for the specific language governing permissions and
  11. limitations under the License.
  12. ############################################################################## */
  13. #ifdef _WIN32
  14. // There's an issue with Python redefining ssize_t resulting in errors - hide their definition
  15. #define ssize_t python_ssize_t
  16. #include "Python.h"
  17. #undef ssize_t
  18. #else
  19. #include "Python.h"
  20. #endif
  21. #include "platform.h"
  22. #include "frameobject.h"
  23. #include "jexcept.hpp"
  24. #include "jutil.hpp"
  25. #include "jthread.hpp"
  26. #include "jregexp.hpp"
  27. #include "hqlplugins.hpp"
  28. #include "deftype.hpp"
  29. #include "eclhelper.hpp"
  30. #include "eclrtl.hpp"
  31. #include "eclrtl_imp.hpp"
  32. #include "rtlds_imp.hpp"
  33. #include "rtlfield.hpp"
  34. #include "nbcd.hpp"
  35. #include "roxiemem.hpp"
  36. #include "enginecontext.hpp"
  37. #if PY_MAJOR_VERSION >=3
  38. #define Py_TPFLAGS_HAVE_ITER 0
  39. #endif
  40. static const char * compatibleVersions[] = {
  41. "Python3.x Embed Helper 1.0.0",
  42. NULL };
  43. static const char *version = "Python3.x Embed Helper 1.0.0";
  44. extern "C" DECL_EXPORT bool getECLPluginDefinition(ECLPluginDefinitionBlock *pb)
  45. {
  46. if (pb->size == sizeof(ECLPluginDefinitionBlockEx))
  47. {
  48. ECLPluginDefinitionBlockEx * pbx = (ECLPluginDefinitionBlockEx *) pb;
  49. pbx->compatibleVersions = compatibleVersions;
  50. }
  51. else if (pb->size != sizeof(ECLPluginDefinitionBlock))
  52. return false;
  53. pb->magicVersion = PLUGIN_VERSION;
  54. pb->version = version;
  55. pb->moduleName = "python3";
  56. pb->ECL = NULL;
  57. pb->flags = PLUGIN_MULTIPLE_VERSIONS;
  58. pb->description = "Python3.x Embed Helper";
  59. return true;
  60. }
  61. namespace py3embed {
  62. // Use class OwnedPyObject for any objects that are not 'borrowed references'
  63. // so that the appropriate Py_DECREF call is made when the OwnedPyObject goes
  64. // out of scope, even if the function returns prematurely (such as via an exception).
  65. // In particular, checkPythonError is a lot easier to call safely if this is used.
  66. class OwnedPyObject
  67. {
  68. PyObject *ptr;
  69. public:
  70. inline OwnedPyObject() : ptr(NULL) {}
  71. inline OwnedPyObject(PyObject *_ptr) : ptr(_ptr) {}
  72. inline ~OwnedPyObject() { if (ptr) Py_DECREF(ptr); }
  73. inline PyObject * get() const { return ptr; }
  74. inline PyObject * getClear() { PyObject *ret = ptr; ptr = NULL; return ret; }
  75. inline PyObject * operator -> () const { return ptr; }
  76. inline operator PyObject *() const { return ptr; }
  77. inline void clear() { if (ptr) Py_DECREF(ptr); ptr = NULL; }
  78. inline void setown(PyObject *_ptr) { clear(); ptr = _ptr; }
  79. inline void set(PyObject *_ptr) { if (_ptr) Py_INCREF(_ptr); clear(); ptr = _ptr; }
  80. inline PyObject *getLink() { if (ptr) Py_INCREF(ptr); return ptr;}
  81. inline PyObject **ref() { return &ptr; }
  82. };
  83. template <class X>
  84. class OwnedPyX
  85. {
  86. X *ptr;
  87. public:
  88. inline OwnedPyX<X>() : ptr(NULL) {}
  89. inline OwnedPyX<X>(X *_ptr) : ptr(_ptr) {}
  90. inline ~OwnedPyX<X>() { if (ptr) Py_DECREF(ptr); }
  91. inline X * get() const { return ptr; }
  92. inline X * getClear() { PyObject *ret = ptr; ptr = NULL; return ret; }
  93. inline X * operator -> () const { return ptr; }
  94. inline operator X *() const { return ptr; }
  95. inline void clear() { if (ptr) Py_DECREF(ptr); ptr = NULL; }
  96. inline void setown(X *_ptr) { clear(); ptr = _ptr; }
  97. inline void set(X *_ptr) { if (_ptr) Py_INCREF(_ptr); clear(); ptr = _ptr; }
  98. inline X *getLink() { if (ptr) Py_INCREF(ptr); return ptr;}
  99. inline X **ref() { return &ptr; }
  100. };
  101. __declspec(noreturn) static void failx(const char *msg, ...) __attribute__((format(printf, 1, 2), noreturn));
  102. static void failx(const char *message, ...)
  103. {
  104. va_list args;
  105. va_start(args,message);
  106. StringBuffer msg;
  107. msg.append("pyembed: ").valist_appendf(message,args);
  108. va_end(args);
  109. rtlFail(0, msg.str());
  110. }
  111. // call checkPythonError to throw an exception if Python error state is set
  112. static void checkPythonError()
  113. {
  114. PyObject* err = PyErr_Occurred();
  115. if (err)
  116. {
  117. OwnedPyObject pType, pValue, pTraceBack;
  118. PyErr_Fetch(pType.ref(), pValue.ref(), pTraceBack.ref());
  119. OwnedPyObject valStr = PyObject_Str(pValue);
  120. PyErr_Clear();
  121. assertex(PyUnicode_Check(valStr));
  122. const char *text = PyUnicode_AsUTF8AndSize(valStr, NULL);
  123. failx("%s", text);
  124. }
  125. }
  126. // The Python Global Interpreter Lock (GIL) won't know about C++-created threads, so we need to
  127. // call PyGILState_Ensure() and PyGILState_Release at the start and end of every function.
  128. // Wrapping them in a class like this ensures that the release always happens even if
  129. // the function exists prematurely
  130. class GILstateWrapper
  131. {
  132. PyGILState_STATE gstate;
  133. public:
  134. GILstateWrapper()
  135. {
  136. gstate = PyGILState_Ensure();
  137. }
  138. ~GILstateWrapper()
  139. {
  140. PyGILState_Release(gstate);
  141. }
  142. };
  143. // There is a singleton PythonThreadContext per thread. This allows us to
  144. // ensure that we can make repeated calls to a Python function efficiently.
  145. class PythonThreadContext
  146. {
  147. public:
  148. PyThreadState *threadState;
  149. public:
  150. PythonThreadContext()
  151. {
  152. threadState = PyEval_SaveThread();
  153. lrutype = NULL;
  154. }
  155. ~PythonThreadContext()
  156. {
  157. PyEval_RestoreThread(threadState);
  158. script.clear();
  159. module.clear();
  160. lru.clear();
  161. }
  162. inline PyObject * importFunction(size32_t lenChars, const char *utf)
  163. {
  164. size32_t bytes = rtlUtf8Size(lenChars, utf);
  165. StringBuffer text(bytes, utf);
  166. if (!prevtext || strcmp(text, prevtext) != 0)
  167. {
  168. prevtext.clear();
  169. // Name should be in the form module.function
  170. const char *funcname = strrchr(text, '.');
  171. if (!funcname)
  172. rtlFail(0, "pyembed: Expected module.function");
  173. StringBuffer modname(funcname-text, text);
  174. funcname++; // skip the '.'
  175. // If the modname is preceded by a path, add it to the python path before importing
  176. const char *pathsep = strrchr(modname, PATHSEPCHAR);
  177. if (pathsep)
  178. {
  179. StringBuffer path(pathsep-modname, modname);
  180. modname.remove(0, 1+pathsep-modname);
  181. PyObject *sys_path = PySys_GetObject((char *) "path");
  182. OwnedPyObject new_path = PyUnicode_FromString(path);
  183. if (sys_path)
  184. {
  185. PyList_Insert(sys_path, 0, new_path);
  186. checkPythonError();
  187. }
  188. }
  189. module.setown(PyImport_ImportModule(modname));
  190. checkPythonError();
  191. PyObject *dict = PyModule_GetDict(module); // this is a borrowed reference and does not need to be released
  192. script.set(PyDict_GetItemString(dict, funcname));
  193. checkPythonError();
  194. if (!script || !PyCallable_Check(script))
  195. rtlFail(0, "pyembed: Object is not callable");
  196. prevtext.set(text);
  197. }
  198. return script.getLink();
  199. }
  200. PyObject *compileEmbeddedScript(size32_t lenChars, const char *utf, const char *argstring);
  201. PyObject *getNamedTupleType(const RtlTypeInfo *type);
  202. private:
  203. GILstateWrapper GILState;
  204. OwnedPyObject module;
  205. OwnedPyObject script;
  206. OwnedPyObject lru;
  207. const RtlTypeInfo *lrutype;
  208. StringAttr prevtext;
  209. };
  210. static __thread PythonThreadContext* threadContext; // We reuse per thread, for speed
  211. static __thread ThreadTermFunc threadHookChain;
  212. static void releaseContext()
  213. {
  214. if (threadContext)
  215. {
  216. delete threadContext;
  217. threadContext = NULL;
  218. }
  219. if (threadHookChain)
  220. {
  221. (*threadHookChain)();
  222. threadHookChain = NULL;
  223. }
  224. }
  225. // Use a global object to ensure that the Python interpreter is initialized on main thread
  226. static HINSTANCE keepLoadedHandle;
  227. static class Python3xGlobalState
  228. {
  229. public:
  230. Python3xGlobalState()
  231. {
  232. pythonLibrary = (HINSTANCE) 0;
  233. #ifndef _WIN32
  234. // If Py_Initialize is called when stdin is set to a directory, it calls exit()
  235. // We don't want that to happen - just disable Python support in such situations
  236. struct stat sb;
  237. if (fstat(fileno(stdin), &sb) == 0 && S_ISDIR(sb.st_mode))
  238. {
  239. initialized = false;
  240. return;
  241. }
  242. #endif
  243. #ifndef _WIN32
  244. // We need to ensure all symbols in the python3.x so are loaded - due to bugs in some distro's python installations
  245. // However this will likely break python2.
  246. // Therefore on systems where both are present, do NOT do this - people using centos systems that suffer from issue
  247. // https://bugs.centos.org/view.php?id=6063 will need to choose which version of python plugin to install but not both
  248. StringBuffer modname, py2modname;
  249. if (findLoadedModule(modname, "libpython3.") && !findLoadedModule(py2modname, "libpython2."))
  250. pythonLibrary = dlopen(modname.str(), RTLD_NOW|RTLD_GLOBAL);
  251. #endif
  252. // Initialize the Python Interpreter
  253. Py_Initialize();
  254. const wchar_t *argv[] = { nullptr };
  255. PySys_SetArgvEx(0, (wchar_t **) argv, 0);
  256. PyEval_InitThreads();
  257. preservedScopes.setown(PyDict_New());
  258. tstate = PyEval_SaveThread();
  259. skipPythonCleanup = queryEnvironmentConf().getPropBool("skipPythonCleanup", true);
  260. initialized = true;
  261. }
  262. ~Python3xGlobalState()
  263. {
  264. if (threadContext)
  265. delete threadContext; // The one on the main thread won't get picked up by the thread hook mechanism
  266. threadContext = NULL;
  267. if (initialized && !skipPythonCleanup)
  268. {
  269. PyEval_RestoreThread(tstate);
  270. // Finish the Python Interpreter
  271. namedtuple.clear();
  272. namedtupleTypes.clear();
  273. compiledScripts.clear();
  274. preservedScopes.clear();
  275. Py_Finalize();
  276. if (pythonLibrary)
  277. FreeSharedObject(pythonLibrary);
  278. }
  279. }
  280. bool isInitialized()
  281. {
  282. return initialized;
  283. }
  284. PyFrameObject *pushDummyFrame()
  285. {
  286. PyThreadState* threadstate = PyThreadState_GET();
  287. if (!threadstate->frame)
  288. {
  289. OwnedPyObject globals = PyDict_New();
  290. OwnedPyObject locals = PyDict_New();
  291. OwnedPyX<PyCodeObject> code = PyCode_NewEmpty("<dummy>","<dummy>", 0);
  292. checkPythonError();
  293. PyFrameObject *frame = PyFrame_New(threadstate, code, globals, locals);
  294. checkPythonError();
  295. threadstate->frame = frame;
  296. return frame;
  297. }
  298. return NULL;
  299. }
  300. void popDummyFrame(PyFrameObject *frame)
  301. {
  302. PyThreadState* threadstate = PyThreadState_GET();
  303. if (threadstate->frame == frame)
  304. threadstate->frame = NULL;
  305. }
  306. PyObject *getActivityContextTupleType()
  307. {
  308. // Note - we do not need (and must not have) a lock protecting this. It is protected by the Python GIL,
  309. // and if we add our own lock we are liable to deadlock as the code within Py_CompileStringFlags may
  310. // temporarily release then re-acquire the GIL.
  311. if (!activityContextTupleType)
  312. activityContextTupleType.setown(getNamedTupleType("isLocal,numSlaves,numStrands,slave,strand"));
  313. return activityContextTupleType.get();
  314. }
  315. PyObject *getNamedTupleType(const char *names)
  316. {
  317. // It seems the customized namedtuple types leak, and they are slow to create, so take care to reuse
  318. // Note - we do not need (and must not have) a lock protecting this. It is protected by the Python GIL,
  319. // and if we add our own lock we are liable to deadlock as the code within Py_CompileStringFlags may
  320. // temporarily release then re-acquire the GIL.
  321. if (!namedtuple)
  322. {
  323. namedtupleTypes.setown(PyDict_New());
  324. OwnedPyObject pName = PyUnicode_FromString("collections");
  325. OwnedPyObject collections = PyImport_Import(pName);
  326. checkPythonError();
  327. namedtuple.setown(PyObject_GetAttrString(collections, "namedtuple"));
  328. checkPythonError();
  329. assertex(PyCallable_Check(namedtuple));
  330. }
  331. OwnedPyObject pnames = PyUnicode_FromString(names);
  332. OwnedPyObject mynamedtupletype;
  333. checkPythonError();
  334. mynamedtupletype.set(PyDict_GetItem(namedtupleTypes, pnames)); // NOTE - returns borrowed reference
  335. if (!mynamedtupletype)
  336. {
  337. OwnedPyObject recname = PyUnicode_FromString("namerec"); // MORE - do we care what the name is?
  338. OwnedPyObject ntargs = PyTuple_Pack(2, recname.get(), pnames.get());
  339. checkPythonError();
  340. OwnedPyX<PyFrameObject> frame = pushDummyFrame();
  341. mynamedtupletype.setown(PyObject_CallObject(namedtuple, ntargs));
  342. popDummyFrame(frame);
  343. checkPythonError();
  344. PyDict_SetItem(namedtupleTypes, pnames, mynamedtupletype);
  345. }
  346. checkPythonError();
  347. assertex(PyCallable_Check(mynamedtupletype));
  348. return mynamedtupletype.getClear();
  349. }
  350. PyObject *getNamedTupleType(const RtlTypeInfo *type)
  351. {
  352. const RtlFieldInfo * const *fields = type->queryFields();
  353. if (!fields && type->queryChildType())
  354. fields = type->queryChildType()->queryFields();
  355. assertex(fields);
  356. StringBuffer names;
  357. while (*fields)
  358. {
  359. const RtlFieldInfo *field = *fields;
  360. if (names.length())
  361. names.append(',');
  362. names.append(field->name);
  363. fields++;
  364. }
  365. return getNamedTupleType(names.str());
  366. }
  367. PyObject *compileScript(const char *text, const char *parameters)
  368. {
  369. // Note - we do not need (and must not have) a lock protecting this. It is protected by the Python GIL,
  370. // and if we add our own lock we are liable to deadlock as the code within Py_CompileStringFlags may
  371. // temporarily release then re-acquire the GIL.
  372. if (!compiledScripts)
  373. compiledScripts.setown(PyDict_New());
  374. OwnedPyObject code;
  375. code.set(PyDict_GetItemString(compiledScripts, text));
  376. if (!code)
  377. {
  378. code.setown(Py_CompileString(text, "", Py_eval_input)); // try compiling as simple expression...
  379. if (!code)
  380. {
  381. PyErr_Clear();
  382. PyCompilerFlags flags = { PyCF_SOURCE_IS_UTF8 };
  383. code.setown(Py_CompileStringFlags(text, "<embed>", Py_file_input, &flags)); // try compiling as global code
  384. if (!code)
  385. {
  386. PyErr_Clear();
  387. StringBuffer wrapped;
  388. wrapPythonText(wrapped, text, parameters);
  389. code.setown(Py_CompileStringFlags(wrapped, "<embed>", Py_file_input, &flags)); // try compiling as a function body
  390. }
  391. }
  392. checkPythonError();
  393. if (code)
  394. PyDict_SetItemString(compiledScripts, text, code);
  395. }
  396. return code.getClear();
  397. }
  398. PyObject *getNamedScope(const char *key, bool &isNew)
  399. {
  400. if (!preservedScopes)
  401. preservedScopes.setown(PyDict_New());
  402. OwnedPyObject scope;
  403. scope.set(PyDict_GetItemString(preservedScopes, key));
  404. if (!scope)
  405. {
  406. scope.setown(PyDict_New());
  407. PyDict_SetItemString(preservedScopes, key, scope);
  408. isNew = true;
  409. }
  410. else
  411. isNew = false;
  412. return scope.getClear();
  413. }
  414. void releaseNamedScope(const char *key)
  415. {
  416. if (preservedScopes)
  417. {
  418. PyDict_DelItemString(preservedScopes, key);
  419. PyErr_Clear(); // Should be present, but ignore the error if it is not
  420. }
  421. }
  422. static void unregister(const char *key);
  423. protected:
  424. static StringBuffer &wrapPythonText(StringBuffer &out, const char *in, const char *params)
  425. {
  426. // Complicated by needing to keep future import lines outside defined function
  427. // Per python spec, a future statement must appear near the top of the module. The only lines that can appear before a future statement are:
  428. // the module docstring (if any),
  429. // comments,
  430. // blank lines, and
  431. // other future statements.
  432. // We don't attempt to parse the python to spot these - instead, we pull all lines up to and including the last future statement out to the global scope.
  433. // Because this is a little unsophisticated it will be fooled by code that includes things that look like future statements inside multiline strings.
  434. // I don't care.
  435. StringArray lines;
  436. lines.appendList(in, "\n", false);
  437. RegExpr expr("^ *from +__future__ +import ");
  438. unsigned leadingLines = 0;
  439. ForEachItemIn(idx, lines)
  440. {
  441. if (expr.find(lines.item(idx)))
  442. leadingLines = idx+1;
  443. }
  444. for (unsigned leadingLine = 0; leadingLine < leadingLines; leadingLine++)
  445. out.append(lines.item(leadingLine)).append('\n');
  446. out.appendf("def __user__(%s):\n", params);
  447. for (unsigned line = leadingLines; line < lines.length(); line++)
  448. out.append(" ").append(lines.item(line)).append('\n');
  449. out.appendf("__result__ = __user__(%s)\n", params);
  450. return out;
  451. }
  452. PyThreadState *tstate = nullptr;
  453. bool initialized = false;
  454. bool skipPythonCleanup = true; // Tensorflow seems to often lockup in the python cleanup process.
  455. HINSTANCE pythonLibrary = 0;
  456. OwnedPyObject namedtuple; // collections.namedtuple
  457. OwnedPyObject namedtupleTypes; // dictionary of return values from namedtuple()
  458. OwnedPyObject compiledScripts; // dictionary of previously compiled scripts
  459. OwnedPyObject preservedScopes; // dictionary of preserved scopes
  460. OwnedPyObject activityContextTupleType; // type used for activity context
  461. } globalState;
  462. MODULE_INIT(INIT_PRIORITY_STANDARD)
  463. {
  464. // Make sure we are never unloaded (as Python may crash if we are)
  465. // we do this by doing a dynamic load of the pyembed library
  466. #ifdef _WIN32
  467. HINSTANCE me = GetModuleHandle("py3embed");
  468. if (me)
  469. {
  470. char helperLibraryName[_MAX_PATH];
  471. ::GetModuleFileName(me, helperLibraryName, _MAX_PATH);
  472. if (strstr(helperLibraryName, "py3embed"))
  473. {
  474. HINSTANCE h = LoadSharedObject(helperLibraryName, false, false);
  475. DBGLOG("LoadSharedObject returned %p", h);
  476. }
  477. }
  478. #else
  479. FILE *diskfp = fopen("/proc/self/maps", "r");
  480. if (diskfp)
  481. {
  482. char ln[_MAX_PATH];
  483. while (fgets(ln, sizeof(ln), diskfp))
  484. {
  485. if (strstr(ln, "libpy3embed"))
  486. {
  487. const char *fullName = strchr(ln, '/');
  488. if (fullName)
  489. {
  490. char *tail = (char *) strstr(fullName, SharedObjectExtension);
  491. if (tail)
  492. {
  493. tail[strlen(SharedObjectExtension)] = 0;
  494. keepLoadedHandle = LoadSharedObject(fullName, false, false);
  495. break;
  496. }
  497. }
  498. }
  499. }
  500. fclose(diskfp);
  501. }
  502. #endif
  503. return true;
  504. }
  505. static void checkThreadContext()
  506. {
  507. if (!threadContext)
  508. {
  509. if (!globalState.isInitialized())
  510. rtlFail(0, "Python not initialized");
  511. threadContext = new PythonThreadContext;
  512. threadHookChain = addThreadTermFunc(releaseContext);
  513. }
  514. }
  515. PyObject *PythonThreadContext::getNamedTupleType(const RtlTypeInfo *type)
  516. {
  517. if (!lru || (type!=lrutype))
  518. {
  519. lru.setown(globalState.getNamedTupleType(type));
  520. lrutype = type;
  521. }
  522. return lru.getLink();
  523. }
  524. PyObject *PythonThreadContext::compileEmbeddedScript(size32_t lenChars, const char *utf, const char *argstring)
  525. {
  526. size32_t bytes = rtlUtf8Size(lenChars, utf);
  527. StringBuffer text(bytes, utf);
  528. if (!prevtext || strcmp(text, prevtext) != 0)
  529. {
  530. prevtext.clear();
  531. text.stripChar('\r');
  532. script.setown(globalState.compileScript(text, argstring));
  533. prevtext.set(utf, bytes);
  534. }
  535. return script.getLink();
  536. }
  537. // Conversions from Python objects to ECL data
  538. __declspec(noreturn) static void typeError(const char *expected, const RtlFieldInfo *field) __attribute__((noreturn));
  539. static void typeError(const char *expected, const RtlFieldInfo *field)
  540. {
  541. VStringBuffer msg("pyembed: type mismatch - %s expected", expected);
  542. if (field)
  543. msg.appendf(" for field %s", field->name);
  544. else
  545. msg.appendf(" for return value");
  546. rtlFail(0, msg.str());
  547. }
  548. static bool getBooleanResult(const RtlFieldInfo *field, PyObject *obj)
  549. {
  550. if (obj && obj != Py_None)
  551. {
  552. if (PyBool_Check(obj))
  553. return obj == Py_True;
  554. }
  555. typeError("boolean", field);
  556. }
  557. static void getDataResult(const RtlFieldInfo *field, PyObject *obj, size32_t &chars, void * &result)
  558. {
  559. if (obj && obj != Py_None && PyByteArray_Check(obj))
  560. rtlStrToDataX(chars, result, PyByteArray_Size(obj), PyByteArray_AsString(obj));
  561. else
  562. typeError("bytearray", field);
  563. }
  564. static double getRealResult(const RtlFieldInfo *field, PyObject *obj)
  565. {
  566. if (obj && obj != Py_None)
  567. {
  568. if (PyFloat_Check(obj))
  569. return PyFloat_AsDouble(obj);
  570. }
  571. typeError("real", field);
  572. }
  573. static __int64 getSignedResult(const RtlFieldInfo *field, PyObject *obj)
  574. {
  575. if (obj && obj != Py_None)
  576. {
  577. if (PyLong_Check(obj))
  578. return (__int64) PyLong_AsLongLong(obj);
  579. }
  580. typeError("integer", field);
  581. }
  582. static unsigned __int64 getUnsignedResult(const RtlFieldInfo *field, PyObject *obj)
  583. {
  584. if (obj && obj != Py_None)
  585. {
  586. if (PyLong_Check(obj))
  587. return (unsigned __int64) PyLong_AsUnsignedLongLong(obj);
  588. }
  589. typeError("integer", field);
  590. }
  591. static void getStringResult(const RtlFieldInfo *field, PyObject *obj, size32_t &chars, char * &result)
  592. {
  593. if (obj && obj != Py_None)
  594. {
  595. if (PyUnicode_Check(obj))
  596. {
  597. OwnedPyObject temp_bytes = PyUnicode_AsEncodedString(obj, ASCII_LIKE_CODEPAGE, "ignore");
  598. checkPythonError();
  599. const char * text = PyBytes_AsString(temp_bytes);
  600. checkPythonError();
  601. size_t lenBytes = PyBytes_Size(temp_bytes);
  602. rtlStrToStrX(chars, result, lenBytes, text);
  603. }
  604. else if (PyBytes_Check(obj))
  605. {
  606. const char * text = PyBytes_AsString(obj);
  607. checkPythonError();
  608. size_t lenBytes = PyBytes_Size(obj);
  609. rtlStrToStrX(chars, result, lenBytes, text);
  610. }
  611. return;
  612. }
  613. typeError("string", field);
  614. }
  615. static void getUTF8Result(const RtlFieldInfo *field, PyObject *obj, size32_t &chars, char * &result)
  616. {
  617. if (obj && obj != Py_None && PyUnicode_Check(obj))
  618. {
  619. Py_ssize_t lenBytes;
  620. const char *text = PyUnicode_AsUTF8AndSize(obj, &lenBytes);
  621. checkPythonError();
  622. size32_t numchars = rtlUtf8Length(lenBytes, text);
  623. rtlUtf8ToUtf8X(chars, result, numchars, text);
  624. }
  625. else
  626. typeError("unicode string", field);
  627. }
  628. static void getSetResult(PyObject *obj, bool & isAllResult, size32_t & resultBytes, void * & result, int elemType, size32_t elemSize)
  629. {
  630. // MORE - should probably recode to use the getResultDataset mechanism
  631. if (!obj || obj == Py_None || (!PyList_Check(obj) && !PySet_Check(obj)))
  632. rtlFail(0, "pyembed: type mismatch - list or set expected");
  633. rtlRowBuilder out;
  634. size32_t outBytes = 0;
  635. byte *outData = NULL;
  636. OwnedPyObject iter = PyObject_GetIter(obj);
  637. OwnedPyObject elem;
  638. for (elem.setown(PyIter_Next(iter)); elem != NULL; elem.setown(PyIter_Next(iter)))
  639. {
  640. if (elemSize != UNKNOWN_LENGTH)
  641. {
  642. out.ensureAvailable(outBytes + elemSize);
  643. outData = out.getbytes() + outBytes;
  644. outBytes += elemSize;
  645. }
  646. switch ((type_t) elemType)
  647. {
  648. case type_int:
  649. rtlWriteInt(outData, py3embed::getSignedResult(NULL, elem), elemSize);
  650. break;
  651. case type_unsigned:
  652. rtlWriteInt(outData, py3embed::getUnsignedResult(NULL, elem), elemSize);
  653. break;
  654. case type_real:
  655. if (elemSize == sizeof(double))
  656. * (double *) outData = (double) py3embed::getRealResult(NULL, elem);
  657. else
  658. {
  659. assertex(elemSize == sizeof(float));
  660. * (float *) outData = (float) py3embed::getRealResult(NULL, elem);
  661. }
  662. break;
  663. case type_boolean:
  664. assertex(elemSize == sizeof(bool));
  665. * (bool *) outData = py3embed::getBooleanResult(NULL, elem);
  666. break;
  667. case type_string:
  668. case type_varstring:
  669. {
  670. if (!PyUnicode_Check(elem))
  671. rtlFail(0, "pyembed: type mismatch - return value in list was not a STRING");
  672. OwnedPyObject temp_bytes = PyUnicode_AsEncodedString(elem, ASCII_LIKE_CODEPAGE, "ignore");
  673. checkPythonError();
  674. const char * text = PyBytes_AsString(temp_bytes);
  675. checkPythonError();
  676. size_t lenBytes = PyBytes_Size(temp_bytes);
  677. if (elemSize == UNKNOWN_LENGTH)
  678. {
  679. if (elemType == type_string)
  680. {
  681. out.ensureAvailable(outBytes + lenBytes + sizeof(size32_t));
  682. outData = out.getbytes() + outBytes;
  683. * (size32_t *) outData = lenBytes;
  684. rtlStrToStr(lenBytes, outData+sizeof(size32_t), lenBytes, text);
  685. outBytes += lenBytes + sizeof(size32_t);
  686. }
  687. else
  688. {
  689. out.ensureAvailable(outBytes + lenBytes + 1);
  690. outData = out.getbytes() + outBytes;
  691. rtlStrToVStr(0, outData, lenBytes, text);
  692. outBytes += lenBytes + 1;
  693. }
  694. }
  695. else
  696. {
  697. if (elemType == type_string)
  698. rtlStrToStr(elemSize, outData, lenBytes, text);
  699. else
  700. rtlStrToVStr(elemSize, outData, lenBytes, text); // Fixed size null terminated strings... weird.
  701. }
  702. break;
  703. }
  704. case type_unicode:
  705. case type_utf8:
  706. {
  707. if (!PyUnicode_Check(elem))
  708. rtlFail(0, "pyembed: type mismatch - return value in list was not a unicode STRING");
  709. Py_ssize_t lenBytes;
  710. const char *text = PyUnicode_AsUTF8AndSize(elem, &lenBytes);
  711. checkPythonError();
  712. size32_t numchars = rtlUtf8Length(lenBytes, text);
  713. if (elemType == type_utf8)
  714. {
  715. assertex (elemSize == UNKNOWN_LENGTH);
  716. out.ensureAvailable(outBytes + lenBytes + sizeof(size32_t));
  717. outData = out.getbytes() + outBytes;
  718. * (size32_t *) outData = numchars;
  719. rtlStrToStr(lenBytes, outData+sizeof(size32_t), lenBytes, text);
  720. outBytes += lenBytes + sizeof(size32_t);
  721. }
  722. else
  723. {
  724. if (elemSize == UNKNOWN_LENGTH)
  725. {
  726. out.ensureAvailable(outBytes + numchars*sizeof(UChar) + sizeof(size32_t));
  727. outData = out.getbytes() + outBytes;
  728. // You can't assume that number of chars in utf8 matches number in unicode16 ...
  729. size32_t numchars16;
  730. rtlDataAttr unicode16;
  731. rtlUtf8ToUnicodeX(numchars16, unicode16.refustr(), numchars, text);
  732. * (size32_t *) outData = numchars16;
  733. rtlUnicodeToUnicode(numchars16, (UChar *) (outData+sizeof(size32_t)), numchars16, unicode16.getustr());
  734. outBytes += numchars16*sizeof(UChar) + sizeof(size32_t);
  735. }
  736. else
  737. rtlUtf8ToUnicode(elemSize / sizeof(UChar), (UChar *) outData, numchars, text);
  738. }
  739. break;
  740. }
  741. case type_data:
  742. {
  743. if (!PyByteArray_Check(elem))
  744. rtlFail(0, "pyembed: type mismatch - return value in list was not a bytearray");
  745. size_t lenBytes = PyByteArray_Size(elem); // Could check does not overflow size32_t
  746. const char *data = PyByteArray_AsString(elem);
  747. if (elemSize == UNKNOWN_LENGTH)
  748. {
  749. out.ensureAvailable(outBytes + lenBytes + sizeof(size32_t));
  750. outData = out.getbytes() + outBytes;
  751. * (size32_t *) outData = lenBytes;
  752. rtlStrToData(lenBytes, outData+sizeof(size32_t), lenBytes, data);
  753. outBytes += lenBytes + sizeof(size32_t);
  754. }
  755. else
  756. rtlStrToData(elemSize, outData, lenBytes, data);
  757. break;
  758. }
  759. default:
  760. rtlFail(0, "pyembed: type mismatch - unsupported return type");
  761. break;
  762. }
  763. checkPythonError();
  764. }
  765. isAllResult = false;
  766. resultBytes = outBytes;
  767. result = out.detachdata();
  768. }
  769. static void getUnicodeResult(const RtlFieldInfo *field, PyObject *obj, size32_t &chars, UChar * &result)
  770. {
  771. if (obj && obj != Py_None && PyUnicode_Check(obj))
  772. {
  773. Py_ssize_t lenBytes;
  774. const char *text = PyUnicode_AsUTF8AndSize(obj, &lenBytes);
  775. checkPythonError();
  776. size32_t numchars = rtlUtf8Length(lenBytes, text);
  777. rtlUtf8ToUnicodeX(chars, result, numchars, text);
  778. }
  779. else
  780. typeError("unicode string", field);
  781. }
  782. // A PythonRowBuilder object is used to construct an ECL row from a python object
  783. class PythonRowBuilder : public CInterfaceOf<IFieldSource>
  784. {
  785. public:
  786. PythonRowBuilder(PyObject *_row)
  787. : iter(NULL), elem(NULL), named(false)
  788. {
  789. pushback.set(_row);
  790. }
  791. virtual bool getBooleanResult(const RtlFieldInfo *field)
  792. {
  793. nextField(field);
  794. return py3embed::getBooleanResult(field, elem);
  795. }
  796. virtual void getDataResult(const RtlFieldInfo *field, size32_t &len, void * &result)
  797. {
  798. nextField(field);
  799. py3embed::getDataResult(field, elem, len, result);
  800. }
  801. virtual double getRealResult(const RtlFieldInfo *field)
  802. {
  803. nextField(field);
  804. return py3embed::getRealResult(field, elem);
  805. }
  806. virtual __int64 getSignedResult(const RtlFieldInfo *field)
  807. {
  808. nextField(field);
  809. return py3embed::getSignedResult(field, elem);
  810. }
  811. virtual unsigned __int64 getUnsignedResult(const RtlFieldInfo *field)
  812. {
  813. nextField(field);
  814. return py3embed::getUnsignedResult(field, elem);
  815. }
  816. virtual void getStringResult(const RtlFieldInfo *field, size32_t &chars, char * &result)
  817. {
  818. nextField(field);
  819. py3embed::getStringResult(field, elem, chars, result);
  820. }
  821. virtual void getUTF8Result(const RtlFieldInfo *field, size32_t &chars, char * &result)
  822. {
  823. nextField(field);
  824. py3embed::getUTF8Result(field, elem, chars, result);
  825. }
  826. virtual void getUnicodeResult(const RtlFieldInfo *field, size32_t &chars, UChar * &result)
  827. {
  828. nextField(field);
  829. py3embed::getUnicodeResult(field, elem, chars, result);
  830. }
  831. virtual void getDecimalResult(const RtlFieldInfo *field, Decimal &value)
  832. {
  833. nextField(field);
  834. double ret = py3embed::getRealResult(field, elem);
  835. value.setReal(ret);
  836. }
  837. virtual void processBeginSet(const RtlFieldInfo * field, bool &isAll)
  838. {
  839. nextField(field);
  840. isAll = false; // No concept of an 'all' set in Python
  841. if (!elem || elem == Py_None || (!PyList_Check(elem) && !PySet_Check(elem)))
  842. typeError("list or set", field);
  843. push();
  844. }
  845. virtual bool processNextSet(const RtlFieldInfo * field)
  846. {
  847. nextField(NULL);
  848. pushback.setown(elem.getClear());
  849. return pushback != NULL;
  850. }
  851. virtual void processBeginDataset(const RtlFieldInfo * field)
  852. {
  853. nextField(field);
  854. if (!PyList_Check(elem))
  855. typeError("list", field);
  856. push();
  857. }
  858. virtual void processBeginRow(const RtlFieldInfo * field)
  859. {
  860. // Expect to see a tuple here, or possibly (if the ECL record has a single field), an arbitrary scalar object
  861. // If it's a tuple, we push it onto our stack as the active object
  862. nextField(field);
  863. if (!PyTuple_Check(elem))
  864. {
  865. if (countFields(field->type->queryFields())==1)
  866. {
  867. // Python doesn't seem to support the concept of a tuple containing a single element.
  868. // If we are expecting a single field in our row, then the 'tuple' layer will be missing
  869. elem.setown(PyTuple_Pack(1, elem.get()));
  870. }
  871. else
  872. typeError("tuple", field);
  873. }
  874. push();
  875. }
  876. virtual bool processNextRow(const RtlFieldInfo * field)
  877. {
  878. nextField(NULL);
  879. pushback.setown(elem.getClear());
  880. return pushback != NULL;
  881. }
  882. virtual void processEndSet(const RtlFieldInfo * field)
  883. {
  884. pop();
  885. }
  886. virtual void processEndDataset(const RtlFieldInfo * field)
  887. {
  888. pop();
  889. }
  890. virtual void processEndRow(const RtlFieldInfo * field)
  891. {
  892. pop();
  893. }
  894. protected:
  895. void pop()
  896. {
  897. iter.setown((PyObject *) iterStack.popGet());
  898. parent.setown((PyObject *) parentStack.popGet());
  899. named = namedStack.popGet();
  900. elem.clear();
  901. }
  902. void push()
  903. {
  904. iterStack.append(iter.getClear());
  905. parentStack.append(parent.getClear());
  906. namedStack.append(named);
  907. parent.set(elem);
  908. iter.setown(PyObject_GetIter(elem));
  909. named = isNamedTuple(elem);
  910. elem.clear();
  911. }
  912. bool isNamedTuple(PyObject *obj)
  913. {
  914. return PyObject_HasAttrString((PyObject *) obj->ob_type, "_fields");
  915. }
  916. void nextField(const RtlFieldInfo * field)
  917. {
  918. if (pushback)
  919. elem.setown(pushback.getClear());
  920. else if (field && named) // If it's named tuple, expect to always resolve fields by name, not position
  921. {
  922. elem.setown(PyObject_GetAttrString(parent, field->name));
  923. }
  924. else if (iter)
  925. elem.setown(PyIter_Next(iter));
  926. else
  927. elem = NULL;
  928. checkPythonError();
  929. }
  930. OwnedPyObject iter;
  931. OwnedPyObject pushback;
  932. OwnedPyObject elem;
  933. OwnedPyObject parent;
  934. bool named;
  935. PointerArray iterStack;
  936. PointerArray parentStack;
  937. BoolArray namedStack;
  938. };
  939. static size32_t getRowResult(PyObject *result, ARowBuilder &builder)
  940. {
  941. PythonRowBuilder pyRowBuilder(result);
  942. const RtlTypeInfo *typeInfo = builder.queryAllocator()->queryOutputMeta()->queryTypeInfo();
  943. assertex(typeInfo);
  944. RtlFieldStrInfo dummyField("<row>", NULL, typeInfo);
  945. return typeInfo->build(builder, 0, &dummyField, pyRowBuilder);
  946. }
  947. // A PythonNamedTupleBuilder object is used to construct a Python named tuple from an ECL row
  948. class PythonNamedTupleBuilder : public CInterfaceOf<IFieldProcessor>
  949. {
  950. public:
  951. PythonNamedTupleBuilder(PythonThreadContext *_sharedCtx, const RtlFieldInfo *_outerRow)
  952. : outerRow(_outerRow), sharedCtx(_sharedCtx)
  953. {
  954. }
  955. virtual void processString(unsigned len, const char *value, const RtlFieldInfo * field)
  956. {
  957. addArg(PyUnicode_FromStringAndSize(value, len));
  958. }
  959. virtual void processBool(bool value, const RtlFieldInfo * field)
  960. {
  961. addArg(PyBool_FromLong(value ? 1 : 0));
  962. }
  963. virtual void processData(unsigned len, const void *value, const RtlFieldInfo * field)
  964. {
  965. addArg(PyByteArray_FromStringAndSize((const char *) value, len));
  966. }
  967. virtual void processInt(__int64 value, const RtlFieldInfo * field)
  968. {
  969. addArg(PyLong_FromLongLong(value));
  970. }
  971. virtual void processUInt(unsigned __int64 value, const RtlFieldInfo * field)
  972. {
  973. addArg(PyLong_FromUnsignedLongLong(value));
  974. }
  975. virtual void processReal(double value, const RtlFieldInfo * field)
  976. {
  977. addArg(PyFloat_FromDouble(value));
  978. }
  979. virtual void processDecimal(const void *value, unsigned digits, unsigned precision, const RtlFieldInfo * field)
  980. {
  981. Decimal val;
  982. val.setDecimal(digits, precision, value);
  983. addArg(PyFloat_FromDouble(val.getReal()));
  984. }
  985. virtual void processUDecimal(const void *value, unsigned digits, unsigned precision, const RtlFieldInfo * field)
  986. {
  987. Decimal val;
  988. val.setUDecimal(digits, precision, value);
  989. addArg(PyFloat_FromDouble(val.getReal()));
  990. }
  991. virtual void processUnicode(unsigned len, const UChar *value, const RtlFieldInfo * field)
  992. {
  993. // You don't really know what size Py_UNICODE is (varies from system to system), so go via utf8
  994. unsigned unicodeChars;
  995. rtlDataAttr unicode;
  996. rtlUnicodeToUtf8X(unicodeChars, unicode.refstr(), len, value);
  997. processUtf8(unicodeChars, unicode.getstr(), field);
  998. }
  999. virtual void processQString(unsigned len, const char *value, const RtlFieldInfo * field)
  1000. {
  1001. size32_t charCount;
  1002. rtlDataAttr text;
  1003. rtlQStrToStrX(charCount, text.refstr(), len, value);
  1004. processString(charCount, text.getstr(), field);
  1005. }
  1006. virtual void processUtf8(unsigned len, const char *value, const RtlFieldInfo * field)
  1007. {
  1008. size32_t sizeBytes = rtlUtf8Size(len, value);
  1009. PyObject *vval = PyUnicode_FromStringAndSize(value, sizeBytes); // NOTE - requires size in bytes not chars
  1010. checkPythonError();
  1011. addArg(vval);
  1012. }
  1013. virtual bool processBeginSet(const RtlFieldInfo * field, unsigned numElements, bool isAll, const byte *data)
  1014. {
  1015. push();
  1016. if (isAll)
  1017. rtlFail(0, "pyembed: ALL sets are not supported");
  1018. return true;
  1019. }
  1020. virtual bool processBeginDataset(const RtlFieldInfo * field, unsigned numRows)
  1021. {
  1022. push();
  1023. return true;
  1024. }
  1025. virtual bool processBeginRow(const RtlFieldInfo * field)
  1026. {
  1027. if (field != outerRow)
  1028. push();
  1029. return true;
  1030. }
  1031. virtual void processEndSet(const RtlFieldInfo * field)
  1032. {
  1033. pop();
  1034. }
  1035. virtual void processEndDataset(const RtlFieldInfo * field)
  1036. {
  1037. pop();
  1038. }
  1039. virtual void processEndRow(const RtlFieldInfo * field)
  1040. {
  1041. if (field != outerRow)
  1042. {
  1043. args.setown(getTuple(field->type));
  1044. pop();
  1045. }
  1046. }
  1047. PyObject *getTuple(const RtlTypeInfo *type)
  1048. {
  1049. OwnedPyObject mynamedtupletype = sharedCtx ? sharedCtx->getNamedTupleType(type) : globalState.getNamedTupleType(type);
  1050. OwnedPyObject argsTuple = PyList_AsTuple(args);
  1051. OwnedPyObject mynamedtuple = PyObject_CallObject(mynamedtupletype, argsTuple); // Creates a namedtuple from the supplied tuple
  1052. checkPythonError();
  1053. return mynamedtuple.getClear();
  1054. }
  1055. protected:
  1056. void push()
  1057. {
  1058. stack.append(args.getClear());
  1059. args.setown(PyList_New(0));
  1060. }
  1061. void pop()
  1062. {
  1063. OwnedPyObject arg = args.getClear();
  1064. args.setown((PyObject *) stack.popGet());
  1065. addArg(arg.getClear());
  1066. }
  1067. void addArg(PyObject *arg)
  1068. {
  1069. if (!args)
  1070. {
  1071. args.setown(PyList_New(0));
  1072. }
  1073. PyList_Append(args, arg);
  1074. Py_DECREF(arg);
  1075. }
  1076. OwnedPyObject args;
  1077. PointerArray stack;
  1078. const RtlFieldInfo *outerRow;
  1079. PythonThreadContext *sharedCtx;
  1080. };
  1081. //----------------------------------------------------------------------
  1082. // Wrap an IRowStream into a Python generator
  1083. struct ECLDatasetIterator
  1084. {
  1085. PyObject_HEAD;
  1086. const RtlTypeInfo *typeInfo; // Not linked (or linkable)
  1087. IRowStream * val; // Linked
  1088. };
  1089. PyObject* ECLDatasetIterator_iter(PyObject *self)
  1090. {
  1091. Py_INCREF(self);
  1092. return self;
  1093. }
  1094. void ECLDatasetIterator_dealloc(PyObject *self)
  1095. {
  1096. ECLDatasetIterator *p = (ECLDatasetIterator *)self;
  1097. if (p->val)
  1098. {
  1099. p->val->stop();
  1100. ::Release(p->val);
  1101. p->val = NULL;
  1102. }
  1103. self->ob_type->tp_free(self);
  1104. }
  1105. PyObject* ECLDatasetIterator_iternext(PyObject *self)
  1106. {
  1107. ECLDatasetIterator *p = (ECLDatasetIterator *)self;
  1108. if (p->val)
  1109. {
  1110. roxiemem::OwnedConstRoxieRow nextRow = p->val->ungroupedNextRow();
  1111. if (!nextRow)
  1112. {
  1113. p->val->stop();
  1114. ::Release(p->val);
  1115. p->val = NULL;
  1116. }
  1117. else
  1118. {
  1119. RtlFieldStrInfo dummyField("<row>", NULL, p->typeInfo);
  1120. PythonNamedTupleBuilder tupleBuilder(NULL, &dummyField);
  1121. const byte *brow = (const byte *) nextRow.get();
  1122. p->typeInfo->process(brow, brow, &dummyField, tupleBuilder);
  1123. return tupleBuilder.getTuple(p->typeInfo);
  1124. }
  1125. }
  1126. // If we get here, it's EOF
  1127. PyErr_SetNone(PyExc_StopIteration);
  1128. return NULL;
  1129. }
  1130. static PyTypeObject ECLDatasetIteratorType =
  1131. {
  1132. PyVarObject_HEAD_INIT(NULL, 0)
  1133. "ECLDatasetIterator._MyIter", /*tp_name*/
  1134. sizeof(ECLDatasetIterator), /*tp_basicsize*/
  1135. 0, /*tp_itemsize*/
  1136. ECLDatasetIterator_dealloc, /*tp_dealloc*/
  1137. 0, /*tp_print*/
  1138. 0, /*tp_getattr*/
  1139. 0, /*tp_setattr*/
  1140. 0, /*tp_compare*/
  1141. 0, /*tp_repr*/
  1142. 0, /*tp_as_number*/
  1143. 0, /*tp_as_sequence*/
  1144. 0, /*tp_as_mapping*/
  1145. 0, /*tp_hash */
  1146. 0, /*tp_call*/
  1147. 0, /*tp_str*/
  1148. 0, /*tp_getattro*/
  1149. 0, /*tp_setattro*/
  1150. 0, /*tp_as_buffer*/
  1151. Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_ITER, /* tp_flags: tell python to use tp_iter and tp_iternext fields. */
  1152. "ECL dataset iterator object.", /* tp_doc */
  1153. 0, /* tp_traverse */
  1154. 0, /* tp_clear */
  1155. 0, /* tp_richcompare */
  1156. 0, /* tp_weaklistoffset */
  1157. ECLDatasetIterator_iter, /* tp_iter: __iter__() method */
  1158. ECLDatasetIterator_iternext /* tp_iternext: next() method */
  1159. };
  1160. static PyObject *createECLDatasetIterator(const RtlTypeInfo *_typeInfo, IRowStream * _val)
  1161. {
  1162. ECLDatasetIteratorType.tp_new = PyType_GenericNew;
  1163. if (PyType_Ready(&ECLDatasetIteratorType) < 0) return NULL;
  1164. ECLDatasetIterator *p = PyObject_New(ECLDatasetIterator, &ECLDatasetIteratorType);
  1165. if (!p)
  1166. {
  1167. checkPythonError();
  1168. rtlFail(0, "pyembed: failed to create dataset iterator");
  1169. }
  1170. p->typeInfo = _typeInfo;
  1171. p->val = _val;
  1172. return (PyObject *)p;
  1173. }
  1174. //-----------------------------------------------------
  1175. // GILBlock ensures the we hold the Python "Global interpreter lock" for the appropriate duration
  1176. class GILBlock
  1177. {
  1178. public:
  1179. GILBlock(PyThreadState * &_state) : state(_state)
  1180. {
  1181. PyEval_RestoreThread(state);
  1182. }
  1183. ~GILBlock()
  1184. {
  1185. state = PyEval_SaveThread();
  1186. }
  1187. private:
  1188. PyThreadState * &state;
  1189. };
  1190. void Python3xGlobalState::unregister(const char *key)
  1191. {
  1192. checkThreadContext();
  1193. GILBlock b(threadContext->threadState);
  1194. globalState.releaseNamedScope(key);
  1195. }
  1196. // A Python function that returns a dataset will return a PythonRowStream object that can be
  1197. // interrogated to return each row of the result in turn
  1198. class PythonRowStream : public CInterfaceOf<IRowStream>
  1199. {
  1200. public:
  1201. PythonRowStream(PyObject *result, IEngineRowAllocator *_resultAllocator)
  1202. : resultIterator(NULL)
  1203. {
  1204. // NOTE - the caller should already have the GIL lock before creating me
  1205. if (!result || result == Py_None)
  1206. typeError("list or generator", NULL);
  1207. resultIterator.setown(PyObject_GetIter(result)); // We allow anything that is iterable to be returned for a row stream
  1208. checkPythonError();
  1209. resultAllocator.set(_resultAllocator);
  1210. }
  1211. ~PythonRowStream()
  1212. {
  1213. if (resultIterator)
  1214. {
  1215. checkThreadContext();
  1216. GILBlock b(threadContext->threadState);
  1217. resultIterator.clear();
  1218. }
  1219. }
  1220. virtual const void *nextRow()
  1221. {
  1222. checkThreadContext();
  1223. GILBlock b(threadContext->threadState);
  1224. if (!resultIterator)
  1225. return NULL;
  1226. OwnedPyObject row = PyIter_Next(resultIterator);
  1227. checkPythonError();
  1228. if (!row)
  1229. return NULL;
  1230. RtlDynamicRowBuilder rowBuilder(resultAllocator);
  1231. size32_t len = py3embed::getRowResult(row, rowBuilder);
  1232. checkPythonError();
  1233. return rowBuilder.finalizeRowClear(len);
  1234. }
  1235. virtual void stop()
  1236. {
  1237. checkThreadContext();
  1238. GILBlock b(threadContext->threadState);
  1239. resultAllocator.clear();
  1240. resultIterator.clear();
  1241. }
  1242. protected:
  1243. Linked<IEngineRowAllocator> resultAllocator;
  1244. OwnedPyObject resultIterator;
  1245. };
  1246. // Each call to a Python function will use a new Python3xEmbedFunctionContext object
  1247. // This takes care of ensuring that the Python GIL is locked while we are executing python code,
  1248. // and released when we are not
  1249. class Python3xEmbedContextBase : public CInterfaceOf<IEmbedFunctionContext>
  1250. {
  1251. public:
  1252. Python3xEmbedContextBase(PythonThreadContext *_sharedCtx)
  1253. : sharedCtx(_sharedCtx)
  1254. {
  1255. PyEval_RestoreThread(sharedCtx->threadState);
  1256. }
  1257. virtual void setActivityOptions(const IThorActivityContext *ctx)
  1258. {
  1259. OwnedPyObject mynamedtupletype = globalState.getActivityContextTupleType();
  1260. OwnedPyObject args = PyTuple_New(5);
  1261. OwnedPyObject isLocal;
  1262. isLocal.set(ctx->isLocal() ? Py_True : Py_False);
  1263. PyTuple_SET_ITEM((PyTupleObject *) args.get(), 0, isLocal.getClear());
  1264. PyTuple_SET_ITEM((PyTupleObject *) args.get(), 1, PyLong_FromLong(ctx->numSlaves()));
  1265. PyTuple_SET_ITEM((PyTupleObject *) args.get(), 2, PyLong_FromLong(ctx->numStrands()));
  1266. PyTuple_SET_ITEM((PyTupleObject *) args.get(), 3, PyLong_FromLong(ctx->querySlave()));
  1267. PyTuple_SET_ITEM((PyTupleObject *) args.get(), 4, PyLong_FromLong(ctx->queryStrand()));
  1268. OwnedPyObject activityTuple = PyObject_CallObject(mynamedtupletype, args); // Creates a namedtuple from the supplied tuple
  1269. checkPythonError();
  1270. PyDict_SetItemString(locals, "__activity__", activityTuple.getClear());
  1271. checkPythonError();
  1272. }
  1273. void setScopes(ICodeContext *codeCtx, const char *_options)
  1274. {
  1275. locals.setown(PyDict_New());
  1276. StringArray options;
  1277. options.appendList(_options, ",");
  1278. StringBuffer scopeKey;
  1279. const char *scopeKey2 = nullptr;
  1280. bool registerCallback = false;
  1281. bool wuidScope = false;
  1282. IEngineContext *engine = nullptr;
  1283. ForEachItemIn(idx, options)
  1284. {
  1285. const char *opt = options.item(idx);
  1286. const char *val = strchr(opt, '=');
  1287. if (val)
  1288. {
  1289. StringBuffer optName(val-opt, opt);
  1290. val++;
  1291. if (strieq(optName, "globalscope"))
  1292. scopeKey2 = val;
  1293. else if (strieq(optName, "persist"))
  1294. {
  1295. if (scopeKey.length())
  1296. failx("persist option specified more than once");
  1297. if (strieq(val, "global"))
  1298. scopeKey.append("global");
  1299. else if (strieq(val, "workunit"))
  1300. {
  1301. engine = codeCtx->queryEngineContext();
  1302. wuidScope = true;
  1303. if (!engine)
  1304. failx("Persist mode 'workunit' not supported here");
  1305. }
  1306. else if (strieq(val, "query"))
  1307. {
  1308. engine = codeCtx->queryEngineContext();
  1309. wuidScope = false;
  1310. if (!engine)
  1311. failx("Persist mode 'query' not supported here");
  1312. }
  1313. else
  1314. failx("Unrecognized persist mode %s", val);
  1315. }
  1316. else
  1317. failx("Unrecognized option %s", optName.str());
  1318. }
  1319. else
  1320. failx("Unrecognized option %s", opt);
  1321. }
  1322. if (engine)
  1323. engine->getQueryId(scopeKey, wuidScope);
  1324. if (scopeKey2)
  1325. scopeKey.append(':').append(scopeKey2);
  1326. if (scopeKey.length())
  1327. {
  1328. bool isNew;
  1329. globals.setown(globalState.getNamedScope(scopeKey, isNew));
  1330. if (isNew && engine)
  1331. engine->onTermination(Python3xGlobalState::unregister, scopeKey.str(), wuidScope);
  1332. }
  1333. else
  1334. globals.setown(PyDict_New());
  1335. PyDict_SetItemString(globals, "__builtins__", PyEval_GetBuiltins()); // required for import to work
  1336. }
  1337. ~Python3xEmbedContextBase()
  1338. {
  1339. // We need to clear these before calling savethread, or we won't own the GIL
  1340. locals.clear();
  1341. globals.clear();
  1342. result.clear();
  1343. script.clear();
  1344. sharedCtx->threadState = PyEval_SaveThread();
  1345. }
  1346. virtual bool getBooleanResult()
  1347. {
  1348. return py3embed::getBooleanResult(NULL, result);
  1349. }
  1350. virtual void getDataResult(size32_t &__chars, void * &__result)
  1351. {
  1352. py3embed::getDataResult(NULL, result, __chars, __result);
  1353. }
  1354. virtual double getRealResult()
  1355. {
  1356. return py3embed::getRealResult(NULL, result);
  1357. }
  1358. virtual __int64 getSignedResult()
  1359. {
  1360. return py3embed::getSignedResult(NULL, result);
  1361. }
  1362. virtual unsigned __int64 getUnsignedResult()
  1363. {
  1364. return py3embed::getUnsignedResult(NULL, result);
  1365. }
  1366. virtual void getStringResult(size32_t &__chars, char * &__result)
  1367. {
  1368. py3embed::getStringResult(NULL, result, __chars, __result);
  1369. }
  1370. virtual void getUTF8Result(size32_t &__chars, char * &__result)
  1371. {
  1372. py3embed::getUTF8Result(NULL, result, __chars, __result);
  1373. }
  1374. virtual void getUnicodeResult(size32_t &__chars, UChar * &__result)
  1375. {
  1376. py3embed::getUnicodeResult(NULL, result, __chars, __result);
  1377. }
  1378. virtual void getSetResult(bool & __isAllResult, size32_t & __resultBytes, void * & __result, int elemType, size32_t elemSize)
  1379. {
  1380. py3embed::getSetResult(result, __isAllResult, __resultBytes, __result, elemType, elemSize);
  1381. }
  1382. virtual IRowStream *getDatasetResult(IEngineRowAllocator * _resultAllocator)
  1383. {
  1384. return new PythonRowStream(result, _resultAllocator);
  1385. }
  1386. virtual byte * getRowResult(IEngineRowAllocator * _resultAllocator)
  1387. {
  1388. RtlDynamicRowBuilder rowBuilder(_resultAllocator);
  1389. size32_t len = py3embed::getRowResult(result, rowBuilder);
  1390. return (byte *) rowBuilder.finalizeRowClear(len);
  1391. }
  1392. virtual size32_t getTransformResult(ARowBuilder & builder)
  1393. {
  1394. return py3embed::getRowResult(result, builder);
  1395. }
  1396. virtual void bindBooleanParam(const char *name, bool val)
  1397. {
  1398. addArg(name, PyBool_FromLong(val ? 1 : 0));
  1399. }
  1400. virtual void bindDataParam(const char *name, size32_t len, const void *val)
  1401. {
  1402. addArg(name, PyByteArray_FromStringAndSize((const char *) val, len));
  1403. }
  1404. virtual void bindFloatParam(const char *name, float val)
  1405. {
  1406. addArg(name, PyFloat_FromDouble((double) val));
  1407. }
  1408. virtual void bindRealParam(const char *name, double val)
  1409. {
  1410. addArg(name, PyFloat_FromDouble(val));
  1411. }
  1412. virtual void bindSignedSizeParam(const char *name, int size, __int64 val)
  1413. {
  1414. addArg(name, PyLong_FromLongLong(val));
  1415. }
  1416. virtual void bindSignedParam(const char *name, __int64 val)
  1417. {
  1418. addArg(name, PyLong_FromLongLong(val));
  1419. }
  1420. virtual void bindUnsignedSizeParam(const char *name, int size, unsigned __int64 val)
  1421. {
  1422. addArg(name, PyLong_FromUnsignedLongLong(val));
  1423. }
  1424. virtual void bindUnsignedParam(const char *name, unsigned __int64 val)
  1425. {
  1426. addArg(name, PyLong_FromUnsignedLongLong(val));
  1427. }
  1428. virtual void bindStringParam(const char *name, size32_t len, const char *val)
  1429. {
  1430. addArg(name, PyUnicode_Decode(val, len, ASCII_LIKE_CODEPAGE, "ignore"));
  1431. }
  1432. virtual void bindVStringParam(const char *name, const char *val)
  1433. {
  1434. addArg(name, PyUnicode_FromString(val));
  1435. }
  1436. virtual void bindUTF8Param(const char *name, size32_t chars, const char *val)
  1437. {
  1438. size32_t sizeBytes = rtlUtf8Size(chars, val);
  1439. PyObject *vval = PyUnicode_FromStringAndSize(val, sizeBytes); // NOTE - requires size in bytes not chars
  1440. checkPythonError();
  1441. addArg(name, vval);
  1442. }
  1443. virtual void bindUnicodeParam(const char *name, size32_t chars, const UChar *val)
  1444. {
  1445. // You don't really know what size Py_UNICODE is (varies from system to system), so go via utf8
  1446. unsigned unicodeChars;
  1447. char *unicode;
  1448. rtlUnicodeToUtf8X(unicodeChars, unicode, chars, val);
  1449. size32_t sizeBytes = rtlUtf8Size(unicodeChars, unicode);
  1450. PyObject *vval = PyUnicode_FromStringAndSize(unicode, sizeBytes); // NOTE - requires size in bytes not chars
  1451. checkPythonError();
  1452. addArg(name, vval);
  1453. rtlFree(unicode);
  1454. }
  1455. virtual void bindSetParam(const char *name, int elemType, size32_t elemSize, bool isAll, size32_t totalBytes, const void *setData)
  1456. {
  1457. if (isAll)
  1458. rtlFail(0, "pyembed: Cannot pass ALL");
  1459. type_t typecode = (type_t) elemType;
  1460. const byte *inData = (const byte *) setData;
  1461. const byte *endData = inData + totalBytes;
  1462. OwnedPyObject vval = PyList_New(0);
  1463. while (inData < endData)
  1464. {
  1465. OwnedPyObject thisElem;
  1466. size32_t thisSize = elemSize;
  1467. switch (typecode)
  1468. {
  1469. case type_int:
  1470. thisElem.setown(PyLong_FromLongLong(rtlReadInt(inData, elemSize)));
  1471. break;
  1472. case type_unsigned:
  1473. thisElem.setown(PyLong_FromUnsignedLongLong(rtlReadUInt(inData, elemSize)));
  1474. break;
  1475. case type_varstring:
  1476. {
  1477. size32_t numChars = strlen((const char *) inData);
  1478. thisElem.setown(PyUnicode_FromStringAndSize((const char *) inData, numChars));
  1479. if (elemSize == UNKNOWN_LENGTH)
  1480. thisSize = numChars + 1;
  1481. break;
  1482. }
  1483. case type_string:
  1484. if (elemSize == UNKNOWN_LENGTH)
  1485. {
  1486. thisSize = * (size32_t *) inData;
  1487. inData += sizeof(size32_t);
  1488. }
  1489. thisElem.setown(PyUnicode_FromStringAndSize((const char *) inData, thisSize));
  1490. break;
  1491. case type_real:
  1492. if (elemSize == sizeof(double))
  1493. thisElem.setown(PyFloat_FromDouble(* (double *) inData));
  1494. else
  1495. thisElem.setown(PyFloat_FromDouble(* (float *) inData));
  1496. break;
  1497. case type_boolean:
  1498. assertex(elemSize == sizeof(bool));
  1499. thisElem.setown(PyBool_FromLong(*(bool*)inData ? 1 : 0));
  1500. break;
  1501. case type_unicode:
  1502. {
  1503. if (elemSize == UNKNOWN_LENGTH)
  1504. {
  1505. thisSize = (* (size32_t *) inData) * sizeof(UChar); // NOTE - it's in chars...
  1506. inData += sizeof(size32_t);
  1507. }
  1508. unsigned unicodeChars;
  1509. rtlDataAttr unicode;
  1510. rtlUnicodeToUtf8X(unicodeChars, unicode.refstr(), thisSize / sizeof(UChar), (const UChar *) inData);
  1511. size32_t sizeBytes = rtlUtf8Size(unicodeChars, unicode.getstr());
  1512. thisElem.setown(PyUnicode_FromStringAndSize(unicode.getstr(), sizeBytes)); // NOTE - requires size in bytes not chars
  1513. checkPythonError();
  1514. break;
  1515. }
  1516. case type_utf8:
  1517. {
  1518. assertex (elemSize == UNKNOWN_LENGTH);
  1519. size32_t numChars = * (size32_t *) inData;
  1520. inData += sizeof(size32_t);
  1521. thisSize = rtlUtf8Size(numChars, inData);
  1522. thisElem.setown(PyUnicode_FromStringAndSize((const char *) inData, thisSize)); // NOTE - requires size in bytes not chars
  1523. break;
  1524. }
  1525. case type_data:
  1526. if (elemSize == UNKNOWN_LENGTH)
  1527. {
  1528. thisSize = * (size32_t *) inData;
  1529. inData += sizeof(size32_t);
  1530. }
  1531. thisElem.setown(PyByteArray_FromStringAndSize((const char *) inData, thisSize));
  1532. break;
  1533. }
  1534. checkPythonError();
  1535. inData += thisSize;
  1536. PyList_Append(vval, thisElem);
  1537. }
  1538. addArg(name, vval.getLink());
  1539. }
  1540. virtual void bindRowParam(const char *name, IOutputMetaData & metaVal, const byte *val) override
  1541. {
  1542. const RtlTypeInfo *typeInfo = metaVal.queryTypeInfo();
  1543. assertex(typeInfo);
  1544. RtlFieldStrInfo dummyField("<row>", NULL, typeInfo);
  1545. PythonNamedTupleBuilder tupleBuilder(sharedCtx, &dummyField);
  1546. typeInfo->process(val, val, &dummyField, tupleBuilder); // Creates a tuple from the incoming ECL row
  1547. addArg(name, tupleBuilder.getTuple(typeInfo));
  1548. }
  1549. virtual void bindDatasetParam(const char *name, IOutputMetaData & metaVal, IRowStream * val)
  1550. {
  1551. addArg(name, createECLDatasetIterator(metaVal.queryTypeInfo(), LINK(val)));
  1552. }
  1553. protected:
  1554. virtual void addArg(const char *name, PyObject *arg) = 0;
  1555. PythonThreadContext *sharedCtx;
  1556. OwnedPyObject locals;
  1557. OwnedPyObject globals;
  1558. OwnedPyObject result;
  1559. OwnedPyObject script;
  1560. };
  1561. class Python3xEmbedScriptContext : public Python3xEmbedContextBase
  1562. {
  1563. public:
  1564. Python3xEmbedScriptContext(PythonThreadContext *_sharedCtx)
  1565. : Python3xEmbedContextBase(_sharedCtx)
  1566. {
  1567. }
  1568. ~Python3xEmbedScriptContext()
  1569. {
  1570. }
  1571. virtual IInterface *bindParamWriter(IInterface *esdl, const char *esdlservice, const char *esdltype, const char *name)
  1572. {
  1573. return NULL;
  1574. }
  1575. virtual void paramWriterCommit(IInterface *writer)
  1576. {
  1577. }
  1578. virtual void writeResult(IInterface *esdl, const char *esdlservice, const char *esdltype, IInterface *writer)
  1579. {
  1580. }
  1581. virtual void importFunction(size32_t lenChars, const char *text)
  1582. {
  1583. throwUnexpected();
  1584. }
  1585. virtual void compileEmbeddedScript(size32_t lenChars, const char *utf)
  1586. {
  1587. script.setown(sharedCtx->compileEmbeddedScript(lenChars, utf, argstring));
  1588. }
  1589. virtual void loadCompiledScript(size32_t chars, const void *_script) override
  1590. {
  1591. throwUnexpected();
  1592. }
  1593. virtual void setActivityOptions(const IThorActivityContext *ctx) override
  1594. {
  1595. Python3xEmbedContextBase::setActivityOptions(ctx);
  1596. argstring.append("__activity__");
  1597. }
  1598. virtual void callFunction()
  1599. {
  1600. result.setown(PyEval_EvalCode(script.get(), globals, locals));
  1601. checkPythonError();
  1602. if (!result || result == Py_None)
  1603. result.set(PyDict_GetItemString(globals, "__result__"));
  1604. if (!result || result == Py_None)
  1605. result.set(PyDict_GetItemString(locals, "__result__"));
  1606. }
  1607. protected:
  1608. virtual void addArg(const char *name, PyObject *arg)
  1609. {
  1610. if (!arg)
  1611. return;
  1612. if (argstring.length())
  1613. argstring.append(',');
  1614. argstring.append(name);
  1615. if (script)
  1616. PyDict_SetItemString(globals, name, arg); // Back compatibility - if compiler did not recognize the prebind flag, we need to use globals
  1617. else
  1618. PyDict_SetItemString(locals, name, arg);
  1619. Py_DECREF(arg);
  1620. checkPythonError();
  1621. }
  1622. StringBuffer argstring;
  1623. };
  1624. class Python3xEmbedImportContext : public Python3xEmbedContextBase
  1625. {
  1626. public:
  1627. Python3xEmbedImportContext(PythonThreadContext *_sharedCtx)
  1628. : Python3xEmbedContextBase(_sharedCtx)
  1629. {
  1630. argcount = 0;
  1631. }
  1632. ~Python3xEmbedImportContext()
  1633. {
  1634. }
  1635. virtual IInterface *bindParamWriter(IInterface *esdl, const char *esdlservice, const char *esdltype, const char *name)
  1636. {
  1637. return NULL;
  1638. }
  1639. virtual void paramWriterCommit(IInterface *writer)
  1640. {
  1641. }
  1642. virtual void writeResult(IInterface *esdl, const char *esdlservice, const char *esdltype, IInterface *writer)
  1643. {
  1644. }
  1645. virtual void importFunction(size32_t lenChars, const char *utf)
  1646. {
  1647. script.setown(sharedCtx->importFunction(lenChars, utf));
  1648. }
  1649. virtual void compileEmbeddedScript(size32_t len, const char *text)
  1650. {
  1651. throwUnexpected();
  1652. }
  1653. virtual void loadCompiledScript(size32_t chars, const void *_script) override
  1654. {
  1655. throwUnexpected();
  1656. }
  1657. virtual void callFunction()
  1658. {
  1659. result.setown(PyObject_CallObject(script, args));
  1660. checkPythonError();
  1661. }
  1662. private:
  1663. virtual void addArg(const char *name, PyObject *arg)
  1664. {
  1665. if (argcount)
  1666. _PyTuple_Resize(args.ref(), argcount+1);
  1667. else
  1668. args.setown(PyTuple_New(1));
  1669. PyTuple_SET_ITEM((PyTupleObject *) args.get(), argcount++, arg); // Note - 'steals' the arg reference
  1670. }
  1671. int argcount;
  1672. OwnedPyObject args;
  1673. };
  1674. class Python3xEmbedContext : public CInterfaceOf<IEmbedContext>
  1675. {
  1676. public:
  1677. virtual IEmbedFunctionContext *createFunctionContext(unsigned flags, const char *options) override
  1678. {
  1679. return createFunctionContextEx(nullptr, nullptr, flags, options);
  1680. }
  1681. virtual IEmbedFunctionContext *createFunctionContextEx(ICodeContext * ctx, const IThorActivityContext *activityCtx, unsigned flags, const char *options) override
  1682. {
  1683. checkThreadContext();
  1684. Owned<Python3xEmbedContextBase> ret;
  1685. if (flags & EFimport)
  1686. ret.setown(new Python3xEmbedImportContext(threadContext));
  1687. else
  1688. ret.setown(new Python3xEmbedScriptContext(threadContext));
  1689. ret->setScopes(ctx, options);
  1690. if (activityCtx)
  1691. ret->setActivityOptions(activityCtx);
  1692. return ret.getClear();
  1693. }
  1694. virtual IEmbedServiceContext *createServiceContext(const char *service, unsigned flags, const char *options) override
  1695. {
  1696. throwUnexpected();
  1697. }
  1698. };
  1699. extern DECL_EXPORT IEmbedContext* getEmbedContext()
  1700. {
  1701. return new Python3xEmbedContext;
  1702. }
  1703. extern DECL_EXPORT bool syntaxCheck(const char *script)
  1704. {
  1705. return true; // MORE
  1706. }
  1707. } // namespace