HPCC-16512 Avoid using global scope to pass parameters to embedded python

Allowing us to consider use of a shared/persistent global scope in the future.

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

ecl/hql/hqlatoms.cpp

@@ -58,6 +58,7 @@ IIdAtom * maxSizeId;
 IIdAtom * __optionsId;
 IIdAtom * outputId;
 IIdAtom * physicalLengthId;
+IIdAtom * prebindId;
 IIdAtom * __queryId;
 IIdAtom * selfId;
 IIdAtom * sharedId;
@@ -330,6 +331,7 @@ IAtom * _payload_Atom;
 IAtom * persistAtom;
 IAtom * physicalFilenameAtom;
 IAtom * pluginAtom;
+IAtom * prebindAtom;
 IAtom * prefetchAtom;
 IAtom * preloadAtom;
 IAtom * priorityAtom;
@@ -510,6 +512,7 @@ MODULE_INIT(INIT_PRIORITY_HQLATOM)
     MAKEID(__options);
     MAKEID(output);
     MAKEID(physicalLength);
+    MAKEID(prebind);
     MAKEID(__query);
     MAKEID(self);
     MAKEID(shared);
@@ -783,6 +786,7 @@ MODULE_INIT(INIT_PRIORITY_HQLATOM)
     MAKEATOM(persist);
     MAKEATOM(physicalFilename);
     MAKEATOM(plugin);
+    MAKEATOM(prebind);
     MAKEATOM(prefetch);
     MAKEATOM(preload);
     MAKEATOM(priority);

ecl/hql/hqlatoms.hpp

@@ -60,6 +60,7 @@ extern HQL_API IIdAtom * maxSizeId;
 extern HQL_API IIdAtom * __optionsId;
 extern HQL_API IIdAtom * outputId;
 extern HQL_API IIdAtom * physicalLengthId;
+extern HQL_API IIdAtom * prebindId;
 extern HQL_API IIdAtom * __queryId;
 extern HQL_API IIdAtom * selfId;
 extern HQL_API IIdAtom * sharedId;
@@ -335,6 +336,7 @@ extern HQL_API IAtom * _payload_Atom;
 extern HQL_API IAtom * persistAtom;
 extern HQL_API IAtom * physicalFilenameAtom;
 extern HQL_API IAtom * pluginAtom;
+extern HQL_API IAtom * prebindAtom;
 extern HQL_API IAtom * prefetchAtom;
 extern HQL_API IAtom * preloadAtom;
 extern HQL_API IAtom * priorityAtom;

ecl/hql/hqlfold.cpp

@@ -1666,11 +1666,13 @@ IHqlExpression * foldEmbeddedCall(IHqlExpression* expr, unsigned foldOptions, IT
     assertex(query);
     StringBuffer queryText;
     query->getUTF8Value(queryText);
-    if (isImport)
-        __ctx->importFunction(queryText.lengthUtf8(), queryText.str());
-    else
-        __ctx->compileEmbeddedScript(queryText.lengthUtf8(), queryText.str());
-
+    if (!body->hasAttribute(prebindAtom))
+    {
+        if (isImport)
+            __ctx->importFunction(queryText.lengthUtf8(), queryText.str());
+        else
+            __ctx->compileEmbeddedScript(queryText.lengthUtf8(), queryText.str());
+    }
     // process all the parameters passed in
     unsigned numParam = expr->numChildren();
     for(unsigned i = 0; i < numParam; i++)
@@ -1778,6 +1780,13 @@ IHqlExpression * foldEmbeddedCall(IHqlExpression* expr, unsigned foldOptions, IT
             return NULL;
         }
     }
+    if (body->hasAttribute(prebindAtom))
+    {
+        if (isImport)
+            __ctx->importFunction(queryText.lengthUtf8(), queryText.str());
+        else
+            __ctx->compileEmbeddedScript(queryText.lengthUtf8(), queryText.str());
+    }
     __ctx->callFunction();
 
     switch (returnType->getTypeCode())

ecl/hql/hqlgram2.cpp

@@ -931,6 +931,9 @@ IHqlExpression * HqlGram::processEmbedBody(const attribute & errpos, IHqlExpress
         OwnedHqlExpr checkSupport = pluginScope->lookupSymbol(isImport ? supportsImportId : supportsScriptId, LSFpublic, lookupCtx);
         if (!matchesBoolean(checkSupport, true))
             reportError(ERR_PluginNoScripting, errpos, "Module %s does not support %s", str(moduleId), isImport ? "import" : "script");
+        OwnedHqlExpr prebind = pluginScope->lookupSymbol(prebindId, LSFpublic, lookupCtx);
+        if (matchesBoolean(prebind, true))
+            args.append(*createAttribute(prebindAtom));
         OwnedHqlExpr syntaxCheckFunc = pluginScope->lookupSymbol(syntaxCheckId, LSFpublic, lookupCtx);
         if (syntaxCheckFunc && !isImport)
         {

ecl/hql/hqlutil.hpp

@@ -212,7 +212,7 @@ extern HQL_API bool isDependentOnParameter(IHqlExpression * expr);
 
 inline bool isInternalEmbedAttr(IAtom *name)
 {
-    return name == languageAtom || name == projectedAtom || name == streamedAtom || name == _linkCounted_Atom || name == importAtom || name==foldAtom || name==timeAtom;
+    return name == languageAtom || name == projectedAtom || name == streamedAtom || name == _linkCounted_Atom || name == importAtom || name==foldAtom || name==timeAtom || name==prebindAtom;
 }
 
 

ecl/hqlcpp/hqlcpp.cpp

@@ -11878,7 +11878,8 @@ void HqlCppTranslator::buildScriptFunctionDefinition(BuildCtx &funcctx, IHqlExpr
         else
             scriptArgs.append(*LINK(bodyCode->queryChild(0)));
     }
-    buildFunctionCall(funcctx, isImport ? importId : compileEmbeddedScriptId, scriptArgs);
+    if (!bodyCode->hasAttribute(prebindAtom))
+        buildFunctionCall(funcctx, isImport ? importId : compileEmbeddedScriptId, scriptArgs);
     ForEachChild(i, formals)
     {
         IHqlExpression * param = formals->queryChild(i);
@@ -11954,6 +11955,8 @@ void HqlCppTranslator::buildScriptFunctionDefinition(BuildCtx &funcctx, IHqlExpr
         args.append(*createActualFromFormal(param));
         buildFunctionCall(funcctx, bindFunc, args);
     }
+    if (bodyCode->hasAttribute(prebindAtom))
+        buildFunctionCall(funcctx, isImport ? importId : compileEmbeddedScriptId, scriptArgs);
     funcctx.addQuotedLiteral("__ctx->callFunction();");
     IIdAtom * returnFunc;
     HqlExprArray retargs;

plugins/pyembed/pyembed.cpp

@@ -199,7 +199,7 @@ public:
         return script.getLink();
     }
 
-    PyObject *compileEmbeddedScript(size32_t lenChars, const char *utf);
+    PyObject *compileEmbeddedScript(size32_t lenChars, const char *utf, const char *argstring);
     PyObject *getNamedTupleType(const RtlTypeInfo *type);
 private:
     GILstateWrapper GILState;
@@ -376,7 +376,7 @@ public:
         assertex(PyCallable_Check(mynamedtupletype));
         return mynamedtupletype.getClear();
     }
-    PyObject *compileScript(const char *text)
+    PyObject *compileScript(const char *text, const char *parameters)
     {
         // Note - we do not need (and must not have) a lock protecting this. It is protected by the Python GIL,
         // and if we add our own lock we are liable to deadlock as the code within Py_CompileStringFlags may
@@ -387,14 +387,19 @@ public:
         code.set(PyDict_GetItemString(compiledScripts, text));
         if (!code)
         {
-            code.setown(Py_CompileString(text, "", Py_eval_input));
+            code.setown(Py_CompileString(text, "", Py_eval_input));   // try compiling as simple expression...
             if (!code)
             {
                 PyErr_Clear();
-                StringBuffer wrapped;
-                wrapPythonText(wrapped, text);
                 PyCompilerFlags flags = { PyCF_SOURCE_IS_UTF8 };
-                code.setown(Py_CompileStringFlags(wrapped, "<embed>", Py_file_input, &flags));
+                code.setown(Py_CompileStringFlags(text, "<embed>", Py_file_input, &flags));  // try compiling as global code
+                if (!code)
+                {
+                    PyErr_Clear();
+                    StringBuffer wrapped;
+                    wrapPythonText(wrapped, text, parameters);
+                    code.setown(Py_CompileStringFlags(wrapped, "<embed>", Py_file_input, &flags)); // try compiling as a function body
+                }
             }
             checkPythonError();
             if (code)
@@ -403,9 +408,9 @@ public:
         return code.getClear();
     }
 protected:
-    static StringBuffer &wrapPythonText(StringBuffer &out, const char *in)
+    static StringBuffer &wrapPythonText(StringBuffer &out, const char *in, const char *params)
     {
-        out.append("def __user__():\n  ");
+        out.appendf("def __user__(%s):\n  ", params);
         char c;
         while ((c = *in++) != '\0')
         {
@@ -413,7 +418,7 @@ protected:
             if (c=='\n')
                 out.append("  ");
         }
-        out.append("\n__result__ = __user__()\n");
+        out.appendf("\n__result__ = __user__(%s)\n", params);
         return out;
     }
     PyThreadState *tstate;
@@ -484,7 +489,7 @@ PyObject *PythonThreadContext::getNamedTupleType(const RtlTypeInfo *type)
     return lru.getLink();
 }
 
-PyObject *PythonThreadContext::compileEmbeddedScript(size32_t lenChars, const char *utf)
+PyObject *PythonThreadContext::compileEmbeddedScript(size32_t lenChars, const char *utf, const char *argstring)
 {
     size32_t bytes = rtlUtf8Size(lenChars, utf);
     StringBuffer text(bytes, utf);
@@ -492,7 +497,7 @@ PyObject *PythonThreadContext::compileEmbeddedScript(size32_t lenChars, const ch
     {
         prevtext.clear();
         text.stripChar('\r');
-        script.setown(globalState.compileScript(text));
+        script.setown(globalState.compileScript(text, argstring));
         prevtext.set(utf, bytes);
     }
     return script.getLink();
@@ -1517,7 +1522,7 @@ public:
     }
     virtual void compileEmbeddedScript(size32_t lenChars, const char *utf)
     {
-        script.setown(sharedCtx->compileEmbeddedScript(lenChars, utf));
+        script.setown(sharedCtx->compileEmbeddedScript(lenChars, utf, argstring));
     }
 
     virtual void callFunction()
@@ -1534,11 +1539,17 @@ protected:
     {
         if (!arg)
             return;
-        assertex(arg);
-        PyDict_SetItemString(globals, name, arg);
+        if (argstring.length())
+            argstring.append(',');
+        argstring.append(name);
+        if (script)
+            PyDict_SetItemString(globals, name, arg);  // Back compatibility - if compiler did not recognize the prebind flag, we need to use globals
+        else
+            PyDict_SetItemString(locals, name, arg);
         Py_DECREF(arg);
         checkPythonError();
     }
+    StringBuffer argstring;
 };
 
 class Python27EmbedImportContext : public Python27EmbedContextBase

plugins/pyembed/python.ecllib

@@ -23,3 +23,4 @@ EXPORT getEmbedContext := Language.getEmbedContext;
 EXPORT syntaxCheck := Language.syntaxCheck;
 EXPORT boolean supportsImport := true;
 EXPORT boolean supportsScript := true;
+EXPORT boolean prebind := true;