HPCC-8030 Java/Python/Javascript language support in ECL

Added boolean parameters/results. Also added an additional test for Java
case (including multithreading).

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

ecl/hql/hqlatoms.cpp

@@ -49,6 +49,7 @@ _ATOM backupAtom;
 _ATOM bcdAtom;
 _ATOM beforeAtom;
 _ATOM bestAtom;
+_ATOM bindBooleanParamAtom;
 _ATOM bindRealParamAtom;
 _ATOM bindSignedParamAtom;
 _ATOM bindStringParamAtom;
@@ -141,6 +142,7 @@ _ATOM globalContextAtom;
 _ATOM gctxmethodAtom;
 _ATOM getAtom;
 _ATOM getEmbedContextAtom;
+_ATOM getBooleanResultAtom;
 _ATOM getRealResultAtom;
 _ATOM getSignedResultAtom;
 _ATOM getStringResultAtom;
@@ -445,6 +447,7 @@ MODULE_INIT(INIT_PRIORITY_HQLATOM)
     MAKEATOM(bcd);
     MAKEATOM(before);
     MAKEATOM(best);
+    MAKEATOM(bindBooleanParam);
     MAKEATOM(bindRealParam);
     MAKEATOM(bindSignedParam);
     MAKEATOM(bindStringParam);
@@ -536,6 +539,7 @@ MODULE_INIT(INIT_PRIORITY_HQLATOM)
     MAKEATOM(gctxmethod);
     MAKEATOM(get);
     MAKEATOM(getEmbedContext);
+    MAKEATOM(getBooleanResult);
     MAKEATOM(getRealResult);
     MAKEATOM(getSignedResult);
     MAKEATOM(getStringResult);

ecl/hql/hqlatoms.hpp

@@ -52,6 +52,7 @@ extern HQL_API _ATOM backupAtom;
 extern HQL_API _ATOM bcdAtom;
 extern HQL_API _ATOM beforeAtom;
 extern HQL_API _ATOM bestAtom;
+extern HQL_API _ATOM bindBooleanParamAtom;
 extern HQL_API _ATOM bindRealParamAtom;
 extern HQL_API _ATOM bindSignedParamAtom;
 extern HQL_API _ATOM bindStringParamAtom;
@@ -143,6 +144,7 @@ extern HQL_API _ATOM _function_Atom;
 extern HQL_API _ATOM gctxmethodAtom;
 extern HQL_API _ATOM getAtom;
 extern HQL_API _ATOM getEmbedContextAtom;
+extern HQL_API _ATOM getBooleanResultAtom;
 extern HQL_API _ATOM getRealResultAtom;
 extern HQL_API _ATOM getSignedResultAtom;
 extern HQL_API _ATOM getStringResultAtom;

ecl/hqlcpp/hqlcpp.cpp

@@ -11482,6 +11482,9 @@ void HqlCppTranslator::buildScriptFunctionDefinition(BuildCtx &funcctx, IHqlExpr
         case type_real:
             bindFunc = bindRealParamAtom;
             break;
+        case type_boolean:
+            bindFunc = bindBooleanParamAtom;
+            break;
         default:
             UNIMPLEMENTED;
         }
@@ -11502,6 +11505,9 @@ void HqlCppTranslator::buildScriptFunctionDefinition(BuildCtx &funcctx, IHqlExpr
     case type_real:
         returnFunc = getRealResultAtom;
         break;
+    case type_boolean:
+        returnFunc = getBooleanResultAtom;
+        break;
     default:
         UNIMPLEMENTED;
     }

ecl/hqlcpp/hqlcppsys.ecl

@@ -805,12 +805,14 @@ const char * cppSystemText[]  = {
     "    boolean dictionaryLookupExists(boolean meta, _linkcounted_ dictionary dict, row key) : eclrtl,include,pure,entrypoint='rtlDictionaryLookupExists';",
 
     // Marshalling parameters to external languages
+    "   bindBooleanParam(const varstring name, boolean val) : method,entrypoint='bindBooleanParam';",
+    "   bindRealParam(const varstring name, real val) : method,entrypoint='bindRealParam';",
     "   bindSignedParam(const varstring name, integer val) : method,entrypoint='bindSignedParam';",
     "   bindUnsignedParam(const varstring name, unsigned val) : method,entrypoint='bindUnsignedParam';",
     "   bindStringParam(const varstring name, const string val) : method,entrypoint='bindStringParam';",
     "   bindVStringParam(const varstring name, const varstring val) : method,entrypoint='bindvStringParam';",  // Not yet used
-    "   bindRealParam(const varstring name, real val) : method,entrypoint='bindRealParam';",
 
+    "   boolean getBooleanResult() : method,entrypoint='getBooleanResult';",
     "   real getRealResult() : method,entrypoint='getRealResult';",
     "   integer getSignedResult() : method,entrypoint='getSignedResult';",
     "   string getStringResult() : method,entrypoint='getStringResult';",

plugins/javaembed/javaembed.cpp

@@ -70,18 +70,23 @@ public:
     JavaGlobalState()
     {
         JavaVMInitArgs vm_args; /* JDK/JRE 6 VM initialization arguments */
-        JavaVMOption* options = new JavaVMOption[2];
+        JavaVMOption* options = new JavaVMOption[3];
         options[0].optionString = (char *) "-Djava.class.path=.";
-        options[1].optionString = (char *) "-verbose:jni";
+        options[1].optionString = (char *) "-verbosegc";
+        options[2].optionString = (char *) "-verbose:jni";
         vm_args.version = JNI_VERSION_1_6;
-        vm_args.nOptions = 1;  // set to 2 if you want the verbose...
+#ifdef _DEBUG
+        vm_args.nOptions = 1;  // set to 3 if you want the verbose...
+#else
+        vm_args.nOptions = 1;
+#endif
         vm_args.options = options;
         vm_args.ignoreUnrecognized = false;
         /* load and initialize a Java VM, return a JNI interface pointer in env */
         JNIEnv *env;       /* receives pointer to native method interface */
         JNI_CreateJavaVM(&javaVM, (void**)&env, &vm_args);
 
-        delete options;
+        delete [] options;
     }
     ~JavaGlobalState()
     {
@@ -109,7 +114,7 @@ public:
     {
     }
 
-    inline jmethodID importFunction(const char *text)
+    inline void importFunction(const char *text)
     {
         if (!prevtext || strcmp(text, prevtext) != 0)
         {
@@ -123,7 +128,7 @@ public:
             funcname++;  // skip the '.'
             StringBuffer methodname(signature-funcname, funcname);
             signature++; // skip the ':'
-            javaClass = JNIenv->FindClass(classname);
+            javaClass = (jclass) JNIenv->NewGlobalRef(JNIenv->FindClass(classname));
             if (!javaClass)
                 throw MakeStringException(MSGAUD_user, 0, "javaembed: Failed to resolve class name %s", classname.str());
             javaMethodID = JNIenv->GetStaticMethodID(javaClass, methodname, signature);
@@ -135,7 +140,6 @@ public:
             returnType.set(returnSig);
             prevtext.set(text);
         }
-        return NULL;
     }
     inline void callFunction(jvalue &result, const jvalue * args)
     {
@@ -146,7 +150,7 @@ public:
         case 'J': result.j = JNIenv->CallStaticLongMethodA(javaClass, javaMethodID, args); break;
         case 'F': result.f = JNIenv->CallStaticFloatMethodA(javaClass, javaMethodID, args); break;
         case 'D': result.d = JNIenv->CallStaticDoubleMethodA(javaClass, javaMethodID, args); break;
-        case 'L': result.l = JNIenv->CallStaticObjectMethodA(javaClass, javaMethodID, args); break;
+        case 'L': result.l = JNIenv->NewGlobalRef(JNIenv->CallStaticObjectMethodA(javaClass, javaMethodID, args)); break;
         case 'I': // Others are all smaller ints, so we can use this for all
         default: result.i = JNIenv->CallStaticIntMethodA(javaClass, javaMethodID, args); break;
         }
@@ -189,6 +193,23 @@ public:
             throw MakeStringException(MSGAUD_user, 0, "javaembed: Type mismatch on result");
         }
     }
+    bool getBooleanResult(jvalue &result)
+    {
+        switch (returnType.get()[0])
+        {
+        case 'Z': return result.z;
+        case 'L':
+            {
+                // Result should be of class 'Boolean'
+                jmethodID getVal = JNIenv->GetMethodID(JNIenv->GetObjectClass(result.l), "booleanValue", "()Z");  // this could probably be cached?
+                if (!getVal)
+                    throw MakeStringException(MSGAUD_user, 0, "javaembed: Type mismatch on result");
+                return JNIenv->CallBooleanMethod(result.l, getVal);
+            }
+        default:
+            throw MakeStringException(MSGAUD_user, 0, "javaembed: Type mismatch on result");
+        }
+    }
 private:
     StringAttr returnType;
     StringAttr prevtext;
@@ -211,6 +232,10 @@ public:
     {
     }
 
+    virtual bool getBooleanResult()
+    {
+        return sharedCtx->getBooleanResult(result);
+    }
     virtual double getRealResult()
     {
         return sharedCtx->getDoubleResult(result);
@@ -228,11 +253,17 @@ public:
         jstring sresult = (jstring) result.l;
         __len = sharedCtx->JNIenv->GetStringUTFLength(sresult);
         const char * chars =  sharedCtx->JNIenv->GetStringUTFChars(sresult, NULL);
-        __result = new char(__len);
+        __result = (char *)rtlMalloc(__len);
         memcpy(__result, chars, __len);
         sharedCtx->JNIenv->ReleaseStringUTFChars(sresult, chars);
     }
 
+    virtual void bindBooleanParam(const char *name, bool val)
+    {
+        jvalue v;
+        v.z = val;
+        addArg(v);
+    }
     virtual void bindRealParam(const char *name, double val)
     {
         jvalue v;

plugins/pyembed/pyembed.cpp

@@ -258,6 +258,13 @@ public:
         sharedCtx->threadState = PyEval_SaveThread();
     }
 
+    virtual bool getBooleanResult()
+    {
+        assertex(result);
+        if (!PyBool_Check(result))
+            throw MakeStringException(MSGAUD_user, 0, "pyembed: Type mismatch on result");
+        return result == Py_True;
+    }
     virtual double getRealResult()
     {
         assertex(result);
@@ -300,6 +307,11 @@ public:
     ~Python27EmbedScriptContext()
     {
     }
+    virtual void bindBooleanParam(const char *name, bool val)
+    {
+        OwnedPyObject vval = PyBool_FromLong(val ? 1 : 0);
+        PyDict_SetItemString(locals, name, vval);
+    }
     virtual void bindRealParam(const char *name, double val)
     {
         OwnedPyObject vval = PyFloat_FromDouble(val);
@@ -357,6 +369,10 @@ public:
     ~Python27EmbedImportContext()
     {
     }
+    virtual void bindBooleanParam(const char *name, bool val)
+    {
+        addArg(PyBool_FromLong(val ? 1 : 0));
+    }
     virtual void bindRealParam(const char *name, double val)
     {
         addArg(PyFloat_FromDouble(val));

plugins/v8embed/v8embed.cpp

@@ -83,6 +83,11 @@ public:
         isolate->Dispose();
     }
 
+    virtual void bindBooleanParam(const char *name, bool val)
+    {
+        v8::HandleScope handle_scope;
+        context->Global()->Set(v8::String::New(name), v8::Boolean::New(val));
+    }
     virtual void bindRealParam(const char *name, double val)
     {
         v8::HandleScope handle_scope;
@@ -111,6 +116,12 @@ public:
         context->Global()->Set(v8::String::New(name), v8::String::New(val));
     }
 
+    virtual bool getBooleanResult()
+    {
+        assertex (!result.IsEmpty());
+        v8::HandleScope handle_scope;
+        return result->BooleanValue();
+    }
     virtual double getRealResult()
     {
         assertex (!result.IsEmpty());

rtl/eclrtl/eclrtl.hpp

@@ -733,12 +733,14 @@ ECLRTL_API bool rtlGPF();
 
 interface IEmbedFunctionContext : extends IInterface
 {
+    virtual void bindBooleanParam(const char *name, bool val) = 0;
     virtual void bindRealParam(const char *name, double val) = 0;
     virtual void bindSignedParam(const char *name, __int64 val) = 0;
     virtual void bindUnsignedParam(const char *name, unsigned __int64 val) = 0;
     virtual void bindStringParam(const char *name, size32_t len, const char *val) = 0;
     virtual void bindVStringParam(const char *name, const char *val) = 0;
 
+    virtual bool getBooleanResult() = 0;
     virtual double getRealResult() = 0;
     virtual __int64 getSignedResult() = 0;
     virtual unsigned __int64 getUnsignedResult() = 0;

testing/ecl/JavaCat.java

@@ -1,5 +1,15 @@
 public class JavaCat
 {
+  public static int add1(int a)
+  {
+    return a + 1;
+  }
+  public static String add2(String a)
+  {
+    return a + '1';
+  }
+
+
   public static int add(int a, int b)
   {
     return a + b;

testing/ecl/embedjava.ecl

@@ -0,0 +1,25 @@
+IMPORT java;
+
+integer add1(integer val) := IMPORT(java, 'JavaCat.add1:(I)I');
+string add2(string val) := IMPORT(java, 'JavaCat.add2:(Ljava/lang/String;)Ljava/lang/String;');
+string add3(varstring val) := IMPORT(java, 'JavaCat.add2:(Ljava/lang/String;)Ljava/lang/String;');
+
+add1(10);
+add2('Hello');
+add3('World');
+
+s1 :=DATASET(250000, TRANSFORM({ integer a }, SELF.a := add1(COUNTER)));
+s2 :=DATASET(250000, TRANSFORM({ integer a }, SELF.a := add1(COUNTER/2)));
+ SUM(NOFOLD(s1 + s2), a);
+
+s1a :=DATASET(250000, TRANSFORM({ integer a }, SELF.a := (integer) add2((STRING)COUNTER)));
+s2a :=DATASET(250000, TRANSFORM({ integer a }, SELF.a := (integer) add3((STRING)(COUNTER/2))));
+ SUM(NOFOLD(s1a + s2a), a);
+
+s1b :=DATASET(250000, TRANSFORM({ integer a }, SELF.a := COUNTER+1));
+s2b :=DATASET(250000, TRANSFORM({ integer a }, SELF.a := (COUNTER/2)+1));
+ SUM(NOFOLD(s1b + s2b), a);
+
+s1c :=DATASET(250000, TRANSFORM({ integer a }, SELF.a := (integer) ((STRING) COUNTER + '1')));
+s2c :=DATASET(250000, TRANSFORM({ integer a }, SELF.a := (integer) ((STRING)(COUNTER/2) + '1')));
+ SUM(NOFOLD(s1c + s2c), a);