HPCC-8030 R/Java/JavaScript/Python language support in ECL

Add tests for exception marshalling, and corresponding fixes in the
exception marshalling implementations.

Check and adjust Java parameters according to the types in the supplied
signature, including handling parameters and results of type char.

Other changes in response to code review.

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

plugins/javaembed/javaembed.cpp

@@ -81,7 +81,10 @@ public:
     {
         JavaVMInitArgs vm_args; /* JDK/JRE 6 VM initialization arguments */
         JavaVMOption* options = new JavaVMOption[3];
-        options[0].optionString = (char *) "-Djava.class.path=.";
+        const char* origPath = getenv("CLASSPATH");
+        StringBuffer newPath;
+        newPath.append("-Djava.class.path=").append(origPath).append(ENVSEPCHAR).append(".");
+        options[0].optionString = (char *) newPath.str();
         options[1].optionString = (char *) "-Xcheck:jni";
         options[2].optionString = (char *) "-verbose:jni";
         vm_args.version = JNI_VERSION_1_6;
@@ -137,6 +140,20 @@ public:
             JNIenv->DeleteGlobalRef(javaClass);
     }
 
+    void checkException()
+    {
+        jthrowable exception = JNIenv->ExceptionOccurred();
+        if (exception)
+        {
+            JNIenv->ExceptionClear();
+            jclass throwableClass = JNIenv->FindClass("java/lang/Throwable");
+            jmethodID throwableToString = JNIenv->GetMethodID(throwableClass, "toString", "()Ljava/lang/String;");
+            jstring cause = (jstring) JNIenv->CallObjectMethod(exception, throwableToString);
+            const char *text = JNIenv->GetStringUTFChars(cause, 0);
+            throw MakeStringException(MSGAUD_user, 0, "javaembed: In method %s: %s", prevtext.get(), text);
+        }
+    }
+
     inline void importFunction(const char *text)
     {
         if (!prevtext || strcmp(text, prevtext) != 0)
@@ -165,14 +182,16 @@ public:
             assertex(returnSig);
             returnSig++;
             returnType.set(returnSig);
+            argsig.set(signature);
             prevtext.set(text);
         }
     }
     inline void callFunction(jvalue &result, const jvalue * args)
     {
+        JNIenv->ExceptionClear();
         switch (returnType.get()[0])
         {
-        case 'C': UNIMPLEMENTED; // jchar has no real ecl equivalent
+        case 'C': result.c = JNIenv->CallStaticCharMethodA(javaClass, javaMethodID, args); break;
         case 'B': result.z = JNIenv->CallStaticBooleanMethodA(javaClass, javaMethodID, args); break;
         case 'J': result.j = JNIenv->CallStaticLongMethodA(javaClass, javaMethodID, args); break;
         case 'F': result.f = JNIenv->CallStaticFloatMethodA(javaClass, javaMethodID, args); break;
@@ -181,6 +200,7 @@ public:
         case 'I': // Others are all smaller ints, so we can use this for all
         default: result.i = JNIenv->CallStaticIntMethodA(javaClass, javaMethodID, args); break;
         }
+        checkException();
     }
     inline __int64 getSignedResult(jvalue & result)
     {
@@ -239,32 +259,73 @@ public:
     }
     inline void getStringResult(jvalue &result, size32_t &__len, char * &__result)
     {
-        jstring sresult = (jstring) result.l;
-        size_t size = JNIenv->GetStringUTFLength(sresult);  // in bytes
-        const char *text =  JNIenv->GetStringUTFChars(sresult, NULL);
-        size32_t chars = rtlUtf8Length(size, text);
-        rtlUtf8ToStrX(__len, __result, chars, text);
-        JNIenv->ReleaseStringUTFChars(sresult, text);
+        switch (returnType.get()[0])
+        {
+        case 'C': // Single char returned, prototyped as STRING or STRING1 in ECL
+            rtlUnicodeToStrX(__len, __result, 1, &result.c);
+            break;
+        case 'L':
+        {
+            jstring sresult = (jstring) result.l;
+            size_t size = JNIenv->GetStringUTFLength(sresult);  // in bytes
+            const char *text =  JNIenv->GetStringUTFChars(sresult, NULL);
+            size32_t chars = rtlUtf8Length(size, text);
+            rtlUtf8ToStrX(__len, __result, chars, text);
+            JNIenv->ReleaseStringUTFChars(sresult, text);
+            break;
+        }
+        default:
+            throwUnexpected();
+        }
     }
     inline void getUTF8Result(jvalue &result, size32_t &__chars, char * &__result)
     {
-        jstring sresult = (jstring) result.l;
-        size_t size = JNIenv->GetStringUTFLength(sresult); // Returns length in bytes (not chars)
-        const char * text =  JNIenv->GetStringUTFChars(sresult, NULL);
-        rtlUtf8ToUtf8X(__chars, __result, rtlUtf8Length(size, text), text);
-        JNIenv->ReleaseStringUTFChars(sresult, text);
+        switch (returnType.get()[0])
+        {
+        case 'C': // Single jchar returned, prototyped as UTF8 in ECL
+            rtlUnicodeToUtf8X(__chars, __result, 1, &result.c);
+            break;
+        case 'L':
+        {
+            jstring sresult = (jstring) result.l;
+            size_t size = JNIenv->GetStringUTFLength(sresult); // Returns length in bytes (not chars)
+            const char * text =  JNIenv->GetStringUTFChars(sresult, NULL);
+            rtlUtf8ToUtf8X(__chars, __result, rtlUtf8Length(size, text), text);
+            JNIenv->ReleaseStringUTFChars(sresult, text);
+            break;
+        }
+        default:
+            throwUnexpected();
+        }
     }
     inline void getUnicodeResult(jvalue &result, size32_t &__chars, UChar * &__result)
     {
-        jstring sresult = (jstring) result.l;
-        size_t size = JNIenv->GetStringUTFLength(sresult);  // in bytes
-        const char *text =  JNIenv->GetStringUTFChars(sresult, NULL);
-        size32_t chars = rtlUtf8Length(size, text);
-        rtlUtf8ToUnicodeX(__chars, __result, chars, text);
-        JNIenv->ReleaseStringUTFChars(sresult, text);
+        switch (returnType.get()[0])
+        {
+        case 'C': // Single jchar returned, prototyped as UNICODE or UNICODE1 in ECL
+            rtlUnicodeToUnicodeX(__chars, __result, 1, &result.c);
+            break;
+        case 'L':
+        {
+            jstring sresult = (jstring) result.l;
+            size_t size = JNIenv->GetStringUTFLength(sresult);  // in bytes
+            const char *text =  JNIenv->GetStringUTFChars(sresult, NULL);
+            size32_t chars = rtlUtf8Length(size, text);
+            rtlUtf8ToUnicodeX(__chars, __result, chars, text);
+            JNIenv->ReleaseStringUTFChars(sresult, text);
+            break;
+        }
+        default:
+            throwUnexpected();
+        }
+    }
+    inline const char *querySignature()
+    {
+        return argsig.get();
     }
 private:
     StringAttr returnType;
+    StringAttr argsig;
     StringAttr prevtext;
     jclass javaClass;
     jmethodID javaMethodID;
@@ -280,6 +341,7 @@ public:
     : sharedCtx(_sharedCtx)
     {
         argcount = 0;
+        argsig = NULL;
     }
     ~JavaEmbedImportContext()
     {
@@ -317,6 +379,9 @@ public:
 
     virtual void bindBooleanParam(const char *name, bool val)
     {
+        if (*argsig != 'B')
+            typeError("BOOLEAN");
+        argsig++;
         jvalue v;
         v.z = val;
         addArg(v);
@@ -324,13 +389,43 @@ public:
     virtual void bindRealParam(const char *name, double val)
     {
         jvalue v;
-        v.d = val;
+        switch(*argsig)
+        {
+        case 'D':
+            v.d = val;
+            break;
+        case 'F':
+            v.f = val;
+            break;
+        default:
+            typeError("REAL");
+            break;
+        }
+        argsig++;
         addArg(v);
     }
     virtual void bindSignedParam(const char *name, __int64 val)
     {
         jvalue v;
-        v.j = val;
+        switch(*argsig)
+        {
+        case 'I':
+            v.i = val;
+            break;
+        case 'J':
+            v.j = val;
+            break;
+        case 'S':
+            v.s = val;
+            break;
+        case 'B':
+            v.b = val;
+            break;
+        default:
+            typeError("INTEGER");
+            break;
+        }
+        argsig++;
         addArg(v);
     }
     virtual void bindUnsignedParam(const char *name, unsigned __int64 val)
@@ -340,43 +435,91 @@ public:
     virtual void bindStringParam(const char *name, size32_t len, const char *val)
     {
         jvalue v;
-        unsigned unicodeChars;
-        UChar *unicode;
-        rtlStrToUnicodeX(unicodeChars, unicode, len, val);
-        v.l = sharedCtx->JNIenv->NewString(unicode, unicodeChars);
-        rtlFree(unicode);
+        switch(*argsig)
+        {
+        case 'C':
+            rtlStrToUnicode(1, &v.c, len, val);
+            argsig++;
+            break;
+        case 'L':
+            if (strncmp(argsig, "Ljava/lang/String;", 18) == 0)
+            {
+                argsig += 18;
+                unsigned unicodeChars;
+                UChar *unicode;
+                rtlStrToUnicodeX(unicodeChars, unicode, len, val);
+                v.l = sharedCtx->JNIenv->NewString(unicode, unicodeChars);
+                rtlFree(unicode);
+                break;
+            }
+            // fall into ...
+        default:
+            typeError("STRING");
+            break;
+        }
         addArg(v);
     }
     virtual void bindVStringParam(const char *name, const char *val)
     {
-        jvalue v;
-        unsigned unicodeChars;
-        UChar *unicode;
-        rtlStrToUnicodeX(unicodeChars, unicode, strlen(val), val);
-        v.l = sharedCtx->JNIenv->NewString(unicode, unicodeChars);
-        rtlFree(unicode);
-        addArg(v);
+        bindStringParam(name, strlen(val), val);
     }
     virtual void bindUTF8Param(const char *name, size32_t numchars, const char *val)
     {
         jvalue v;
-        unsigned unicodeChars;
-        UChar *unicode;
-        rtlUtf8ToUnicodeX(unicodeChars, unicode, numchars, val);
-        v.l = sharedCtx->JNIenv->NewString(unicode, unicodeChars);
-        rtlFree(unicode);
+        switch(*argsig)
+        {
+        case 'C':
+            rtlUtf8ToUnicode(1, &v.c, numchars, val);
+            argsig++;
+            break;
+        case 'L':
+            if (strncmp(argsig, "Ljava/lang/String;", 18) == 0)
+            {
+                argsig += 18;
+                unsigned unicodeChars;
+                UChar *unicode;
+                rtlUtf8ToUnicodeX(unicodeChars, unicode, numchars, val);
+                v.l = sharedCtx->JNIenv->NewString(unicode, unicodeChars);
+                rtlFree(unicode);
+                break;
+            }
+            // fall into ...
+        default:
+            typeError("UTF8");
+            break;
+        }
         addArg(v);
     }
     virtual void bindUnicodeParam(const char *name, size32_t numchars, const UChar *val)
     {
         jvalue v;
-        v.l = sharedCtx->JNIenv->NewString(val, numchars);
+        switch(*argsig)
+        {
+        case 'C':
+            rtlUnicodeToUnicode(1, &v.c, numchars, val);
+            argsig++;
+            break;
+        case 'L':
+            if (strncmp(argsig, "Ljava/lang/String;", 18) == 0)
+            {
+                argsig += 18;
+                v.l = sharedCtx->JNIenv->NewString(val, numchars);
+                break;
+            }
+            // fall into ...
+        default:
+            typeError("UNICODE");
+            break;
+        }
         addArg(v);
     }
 
     virtual void importFunction(const char *text)
     {
         sharedCtx->importFunction(text);
+        argsig = sharedCtx->querySignature();
+        assertex(*argsig == '(');
+        argsig++;
     }
     virtual void callFunction()
     {
@@ -391,6 +534,37 @@ protected:
     JavaThreadContext *sharedCtx;
     jvalue result;
 private:
+    void typeError(const char *ECLtype)
+    {
+        const char *javaType;
+        int javaLen = 0;
+        switch (*argsig)
+        {
+        case 'Z': javaType = "boolean"; break;
+        case 'B': javaType = "byte"; break;
+        case 'C': javaType = "char"; break;
+        case 'S': javaType = "short"; break;
+        case 'I': javaType = "int"; break;
+        case 'J': javaType = "long"; break;
+        case 'F': javaType = "float"; break;
+        case 'D': javaType = "double"; break;
+        case 'L':
+            {
+                javaType = argsig+1;
+                const char *semi = strchr(argsig, ';');
+                if (semi)
+                    javaLen = semi - javaType;
+                break;
+            }
+        case ')':
+            throw MakeStringException(0, "javaembed: Too many ECL parameters passed for Java signature %s", sharedCtx->querySignature());
+        default:
+            throw MakeStringException(0, "javaembed: Unrecognized character %c in java signature %s", *argsig, sharedCtx->querySignature());
+        }
+        if (!javaLen)
+            javaLen = strlen(argsig);
+        throw MakeStringException(0, "javaembed: ECL type %s cannot be passed to Java type %.*s", ECLtype, javaLen, javaType);
+    }
     void addArg(jvalue &arg)
     {
         assertex(argcount < MAX_JNI_ARGS);
@@ -399,6 +573,7 @@ private:
     }
     jvalue args[MAX_JNI_ARGS];
     int argcount;
+    const char *argsig;
 };
 
 static __thread JavaThreadContext* threadContext;  // We reuse per thread, for speed

plugins/pyembed/pyembed.cpp

@@ -94,11 +94,10 @@ static void checkPythonError()
     if (err)
     {
         OwnedPyObject pType, pValue, pTraceBack;
-        PyErr_Fetch(pType.ref(), pType.ref(), pType.ref());
-        OwnedPyObject errStr = PyObject_Str(err);
+        PyErr_Fetch(pType.ref(), pValue.ref(), pTraceBack.ref());
         OwnedPyObject valStr = PyObject_Str(pValue);
         PyErr_Clear();
-        VStringBuffer errMessage("%s: %s", PyString_AsString(errStr), PyString_AsString(valStr));
+        VStringBuffer errMessage("pyembed: %s", PyString_AsString(valStr));
         rtlFail(0, errMessage.str());
     }
 }

plugins/v8embed/v8embed.cpp

@@ -189,7 +189,14 @@ public:
     {
         assertex (!script.IsEmpty());
         v8::HandleScope handle_scope;
+        v8::TryCatch tryCatch;
         result = v8::Persistent<v8::Value>::New(script->Run());
+        v8::Handle<v8::Value> exception = tryCatch.Exception();
+        if (!exception.IsEmpty())
+        {
+            v8::String::AsciiValue msg(exception);
+            throw MakeStringException(MSGAUD_user, 0, "v8embed: %s", *msg);
+        }
     }
 
 protected:

testing/ecl/JavaCat.java

@@ -8,6 +8,14 @@ public class JavaCat
   {
     return a + '1';
   }
+  public static char addChar(char a)
+  {
+    return ++a;
+  }
+  public static int testThrow(int a) throws Exception
+  {
+    throw new Exception("Exception from Java");
+  }
 
 
   public static int add(int a, int b)

testing/ecl/embedjava.ecl

@@ -5,12 +5,29 @@ string add2(string val) := IMPORT(java, 'JavaCat.add2:(Ljava/lang/String;)Ljava/
 string add3(varstring val) := IMPORT(java, 'JavaCat.add2:(Ljava/lang/String;)Ljava/lang/String;');
 utf8 add4(utf8 val) := IMPORT(java, 'JavaCat.add2:(Ljava/lang/String;)Ljava/lang/String;');
 unicode add5(unicode val) := IMPORT(java, 'JavaCat.add2:(Ljava/lang/String;)Ljava/lang/String;');
+integer testThrow(integer p) := IMPORT(java, 'JavaCat.testThrow:(I)I');
+
+string addChar(string c) := IMPORT(java, 'JavaCat.addChar:(C)C');
+string cat(string s1, string s2) := IMPORT(java, 'JavaCat.cat:(Ljava/lang/String;Ljava/lang/String;)Ljava/lang/String;');
 
 add1(10);
 add2('Hello');
 add3('World');
 add4(U'Leovenaðes');
 add5(U'Стоял');
+addChar('A');
+
+cat('Hello', ' world');
+// Can't catch an expression(only a dataset)
+d := dataset([{ 1, '' }], { integer a, string m} ) : stored('nofold');
+
+d t := transform
+  self.a := FAILCODE;
+  self.m := FAILMESSAGE;
+  self := [];
+end;
+
+catch(d(testThrow(a) = a), onfail(t));
 
 s1 :=DATASET(250000, TRANSFORM({ integer a }, SELF.a := add1(COUNTER)));
 s2 :=DATASET(250000, TRANSFORM({ integer a }, SELF.a := add1(COUNTER/2)));

testing/ecl/embedjs.ecl

@@ -8,6 +8,8 @@ string add3(varstring val) := EMBED(javascript) val+'1'; ENDEMBED;
 utf8 add4(utf8 val) := EMBED(javascript) val+'1'; ENDEMBED;
 unicode add5(unicode val) := EMBED(javascript) val+'1'; ENDEMBED;
 
+integer testThrow(integer val) := EMBED(javascript) throw new Error("Error from JavaScript"); ENDEMBED;
+
 
 add1(10);
 add2('Hello');
@@ -17,6 +19,17 @@ add5(U'Стоял');
 
 add2('Oh là là Straße');  // Passing latin chars - should be untranslated
 
+// Test exception throwing/catching
+d := dataset([{ 1, '' }], { integer a, string m} ) : stored('nofold');
+
+d t := transform
+  self.a := FAILCODE;
+  self.m := FAILMESSAGE;
+  self := [];
+end;
+
+catch(d(testThrow(a) = a), onfail(t));
+
 s1 :=DATASET(250000, TRANSFORM({ integer a }, SELF.a := add1(COUNTER)));
 s2 :=DATASET(250000, TRANSFORM({ integer a }, SELF.a := add1(COUNTER/2)));
 SUM(NOFOLD(s1 + s2), a);

testing/ecl/embedpy.ecl

@@ -30,6 +30,10 @@ unicode add7(unicode val) := EMBED(Python)
 return val+'1'
 ENDEMBED;
 
+integer testThrow(integer val) := EMBED(Python)
+raise Exception('Error from Python')
+ENDEMBED;
+
 add1(10);
 add2('Hello');
 add3('World');
@@ -40,6 +44,17 @@ add7(U'Стоял');
 
 add2('Oh là là Straße');  // Passing latin chars - should be untranslated
 
+// Can't catch an expression(only a dataset)
+d := dataset([{ 1, '' }], { integer a, string m} ) : stored('nofold');
+
+d t := transform
+  self.a := FAILCODE;
+  self.m := FAILMESSAGE;
+  self := [];
+end;
+
+catch(d(testThrow(a) = a), onfail(t));
+
 s1 :=DATASET(250000, TRANSFORM({ integer a }, SELF.a := add1(COUNTER)));
 s2 :=DATASET(250000, TRANSFORM({ integer a }, SELF.a := add1(COUNTER/2)));
  SUM(NOFOLD(s1 + s2), a);

testing/ecl/key/embedjava.xml

@@ -1 +1,36 @@
-<Error><source>eclagent</source><code>0</code><message>System error: 0: javaembed: Failed to resolve class name JavaCat</message></Error>
+<Dataset name='Result 1'>
+ <Row><Result_1>11</Result_1></Row>
+</Dataset>
+<Dataset name='Result 2'>
+ <Row><Result_2>Hello1</Result_2></Row>
+</Dataset>
+<Dataset name='Result 3'>
+ <Row><Result_3>World1</Result_3></Row>
+</Dataset>
+<Dataset name='Result 4'>
+ <Row><Result_4>Leovenaðes1</Result_4></Row>
+</Dataset>
+<Dataset name='Result 5'>
+ <Row><Result_5>Стоял1</Result_5></Row>
+</Dataset>
+<Dataset name='Result 6'>
+ <Row><Result_6>B</Result_6></Row>
+</Dataset>
+<Dataset name='Result 7'>
+ <Row><Result_7>Hello world</Result_7></Row>
+</Dataset>
+<Dataset name='Result 8'>
+ <Row><a>0</a><m>javaembed: In method JavaCat.testThrow:(I)I: java.lang.Exception: Exception from Java</m></Row>
+</Dataset>
+<Dataset name='Result 9'>
+ <Row><Result_9>46875625000</Result_9></Row>
+</Dataset>
+<Dataset name='Result 10'>
+ <Row><Result_10>328126500000</Result_10></Row>
+</Dataset>
+<Dataset name='Result 11'>
+ <Row><Result_11>46875625000</Result_11></Row>
+</Dataset>
+<Dataset name='Result 12'>
+ <Row><Result_12>328126500000</Result_12></Row>
+</Dataset>

testing/ecl/key/embedjs.xml

@@ -20,14 +20,17 @@
  <Row><Result_7>Oh l&#253; l&#253; Stra&#253;e1</Result_7></Row>
 </Dataset>
 <Dataset name='Result 8'>
- <Row><Result_8>46875625000</Result_8></Row>
+ <Row><a>0</a><m>v8embed: Error: Error from JavaScript</m></Row>
 </Dataset>
 <Dataset name='Result 9'>
- <Row><Result_9>328126500000</Result_9></Row>
+ <Row><Result_9>46875625000</Result_9></Row>
 </Dataset>
 <Dataset name='Result 10'>
- <Row><Result_10>46875625000</Result_10></Row>
+ <Row><Result_10>328126500000</Result_10></Row>
 </Dataset>
 <Dataset name='Result 11'>
- <Row><Result_11>328126500000</Result_11></Row>
+ <Row><Result_11>46875625000</Result_11></Row>
+</Dataset>
+<Dataset name='Result 12'>
+ <Row><Result_12>328126500000</Result_12></Row>
 </Dataset>

testing/ecl/key/embedpy.xml

@@ -26,14 +26,17 @@
  <Row><Result_9>Oh l&#224; l&#224; Stra&#223;e1</Result_9></Row>
 </Dataset>
 <Dataset name='Result 10'>
- <Row><Result_10>46875625000</Result_10></Row>
+ <Row><a>0</a><m>pyembed: Error from Python</m></Row>
 </Dataset>
 <Dataset name='Result 11'>
- <Row><Result_11>328126500000</Result_11></Row>
+ <Row><Result_11>46875625000</Result_11></Row>
 </Dataset>
 <Dataset name='Result 12'>
- <Row><Result_12>46875625000</Result_12></Row>
+ <Row><Result_12>328126500000</Result_12></Row>
 </Dataset>
 <Dataset name='Result 13'>
- <Row><Result_13>328126500000</Result_13></Row>
+ <Row><Result_13>46875625000</Result_13></Row>
+</Dataset>
+<Dataset name='Result 14'>
+ <Row><Result_14>328126500000</Result_14></Row>
 </Dataset>