HPCC-8030 R/Java/JavaScript/Python language support in ECL

Add support for DATA params and return values.

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

ecl/hql/hqlatoms.cpp

@@ -50,6 +50,7 @@ _ATOM bcdAtom;
 _ATOM beforeAtom;
 _ATOM bestAtom;
 _ATOM bindBooleanParamAtom;
+_ATOM bindDataParamAtom;
 _ATOM bindRealParamAtom;
 _ATOM bindSignedParamAtom;
 _ATOM bindStringParamAtom;
@@ -146,6 +147,7 @@ _ATOM gctxmethodAtom;
 _ATOM getAtom;
 _ATOM getEmbedContextAtom;
 _ATOM getBooleanResultAtom;
+_ATOM getDataResultAtom;
 _ATOM getRealResultAtom;
 _ATOM getSignedResultAtom;
 _ATOM getStringResultAtom;
@@ -453,6 +455,7 @@ MODULE_INIT(INIT_PRIORITY_HQLATOM)
     MAKEATOM(before);
     MAKEATOM(best);
     MAKEATOM(bindBooleanParam);
+    MAKEATOM(bindDataParam);
     MAKEATOM(bindRealParam);
     MAKEATOM(bindSignedParam);
     MAKEATOM(bindStringParam);
@@ -548,6 +551,7 @@ MODULE_INIT(INIT_PRIORITY_HQLATOM)
     MAKEATOM(get);
     MAKEATOM(getEmbedContext);
     MAKEATOM(getBooleanResult);
+    MAKEATOM(getDataResult);
     MAKEATOM(getRealResult);
     MAKEATOM(getSignedResult);
     MAKEATOM(getStringResult);

ecl/hql/hqlatoms.hpp

@@ -53,6 +53,7 @@ extern HQL_API _ATOM bcdAtom;
 extern HQL_API _ATOM beforeAtom;
 extern HQL_API _ATOM bestAtom;
 extern HQL_API _ATOM bindBooleanParamAtom;
+extern HQL_API _ATOM bindDataParamAtom;
 extern HQL_API _ATOM bindRealParamAtom;
 extern HQL_API _ATOM bindSignedParamAtom;
 extern HQL_API _ATOM bindStringParamAtom;
@@ -148,6 +149,7 @@ extern HQL_API _ATOM gctxmethodAtom;
 extern HQL_API _ATOM getAtom;
 extern HQL_API _ATOM getEmbedContextAtom;
 extern HQL_API _ATOM getBooleanResultAtom;
+extern HQL_API _ATOM getDataResultAtom;
 extern HQL_API _ATOM getRealResultAtom;
 extern HQL_API _ATOM getSignedResultAtom;
 extern HQL_API _ATOM getStringResultAtom;

ecl/hqlcpp/hqlcpp.cpp

@@ -11494,6 +11494,9 @@ void HqlCppTranslator::buildScriptFunctionDefinition(BuildCtx &funcctx, IHqlExpr
         case type_unicode:
             bindFunc = bindUnicodeParamAtom;
             break;
+        case type_data:
+            bindFunc = bindDataParamAtom;
+            break;
         default:
             StringBuffer typeText;
             getFriendlyTypeStr(paramType, typeText);
@@ -11525,6 +11528,9 @@ void HqlCppTranslator::buildScriptFunctionDefinition(BuildCtx &funcctx, IHqlExpr
     case type_utf8:
         returnFunc = getUTF8ResultAtom;
         break;
+    case type_data:
+        returnFunc = getDataResultAtom;
+        break;
     default:
         StringBuffer typeText;
         getFriendlyTypeStr(returnType, typeText);

ecl/hqlcpp/hqlcppsys.ecl

@@ -806,6 +806,7 @@ const char * cppSystemText[]  = {
 
     // Marshalling parameters to external languages
     "   bindBooleanParam(const varstring name, boolean val) : method,entrypoint='bindBooleanParam';",
+    "   bindDataParam(const varstring name, data val) : method,entrypoint='bindDataParam';",
     "   bindRealParam(const varstring name, real val) : method,entrypoint='bindRealParam';",
     "   bindSignedParam(const varstring name, integer val) : method,entrypoint='bindSignedParam';",
     "   bindUnsignedParam(const varstring name, unsigned val) : method,entrypoint='bindUnsignedParam';",
@@ -815,6 +816,7 @@ const char * cppSystemText[]  = {
     "   bindUnicodeParam(const varstring name, const unicode val) : method,entrypoint='bindUnicodeParam';",
 
     "   boolean getBooleanResult() : method,entrypoint='getBooleanResult';",
+    "   data getDataResult() : method,entrypoint='getDataResult';",
     "   real getRealResult() : method,entrypoint='getRealResult';",
     "   integer getSignedResult() : method,entrypoint='getSignedResult';",
     "   string getStringResult() : method,entrypoint='getStringResult';",

plugins/Rembed/Rembed.cpp

@@ -129,6 +129,11 @@ public:
     {
         return ::Rcpp::as<bool>(result);
     }
+    virtual void getDataResult(size32_t &__len, void * &__result)
+    {
+        std::vector<byte> vval = ::Rcpp::as<std::vector<byte> >(result);;
+        rtlStrToDataX(__len, __result, vval.size(), vval.data());
+    }
     virtual double getRealResult()
     {
         return ::Rcpp::as<double>(result);
@@ -159,6 +164,13 @@ public:
     {
         R[name] = val;
     }
+    virtual void bindDataParam(const char *name, size32_t len, const void *val)
+    {
+        std::vector<byte> vval;
+        const byte *cval = (const byte *) val;
+        vval.assign(cval, cval+len);
+        R[name] = vval;
+    }
     virtual void bindRealParam(const char *name, double val)
     {
         R[name] = val;

plugins/javaembed/javaembed.cpp

@@ -194,13 +194,18 @@ public:
         switch (returnType.get()[0])
         {
         case 'C': result.c = JNIenv->CallStaticCharMethodA(javaClass, javaMethodID, args); break;
-        case 'B': result.z = JNIenv->CallStaticBooleanMethodA(javaClass, javaMethodID, args); break;
+        case 'Z': result.z = JNIenv->CallStaticBooleanMethodA(javaClass, javaMethodID, args); break;
         case 'J': result.j = JNIenv->CallStaticLongMethodA(javaClass, javaMethodID, args); break;
         case 'F': result.f = JNIenv->CallStaticFloatMethodA(javaClass, javaMethodID, args); break;
         case 'D': result.d = JNIenv->CallStaticDoubleMethodA(javaClass, javaMethodID, args); break;
+        case 'I': result.i = JNIenv->CallStaticIntMethodA(javaClass, javaMethodID, args); break;
+        case 'S': result.s = JNIenv->CallStaticShortMethodA(javaClass, javaMethodID, args); break;
+        case 'B': result.s = JNIenv->CallStaticByteMethodA(javaClass, javaMethodID, args); break;
+
+        case '[':
         case 'L': result.l = JNIenv->CallStaticObjectMethodA(javaClass, javaMethodID, args); break;
-        case 'I': // Others are all smaller ints, so we can use this for all
-        default: result.i = JNIenv->CallStaticIntMethodA(javaClass, javaMethodID, args); break;
+
+        default: throwUnexpected();
         }
         checkException();
     }
@@ -259,6 +264,15 @@ public:
             throw MakeStringException(MSGAUD_user, 0, "javaembed: Type mismatch on result");
         }
     }
+    inline void getDataResult(jvalue &result, size32_t &__len, void * &__result)
+    {
+        if (strcmp(returnType, "[B")!=0)
+            throw MakeStringException(MSGAUD_user, 0, "javaembed: Type mismatch on result");
+        jbyteArray array = (jbyteArray) result.l;
+        __len = JNIenv->GetArrayLength(array);
+        __result = rtlMalloc(__len);
+        JNIenv->GetByteArrayRegion(array, 0, __len, (jbyte *) __result);
+    }
     inline void getStringResult(jvalue &result, size32_t &__len, char * &__result)
     {
         switch (returnType.get()[0])
@@ -353,6 +367,10 @@ public:
     {
         return sharedCtx->getBooleanResult(result);
     }
+    virtual void getDataResult(size32_t &__len, void * &__result)
+    {
+        sharedCtx->getDataResult(result, __len, __result);
+    }
     virtual double getRealResult()
     {
         return sharedCtx->getDoubleResult(result);
@@ -388,6 +406,17 @@ public:
         v.z = val;
         addArg(v);
     }
+    virtual void bindDataParam(const char *name, size32_t len, const void *val)
+    {
+        if (argsig[0] != '[' || argsig[1] != 'B')
+            typeError("DATA");
+        argsig += 2;
+        jvalue v;
+        jbyteArray javaData = sharedCtx->JNIenv->NewByteArray(len);
+        sharedCtx->JNIenv->SetByteArrayRegion(javaData, 0, len, (jbyte *) val);
+        v.l = javaData;
+        addArg(v);
+    }
     virtual void bindRealParam(const char *name, double val)
     {
         jvalue v;

plugins/pyembed/pyembed.cpp

@@ -287,6 +287,13 @@ public:
             throw MakeStringException(MSGAUD_user, 0, "pyembed: Type mismatch on result");
         return result == Py_True;
     }
+    virtual void getDataResult(size32_t &__chars, void * &__result)
+    {
+        assertex(result && result != Py_None);
+        if (!PyByteArray_Check(result))
+            throw MakeStringException(MSGAUD_user, 0, "pyembed: Type mismatch on result");
+        rtlStrToDataX(__chars, __result, PyByteArray_Size(result), PyByteArray_AsString(result));
+    }
     virtual double getRealResult()
     {
         assertex(result && result != Py_None);
@@ -372,6 +379,11 @@ public:
         OwnedPyObject vval = PyBool_FromLong(val ? 1 : 0);
         PyDict_SetItemString(locals, name, vval);
     }
+    virtual void bindDataParam(const char *name, size32_t len, const void *val)
+    {
+        OwnedPyObject vval = PyByteArray_FromStringAndSize((const char *) val, len);
+        PyDict_SetItemString(locals, name, vval);
+    }
     virtual void bindRealParam(const char *name, double val)
     {
         OwnedPyObject vval = PyFloat_FromDouble(val);
@@ -451,6 +463,10 @@ public:
     {
         addArg(PyBool_FromLong(val ? 1 : 0));
     }
+    virtual void bindDataParam(const char *name, size32_t len, const void *val)
+    {
+        addArg(PyByteArray_FromStringAndSize((const char *) val, len));
+    }
     virtual void bindRealParam(const char *name, double val)
     {
         addArg(PyFloat_FromDouble(val));

plugins/v8embed/v8embed.cpp

@@ -88,6 +88,17 @@ public:
         v8::HandleScope handle_scope;
         context->Global()->Set(v8::String::New(name), v8::Boolean::New(val));
     }
+    virtual void bindDataParam(const char *name, size32_t len, const void *val)
+    {
+        v8::HandleScope handle_scope;
+        v8::Local<v8::Array> array = v8::Array::New(len);
+        const byte *vval = (const byte *) val;
+        for (int i = 0; i < len; i++)
+        {
+            array->Set(v8::Number::New(i), v8::Integer::New(vval[i])); // feels horridly inefficient, but seems to be the expected approach
+        }
+        context->Global()->Set(v8::String::New(name), array);
+    }
     virtual void bindRealParam(const char *name, double val)
     {
         v8::HandleScope handle_scope;
@@ -134,6 +145,19 @@ public:
         v8::HandleScope handle_scope;
         return result->BooleanValue();
     }
+    virtual void getDataResult(size32_t &__len, void * &__result)
+    {
+        assertex (!result.IsEmpty() && result->IsArray());
+        v8::HandleScope handle_scope;
+        v8::Handle<v8::Array> array = v8::Handle<v8::Array>::Cast(result);
+        __len = array->Length();
+        __result = rtlMalloc(__len);
+        byte *bresult = (byte *) __result;
+        for (size32_t i = 0; i < __len; i++)
+        {
+            bresult[i] = v8::Integer::Cast(*array->Get(i))->Value(); // feels horridly inefficient, but seems to be the expected approach
+        }
+    }
     virtual double getRealResult()
     {
         assertex (!result.IsEmpty());

rtl/eclrtl/eclrtl.hpp

@@ -734,6 +734,7 @@ ECLRTL_API bool rtlGPF();
 interface IEmbedFunctionContext : extends IInterface
 {
     virtual void bindBooleanParam(const char *name, bool val) = 0;
+    virtual void bindDataParam(const char *name, size32_t len, const void *val) = 0;
     virtual void bindRealParam(const char *name, double val) = 0;
     virtual void bindSignedParam(const char *name, __int64 val) = 0;
     virtual void bindUnsignedParam(const char *name, unsigned __int64 val) = 0;
@@ -743,6 +744,7 @@ interface IEmbedFunctionContext : extends IInterface
     virtual void bindUnicodeParam(const char *name, size32_t chars, const UChar *val) = 0;
 
     virtual bool getBooleanResult() = 0;
+    virtual void getDataResult(size32_t &len, void * &result) = 0;
     virtual double getRealResult() = 0;
     virtual __int64 getSignedResult() = 0;
     virtual unsigned __int64 getUnsignedResult() = 0;

testing/ecl/JavaCat.java

@@ -17,6 +17,11 @@ public class JavaCat
     throw new Exception("Exception from Java");
   }
 
+  public static byte[] testData(byte [] indata)
+  {
+    indata[0]++;
+    return indata;
+  }
 
   public static int add(int a, int b)
   {

testing/ecl/embedR.ecl

@@ -8,8 +8,14 @@ string cat(varstring what, string who) := EMBED(R)
 paste(what,who)
 ENDEMBED;
 
+data testData(data val) := EMBED(R)
+val[1] = val[2];
+val;
+ENDEMBED;
+
 add1(10);
 cat('Hello', 'World');
+testData(D'ab');
 
 s1 :=DATASET(250000, TRANSFORM({ integer a }, SELF.a := add1(COUNTER)));
 s2 :=DATASET(250000, TRANSFORM({ integer a }, SELF.a := add1(COUNTER/2)));

testing/ecl/embedjava.ecl

@@ -9,6 +9,7 @@ integer testThrow(integer p) := IMPORT(java, 'JavaCat.testThrow:(I)I');
 
 string addChar(string c) := IMPORT(java, 'JavaCat.addChar:(C)C');
 string cat(string s1, string s2) := IMPORT(java, 'JavaCat.cat:(Ljava/lang/String;Ljava/lang/String;)Ljava/lang/String;');
+data testData(data indata) := IMPORT(java, 'JavaCat.testData:([B)[B');
 
 add1(10);
 add2('Hello');
@@ -28,6 +29,7 @@ d t := transform
 end;
 
 catch(d(testThrow(a) = a), onfail(t));
+testData(d'aa');
 
 s1 :=DATASET(250000, TRANSFORM({ integer a }, SELF.a := add1(COUNTER)));
 s2 :=DATASET(250000, TRANSFORM({ integer a }, SELF.a := add1(COUNTER/2)));

testing/ecl/embedjs.ecl

@@ -9,7 +9,7 @@ utf8 add4(utf8 val) := EMBED(javascript) val+'1'; ENDEMBED;
 unicode add5(unicode val) := EMBED(javascript, U' val+\' at Oh là là Straße\';');
 
 integer testThrow(integer val) := EMBED(javascript) throw new Error("Error from JavaScript"); ENDEMBED;
-
+data testData(data val) := EMBED(javascript) val[0] = val[0] + 1; val; ENDEMBED;
 
 add1(10);
 add2('Hello');
@@ -29,6 +29,7 @@ d t := transform
 end;
 
 catch(d(testThrow(a) = a), onfail(t));
+testdata(D'aa');
 
 s1 :=DATASET(250000, TRANSFORM({ integer a }, SELF.a := add1(COUNTER)));
 s2 :=DATASET(250000, TRANSFORM({ integer a }, SELF.a := add1(COUNTER/2)));

testing/ecl/embedpy.ecl

@@ -34,6 +34,11 @@ integer testThrow(integer val) := EMBED(Python)
 raise Exception('Error from Python')
 ENDEMBED;
 
+data testData(data val) := EMBED(Python)
+val[0] = val[0] + 1
+return val
+ENDEMBED;
+
 add1(10);
 add2('Hello');
 add3('World');
@@ -54,6 +59,7 @@ d t := transform
 end;
 
 catch(d(testThrow(a) = a), onfail(t));
+testData(D'aa');
 
 s1 :=DATASET(250000, TRANSFORM({ integer a }, SELF.a := add1(COUNTER)));
 s2 :=DATASET(250000, TRANSFORM({ integer a }, SELF.a := add1(COUNTER/2)));

testing/ecl/key/embedR.xml

@@ -5,8 +5,11 @@
  <Row><Result_2>Hello World</Result_2></Row>
 </Dataset>
 <Dataset name='Result 3'>
- <Row><Result_3>46875625000</Result_3></Row>
+ <Row><Result_3>6262</Result_3></Row>
 </Dataset>
 <Dataset name='Result 4'>
  <Row><Result_4>46875625000</Result_4></Row>
 </Dataset>
+<Dataset name='Result 5'>
+ <Row><Result_5>46875625000</Result_5></Row>
+</Dataset>

testing/ecl/key/embedjava.xml

@@ -23,14 +23,17 @@
  <Row><a>0</a><m>javaembed: In method JavaCat.testThrow:(I)I: java.lang.Exception: Exception from Java</m></Row>
 </Dataset>
 <Dataset name='Result 9'>
- <Row><Result_9>46875625000</Result_9></Row>
+ <Row><Result_9>6261</Result_9></Row>
 </Dataset>
 <Dataset name='Result 10'>
- <Row><Result_10>328126500000</Result_10></Row>
+ <Row><Result_10>46875625000</Result_10></Row>
 </Dataset>
 <Dataset name='Result 11'>
- <Row><Result_11>46875625000</Result_11></Row>
+ <Row><Result_11>328126500000</Result_11></Row>
 </Dataset>
 <Dataset name='Result 12'>
- <Row><Result_12>328126500000</Result_12></Row>
+ <Row><Result_12>46875625000</Result_12></Row>
+</Dataset>
+<Dataset name='Result 13'>
+ <Row><Result_13>328126500000</Result_13></Row>
 </Dataset>

testing/ecl/key/embedjs.xml

@@ -23,14 +23,17 @@
  <Row><a>0</a><m>v8embed: Error: Error from JavaScript</m></Row>
 </Dataset>
 <Dataset name='Result 9'>
- <Row><Result_9>46875625000</Result_9></Row>
+ <Row><Result_9>6261</Result_9></Row>
 </Dataset>
 <Dataset name='Result 10'>
- <Row><Result_10>328126500000</Result_10></Row>
+ <Row><Result_10>46875625000</Result_10></Row>
 </Dataset>
 <Dataset name='Result 11'>
- <Row><Result_11>46875625000</Result_11></Row>
+ <Row><Result_11>328126500000</Result_11></Row>
 </Dataset>
 <Dataset name='Result 12'>
- <Row><Result_12>328126500000</Result_12></Row>
+ <Row><Result_12>46875625000</Result_12></Row>
+</Dataset>
+<Dataset name='Result 13'>
+ <Row><Result_13>328126500000</Result_13></Row>
 </Dataset>

testing/ecl/key/embedpy.xml

@@ -29,14 +29,17 @@
  <Row><a>0</a><m>pyembed: Error from Python</m></Row>
 </Dataset>
 <Dataset name='Result 11'>
- <Row><Result_11>46875625000</Result_11></Row>
+ <Row><Result_11>6261</Result_11></Row>
 </Dataset>
 <Dataset name='Result 12'>
- <Row><Result_12>328126500000</Result_12></Row>
+ <Row><Result_12>46875625000</Result_12></Row>
 </Dataset>
 <Dataset name='Result 13'>
- <Row><Result_13>46875625000</Result_13></Row>
+ <Row><Result_13>328126500000</Result_13></Row>
 </Dataset>
 <Dataset name='Result 14'>
- <Row><Result_14>328126500000</Result_14></Row>
+ <Row><Result_14>46875625000</Result_14></Row>
+</Dataset>
+<Dataset name='Result 15'>
+ <Row><Result_15>328126500000</Result_15></Row>
 </Dataset>