HPCC-8030 R/Java/JavaScript/Python language support in ECL

Fixes to support for unicode/utf8 parameters and results, after code review.

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

plugins/pyembed/pyembed.cpp

@@ -302,32 +302,47 @@ public:
     virtual void getStringResult(size32_t &__chars, char * &__result)
     {
         assertex(result && result != Py_None);
-        const char * text =  PyString_AsString(result);
-        checkPythonError();
-        size_t lenBytes = PyString_Size(result);
-        rtlUtf8ToStrX(__chars, __result, lenBytes, text);
+        if (PyString_Check(result))
+        {
+            const char * text =  PyString_AsString(result);
+            checkPythonError();
+            size_t lenBytes = PyString_Size(result);
+            rtlStrToStrX(__chars, __result, lenBytes, text);
+        }
+        else
+            rtlFail(0, "Python type mismatch - return value was not a string");
     }
     virtual void getUTF8Result(size32_t &__chars, char * &__result)
     {
         assertex(result && result != Py_None);
-        OwnedPyObject utf8 = PyUnicode_AsUTF8String(result);
-        checkPythonError();
-        size_t lenBytes = PyString_Size(utf8);
-        const char * text =  PyString_AsString(utf8);
-        checkPythonError();
-        size32_t numchars = rtlUtf8Length(lenBytes, text);
-        rtlUtf8ToUtf8X(__chars, __result, numchars, text);
+        if (PyUnicode_Check(result))
+        {
+            OwnedPyObject utf8 = PyUnicode_AsUTF8String(result);
+            checkPythonError();
+            size_t lenBytes = PyString_Size(utf8);
+            const char * text =  PyString_AsString(utf8);
+            checkPythonError();
+            size32_t numchars = rtlUtf8Length(lenBytes, text);
+            rtlUtf8ToUtf8X(__chars, __result, numchars, text);
+        }
+        else
+            rtlFail(0, "Python type mismatch - return value was not a unicode string");
     }
     virtual void getUnicodeResult(size32_t &__chars, UChar * &__result)
     {
         assertex(result && result != Py_None);
-        OwnedPyObject utf8 = PyUnicode_AsUTF8String(result);
-        checkPythonError();
-        size_t lenBytes = PyString_Size(utf8);
-        const char * text =  PyString_AsString(utf8);
-        checkPythonError();
-        size32_t numchars = rtlUtf8Length(lenBytes, text);
-        rtlUtf8ToUnicodeX(__chars, __result, numchars, text);
+        if (PyUnicode_Check(result))
+        {
+            OwnedPyObject utf8 = PyUnicode_AsUTF8String(result);
+            checkPythonError();
+            size_t lenBytes = PyString_Size(utf8);
+            const char * text =  PyString_AsString(utf8);
+            checkPythonError();
+            size32_t numchars = rtlUtf8Length(lenBytes, text);
+            rtlUtf8ToUnicodeX(__chars, __result, numchars, text);
+        }
+        else
+            rtlFail(0, "Python type mismatch - return value was not a unicode string");
     }
 
 

plugins/v8embed/v8embed.cpp

@@ -118,12 +118,12 @@ public:
     virtual void bindUTF8Param(const char *name, size32_t chars, const char *val)
     {
         v8::HandleScope handle_scope;
-        context->Global()->Set(v8::String::New(name), v8::String::New(val, chars));   // more - should this be chars or bytes?
+        context->Global()->Set(v8::String::New(name), v8::String::New(val, rtlUtf8Size(chars, val)));
     }
     virtual void bindUnicodeParam(const char *name, size32_t chars, const UChar *val)
     {
         v8::HandleScope handle_scope;
-        context->Global()->Set(v8::String::New(name), v8::String::New(val, chars));   // more - should this be chars or bytes?
+        context->Global()->Set(v8::String::New(name), v8::String::New(val, chars));
     }
 
     virtual bool getBooleanResult()
@@ -154,7 +154,6 @@ public:
     {
         assertex (!result.IsEmpty() && result->IsString());
         v8::HandleScope handle_scope;
-        v8::Handle<v8::String> s = result->ToString();
         v8::String::AsciiValue ascii(result);
         rtlStrToStrX(__chars, __result, ascii.length(), *ascii);
     }
@@ -162,7 +161,6 @@ public:
     {
         assertex (!result.IsEmpty() && result->IsString());
         v8::HandleScope handle_scope;
-        v8::Handle<v8::String> s = result->ToString();
         v8::String::Utf8Value utf8(result);
         unsigned numchars = rtlUtf8Length(utf8.length(), *utf8);
         rtlUtf8ToUtf8X(__chars, __result, numchars, *utf8);
@@ -171,7 +169,6 @@ public:
     {
         assertex (!result.IsEmpty() && result->IsString());
         v8::HandleScope handle_scope;
-        v8::Handle<v8::String> s = result->ToString();
         v8::String::Utf8Value utf8(result);
         unsigned numchars = rtlUtf8Length(utf8.length(), *utf8);
         rtlUtf8ToUnicodeX(__chars, __result, numchars, *utf8);

testing/ecl/embedjs.ecl

@@ -8,12 +8,15 @@ string add3(varstring val) := EMBED(javascript) val+'1'; ENDEMBED;
 utf8 add4(utf8 val) := EMBED(javascript) val+'1'; ENDEMBED;
 unicode add5(unicode val) := EMBED(javascript) val+'1'; ENDEMBED;
 
+
 add1(10);
 add2('Hello');
 add3('World');
-add4(U'Leovenaðes');
+add4(U'Oh là là Straße');
 add5(U'Стоял');
 
+add2('Oh là là Straße');  // Passing latin chars - should be untranslated
+
 s1 :=DATASET(250000, TRANSFORM({ integer a }, SELF.a := add1(COUNTER)));
 s2 :=DATASET(250000, TRANSFORM({ integer a }, SELF.a := add1(COUNTER/2)));
 SUM(NOFOLD(s1 + s2), a);

testing/ecl/embedpy.ecl

@@ -33,11 +33,13 @@ ENDEMBED;
 add1(10);
 add2('Hello');
 add3('World');
-add4(U'Leovenaðes');
+add4(U'Oh là là Straße');
 add5(U'Стоял');
-add6(U'Leovenaðes');
+add6(U'Oh là là Straße');
 add7(U'Стоял');
 
+add2('Oh là là Straße');  // Passing latin chars - should be untranslated
+
 s1 :=DATASET(250000, TRANSFORM({ integer a }, SELF.a := add1(COUNTER)));
 s2 :=DATASET(250000, TRANSFORM({ integer a }, SELF.a := add1(COUNTER/2)));
  SUM(NOFOLD(s1 + s2), a);

testing/ecl/key/embedjs.xml

@@ -11,20 +11,23 @@
  <Row><Result_4>World1</Result_4></Row>
 </Dataset>
 <Dataset name='Result 5'>
- <Row><Result_5>Leovenaðe1</Result_5></Row>
+ <Row><Result_5>Oh là là Straße1</Result_5></Row>
 </Dataset>
 <Dataset name='Result 6'>
  <Row><Result_6>Стоял1</Result_6></Row>
 </Dataset>
 <Dataset name='Result 7'>
- <Row><Result_7>46875625000</Result_7></Row>
+ <Row><Result_7>Oh l&#253; l&#253; Stra&#253;e1</Result_7></Row>
 </Dataset>
 <Dataset name='Result 8'>
- <Row><Result_8>328126500000</Result_8></Row>
+ <Row><Result_8>46875625000</Result_8></Row>
 </Dataset>
 <Dataset name='Result 9'>
- <Row><Result_9>46875625000</Result_9></Row>
+ <Row><Result_9>328126500000</Result_9></Row>
 </Dataset>
 <Dataset name='Result 10'>
- <Row><Result_10>328126500000</Result_10></Row>
+ <Row><Result_10>46875625000</Result_10></Row>
+</Dataset>
+<Dataset name='Result 11'>
+ <Row><Result_11>328126500000</Result_11></Row>
 </Dataset>

testing/ecl/key/embedpy.xml

@@ -11,26 +11,29 @@
  <Row><Result_4>World1</Result_4></Row>
 </Dataset>
 <Dataset name='Result 5'>
- <Row><Result_5>Leovenaðes1</Result_5></Row>
+ <Row><Result_5>Oh là là Straße1</Result_5></Row>
 </Dataset>
 <Dataset name='Result 6'>
  <Row><Result_6>Стоял1</Result_6></Row>
 </Dataset>
 <Dataset name='Result 7'>
- <Row><Result_7>Leovenaðes1</Result_7></Row>
+ <Row><Result_7>Oh là là Straße1</Result_7></Row>
 </Dataset>
 <Dataset name='Result 8'>
  <Row><Result_8>Стоял1</Result_8></Row>
 </Dataset>
 <Dataset name='Result 9'>
- <Row><Result_9>46875625000</Result_9></Row>
+ <Row><Result_9>Oh l&#224; l&#224; Stra&#223;e1</Result_9></Row>
 </Dataset>
 <Dataset name='Result 10'>
- <Row><Result_10>328126500000</Result_10></Row>
+ <Row><Result_10>46875625000</Result_10></Row>
 </Dataset>
 <Dataset name='Result 11'>
- <Row><Result_11>46875625000</Result_11></Row>
+ <Row><Result_11>328126500000</Result_11></Row>
 </Dataset>
 <Dataset name='Result 12'>
- <Row><Result_12>328126500000</Result_12></Row>
+ <Row><Result_12>46875625000</Result_12></Row>
+</Dataset>
+<Dataset name='Result 13'>
+ <Row><Result_13>328126500000</Result_13></Row>
 </Dataset>