HPCC-8749 Support common directory for loading java classes

Regression suite tests for embedded Python/Java struggle because of classpath
issues.

Python case can be managed via a path on the IMPORT parameter (though it
should prepend rather than append the supplied path) but Java requires a
standard location to be selected up front.

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

ecl/eclcc/eclcc.cpp

@@ -378,6 +378,8 @@ static int doMain(int argc, const char *argv[])
 
 int main(int argc, const char *argv[])
 {
+    EnableSEHtoExceptionMapping();
+    setTerminateOnSEH(true);
     InitModuleObjects();
     queryStderrLogMsgHandler()->setMessageFields(0);
     // Turn logging down (we turn it back up if -v option seen)

initfiles/CMakeLists.txt

@@ -47,3 +47,4 @@ if ( PLATFORM AND UNIX )
 endif ()
 
 Install ( DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/examples DESTINATION "." COMPONENT Runtime USE_SOURCE_PERMISSIONS PATTERN ".svn" EXCLUDE )
+Install ( DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/classes DESTINATION "." COMPONENT Runtime USE_SOURCE_PERMISSIONS PATTERN ".svn" EXCLUDE )

initfiles/etc/DIR_NAME/environment.conf.in

@@ -3,6 +3,7 @@
 [DEFAULT]
 configs=${CONFIG_DIR}
 path=${INSTALL_DIR}
+classpath=${INSTALL_DIR}/classes
 runtime=${RUNTIME_PATH}
 lock=${LOCK_PATH}
 pid=${PID_PATH}

initfiles/examples/embed/JavaCat.java

@@ -0,0 +1,82 @@
+public class JavaCat
+{
+  public static int add1(int a)
+  {
+    return a + 1;
+  }
+  public static String add2(String a)
+  {
+    return a + '1';
+  }
+  public static char addChar(char a)
+  {
+    return ++a;
+  }
+  public static int testThrow(int a) throws Exception
+  {
+    throw new Exception("Exception from Java");
+  }
+
+  public static byte[] testData(byte [] indata)
+  {
+    indata[0]++;
+    return indata;
+  }
+
+  public static int add(int a, int b)
+  {
+    return a + b;
+  }
+  public static long addL(int a, int b)
+  {
+    return a + b;
+  }
+  public static Integer addI(int a, int b)
+  {
+    return a + b;
+  }
+  public static float fadd(float a, float b)
+  {
+    System.out.print("fadd(");
+    System.out.print(a);
+    System.out.print(",");
+    System.out.print(b);
+    System.out.println(")");
+    return a + b;
+  }
+  public static double dadd(double a, double b)
+  {
+    System.out.print("fadd(");
+    System.out.print(a);
+    System.out.print(",");
+    System.out.print(b);
+    System.out.println(")");
+    return a + b;
+  }
+  public static Double daddD(double a, double b)
+  {
+    System.out.print("fadd(");
+    System.out.print(a);
+    System.out.print(",");
+    System.out.print(b);
+    System.out.println(")");
+    return a + b;
+  }
+  public static String cat(String a, String b)
+  {
+    return a + b;
+  }
+
+  public static int testArrays(boolean[] b, short[] s, int[] i, double[] d)
+  {
+    return b.length + s.length + i.length + d.length;
+  }
+
+  public static String[] testStringArray(String[] in)
+  {
+    String t = in[0];
+    in[0] = in[1];
+    in[1] = t;
+    return in;
+  }
+}

initfiles/examples/embed/embedjava.ecl

@@ -0,0 +1,76 @@
+IMPORT java;
+
+/*
+ This example illustrates various calls to Java functions defined in the Java module JavaCat.
+ The source of JavaCat can be found in the examples directory - it can be compiled to JavaCat.class
+ using
+
+   javac JavaCat
+
+ and the resulting file JavaCat.class should be placed in /opt/HPCCSystems/classes (or somewhere else
+ where it can be located via the standatd Java CLASSPATH environment variable.
+ */
+
+integer add1(integer val) := IMPORT(java, 'JavaCat.add1:(I)I');
+string add2(string val) := IMPORT(java, 'JavaCat.add2:(Ljava/lang/String;)Ljava/lang/String;');
+string add3(varstring val) := IMPORT(java, 'JavaCat.add2:(Ljava/lang/String;)Ljava/lang/String;');
+utf8 add4(utf8 val) := IMPORT(java, 'JavaCat.add2:(Ljava/lang/String;)Ljava/lang/String;');
+unicode add5(unicode val) := IMPORT(java, 'JavaCat.add2:(Ljava/lang/String;)Ljava/lang/String;');
+integer testThrow(integer p) := IMPORT(java, 'JavaCat.testThrow:(I)I');
+
+string addChar(string c) := IMPORT(java, 'JavaCat.addChar:(C)C');
+string cat(string s1, string s2) := IMPORT(java, 'JavaCat.cat:(Ljava/lang/String;Ljava/lang/String;)Ljava/lang/String;');
+data testData(data indata) := IMPORT(java, 'JavaCat.testData:([B)[B');
+integer testArrays(set of boolean b, set of integer2 s, set of integer4 i, set of real8 d) := IMPORT(java, 'JavaCat.testArrays:([Z[S[I[D)I');
+set of string testStringArray1(set of string s) := IMPORT(java, 'JavaCat.testStringArray:([Ljava/lang/String;)[Ljava/lang/String;');
+set of varstring testStringArray2(set of varstring s) := IMPORT(java, 'JavaCat.testStringArray:([Ljava/lang/String;)[Ljava/lang/String;');
+set of string8 testStringArray3(set of string8 s) := IMPORT(java, 'JavaCat.testStringArray:([Ljava/lang/String;)[Ljava/lang/String;');
+set of varstring8 testStringArray4(set of varstring8 s) := IMPORT(java, 'JavaCat.testStringArray:([Ljava/lang/String;)[Ljava/lang/String;');
+set of utf8 testStringArray5(set of utf8 s) := IMPORT(java, 'JavaCat.testStringArray:([Ljava/lang/String;)[Ljava/lang/String;');
+set of unicode8 testStringArray6(set of unicode8 s) := IMPORT(java, 'JavaCat.testStringArray:([Ljava/lang/String;)[Ljava/lang/String;');
+set of unicode testStringArray7(set of unicode s) := IMPORT(java, 'JavaCat.testStringArray:([Ljava/lang/String;)[Ljava/lang/String;');
+
+add1(10);
+add2('Hello');
+add3('World');
+add4(U'Leovenaðes');
+add5(U'Стоял');
+addChar('A');
+
+cat('Hello', ' world');
+// Can't catch an expression(only a dataset)
+d := dataset([{ 1, '' }], { integer a, string m} ) : stored('nofold');
+
+d t := transform
+  self.a := FAILCODE;
+  self.m := FAILMESSAGE;
+  self := [];
+end;
+
+catch(d(testThrow(a) = a), onfail(t));
+testData(d'aa');
+testArrays([true],[2,3],[4,5,6,7],[8.0,9.0]);
+testArrays([],[],[],[]);
+testStringArray1(['one', 'two', 'three']);
+testStringArray2(['one', 'two', 'three']);
+testStringArray3(['one', 'two', 'three']);
+testStringArray4(['one', 'two', 'three']);
+testStringArray5(['one', 'two', 'three']);
+testStringArray6(['one', 'two', 'three']);
+testStringArray7(['one', 'two', 'three']);
+
+s1 :=DATASET(250000, TRANSFORM({ integer a }, SELF.a := add1(COUNTER)));
+s2 :=DATASET(250000, TRANSFORM({ integer a }, SELF.a := add1(COUNTER/2)));
+ SUM(NOFOLD(s1 + s2), a);
+
+s1a :=DATASET(250000, TRANSFORM({ integer a }, SELF.a := (integer) add2((STRING)COUNTER)));
+s2a :=DATASET(250000, TRANSFORM({ integer a }, SELF.a := (integer) add3((STRING)(COUNTER/2))));
+ SUM(NOFOLD(s1a + s2a), a);
+
+s1b :=DATASET(250000, TRANSFORM({ integer a }, SELF.a := COUNTER+1));
+s2b :=DATASET(250000, TRANSFORM({ integer a }, SELF.a := (COUNTER/2)+1));
+ SUM(NOFOLD(s1b + s2b), a);
+
+s1c :=DATASET(250000, TRANSFORM({ integer a }, SELF.a := (integer) ((STRING) COUNTER + '1')));
+s2c :=DATASET(250000, TRANSFORM({ integer a }, SELF.a := (integer) ((STRING)(COUNTER/2) + '1')));
+ SUM(NOFOLD(s1c + s2c), a);

initfiles/examples/embed/javaimport.ecl

@@ -0,0 +1,32 @@
+import java;
+
+/*
+ This example illustrates various calls to Java functions defined in the Java module JavaCat.
+ The source of JavaCat can be found in the examples directory - it can be compiled to JavaCat.class
+ using
+
+   javac JavaCat
+
+ and the resulting file JavaCat.class should be placed in /opt/HPCCSystems/classes (or somewhere else
+ where it can be located via the standatd Java CLASSPATH environment variable.
+ */
+
+
+string jcat(string a, string b) := IMPORT(java, 'JavaCat.cat:(Ljava/lang/String;Ljava/lang/String;)Ljava/lang/String;');
+
+integer jadd(integer a, integer b) := IMPORT(java, 'JavaCat.add:(II)I');
+integer jaddl(integer a, integer b) := IMPORT(java, 'JavaCat.addL:(II)J');
+integer jaddi(integer a, integer b) := IMPORT(java, 'JavaCat.addI:(II)Ljava/lang/Integer;');
+
+real jfadd(real4 a, real4 b) := IMPORT(java, 'JavaCat.fadd:(FF)F');
+real jdadd(real a, real b) := IMPORT(java, 'JavaCat.dadd:(DD)D');
+real jdaddD(real a, real b) := IMPORT(java, 'JavaCat.daddD:(DD)Ljava/lang/Double;');
+
+jcat('Hello ', 'world!');
+jadd(1,2);
+jaddl(3,4);
+jaddi(5,6);
+
+jfadd(1,2);
+jdadd(3,4);
+jdaddD(5,6);

initfiles/examples/embed/pyimport.ecl

@@ -0,0 +1,14 @@
+IMPORT python;
+
+/*
+ This example illustrates a call to a Python functions defined in the Python module python_cat.py
+
+ The python file must be reachable at runtime on the Roxie/Thor clusters via the standard Python
+ search path. If a path is specified as in this example, then that path will be added to the start
+ of the Python search path before the module load is attempted. You should ensure that the .py
+ file exists at the specified location on every machine in the cluster.
+ */
+
+
+STRING pcat(STRING a, STRING b) := IMPORT(Python, '/opt/HPCCSystems/examples/embed/python_cat.cat');
+pcat('Hello ', 'world!');

initfiles/examples/embed/python_cat.py

@@ -0,0 +1,7 @@
+#!/usr/bin/python
+
+def cat(a, b):
+  return a + b
+
+if __name__=="__main__":
+   print cat("Hello", " world")

plugins/javaembed/CMakeLists.txt

@@ -41,6 +41,7 @@ include_directories (
          ./../../rtl/include
          ./../../common/deftype
          ./../../system/jlib
+         ${CMAKE_BINARY_DIR}
     )
 
 ADD_DEFINITIONS( -D_USRDLL -DJAVAEMBED_EXPORTS )

plugins/javaembed/javaembed.cpp

@@ -23,6 +23,8 @@
 #include "deftype.hpp"
 #include "eclrtl.hpp"
 #include "eclrtl_imp.hpp"
+#include "jprop.hpp"
+#include "build-config.h"
 
 #ifdef _WIN32
 #define EXPORT __declspec(dllexport)
@@ -85,7 +87,18 @@ public:
         JavaVMOption* options = new JavaVMOption[3];
         const char* origPath = getenv("CLASSPATH");
         StringBuffer newPath;
-        newPath.append("-Djava.class.path=").append(origPath).append(ENVSEPCHAR).append(".");
+        newPath.append("-Djava.class.path=").append(origPath);
+        StringBuffer envConf;
+        envConf.append(CONFIG_DIR).append(PATHSEPSTR).append("environment.conf");
+        Owned<IProperties> conf = createProperties(envConf.str(), true);
+        if (conf && conf->hasProp("classpath"))
+        {
+            newPath.append(ENVSEPCHAR);
+            conf->getProp("classpath", newPath);
+        }
+        newPath.append(ENVSEPCHAR).append(".");
+
+
         options[0].optionString = (char *) newPath.str();
         options[1].optionString = (char *) "-Xcheck:jni";
         options[2].optionString = (char *) "-verbose:jni";

plugins/pyembed/pyembed.cpp

@@ -164,7 +164,7 @@ public:
                 OwnedPyObject new_path = PyString_FromString(path);
                 if (sys_path)
                 {
-                    PyList_Append(sys_path, new_path);
+                    PyList_Insert(sys_path, 0, new_path);
                     checkPythonError();
                 }
             }

testing/ecl/key/embedR.xml

@@ -11,32 +11,35 @@
  <Row><Result_4><Item>2</Item><Item>1</Item><Item>3</Item></Result_4></Row>
 </Dataset>
 <Dataset name='Result 5'>
- <Row><Result_5><Item>two</Item><Item>one</Item><Item>three</Item></Result_5></Row>
+ <Row><Result_5><Item>30000</Item><Item>40000</Item><Item>50000</Item></Result_5></Row>
 </Dataset>
 <Dataset name='Result 6'>
- <Row><Result_6><Item>dos     </Item><Item>uno     </Item><Item>tre     </Item></Result_6></Row>
+ <Row><Result_6><Item>two</Item><Item>one</Item><Item>three</Item></Result_6></Row>
 </Dataset>
 <Dataset name='Result 7'>
- <Row><Result_7><Item>deux</Item><Item>un</Item><Item>trois</Item></Result_7></Row>
+ <Row><Result_7><Item>dos     </Item><Item>uno     </Item><Item>tre     </Item></Result_7></Row>
 </Dataset>
 <Dataset name='Result 8'>
- <Row><Result_8><Item>zwei</Item><Item>ein</Item><Item>drei</Item></Result_8></Row>
+ <Row><Result_8><Item>deux</Item><Item>un</Item><Item>trois</Item></Result_8></Row>
 </Dataset>
 <Dataset name='Result 9'>
- <Row><Result_9><Item>true</Item><Item>false</Item><Item>false</Item><Item>true</Item></Result_9></Row>
+ <Row><Result_9><Item>zwei</Item><Item>ein</Item><Item>drei</Item></Result_9></Row>
 </Dataset>
 <Dataset name='Result 10'>
- <Row><Result_10><Item>2.200000047683716</Item><Item>1.100000023841858</Item><Item>3.299999952316284</Item></Result_10></Row>
+ <Row><Result_10><Item>true</Item><Item>false</Item><Item>false</Item><Item>true</Item></Result_10></Row>
 </Dataset>
 <Dataset name='Result 11'>
- <Row><Result_11><Item>2.3</Item><Item>1.2</Item><Item>3.4</Item></Result_11></Row>
+ <Row><Result_11><Item>2.200000047683716</Item><Item>1.100000023841858</Item><Item>3.299999952316284</Item></Result_11></Row>
 </Dataset>
 <Dataset name='Result 12'>
- <Row><Result_12><Item>12</Item><Item>11</Item><Item>13</Item></Result_12></Row>
+ <Row><Result_12><Item>2.3</Item><Item>1.2</Item><Item>3.4</Item></Result_12></Row>
 </Dataset>
 <Dataset name='Result 13'>
- <Row><Result_13>46875625000</Result_13></Row>
+ <Row><Result_13><Item>-111</Item><Item>0</Item><Item>113</Item></Result_13></Row>
 </Dataset>
 <Dataset name='Result 14'>
  <Row><Result_14>46875625000</Result_14></Row>
 </Dataset>
+<Dataset name='Result 15'>
+ <Row><Result_15>46875625000</Result_15></Row>
+</Dataset>

testing/ecl/key/javaimport.xml

@@ -0,0 +1,21 @@
+<Dataset name='Result 1'>
+ <Row><Result_1>Hello world!</Result_1></Row>
+</Dataset>
+<Dataset name='Result 2'>
+ <Row><Result_2>3</Result_2></Row>
+</Dataset>
+<Dataset name='Result 3'>
+ <Row><Result_3>7</Result_3></Row>
+</Dataset>
+<Dataset name='Result 4'>
+ <Row><Result_4>11</Result_4></Row>
+</Dataset>
+<Dataset name='Result 5'>
+ <Row><Result_5>3.0</Result_5></Row>
+</Dataset>
+<Dataset name='Result 6'>
+ <Row><Result_6>7.0</Result_6></Row>
+</Dataset>
+<Dataset name='Result 7'>
+ <Row><Result_7>11.0</Result_7></Row>
+</Dataset>

testing/ecl/key/pyimport.xml

@@ -0,0 +1,3 @@
+<Dataset name='Result 1'>
+ <Row><Result_1>Hello world!</Result_1></Row>
+</Dataset>

testing/ecl/pyimport.ecl

@@ -1,3 +1,3 @@
 import python;
-string pcat(string a, string b) := IMPORT(Python, '/opt/HPCCSystems/examples/python/python_cat.cat');
+string pcat(string a, string b) := IMPORT(Python, '/opt/HPCCSystems/examples/embed/python_cat.cat');
 pcat('Hello ', 'world!');