HPCC-12124 Examples for embedded language support weak

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

initfiles/examples/embed/JavaCat.java

@@ -38,29 +38,14 @@ public class JavaCat
   }
   public static float fadd(float a, float b)
   {
-    System.out.print("fadd(");
-    System.out.print(a);
-    System.out.print(",");
-    System.out.print(b);
-    System.out.println(")");
     return a + b;
   }
   public static double dadd(double a, double b)
   {
-    System.out.print("fadd(");
-    System.out.print(a);
-    System.out.print(",");
-    System.out.print(b);
-    System.out.println(")");
     return a + b;
   }
   public static Double daddD(double a, double b)
   {
-    System.out.print("fadd(");
-    System.out.print(a);
-    System.out.print(",");
-    System.out.print(b);
-    System.out.println(")");
     return a + b;
   }
   public static String cat(String a, String b)
@@ -155,8 +140,6 @@ public class JavaCat
     while (d.hasNext())
     {
       JavaCat r = d.next();
-      System.out.print(r.lfield);
-      System.out.println("");
       sum += r.lfield;
     }
     return sum;

initfiles/examples/embed/R-simple.ecl

@@ -0,0 +1,157 @@
+IMPORT R;
+
+/*
+ This example illustrates and tests the use of embedded JavaScript
+ */
+
+// Scalar parameters and resuls
+
+integer add1(integer VAL) := EMBED(R)
+VAL+1
+ENDEMBED;
+
+string cat(varstring what, string who) := EMBED(R)
+paste(what,who)
+ENDEMBED;
+
+data testData(data val) := EMBED(R)
+val[1] = val[2];
+val;
+ENDEMBED;
+
+// Sets
+
+set of integer testSet(set of integer val) := EMBED(R)
+t = val [1];
+val[1] = val[2];
+val[2] = t;
+val;
+ENDEMBED;
+
+set of unsigned2 testSet0(set of unsigned2 val) := EMBED(R)
+sort(val);
+ENDEMBED;
+
+set of string testSet2(set of string val) := EMBED(R)
+t = val [1];
+val[1] = val[2];
+val[2] = t;
+val;
+ENDEMBED;
+
+set of string testSet3(set of string8 val) := EMBED(R)
+t = val [1];
+val[1] = val[2];
+val[2] = t;
+val;
+ENDEMBED;
+
+set of varstring testSet4(set of varstring val) := EMBED(R)
+t = val [1];
+val[1] = val[2];
+val[2] = t;
+val;
+ENDEMBED;
+
+set of varstring8 testSet5(set of varstring8 val) := EMBED(R)
+t = val [1];
+val[1] = val[2];
+val[2] = t;
+val;
+ENDEMBED;
+
+set of boolean testSet6(set of boolean val) := EMBED(R)
+t = val [1];
+val[1] = val[2];
+val[2] = t;
+val;
+ENDEMBED;
+
+set of real4 testSet7(set of real4 val) := EMBED(R)
+t = val [1];
+val[1] = val[2];
+val[2] = t;
+val;
+ENDEMBED;
+
+set of real8 testSet8(set of real8 val) := EMBED(R)
+t = val [1];
+val[1] = val[2];
+val[2] = t;
+val;
+ENDEMBED;
+
+set of integer2 testSet9(set of integer2 val) := EMBED(R)
+sort(val);
+ENDEMBED;
+
+// datasets - fields are mapped by name
+
+mtcarsrec := RECORD
+  real8     mpg;
+  unsigned1 cyl;
+  real8     disp;
+  unsigned2 hp;
+  real8     drat;
+  real8     wt;
+  real8     qsec;
+  boolean   vs;
+  boolean   am;
+  unsigned1 gear;
+  unsigned1 carb;
+END;
+
+// returning a dataset
+
+DATASET(mtcarsrec) testDsOut() := EMBED(R)
+mtcars;
+ENDEMBED;
+
+// returning a record
+
+mtcarsrec testRecordOut() := EMBED(R)
+mtcars[t,];
+ENDEMBED;
+
+mtcarsrec testRecordOut2() := EMBED(R)
+list( 1,2,3,4,5,6,7,1,0,11,12);
+ENDEMBED;
+
+// Dataset parameters
+
+DATASET(mtcarsrec) testDsIn(DATASET(mtcarsrec) l) := EMBED(R)
+l;
+ENDEMBED;
+
+// Now do the tests
+
+add1(10);
+cat('Hello', 'World');
+testData(D'ab');
+testSet([1,2,3]);
+testSet0([30000,40000,50000]);
+testSet2(['one','two','three']);
+testSet3(['uno','dos','tre']);
+testSet4(['un','deux','trois']);
+testSet5(['ein','zwei','drei']);
+testSet6([false,true,false,true]);
+testSet7([1.1,2.2,3.3]);
+testSet8([1.2,2.3,3.4]);
+testSet9([-111,0,113]);
+
+testDsOut();
+testRecordOut();
+testRecordOut2();
+testDsIn(SORT(testDsOut(), mpg));
+
+// Test some performance and multithreading - the + operation on datasets executes the two sides in parallel
+
+s1 :=DATASET(250000, TRANSFORM({ integer a }, SELF.a := add1(COUNTER)));
+s2 :=DATASET(250000, TRANSFORM({ integer a }, SELF.a := add1(COUNTER/2)));
+SUM(NOFOLD(s1 + s2), a);
+
+// For comparison, these versions do comparable operations but using pure ECL, to give an idea of the overhead
+
+s1b :=DATASET(250000, TRANSFORM({ integer a }, SELF.a := COUNTER+1));
+s2b :=DATASET(250000, TRANSFORM({ integer a }, SELF.a := (COUNTER/2)+1));
+SUM(NOFOLD(s1b + s2b), a);

initfiles/examples/embed/embedjava.ecl

@@ -8,19 +8,32 @@ IMPORT java;
    javac JavaCat
 
  and the resulting file JavaCat.class should be placed in /opt/HPCCSystems/classes (or somewhere else
- where it can be located via the standatd Java CLASSPATH environment variable.
+ where it can be located via the standard Java CLASSPATH environment variable.
  */
 
+// Passing and returning simple types
+
+integer jadd(integer a, integer b) := IMPORT(java, 'JavaCat.add:(II)I');
+integer jaddl(integer a, integer b) := IMPORT(java, 'JavaCat.addL:(II)J');
+integer jaddi(integer a, integer b) := IMPORT(java, 'JavaCat.addI:(II)Ljava/lang/Integer;');
+
+real jfadd(real4 a, real4 b) := IMPORT(java, 'JavaCat.fadd:(FF)F');
+real jdadd(real a, real b) := IMPORT(java, 'JavaCat.dadd:(DD)D');
+real jdaddD(real a, real b) := IMPORT(java, 'JavaCat.daddD:(DD)Ljava/lang/Double;');
+
 integer add1(integer val) := IMPORT(java, 'JavaCat.add1:(I)I');
 string add2(string val) := IMPORT(java, 'JavaCat.add2:(Ljava/lang/String;)Ljava/lang/String;');
 string add3(varstring val) := IMPORT(java, 'JavaCat.add2:(Ljava/lang/String;)Ljava/lang/String;');
 utf8 add4(utf8 val) := IMPORT(java, 'JavaCat.add2:(Ljava/lang/String;)Ljava/lang/String;');
 unicode add5(unicode val) := IMPORT(java, 'JavaCat.add2:(Ljava/lang/String;)Ljava/lang/String;');
-integer testThrow(integer p) := IMPORT(java, 'JavaCat.testThrow:(I)I');
-
 string addChar(string c) := IMPORT(java, 'JavaCat.addChar:(C)C');
 string cat(string s1, string s2) := IMPORT(java, 'JavaCat.cat:(Ljava/lang/String;Ljava/lang/String;)Ljava/lang/String;');
 data testData(data indata) := IMPORT(java, 'JavaCat.testData:([B)[B');
+
+// Exceptions
+integer testThrow(integer p) := IMPORT(java, 'JavaCat.testThrow:(I)I');
+
+// Arrays and sets
 integer testArrays(set of boolean b, set of integer2 s, set of integer4 i, set of real8 d) := IMPORT(java, 'JavaCat.testArrays:([Z[S[I[D)I');
 set of string testStringArray1(set of string s) := IMPORT(java, 'JavaCat.testStringArray:([Ljava/lang/String;)[Ljava/lang/String;');
 set of varstring testStringArray2(set of varstring s) := IMPORT(java, 'JavaCat.testStringArray:([Ljava/lang/String;)[Ljava/lang/String;');
@@ -30,14 +43,24 @@ set of utf8 testStringArray5(set of utf8 s) := IMPORT(java, 'JavaCat.testStringA
 set of unicode8 testStringArray6(set of unicode8 s) := IMPORT(java, 'JavaCat.testStringArray:([Ljava/lang/String;)[Ljava/lang/String;');
 set of unicode testStringArray7(set of unicode s) := IMPORT(java, 'JavaCat.testStringArray:([Ljava/lang/String;)[Ljava/lang/String;');
 
+// Now run the tests...
+
+jadd(1,2);
+jaddl(3,4);
+jaddi(5,6);
+
+jfadd(1,2);
+jdadd(3,4);
+jdaddD(5,6);
+
 add1(10);
 add2('Hello');
 add3('World');
 add4(U'Leovenaðes');
 add5(U'Стоял');
 addChar('A');
-
 cat('Hello', ' world');
+
 // Can't catch an expression(only a dataset)
 d := dataset([{ 1, '' }], { integer a, string m} ) : stored('nofold');
 
@@ -48,6 +71,7 @@ d t := transform
 end;
 
 catch(d(testThrow(a) = a), onfail(t));
+
 testData(d'aa');
 testArrays([true],[2,3],[4,5,6,7],[8.0,9.0]);
 testArrays([],[],[],[]);
@@ -58,19 +82,3 @@ testStringArray4(['one', 'two', 'three']);
 testStringArray5(['one', 'two', 'three']);
 testStringArray6(['one', 'two', 'three']);
 testStringArray7(['one', 'two', 'three']);
-
-s1 :=DATASET(250000, TRANSFORM({ integer a }, SELF.a := add1(COUNTER)));
-s2 :=DATASET(250000, TRANSFORM({ integer a }, SELF.a := add1(COUNTER/2)));
- SUM(NOFOLD(s1 + s2), a);
-
-s1a :=DATASET(250000, TRANSFORM({ integer a }, SELF.a := (integer) add2((STRING)COUNTER)));
-s2a :=DATASET(250000, TRANSFORM({ integer a }, SELF.a := (integer) add3((STRING)(COUNTER/2))));
- SUM(NOFOLD(s1a + s2a), a);
-
-s1b :=DATASET(250000, TRANSFORM({ integer a }, SELF.a := COUNTER+1));
-s2b :=DATASET(250000, TRANSFORM({ integer a }, SELF.a := (COUNTER/2)+1));
- SUM(NOFOLD(s1b + s2b), a);
-
-s1c :=DATASET(250000, TRANSFORM({ integer a }, SELF.a := (integer) ((STRING) COUNTER + '1')));
-s2c :=DATASET(250000, TRANSFORM({ integer a }, SELF.a := (integer) ((STRING)(COUNTER/2) + '1')));
- SUM(NOFOLD(s1c + s2c), a);

initfiles/examples/embed/java-stream.ecl

@@ -0,0 +1,76 @@
+import java;
+
+/*
+ This example illustrates various calls to Java functions defined in the Java module JavaCat.
+ The source of JavaCat can be found in the examples directory - it can be compiled to JavaCat.class
+ using
+
+   javac JavaCat
+
+ and the resulting file JavaCat.class should be placed in /opt/HPCCSystems/classes (or somewhere else
+ where it can be located via the standard Java CLASSPATH environment variable.
+ */
+
+// Passing and returning records and datasets
+// When passing/returning a record, the corresponding Java function should take/return an object as a parameter whose fields
+// can be mapped by name to the ECL record fields
+
+nrec := record
+  utf8 ufield;
+end;
+
+jret := RECORD
+  boolean bfield;
+  integer4 ifield;
+  integer8 lfield;
+  real8 dfield;
+  real4 ffield;
+  string1 cfield1;
+  string1 cfield2;
+  string sfield;
+  nrec n;
+  set of boolean bset;
+  set of data dset;
+  set of string sset;
+  LINKCOUNTED DATASET(nrec) sub;
+end;
+
+jret jreturnrec(boolean b, integer i, real8 d) := IMPORT(java, 'JavaCat.returnrec:(ZID)LJavaCat;');
+STRING jpassrec(jret r) := IMPORT(java, 'JavaCat.passrec:(LJavaCat;)Ljava/lang/String;');
+
+ret := jreturnrec(false, 10, 2.345);
+
+OUTPUT(ret);            // Calls a Java function that returns an ECL record
+OUTPUT(jpassrec(ret));  // Passes an ECL record to a Java function
+
+// When passing a dataset to a Java function, the Java function should take either an array or an iterator of objects,
+// where the fields of the object in question are mapped by name to the fields in the ECL record.
+// When passing an iterator, we use a modified form of the function signature in the IMPORT statement, using a < to indicate "Iterator of"
+// followed by the name of the class of the objects that the iterator is to return. This is the one case where the output of javap -s cannot be used
+// directly to provide the signature of the java function being called.
+// To return a dataset, an iterator must be returned.
+
+INTEGER passDataset(LINKCOUNTED DATASET(jret) d) :=
+  IMPORT(java, 'JavaCat.passDataset:(<LJavaCat;)I'); // Calls int passDataset(Iterator<JavaCat> d)
+
+DATASET(jret) passDataset2(LINKCOUNTED DATASET(jret) d) :=
+  IMPORT(java, 'JavaCat.passDataset2:([LJavaCat;)Ljava/util/Iterator;'); // Calls Iterator<JavaCat> passDataset2(JavaCat d[])
+
+ds := DATASET(
+  [
+     {true, 1,2,3,4,'a', 'b', 'cd', u'ef', [true,false], [], ['Hello from ECL'], [{'1'},{'2'},{'3'},{'4'},{'5'}]}
+    ,{true, 2,4,3,4,'a', 'b', 'cd', u'ef', [true,false], [], [], []}
+    ,{true, 3,6,3,4,'a', 'b', 'cd', u'ef', [true,false], [], [], []}
+    ,{true, 8,8,3,4,'a', 'b', 'cd', u'ef', [true,false], [d'AA55'], [], []}
+  ], jret);
+
+output(passDataset(ds));  // Using an iterator
+output(passDataset2(ds)); // using an array, and illustrating the return of a dataset
+
+// It is also possible to code a traonsform function in Java - both the parameter and the return type should be a
+// Java object type that maps the fields of the ECL record by name.
+
+transform(jret) testTransform(jret in, integer lim) := IMPORT(java, 'JavaCat.transform:(LJavaCat;I)LJavaCat;');
+
+output(passDataset2(ds));
+output(project(ds, testTransform(LEFT, COUNTER)));

initfiles/examples/embed/javaimport.ecl

@@ -1,32 +0,0 @@
-import java;
-
-/*
- This example illustrates various calls to Java functions defined in the Java module JavaCat.
- The source of JavaCat can be found in the examples directory - it can be compiled to JavaCat.class
- using
-
-   javac JavaCat
-
- and the resulting file JavaCat.class should be placed in /opt/HPCCSystems/classes (or somewhere else
- where it can be located via the standatd Java CLASSPATH environment variable.
- */
-
-
-string jcat(string a, string b) := IMPORT(java, 'JavaCat.cat:(Ljava/lang/String;Ljava/lang/String;)Ljava/lang/String;');
-
-integer jadd(integer a, integer b) := IMPORT(java, 'JavaCat.add:(II)I');
-integer jaddl(integer a, integer b) := IMPORT(java, 'JavaCat.addL:(II)J');
-integer jaddi(integer a, integer b) := IMPORT(java, 'JavaCat.addI:(II)Ljava/lang/Integer;');
-
-real jfadd(real4 a, real4 b) := IMPORT(java, 'JavaCat.fadd:(FF)F');
-real jdadd(real a, real b) := IMPORT(java, 'JavaCat.dadd:(DD)D');
-real jdaddD(real a, real b) := IMPORT(java, 'JavaCat.daddD:(DD)Ljava/lang/Double;');
-
-jcat('Hello ', 'world!');
-jadd(1,2);
-jaddl(3,4);
-jaddi(5,6);
-
-jfadd(1,2);
-jdadd(3,4);
-jdaddD(5,6);

initfiles/examples/embed/javascript-catch.ecl

@@ -0,0 +1,19 @@
+IMPORT javascript;
+
+/*
+ This example illustrates and tests the use of embedded JavaScript
+*/
+
+// Mapping of exceptions from JavaScript to ECL
+
+integer testThrow(integer val) := EMBED(javascript) throw new Error("Error from JavaScript"); ENDEMBED;
+
+d := dataset([{ 1, '' }], { integer a, string m} ) : stored('nofold');
+
+d t := transform
+  self.a := FAILCODE;
+  self.m := FAILMESSAGE;
+  self := [];
+end;
+
+catch(d(testThrow(a) = a), onfail(t));

initfiles/examples/embed/javascript-simple.ecl

@@ -0,0 +1,109 @@
+IMPORT javascript;
+
+/*
+ This example illustrates and tests the use of embedded JavaScript
+ */
+
+// Scalar parameters and resuls
+
+integer add1(integer val) := EMBED(javascript) val+1; ENDEMBED;
+string add2(string val) := EMBED(javascript) val+'1'; ENDEMBED;
+string add3(varstring val) := EMBED(javascript) val+'1'; ENDEMBED;
+utf8 add4(utf8 val) := EMBED(javascript) val+'1'; ENDEMBED;
+unicode add5(unicode val) := EMBED(javascript, U' val+\' at Oh là là Straße\';');
+
+data testData(data val) := EMBED(javascript) val[0] = val[0] + 1; val; ENDEMBED;
+
+// Sets
+
+set of integer testSet(set of integer val) := EMBED(javascript)
+t = val [1];
+val[1] = val[2];
+val[2] = t;
+val;
+ENDEMBED;
+
+set of unsigned2 testSet0(set of unsigned2 val) := EMBED(javascript)
+t = val [1];
+val[1] = val[2];
+val[2] = t;
+val;
+ENDEMBED;
+
+set of string testSet2(set of string val) := EMBED(javascript)
+t = val [1];
+val[1] = val[2];
+val[2] = t;
+val;
+ENDEMBED;
+
+set of string testSet3(set of string8 val) := EMBED(javascript)
+t = val [1];
+val[1] = val[2];
+val[2] = t;
+val;
+ENDEMBED;
+
+set of varstring testSet4(set of varstring val) := EMBED(javascript)
+t = val [1];
+val[1] = val[2];
+val[2] = t;
+val;
+ENDEMBED;
+
+set of varstring8 testSet5(set of varstring8 val) := EMBED(javascript)
+t = val [1];
+val[1] = val[2];
+val[2] = t;
+val;
+ENDEMBED;
+
+set of boolean testSet6(set of boolean val) := EMBED(javascript)
+t = val [1];
+val[1] = val[2];
+val[2] = t;
+val;
+ENDEMBED;
+
+set of real4 testSet7(set of real4 val) := EMBED(javascript)
+t = val [1];
+val[1] = val[2];
+val[2] = t;
+val;
+ENDEMBED;
+
+set of real8 testSet8(set of real8 val) := EMBED(javascript)
+t = val [1];
+val[1] = val[2];
+val[2] = t;
+val;
+ENDEMBED;
+
+set of integer2 testSet9(set of integer2 val) := EMBED(javascript)
+t = val [1];
+val[1] = val[2];
+val[2] = t;
+val;
+ENDEMBED;
+
+// Now run the tests...
+
+add1(10);
+add2('Hello');
+add3('World');
+add4(U'Oh là là Straße');
+add5(U'Стоял');
+
+add2('Oh là là Straße');  // Passing latin chars - should be untranslated
+
+testdata(D'aa');
+testSet([1,2,3]);
+testSet0([30000,40000,50000]);
+testSet2(['one','two','three']);
+testSet3(['uno','dos','tre']);
+testSet4(['un','deux','trois']);
+testSet5(['ein','zwei','drei']);
+testSet6([false,true,false,true]);
+testSet7([1.1,2.2,3.3]);
+testSet8([1.2,2.3,3.4]);
+testSet9([-111,0,113]);

initfiles/examples/embed/javascript-simple2.ecl

@@ -0,0 +1,21 @@
+import Javascript;
+/*
+ This example illustrates and tests the use of embedded JavaScript.
+ In this example the javascript that is embedded is more complex, including a definition of a function
+ */
+
+string anagram(string word) := EMBED(Javascript)
+
+function anagram(word)
+{
+  if (word == 'cat')
+    return 'act';
+  else
+    return word;
+}
+
+anagram(word)
+ENDEMBED;
+
+anagram('dog');
+anagram('cat');

initfiles/examples/embed/mysql-simple.ecl

@@ -0,0 +1,144 @@
+IMPORT mysql;
+
+/*
+ This example illustrates various calls to embdded MySQL code
+ */
+
+// This is the record structure in ECL that will correspond to the rows in the MySQL dataset
+
+childrec := RECORD
+   string name,
+   integer value,
+   boolean boolval,
+   real8 r8,
+   real4 r4,
+   DATA d,
+   DECIMAL10_2 ddd,
+   UTF8 u1,
+   UNICODE8 u2
+END;
+
+// Some data we will use to initialize the MySQL table
+
+init := DATASET([{'name1', 1, true, 1.2, 3.4, D'aa55aa55', 1234567.89, U'Straße', U'Straße'},
+                 {'name2', 2, false, 5.6, 7.8, D'00', -1234567.89, U'là', U'là'}], childrec);
+
+// Set up the MySQL database
+
+drop() := EMBED(mysql : user('rchapman'),database('test'))
+  DROP TABLE IF EXISTS tbl1;
+ENDEMBED;
+
+create() := EMBED(mysql : user('rchapman'),database('test'))
+  CREATE TABLE tbl1 ( name VARCHAR(20), value INT, boolval TINYINT, r8 DOUBLE, r4 FLOAT, d BLOB, ddd DECIMAL(10,2), u1 VARCHAR(10), u2 VARCHAR(10) );
+ENDEMBED;
+
+// Initialize the MySQL table, passing in the ECL dataset to provide the rows
+
+initialize(dataset(childrec) values) := EMBED(mysql : user('rchapman'),database('test'))
+  INSERT INTO tbl1 values (?, ?, ?, ?, ?, ?, ?, ?, ?);
+ENDEMBED;
+
+// Add an additional row containing NULL values, to test that they are handled properly when read in ECL
+
+initializeNulls() := EMBED(mysql : user('rchapman'),database('test'))
+  INSERT INTO tbl1 (name) values ('nulls');
+ENDEMBED;
+
+// Returning a dataset
+
+dataset(childrec) testMySQLDS() := EMBED(mysql : user('rchapman'),database('test'))
+  SELECT * from tbl1;
+ENDEMBED;
+
+// Returning a single row
+
+childrec testMySQLRow() := EMBED(mysql : user('rchapman'),database('test'))
+  SELECT * from tbl1 LIMIT 1;
+ENDEMBED;
+
+// Passing in parameters
+
+childrec testMySQLParms(
+   string name,
+   integer value,
+   boolean boolval,
+   real8 r8,
+   real4 r4,
+   DATA d,
+   UTF8 u1,
+   UNICODE8 u2) := EMBED(mysql : user('rchapman'),database('test'))
+  SELECT * from tbl1 WHERE name=? AND value=? AND boolval=? AND r8=? AND r4=? AND d=? AND u1=? AND u2=?;
+ENDEMBED;
+
+// Returning scalars
+
+string testMySQLString() := EMBED(mysql : user('rchapman'),database('test'))
+  SELECT max(name) from tbl1;
+ENDEMBED;
+
+dataset(childrec) testMySQLStringParam(string filter) := EMBED(mysql : user('rchapman'),database('test'))
+  SELECT * from tbl1 where name = ?;
+ENDEMBED;
+
+integer testMySQLInt() := EMBED(mysql : user('rchapman'),database('test'))
+  SELECT max(value) from tbl1;
+ENDEMBED;
+
+boolean testMySQLBool() := EMBED(mysql : user('rchapman'),database('test'))
+  SELECT max(boolval) from tbl1;
+ENDEMBED;
+
+real8 testMySQLReal8() := EMBED(mysql : user('rchapman'),database('test'))
+  SELECT max(r8) from tbl1;
+ENDEMBED;
+
+real4 testMySQLReal4() := EMBED(mysql : user('rchapman'),database('test'))
+  SELECT max(r4) from tbl1;
+ENDEMBED;
+
+data testMySQLData() := EMBED(mysql : user('rchapman'),database('test'))
+  SELECT max(d) from tbl1;
+ENDEMBED;
+
+UTF8 testMySQLUtf8() := EMBED(mysql : user('rchapman'),database('test'))
+  SELECT max(u1) from tbl1;
+ENDEMBED;
+
+UNICODE testMySQLUnicode() := EMBED(mysql : user('rchapman'),database('test'))
+  SELECT max(u2) from tbl1;
+ENDEMBED;
+
+// Passing in AND returning a dataset - this ends up acting a bit like a keyed join...
+
+stringrec := RECORD
+   string name
+END;
+
+stringrec extractName(childrec l) := TRANSFORM
+  SELF := l;
+END;
+
+dataset(childrec) testMySQLDSParam(dataset(stringrec) inrecs) := EMBED(mysql : user('rchapman'),database('test'))
+  SELECT * from tbl1 where name = ?;
+ENDEMBED;
+
+sequential (
+  drop(),
+  create(),
+  initialize(init),
+  initializeNulls(),
+  OUTPUT(testMySQLDS()),
+  OUTPUT(testMySQLRow().name),
+  OUTPUT(testMySQLParms('name1', 1, true, 1.2, 3.4, D'aa55aa55', U'Straße', U'Straße')),
+  OUTPUT(testMySQLString()),
+  OUTPUT(testMySQLStringParam(testMySqlString())),
+  OUTPUT(testMySQLInt()),
+  OUTPUT(testMySQLBool()),
+  OUTPUT(testMySQLReal8()),
+  OUTPUT(testMySQLReal4()),
+  OUTPUT(testMySQLData()),
+  OUTPUT(testMySQLUtf8()),
+  OUTPUT(testMySQLUnicode()),
+  OUTPUT(testMySQLDSParam(PROJECT(init, extractName(LEFT))))
+);

initfiles/examples/embed/python-catch.ecl

@@ -0,0 +1,22 @@
+IMPORT Python;
+
+/*
+ This example illustrates and tests the use of embedded Python
+*/
+
+// Mapping of exceptions from Python to ECL
+
+integer testThrow(integer val) := EMBED(Python)
+raise Exception('Error from Python')
+ENDEMBED;
+
+// Can't catch an expression(only a dataset)
+d := dataset([{ 1, '' }], { integer a, string m} ) : stored('nofold');
+
+d t := transform
+  self.a := FAILCODE;
+  self.m := FAILMESSAGE;
+  self := [];
+end;
+
+catch(d(testThrow(a) = a), onfail(t));

initfiles/examples/embed/python-simple.ecl

@@ -0,0 +1,101 @@
+IMPORT Python;
+
+/*
+ This example illustrates and tests the use of embedded Python
+ */
+
+// Scalar parameters and resuls
+
+integer add1(integer val) := EMBED(Python)
+val+1
+ENDEMBED;
+
+string add2(string val) := EMBED(Python)
+val+'1'
+ENDEMBED;
+
+string add3(varstring val) := EMBED(Python)
+val+'1'
+ENDEMBED;
+
+utf8 add4(utf8 val) := EMBED(Python)
+val+'1'
+ENDEMBED;
+
+unicode add5(unicode val) := EMBED(Python)
+val+'1'
+ENDEMBED;
+
+utf8 add6(utf8 val) := EMBED(Python)
+return val+'1'
+ENDEMBED;
+
+unicode add7(unicode val) := EMBED(Python)
+return val+'1'
+ENDEMBED;
+
+data testData(data val) := EMBED(Python)
+val[0] = val[0] + 1
+return val
+ENDEMBED;
+
+// Sets in ECL map to Python lists
+
+set of integer testSet(set of integer val) := EMBED(Python)
+return sorted(val)
+ENDEMBED;
+
+set of string testSet2(set of string val) := EMBED(Python)
+return sorted(val)
+ENDEMBED;
+
+set of string testSet3(set of string8 val) := EMBED(Python)
+return sorted(val)
+ENDEMBED;
+
+set of utf8 testSet4(set of utf8 val) := EMBED(Python)
+return sorted(val)
+ENDEMBED;
+
+set of varstring testSet5(set of varstring val) := EMBED(Python)
+return sorted(val)
+ENDEMBED;
+
+set of varstring8 testSet6(set of varstring8 val) := EMBED(Python)
+return sorted(val)
+ENDEMBED;
+
+set of unicode testSet7(set of unicode val) := EMBED(Python)
+return sorted(val)
+ENDEMBED;
+
+set of unicode8 testSet8(set of unicode8 val) := EMBED(Python)
+return sorted(val)
+ENDEMBED;
+
+set of data testSet9(set of data val) := EMBED(Python)
+return val
+ENDEMBED;
+
+// Now run the tests
+
+add1(10);
+add2('Hello');
+add3('World');
+add4(U'Oh là là Straße');
+add5(U'Стоял');
+add6(U'Oh là là Straße');
+add7(U'Стоял');
+
+add2('Oh là là Straße');  // Passing latin chars - should be untranslated
+
+testData(D'aa');
+testSet([1,3,2]);
+testSet2(['red','green','yellow']);
+testSet3(['one','two','three']);
+testSet4([U'Oh', U'là', U'Straße']);
+testSet5(['Un','Deux','Trois']);
+testSet6(['Uno','Dos','Tre']);
+testSet7([U'On', U'der', U'Straße']);
+testSet8([U'Aus', U'zum', U'Straße']);
+testSet9([D'Aus', D'zum', D'Strade']);

initfiles/examples/embed/python-simple2.ecl

@@ -0,0 +1,19 @@
+import python;
+/*
+ This example illustrates and tests the use of embedded Python.
+ In this example the python that is embedded is more complex, including a definition of a function
+ */
+
+
+string anagram(string word) := EMBED(Python)
+  def anagram(w):
+    if word == 'cat':
+      return 'act'
+    else:
+      return w
+
+  return anagram(word)
+ENDEMBED;
+
+anagram('dog');
+anagram('cat');

initfiles/examples/embed/python-stream.ecl

@@ -0,0 +1,132 @@
+IMPORT Python;
+
+/*
+ This example illustrates and tests the use of embedded Python.
+ */
+
+// This example illustrates returning datasets from Python
+
+// These are the record structures we will be returning - note the child records, datasets and dictionaries in it
+
+childrec := RECORD
+   string name => unsigned value;
+END;
+
+eclRecord := RECORD
+    STRING name1;
+    STRING10 name2;
+    LINKCOUNTED DATASET(childrec) childnames;
+    LINKCOUNTED DICTIONARY(childrec) childdict{linkcounted};
+    childrec r;
+    unsigned1 val1;
+    integer1   val2;
+    UTF8 u1;
+    UNICODE u2;
+    UNICODE8 u3;
+    BIG_ENDIAN unsigned4 val3;
+    DATA d;
+    BOOLEAN b;
+    SET OF STRING ss1;
+END;
+
+namerec := RECORD
+   string name;
+END;
+
+namerec2 := RECORD
+   string name;
+   string name2;
+END;
+
+// To return a dataset, we can return a list of tuples, each one correponding to a field in the resulting ECL record
+// In this example, the fields are mapped by position
+// Just to spice things up we proved a couple of parameters too
+
+dataset(eclRecord) streamedNames(data d, utf8 u) := EMBED(Python)
+  return [  \
+     ("Gavin", "Halliday", [("a", 1),("b", 2),("c", 3)], [("aa", 11)], ("aaa", 111), 250, -1,  U'là',  U'là',  U'là', 0x01000000, d, False, {"1","2"}), \
+     ("John", "Smith", [], [], ("c", 3), 250, -1,  U'là',  U'là',  u, 0x02000000, d, True, []) \
+     ]
+ENDEMBED;
+
+output(streamedNames(d'AA', u'là'));
+
+// We can also return a dataset by using a Python generator, which will be lazy-evaluated as the records are required by ECL code...
+
+dataset(childrec) testGenerator(unsigned lim) := EMBED(Python)
+  num = 0
+  while num < lim:
+    yield ("Generated", num)
+    num += 1
+ENDEMBED;
+
+output (testGenerator(10));
+
+// If the returned tuples are namedtuples, we map fields by name rather than by position
+
+// Test use of Python named tuple...
+
+dataset(childrec) testNamedTuples() := EMBED(Python)
+  import collections
+  ChildRec = collections.namedtuple("childrec", "value, name") # Note - order is reverse of childrec - but works as we get fields by name
+  c1 = ChildRec(1, "name1")
+  c2 = ChildRec(name="name2", value=2)
+  return [ c1, c2 ]
+ENDEMBED;
+
+output(testNamedTuples());
+
+// To return a record, just return a tuple (or namedtuple)
+
+childrec testRecord(integer value, string s) := EMBED(Python)
+  return (s, value)
+ENDEMBED;
+
+output(testRecord(1,'Hello').value);
+output(testRecord(1,'Hello').name);
+
+// If the record has a single field, you don't need to put the field into a tuple...
+
+dataset(namerec) testMissingTuple1(unsigned lim) := EMBED(Python)
+  return [ '1', '2', '3' ]
+ENDEMBED;
+output (testMissingTuple1(10));
+
+// ... but you can if you want
+
+dataset(namerec) testMissingTuple2(unsigned lim) := EMBED(Python)
+  return [ ('1'), ('2'), ('3') ]
+ENDEMBED;
+output (testMissingTuple2(10));
+
+// You can define a transform in Python, using a function that returns a record (i.e. a Python tuple)
+// Note that the tuple we pass to Python is a namedtuple
+
+transform(childrec) testTransform(namerec inrec, unsigned c) := EMBED(Python)
+  return (inrec.name, c)
+ENDEMBED;
+
+d := dataset([{'Richard'},{'Gavin'}], namerec);
+
+output(project(d, testTransform(LEFT, COUNTER)));
+
+// Most transforms take a record as the input, but it's not a requirement
+
+transform(childrec) testTransformNoRow(unsigned lim) := EMBED(Python)
+  return ("Hello", lim)
+ENDEMBED;
+
+output(row(testTransformNoRow(10)));
+
+// When passing datasets to Python, we get an iterator of named tuples
+// They are actually implemented as generators, meaning they are lazy-evaluated
+
+names := DATASET([{'Richard'}, {'James'}, {'Andrew'}], namerec);
+
+string datasetAsIterator(dataset(namerec) input) := EMBED(Python)
+  s = ''
+  for n in input:
+    s = s + ' ' + n.name
+  return s;
+ENDEMBED;
+output(datasetAsIterator(names));

initfiles/examples/embed/sqlite-simple.ecl

@@ -0,0 +1,38 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2013 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+IMPORT SqLite3;
+
+childrec := RECORD
+   string name => unsigned value;
+END;
+
+dataset(childrec) testSqLite(unsigned lim) := EMBED(SqLite3 : file('test.db'))
+  SELECT * from tbl1 where two = :lim;
+ENDEMBED;
+
+dataset(childrec) testSqLite2(string v) := EMBED(SqLite3 : file('test.db'))
+  SELECT * from tbl1 where one = :v;
+ENDEMBED;
+
+unsigned testSqLite3(string v) := EMBED(SqLite3 : file('test.db'))
+  SELECT count(*) from tbl1 where one = :v;
+ENDEMBED;
+
+output(testSqLite(20));
+output(testSqLite2('hello!'));
+output(testSqLite3('hello!'));