HPCC-10617 Add support for Cassandra

Improved examples

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

initfiles/examples/embed/mysql-simple.ecl

@@ -5,17 +5,19 @@ IMPORT mysql;
  */
 
 // This is the record structure in ECL that will correspond to the rows in the MySQL dataset
+// Note that the default values specified in the fields will be used when a NULL value is being
+// returned from MySQL
 
 childrec := RECORD
    string name,
-   integer value,
-   boolean boolval,
-   real8 r8,
-   real4 r4,
-   DATA d,
-   DECIMAL10_2 ddd,
-   UTF8 u1,
-   UNICODE8 u2
+   integer4 value { default(99999) },
+   boolean boolval { default(true) },
+   real8 r8 {default(99.99)},
+   real4 r4 {default(999.99)},
+   DATA d {default (D'999999')},
+   DECIMAL10_2 ddd {default(9.99)},
+   UTF8 u1 {default(U'9999 ß')},
+   UNICODE8 u2 {default(U'9999 ßßßß')}
 END;
 
 // Some data we will use to initialize the MySQL table
@@ -45,6 +47,12 @@ initializeNulls() := EMBED(mysql : user('rchapman'),database('test'))
   INSERT INTO tbl1 (name) values ('nulls');
 ENDEMBED;
 
+// Note that the query string is encoded in utf8
+
+initializeUtf8() := EMBED(mysql : user('rchapman'),database('test'))
+  INSERT INTO tbl1 values ('utf8test', 1, 1, 1.2, 3.4, 'aa55aa55', 1234567.89, 'Straße', 'Straße');
+ENDEMBED;
+
 // Returning a dataset
 
 dataset(childrec) testMySQLDS() := EMBED(mysql : user('rchapman'),database('test'))
@@ -60,6 +68,7 @@ ENDEMBED;
 // Passing in parameters
 
 childrec testMySQLParms(
+
    string name,
    integer value,
    boolean boolval,
@@ -128,6 +137,7 @@ sequential (
   create(),
   initialize(init),
   initializeNulls(),
+  initializeUtf8(),
   OUTPUT(testMySQLDS()),
   OUTPUT(testMySQLRow().name),
   OUTPUT(testMySQLParms('name1', 1, true, 1.2, 3.4, D'aa55aa55', U'Straße', U'Straße')),

initfiles/examples/embed/sqlite-simple.ecl

@@ -1,38 +1,129 @@
-/*##############################################################################
+IMPORT SqLite3;
 
-    HPCC SYSTEMS software Copyright (C) 2013 HPCC Systems.
+/*
+ This example illustrates various calls to embdded SQLite code
+ */
 
-    Licensed under the Apache License, Version 2.0 (the "License");
-    you may not use this file except in compliance with the License.
-    You may obtain a copy of the License at
+// This is the record structure in ECL that will correspond to the rows in the SQLite dataset
+// Note that the default values specified in the fields will be used when a NULL value is being
+// returned from MySQL
 
-       http://www.apache.org/licenses/LICENSE-2.0
+childrec := RECORD
+   string name,
+   integer4 value { default(99999) },
+   boolean boolval { default(true) },
+   real8 r8 {default(99.99)},
+   real4 r4 {default(999.99)},
+   DATA d {default (D'999999')},
+   DECIMAL10_2 ddd {default(9.99)},
+   UTF8 u1 {default(U'9999 ß')},
+   UNICODE8 u2 {default(U'9999 ßßßß')}
+END;
 
-    Unless required by applicable law or agreed to in writing, software
-    distributed under the License is distributed on an "AS IS" BASIS,
-    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-    See the License for the specific language governing permissions and
-    limitations under the License.
-############################################################################## */
+// Set up the SQLite database
 
-IMPORT SqLite3;
+drop() := EMBED(SqLite3 : file('test.db'))
+  DROP TABLE IF EXISTS tbl1;
+ENDEMBED;
 
-childrec := RECORD
-   string name => unsigned value;
-END;
+create() := EMBED(SqLite3 : file('test.db'))
+  CREATE TABLE tbl1 ( name VARCHAR(20), value INT, boolval TINYINT, r8 DOUBLE, r4 FLOAT, d BLOB, ddd DECIMAL(10,2), u1 VARCHAR(10), u2 VARCHAR(10) );
+ENDEMBED;
 
-dataset(childrec) testSqLite(unsigned lim) := EMBED(SqLite3 : file('test.db'))
-  SELECT * from tbl1 where two = :lim;
+// Add a row
+
+initialize() := EMBED(SqLite3 : file('test.db'))
+  INSERT INTO tbl1 values ('name1', 1, 1, 1.2, 3.4, 'aa55aa55', 1234567.89, 'Straße', 'Straße');
 ENDEMBED;
 
-dataset(childrec) testSqLite2(string v) := EMBED(SqLite3 : file('test.db'))
-  SELECT * from tbl1 where one = :v;
+// Add an additional row containing NULL values, to test that they are handled properly when read in ECL
+
+initializeNulls() := EMBED(SqLite3 : file('test.db'))
+  INSERT INTO tbl1 (name) values ('nulls');
 ENDEMBED;
 
-unsigned testSqLite3(string v) := EMBED(SqLite3 : file('test.db'))
-  SELECT count(*) from tbl1 where one = :v;
+// Test various types of parameters
+
+testSqLiteParms(
+   string name,
+   integer4 value,
+   boolean boolval,
+   real8 r8,
+   real4 r4,
+   DATA d,
+   REAL8 ddd, // Decimal parms not supported.
+   UTF8 u1,
+   UNICODE8 u2) := EMBED(SqLite3 : file('test.db'))
+  INSERT INTO tbl1 (name, value, boolval, r8, r4,d,ddd,u1,u2) values (:name, :value, :boolval, :r8, :r4,:d,:ddd,:u1,:u2);
 ENDEMBED;
 
-output(testSqLite(20));
-output(testSqLite2('hello!'));
-output(testSqLite3('hello!'));
+// Returning a dataset
+
+dataset(childrec) testSQLiteDS() := EMBED(SqLite3 : file('test.db'))
+  SELECT * from tbl1;
+ENDEMBED;
+
+// Returning a single row
+
+childrec testSQLiteRow() := EMBED(SqLite3 : file('test.db'))
+  SELECT * from tbl1 LIMIT 1;
+ENDEMBED;
+
+// Returning scalars
+
+string testSQLiteString() := EMBED(SqLite3 : file('test.db'))
+  SELECT max(name) from tbl1;
+ENDEMBED;
+
+dataset(childrec) testSQLiteStringParam(string filter) := EMBED(SqLite3 : file('test.db'))
+  SELECT * from tbl1 where name = :filter;
+ENDEMBED;
+
+integer testSQLiteInt() := EMBED(SqLite3 : file('test.db'))
+  SELECT max(value) from tbl1;
+ENDEMBED;
+
+boolean testSQLiteBool() := EMBED(SqLite3 : file('test.db'))
+  SELECT max(boolval) from tbl1;
+ENDEMBED;
+
+real8 testSQLiteReal8() := EMBED(SqLite3 : file('test.db'))
+  SELECT max(r8) from tbl1;
+ENDEMBED;
+
+real4 testSQLiteReal4() := EMBED(SqLite3 : file('test.db'))
+  SELECT max(r4) from tbl1;
+ENDEMBED;
+
+data testSQLiteData() := EMBED(SqLite3 : file('test.db'))
+  SELECT max(d) from tbl1;
+ENDEMBED;
+
+UTF8 testSQLiteUtf8() := EMBED(SqLite3 : file('test.db'))
+  SELECT max(u1) from tbl1;
+ENDEMBED;
+
+UNICODE testSQLiteUnicode() := EMBED(SqLite3 : file('test.db'))
+  SELECT max(u2) from tbl1;
+ENDEMBED;
+
+// Run the tests
+
+sequential (
+  drop(),
+  create(),
+  initialize(),
+  initializeNulls(),
+  testSQLiteParms('name2', 1, true, 1.2, 3.4, D'aa55aa55', 23.45, U'Straßaße', U'Straßßßße'),
+  OUTPUT(testSQLiteDS()),
+  OUTPUT(testSQLiteRow().name),
+  OUTPUT(testSQLiteString()),
+  OUTPUT(testSQLiteStringParam(testSQLiteString())),
+  OUTPUT(testSQLiteInt()),
+  OUTPUT(testSQLiteBool()),
+  OUTPUT(testSQLiteReal8()),
+  OUTPUT(testSQLiteReal4()),
+  OUTPUT(testSQLiteData()),
+  OUTPUT(testSQLiteUtf8()),
+  OUTPUT(testSQLiteUnicode())
+);

testing/regress/ecl/cassandraembed.ecl

@@ -17,6 +17,14 @@
 
 IMPORT cassandra;
 
+/*
+ This example illustrates various calls to embdded Cassandra CQL code
+ */
+
+// This is the record structure in ECL that will correspond to the rows in the Cassabdra dataset
+// Note that the default values specified in the fields will be used when a NULL value is being
+// returned from Cassandra
+
 childrec := RECORD
    string name,
    integer4 value { default(99999) },
@@ -29,19 +37,17 @@ childrec := RECORD
    UNICODE8 u2 {default(U'9999 ßßßß')}
 END;
 
-stringrec := RECORD
-   string name
-END;
-
-stringrec extractName(childrec l) := TRANSFORM
-  SELF := l;
-END;
+// Some data we will use to initialize the Cassandra table
 
-init := DATASET([{'name1' , 1, true, 1.2, 3.4, D'aa55aa55', 1234567.89, U'Straße', U'Straße'},
+init := DATASET([{'name1', 1, true, 1.2, 3.4, D'aa55aa55', 1234567.89, U'Straße', U'Straße'},
                  {'name2', 2, false, 5.6, 7.8, D'00', -1234567.89, U'là', U'là'}], childrec);
 
 init2 := ROW({'name4' , 3, true, 9.10, 11.12, D'aa55aa55', 987.65, U'Baße', U'Baße'}, childrec);
 
+// Set up the Cassandra database
+// Note that we can execute multiple statements in a single embed, provided that there are
+// no parameters and no result type
+
 createks() := EMBED(cassandra : user('rchapman'))
   CREATE KEYSPACE IF NOT EXISTS test WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '3' } ;
 ENDEMBED;
@@ -49,12 +55,18 @@ ENDEMBED;
 createTables() := EMBED(cassandra : user('rchapman'),keyspace('test'))
   DROP TABLE IF EXISTS tbl1;
 
-  CREATE TABLE tbl1 ( name VARCHAR, value INT, boolval boolean , r8 DOUBLE, r4 FLOAT, d BLOB, ddd VARCHAR, u1 VARCHAR, u2 VARCHAR,   PRIMARY KEY (name) );
+  CREATE TABLE tbl1 ( name VARCHAR, value INT, boolval boolean , r8 DOUBLE, r4 FLOAT, d BLOB, ddd VARCHAR, u1 VARCHAR, u2 VARCHAR, PRIMARY KEY (name) );
   CREATE INDEX tbl1_value  ON tbl1 (value);
   CREATE INDEX tbl1_boolval  ON tbl1 (boolval);
-  INSERT INTO tbl1 (name,u1) values ('; /* // ?', 'Straßßßßßße');
+  INSERT INTO tbl1 (name, u1) values ('nulls', 'ß');  // Creates some null fields. Also note that query is utf8
 ENDEMBED;
 
+// Initialize the Cassandra table, passing in the ECL dataset to provide the rows
+// Note the batch option to control how cassandra inserts are batched
+// If not supplied, each insert is executed individually - because Cassandra
+// has restrictions about what can be done in a batch, we can't default to using batch
+// unless told to...
+
 initialize(dataset(childrec) values) := EMBED(cassandra : user('rchapman'),keyspace('test'),batch('unlogged'))
   INSERT INTO tbl1 (name, value, boolval, r8, r4,d,ddd,u1,u2) values (?,?,?,?,?,?,?,?,?);
 ENDEMBED;
@@ -63,14 +75,20 @@ initialize2(row(childrec) values) := EMBED(cassandra : user('rchapman'),keyspace
   INSERT INTO tbl1 (name, value, boolval, r8, r4,d,ddd,u1,u2) values (?,?,?,?,?,?,?,?,?);
 ENDEMBED;
 
+// Returning a dataset
+
 dataset(childrec) testCassandraDS() := EMBED(cassandra : user('rchapman'),keyspace('test'))
   SELECT name, value, boolval, r8, r4,d,ddd,u1,u2 from tbl1;
 ENDEMBED;
 
+// Returning a single row
+
 childrec testCassandraRow() := EMBED(cassandra : user('rchapman'),keyspace('test'))
   SELECT name, value, boolval, r8, r4,d,ddd,u1,u2 from tbl1 LIMIT 1;
 ENDEMBED;
 
+// Passing in parameters
+
 testCassandraParms(
    string name,
    integer4 value,
@@ -84,6 +102,8 @@ testCassandraParms(
   INSERT INTO tbl1 (name, value, boolval, r8, r4,d,ddd,u1,u2) values (?,?,?,?,?,?,'8.76543',?,?);
 ENDEMBED;
 
+// Returning scalars
+
 string testCassandraString() := EMBED(cassandra : user('rchapman'),keyspace('test'))
   SELECT name from tbl1 LIMIT 1;
 ENDEMBED;
@@ -92,10 +112,6 @@ dataset(childrec) testCassandraStringParam(string filter) := EMBED(cassandra : u
   SELECT name, value, boolval, r8, r4,d,ddd,u1,u2 from tbl1 where name = ?;
 ENDEMBED;
 
-dataset(childrec) testCassandraDSParam(dataset(stringrec) inrecs) := EMBED(cassandra : user('rchapman'),keyspace('test'))
-  SELECT name, value, boolval, r8, r4,d,ddd,u1,u2 from tbl1 where name = ?;
-ENDEMBED;
-
 integer testCassandraInt() := EMBED(cassandra : user('rchapman'),keyspace('test'))
   SELECT value from tbl1 LIMIT 1;
 ENDEMBED;
@@ -124,23 +140,33 @@ UNICODE testCassandraUnicode() := EMBED(cassandra : user('rchapman'),keyspace('t
   SELECT u2 from tbl1 WHERE name='name1';
 ENDEMBED;
 
-searchrec := RECORD
-  STRING name;
+// Coding a TRANSFORM to call Cassandra - this ends up acting a little like a join
+
+stringrec := RECORD
+   string name
 END;
 
-TRANSFORM(childrec) t(searchrec L) := EMBED(cassandra : user('rchapman'),keyspace('test'))
+TRANSFORM(childrec) t(stringrec L) := EMBED(cassandra : user('rchapman'),keyspace('test'))
   SELECT name, value, boolval, r8, r4,d,ddd,u1,u2 from tbl1 where name = ?;
 ENDEMBED;
 
-integer testCassandraCount() := EMBED(cassandra : user('rchapman'),keyspace('test'))
-  SELECT COUNT(*) from tbl1;
-ENDEMBED;
-
 init3 := DATASET([{'name1'},
-                  {'name2'}], searchrec);
+                  {'name2'}], stringrec);
 
 testCassandraTransform := PROJECT(init3, t(LEFT));
 
+// Passing in AND returning a dataset - this also ends up acting a bit like a keyed join...
+
+stringrec extractName(childrec l) := TRANSFORM
+  SELF := l;
+END;
+
+dataset(childrec) testCassandraDSParam(dataset(stringrec) inrecs) := EMBED(cassandra : user('rchapman'),keyspace('test'))
+  SELECT name, value, boolval, r8, r4,d,ddd,u1,u2 from tbl1 where name = ?;
+ENDEMBED;
+
+// Testing performance of batch inserts
+
 s1 :=DATASET(25000, TRANSFORM(childrec, SELF.name := 'name'+COUNTER,
                                          SELF.value:=COUNTER,
                                          SELF.boolval:=COUNTER % 2 =1,
@@ -150,6 +176,15 @@ s1 :=DATASET(25000, TRANSFORM(childrec, SELF.name := 'name'+COUNTER,
 
 testCassandraBulk := initialize(s1);
 
+// Check that 25000 got inserted
+
+integer testCassandraCount() := EMBED(cassandra : user('rchapman'),keyspace('test'))
+  SELECT COUNT(*) from tbl1;
+ENDEMBED;
+
+
+// Execute the tests
+
 sequential (
   createks(),
   createTables(),
@@ -161,7 +196,6 @@ sequential (
   OUTPUT(testCassandraRow().name),
   OUTPUT(testCassandraString()),
   OUTPUT(testCassandraStringParam(testCassandraString())),
-  OUTPUT(testCassandraDSParam(PROJECT(init, extractName(LEFT)))),
   OUTPUT(testCassandraInt()),
   OUTPUT(testCassandraBool()),
   OUTPUT(testCassandraReal8()),
@@ -170,6 +204,7 @@ sequential (
   OUTPUT(testCassandraUtf8()),
   OUTPUT(testCassandraUnicode()),
   OUTPUT(testCassandraTransform()),
+  OUTPUT(testCassandraDSParam(PROJECT(init, extractName(LEFT)))),
   testCassandraBulk,
   OUTPUT(testCassandraCount())
-);
+);

testing/regress/ecl/key/cassandraembed.xml

@@ -1,9 +1,9 @@
 <Dataset name='Result 1'>
- <Row><name>; /* // ?</name><value>99999</value><boolval>true</boolval><r8>99.98999999999999</r8><r4>999.989990234375</r4><d>393939393939</d><ddd>9.99</ddd><u1>Straßßßßßße</u1><u2>9999 ßßß</u2></Row>
  <Row><name>name1</name><value>1</value><boolval>true</boolval><r8>1.2</r8><r4>3.400000095367432</r4><d>6161353561613535</d><ddd>1234567.89</ddd><u1>Straße</u1><u2>Straße  </u2></Row>
  <Row><name>name2</name><value>2</value><boolval>false</boolval><r8>5.6</r8><r4>7.800000190734863</r4><d>3030</d><ddd>-1234567.89</ddd><u1>là</u1><u2>là      </u2></Row>
  <Row><name>name3</name><value>1</value><boolval>true</boolval><r8>1.2</r8><r4>3.400000095367432</r4><d>6161353561613535</d><ddd>8.76</ddd><u1>Straße</u1><u2>Straße  </u2></Row>
  <Row><name>name4</name><value>3</value><boolval>true</boolval><r8>9.1</r8><r4>11.11999988555908</r4><d>6161353561613535</d><ddd>987.65</ddd><u1>Baße</u1><u2>Baße    </u2></Row>
+ <Row><name>nulls</name><value>99999</value><boolval>true</boolval><r8>99.98999999999999</r8><r4>999.989990234375</r4><d>393939393939</d><ddd>9.99</ddd><u1>ß</u1><u2>9999 ßßß</u2></Row>
 </Dataset>
 <Dataset name='Result 2'>
  <Row><Result_2>name1</Result_2></Row>
@@ -15,32 +15,32 @@
  <Row><name>name1</name><value>1</value><boolval>true</boolval><r8>1.2</r8><r4>3.400000095367432</r4><d>6161353561613535</d><ddd>1234567.89</ddd><u1>Straße</u1><u2>Straße  </u2></Row>
 </Dataset>
 <Dataset name='Result 5'>
+ <Row><Result_5>1</Result_5></Row>
 </Dataset>
 <Dataset name='Result 6'>
- <Row><Result_6>1</Result_6></Row>
+ <Row><Result_6>true</Result_6></Row>
 </Dataset>
 <Dataset name='Result 7'>
- <Row><Result_7>true</Result_7></Row>
+ <Row><Result_7>1.2</Result_7></Row>
 </Dataset>
 <Dataset name='Result 8'>
- <Row><Result_8>1.2</Result_8></Row>
+ <Row><Result_8>3.400000095367432</Result_8></Row>
 </Dataset>
 <Dataset name='Result 9'>
- <Row><Result_9>3.400000095367432</Result_9></Row>
+ <Row><Result_9>6161353561613535</Result_9></Row>
 </Dataset>
 <Dataset name='Result 10'>
- <Row><Result_10>6161353561613535</Result_10></Row>
+ <Row><Result_10>Straße</Result_10></Row>
 </Dataset>
 <Dataset name='Result 11'>
- <Row><Result_11>Straße</Result_11></Row>
+ <Row><Result_11>Straße  </Result_11></Row>
 </Dataset>
 <Dataset name='Result 12'>
- <Row><Result_12>Straße  </Result_12></Row>
-</Dataset>
-<Dataset name='Result 13'>
  <Row><name>name1</name><value>1</value><boolval>true</boolval><r8>1.2</r8><r4>3.400000095367432</r4><d>6161353561613535</d><ddd>1234567.89</ddd><u1>Straße</u1><u2>Straße  </u2></Row>
  <Row><name>name2</name><value>2</value><boolval>false</boolval><r8>5.6</r8><r4>7.800000190734863</r4><d>3030</d><ddd>-1234567.89</ddd><u1>là</u1><u2>là      </u2></Row>
 </Dataset>
+<Dataset name='Result 13'>
+</Dataset>
 <Dataset name='Result 14'>
  <Row><Result_14>25001</Result_14></Row>
 </Dataset>

testing/regress/ecl/sqlite.ecl

@@ -1,21 +1,12 @@
-/*##############################################################################
-
-    HPCC SYSTEMS software Copyright (C) 2013 HPCC Systems.
-
-    Licensed under the Apache License, Version 2.0 (the "License");
-    you may not use this file except in compliance with the License.
-    You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
+IMPORT SqLite3;
 
-    Unless required by applicable law or agreed to in writing, software
-    distributed under the License is distributed on an "AS IS" BASIS,
-    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-    See the License for the specific language governing permissions and
-    limitations under the License.
-############################################################################## */
+/*
+ This example illustrates various calls to embdded SQLite code
+ */
 
-IMPORT SqLite3;
+// This is the record structure in ECL that will correspond to the rows in the SQLite dataset
+// Note that the default values specified in the fields will be used when a NULL value is being
+// returned from MySQL
 
 childrec := RECORD
    string name,
@@ -29,13 +20,7 @@ childrec := RECORD
    UNICODE8 u2 {default(U'9999 ßßßß')}
 END;
 
-stringrec := RECORD
-   string name
-END;
-
-stringrec extractName(childrec l) := TRANSFORM
-  SELF := l;
-END;
+// Set up the SQLite database
 
 drop() := EMBED(SqLite3 : file('test.db'))
   DROP TABLE IF EXISTS tbl1;
@@ -45,14 +30,20 @@ create() := EMBED(SqLite3 : file('test.db'))
   CREATE TABLE tbl1 ( name VARCHAR(20), value INT, boolval TINYINT, r8 DOUBLE, r4 FLOAT, d BLOB, ddd DECIMAL(10,2), u1 VARCHAR(10), u2 VARCHAR(10) );
 ENDEMBED;
 
+// Add a row
+
 initialize() := EMBED(SqLite3 : file('test.db'))
   INSERT INTO tbl1 values ('name1', 1, 1, 1.2, 3.4, 'aa55aa55', 1234567.89, 'Straße', 'Straße');
 ENDEMBED;
 
+// Add an additional row containing NULL values, to test that they are handled properly when read in ECL
+
 initializeNulls() := EMBED(SqLite3 : file('test.db'))
   INSERT INTO tbl1 (name) values ('nulls');
 ENDEMBED;
 
+// Test various types of parameters
+
 testSqLiteParms(
    string name,
    integer4 value,
@@ -66,14 +57,20 @@ testSqLiteParms(
   INSERT INTO tbl1 (name, value, boolval, r8, r4,d,ddd,u1,u2) values (:name, :value, :boolval, :r8, :r4,:d,:ddd,:u1,:u2);
 ENDEMBED;
 
+// Returning a dataset
+
 dataset(childrec) testSQLiteDS() := EMBED(SqLite3 : file('test.db'))
   SELECT * from tbl1;
 ENDEMBED;
 
+// Returning a single row
+
 childrec testSQLiteRow() := EMBED(SqLite3 : file('test.db'))
   SELECT * from tbl1 LIMIT 1;
 ENDEMBED;
 
+// Returning scalars
+
 string testSQLiteString() := EMBED(SqLite3 : file('test.db'))
   SELECT max(name) from tbl1;
 ENDEMBED;
@@ -110,6 +107,8 @@ UNICODE testSQLiteUnicode() := EMBED(SqLite3 : file('test.db'))
   SELECT max(u2) from tbl1;
 ENDEMBED;
 
+// Run the tests
+
 sequential (
   drop(),
   create(),