Merge pull request #6357 from richardkchapman/cassandra

HPCC-10617 Add support for Cassandra

Reviewed-by: Gavin Halliday <ghalliday@hpccsystems.com>

.gitmodules

@@ -25,3 +25,9 @@
 [submodule "esp/src/util"]
 	path = esp/src/util
 	url = https://github.com/hpcc-systems/util.git
+[submodule "plugins/cassandra/libuv"]
+	path = plugins/cassandra/libuv
+	url = https://github.com/hpcc-systems/libuv.git
+[submodule "plugins/cassandra/cpp-driver"]
+	path = plugins/cassandra/cpp-driver
+	url = https://github.com/hpcc-systems/cpp-driver.git

cmake_modules/commonSetup.cmake

@@ -76,6 +76,7 @@ IF ("${COMMONSETUP_DONE}" STREQUAL "")
   option(USE_UNSIGNED_CHAR "Build system with default char type is unsigned" OFF)
 
   option(USE_MYSQL "Enable MySQL support" ON)
+  option(USE_CASSANDRA "Enable Cassandra support" ON)
   option(USE_SQLITE3 "Enable SqLite3 support" ON)
   option(USE_PYTHON "Enable Python support" ON)
   option(USE_V8 "Enable V8 JavaScript support" ON)
@@ -84,7 +85,7 @@ IF ("${COMMONSETUP_DONE}" STREQUAL "")
 
   option(USE_OPTIONAL "Automatically disable requested features with missing dependencies" ON)
 
-  if ( USE_PYTHON OR USE_V8 OR USE_JNI OR USE_RINSIDE OR USE_SQLITE3 OR USE_MYSQL)
+  if ( USE_PYTHON OR USE_V8 OR USE_JNI OR USE_RINSIDE OR USE_SQLITE3 OR USE_MYSQL OR USE_CASSANDRA)
       set( WITH_PLUGINS ON )
   endif()
 

ecl/eclagent/eclagent.cpp

@@ -1229,10 +1229,10 @@ void EclAgent::setResultUnicode(const char * name, unsigned sequence, int len, U
     {
         if (outputFmt == ofSTD)
         {
-            char * buff = 0;
+            rtlDataAttr buff;
             unsigned bufflen = 0;
-            rtlUnicodeToCodepageX(bufflen, buff, len, val, "utf-8");
-            outputSerializer->fwrite(sequence, (const void*)buff, 1, bufflen);
+            rtlUnicodeToCodepageX(bufflen, buff.refstr(), len, val, "utf-8");
+            outputSerializer->fwrite(sequence, buff.getdata(), 1, bufflen);
             outputSerializer->close(sequence, true);
         }
         else

ecl/hql/hqlatoms.cpp

@@ -23,14 +23,17 @@ IIdAtom * failId;
 IIdAtom * bindBooleanParamId;
 IIdAtom * bindDataParamId;
 IIdAtom * bindDatasetParamId;
+IIdAtom * bindFloatParamId;
 IIdAtom * bindRealParamId;
 IIdAtom * bindRowParamId;
 IIdAtom * bindSetParamId;
 IIdAtom * bindSignedParamId;
+IIdAtom * bindSignedSizeParamId;
 IIdAtom * bindStringParamId;
 IIdAtom * bindVStringParamId;
 IIdAtom * bindUnicodeParamId;
 IIdAtom * bindUnsignedParamId;
+IIdAtom * bindUnsignedSizeParamId;
 IIdAtom * bindUtf8ParamId;
 IIdAtom * compileEmbeddedScriptId;
 IIdAtom * getEmbedContextId;
@@ -436,14 +439,17 @@ MODULE_INIT(INIT_PRIORITY_HQLATOM)
     MAKEID(bindBooleanParam);
     MAKEID(bindDataParam);
     MAKEID(bindDatasetParam);
+    MAKEID(bindFloatParam);
     MAKEID(bindRealParam);
     MAKEID(bindRowParam);
     MAKEID(bindSetParam);
     MAKEID(bindSignedParam);
+    MAKEID(bindSignedSizeParam);
     MAKEID(bindStringParam);
     MAKEID(bindVStringParam);
     MAKEID(bindUnicodeParam);
     MAKEID(bindUnsignedParam);
+    MAKEID(bindUnsignedSizeParam);
     MAKEID(bindUtf8Param);
     MAKEID(compileEmbeddedScript);
     defaultFieldNameId = createIdAtom("__f1__");

ecl/hql/hqlatoms.hpp

@@ -23,14 +23,17 @@ extern HQL_API IIdAtom * atId;
 extern HQL_API IIdAtom * bindBooleanParamId;
 extern HQL_API IIdAtom * bindDataParamId;
 extern HQL_API IIdAtom * bindDatasetParamId;
+extern HQL_API IIdAtom * bindFloatParamId;
 extern HQL_API IIdAtom * bindRealParamId;
 extern HQL_API IIdAtom * bindRowParamId;
 extern HQL_API IIdAtom * bindSetParamId;
 extern HQL_API IIdAtom * bindSignedParamId;
+extern HQL_API IIdAtom * bindSignedSizeParamId;
 extern HQL_API IIdAtom * bindStringParamId;
 extern HQL_API IIdAtom * bindVStringParamId;
 extern HQL_API IIdAtom * bindUnicodeParamId;
 extern HQL_API IIdAtom * bindUnsignedParamId;
+extern HQL_API IIdAtom * bindUnsignedSizeParamId;
 extern HQL_API IIdAtom * bindUtf8ParamId;
 extern HQL_API IIdAtom * compileEmbeddedScriptId;
 extern HQL_API IIdAtom * defaultFieldNameId;

ecl/hql/hqlutil.cpp

@@ -7924,13 +7924,13 @@ public:
     ConstantRowCreator(MemoryBuffer & _out) : out(_out) { expectedIndex = 0; }
 
     bool buildTransformRow(IHqlExpression * transform);
+    bool processFieldValue(IHqlExpression * optField, ITypeInfo * lhsType, IHqlExpression * rhs);
 
 protected:
     bool expandAssignChildren(IHqlExpression * expr);
     bool expandAssignElement(IHqlExpression * expr);
 
     bool processElement(IHqlExpression * expr, IHqlExpression * parentSelector);
-    bool processFieldValue(IHqlExpression * optField, ITypeInfo * lhsType, IHqlExpression * rhs);
     bool processRecord(IHqlExpression * record, IHqlExpression * parentSelector);
 
     IHqlExpression * queryMatchingAssign(IHqlExpression * self, IHqlExpression * search);
@@ -8131,12 +8131,13 @@ bool ConstantRowCreator::processFieldValue(IHqlExpression * optLhs, ITypeInfo *
             }
             return true;
         }
+    case type_utf8:
     case type_unicode:
     case type_qstring:
         {
             if (lenLhs == UNKNOWN_LENGTH)
                 rtlWriteInt4(out.reserve(sizeof(size32_t)), lenValue);
-            castValue->toMem(out.reserve(castValueType->getSize()));
+            castValue->toMem(out.reserve(castValue->getSize()));
             return true;
         }
     //MORE:
@@ -8217,6 +8218,12 @@ bool createConstantRow(MemoryBuffer & target, IHqlExpression * transform)
     return builder.buildTransformRow(transform);
 }
 
+bool createConstantField(MemoryBuffer & target, IHqlExpression * field, IHqlExpression * value)
+{
+    ConstantRowCreator builder(target);
+    return builder.processFieldValue(field, field->queryType(), value);
+}
+
 IHqlExpression * createConstantRowExpr(IHqlExpression * transform)
 {
     MemoryBuffer rowData;

ecl/hql/hqlutil.hpp

@@ -648,6 +648,7 @@ extern HQL_API IHqlExpression * createSizeof(IHqlExpression * expr);
 extern HQL_API bool allParametersHaveDefaults(IHqlExpression * function);
 extern HQL_API bool expandMissingDefaultsAsStoreds(HqlExprArray & args, IHqlExpression * function);
 
+extern HQL_API bool createConstantField(MemoryBuffer & target, IHqlExpression * field, IHqlExpression * value);
 extern HQL_API bool createConstantRow(MemoryBuffer & target, IHqlExpression * transform);
 extern HQL_API bool createConstantNullRow(MemoryBuffer & target, IHqlExpression * record);
 extern HQL_API IHqlExpression * createConstantRowExpr(IHqlExpression * transform);

ecl/hqlcpp/hqlcerrors.hpp

@@ -211,6 +211,7 @@
 #define HQLERR_UserCodeNotAllowed               4191
 #define HQLERR_StreamInputUsedDirectly          4192
 #define HQLERR_MaxlengthExceedsLimit            4193
+#define HQLERR_CouldNotGenerateDefault          4194
 
 //Warnings....
 #define HQLWRN_PersistDataNotLikely             4500
@@ -580,6 +581,7 @@
 #define HQLERR_DependencyWithinGraph_Text       "INTERNAL: Dependency within a graph incorrectly generated for hThor (%u)"
 #define HQLERR_UnknownCompoundAssign_Text       "INTERNAL: Unrecognised compound assign %s"
 #define HQLERR_ReadSpillBeforeWriteFix_Text     "INTERNAL: Attempt to read spill file %s before it is written.  Try adding #option ('allowThroughSpill', false); to the query."
+#define HQLERR_CouldNotGenerateDefault_Text     "INTERNAL: Could not generate default value for field %s"
 
 #define WARNINGAT(cat, e, x)                 reportWarning(cat, e, x, x##_Text)
 #define WARNINGAT1(cat, e, x, a)             reportWarning(cat, e, x, x##_Text, a)

ecl/hqlcpp/hqlcpp.cpp

@@ -11590,6 +11590,7 @@ void HqlCppTranslator::buildScriptFunctionDefinition(BuildCtx &funcctx, IHqlExpr
 {
     ITypeInfo * returnType = funcdef->queryType()->queryChildType();
     IHqlExpression * outofline = funcdef->queryChild(0);
+    IHqlExpression * formals = funcdef->queryChild(1);
     assertex(outofline->getOperator() == no_outofline);
     IHqlExpression * bodyCode = outofline->queryChild(0);
     IHqlExpression *language = queryAttributeChild(bodyCode, languageAtom, 0);
@@ -11603,7 +11604,11 @@ void HqlCppTranslator::buildScriptFunctionDefinition(BuildCtx &funcctx, IHqlExpr
     buildAssignToTemp(funcctx, pluginPtr, getPlugin);
     StringBuffer createParam;
     createParam.append("Owned<IEmbedFunctionContext> __ctx = __plugin->createFunctionContext(");
-    createParam.append(isImport ? "true" : "false");
+    createParam.append(isImport ? "EFimport" : "EFembed");
+    if (returnType->getTypeCode()==type_void)
+        createParam.append("|EFnoreturn");
+    if (formals->numChildren()==0)
+        createParam.append("|EFnoparams");
     StringBuffer attrParam;
     ForEachChild(idx, bodyCode)
     {
@@ -11632,7 +11637,6 @@ void HqlCppTranslator::buildScriptFunctionDefinition(BuildCtx &funcctx, IHqlExpr
     scriptArgs.append(*LINK(ctxVar));
     scriptArgs.append(*LINK(bodyCode->queryChild(0)));
     buildFunctionCall(funcctx, isImport ? importId : compileEmbeddedScriptId, scriptArgs);
-    IHqlExpression *formals = funcdef->queryChild(1);
     ForEachChild(i, formals)
     {
         HqlExprArray args;
@@ -11648,7 +11652,13 @@ void HqlCppTranslator::buildScriptFunctionDefinition(BuildCtx &funcctx, IHqlExpr
         switch (paramType->getTypeCode())
         {
         case type_int:
-            bindFunc = paramType->isSigned() ? bindSignedParamId : bindUnsignedParamId;
+            if (paramType->getSize()<8)
+            {
+                bindFunc = paramType->isSigned() ? bindSignedSizeParamId : bindUnsignedSizeParamId;
+                args.append(*createIntConstant(paramType->getSize()));
+            }
+            else
+                bindFunc = paramType->isSigned() ? bindSignedParamId : bindUnsignedParamId;
             break;
         case type_varstring:
             bindFunc = bindVStringParamId;
@@ -11657,7 +11667,10 @@ void HqlCppTranslator::buildScriptFunctionDefinition(BuildCtx &funcctx, IHqlExpr
             bindFunc = bindStringParamId;
             break;
         case type_real:
-            bindFunc = bindRealParamId;
+            if (paramType->getSize()==4)
+                bindFunc = bindFloatParamId;
+            else
+                bindFunc = bindRealParamId;
             break;
         case type_boolean:
             bindFunc = bindBooleanParamId;

ecl/hqlcpp/hqlcppsys.ecl

@@ -853,9 +853,12 @@ const char * cppSystemText[]  = {
     "   bindDataParam(const varstring name, data val) : method,entrypoint='bindDataParam';",
     "   bindDatasetParam(const varstring name, streamed dataset val) : method,entrypoint='bindDatasetParam',passParameterMeta(true);",
     "   bindRealParam(const varstring name, real val) : method,entrypoint='bindRealParam';",
+    "   bindFloatParam(const varstring name, real4 val) : method,entrypoint='bindFloatParam';",
     "   bindRowParam(const varstring name, _linkcounted_ row row) : method,entrypoint='bindRowParam',passParameterMeta(true);",
     "   bindSignedParam(const varstring name, integer val) : method,entrypoint='bindSignedParam';",
     "   bindUnsignedParam(const varstring name, unsigned val) : method,entrypoint='bindUnsignedParam';",
+    "   bindSignedSizeParam(const varstring name, integer4 size, integer val) : method,entrypoint='bindSignedSizeParam';",
+    "   bindUnsignedSizeParam(const varstring name, integer4 size, unsigned val) : method,entrypoint='bindUnsignedSizeParam';",
     "   bindStringParam(const varstring name, const string val) : method,entrypoint='bindStringParam';",
     "   bindVStringParam(const varstring name, const varstring val) : method,entrypoint='bindVStringParam';",
     "   bindUTF8Param(const varstring name, const utf8 val) : method,entrypoint='bindUTF8Param';",

ecl/hqlcpp/hqlhtcpp.cpp

@@ -3619,8 +3619,22 @@ unsigned HqlCppTranslator::buildRtlField(StringBuffer * instanceName, IHqlExpres
         else
             xpathCppText.append("NULL");
 
+        StringBuffer defaultInitializer;
+        IHqlExpression *defaultValue = queryAttributeChild(field, defaultAtom, 0);
+        if (defaultValue)
+        {
+            MemoryBuffer target;
+            if (createConstantField(target, field, defaultValue))
+                appendStringAsQuotedCPP(defaultInitializer, target.length(), target.toByteArray(), false);
+            else
+                throwError1(HQLERR_CouldNotGenerateDefault, field->queryName()->str());
+        }
+
         StringBuffer definition;
-        definition.append("const RtlFieldStrInfo ").append(name).append("(\"").append(lowerName).append("\",").append(xpathCppText).append(",&").append(typeName).append(");");
+        definition.append("const RtlFieldStrInfo ").append(name).append("(\"").append(lowerName).append("\",").append(xpathCppText).append(",&").append(typeName);
+        if (defaultInitializer.length())
+            definition.append(',').append(defaultInitializer);
+        definition.append(");");
 
         BuildCtx fieldctx(declarectx);
         fieldctx.setNextPriority(TypeInfoPrio);

initfiles/examples/embed/mysql-simple.ecl

@@ -5,17 +5,19 @@ IMPORT mysql;
  */
 
 // This is the record structure in ECL that will correspond to the rows in the MySQL dataset
+// Note that the default values specified in the fields will be used when a NULL value is being
+// returned from MySQL
 
 childrec := RECORD
    string name,
-   integer value,
-   boolean boolval,
-   real8 r8,
-   real4 r4,
-   DATA d,
-   DECIMAL10_2 ddd,
-   UTF8 u1,
-   UNICODE8 u2
+   integer4 value { default(99999) },
+   boolean boolval { default(true) },
+   real8 r8 {default(99.99)},
+   real4 r4 {default(999.99)},
+   DATA d {default (D'999999')},
+   DECIMAL10_2 ddd {default(9.99)},
+   UTF8 u1 {default(U'9999 ß')},
+   UNICODE8 u2 {default(U'9999 ßßßß')}
 END;
 
 // Some data we will use to initialize the MySQL table
@@ -45,6 +47,12 @@ initializeNulls() := EMBED(mysql : user('rchapman'),database('test'))
   INSERT INTO tbl1 (name) values ('nulls');
 ENDEMBED;
 
+// Note that the query string is encoded in utf8
+
+initializeUtf8() := EMBED(mysql : user('rchapman'),database('test'))
+  INSERT INTO tbl1 values ('utf8test', 1, 1, 1.2, 3.4, 'aa55aa55', 1234567.89, 'Straße', 'Straße');
+ENDEMBED;
+
 // Returning a dataset
 
 dataset(childrec) testMySQLDS() := EMBED(mysql : user('rchapman'),database('test'))
@@ -60,6 +68,7 @@ ENDEMBED;
 // Passing in parameters
 
 childrec testMySQLParms(
+
    string name,
    integer value,
    boolean boolval,
@@ -128,6 +137,7 @@ sequential (
   create(),
   initialize(init),
   initializeNulls(),
+  initializeUtf8(),
   OUTPUT(testMySQLDS()),
   OUTPUT(testMySQLRow().name),
   OUTPUT(testMySQLParms('name1', 1, true, 1.2, 3.4, D'aa55aa55', U'Straße', U'Straße')),

initfiles/examples/embed/sqlite-simple.ecl

@@ -1,38 +1,129 @@
-/*##############################################################################
+IMPORT SqLite3;
 
-    HPCC SYSTEMS software Copyright (C) 2013 HPCC Systems.
+/*
+ This example illustrates various calls to embdded SQLite code
+ */
 
-    Licensed under the Apache License, Version 2.0 (the "License");
-    you may not use this file except in compliance with the License.
-    You may obtain a copy of the License at
+// This is the record structure in ECL that will correspond to the rows in the SQLite dataset
+// Note that the default values specified in the fields will be used when a NULL value is being
+// returned from MySQL
 
-       http://www.apache.org/licenses/LICENSE-2.0
+childrec := RECORD
+   string name,
+   integer4 value { default(99999) },
+   boolean boolval { default(true) },
+   real8 r8 {default(99.99)},
+   real4 r4 {default(999.99)},
+   DATA d {default (D'999999')},
+   DECIMAL10_2 ddd {default(9.99)},
+   UTF8 u1 {default(U'9999 ß')},
+   UNICODE8 u2 {default(U'9999 ßßßß')}
+END;
 
-    Unless required by applicable law or agreed to in writing, software
-    distributed under the License is distributed on an "AS IS" BASIS,
-    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-    See the License for the specific language governing permissions and
-    limitations under the License.
-############################################################################## */
+// Set up the SQLite database
 
-IMPORT SqLite3;
+drop() := EMBED(SqLite3 : file('test.db'))
+  DROP TABLE IF EXISTS tbl1;
+ENDEMBED;
 
-childrec := RECORD
-   string name => unsigned value;
-END;
+create() := EMBED(SqLite3 : file('test.db'))
+  CREATE TABLE tbl1 ( name VARCHAR(20), value INT, boolval TINYINT, r8 DOUBLE, r4 FLOAT, d BLOB, ddd DECIMAL(10,2), u1 VARCHAR(10), u2 VARCHAR(10) );
+ENDEMBED;
 
-dataset(childrec) testSqLite(unsigned lim) := EMBED(SqLite3 : file('test.db'))
-  SELECT * from tbl1 where two = :lim;
+// Add a row
+
+initialize() := EMBED(SqLite3 : file('test.db'))
+  INSERT INTO tbl1 values ('name1', 1, 1, 1.2, 3.4, 'aa55aa55', 1234567.89, 'Straße', 'Straße');
 ENDEMBED;
 
-dataset(childrec) testSqLite2(string v) := EMBED(SqLite3 : file('test.db'))
-  SELECT * from tbl1 where one = :v;
+// Add an additional row containing NULL values, to test that they are handled properly when read in ECL
+
+initializeNulls() := EMBED(SqLite3 : file('test.db'))
+  INSERT INTO tbl1 (name) values ('nulls');
 ENDEMBED;
 
-unsigned testSqLite3(string v) := EMBED(SqLite3 : file('test.db'))
-  SELECT count(*) from tbl1 where one = :v;
+// Test various types of parameters
+
+testSqLiteParms(
+   string name,
+   integer4 value,
+   boolean boolval,
+   real8 r8,
+   real4 r4,
+   DATA d,
+   REAL8 ddd, // Decimal parms not supported.
+   UTF8 u1,
+   UNICODE8 u2) := EMBED(SqLite3 : file('test.db'))
+  INSERT INTO tbl1 (name, value, boolval, r8, r4,d,ddd,u1,u2) values (:name, :value, :boolval, :r8, :r4,:d,:ddd,:u1,:u2);
 ENDEMBED;
 
-output(testSqLite(20));
-output(testSqLite2('hello!'));
-output(testSqLite3('hello!'));
+// Returning a dataset
+
+dataset(childrec) testSQLiteDS() := EMBED(SqLite3 : file('test.db'))
+  SELECT * from tbl1;
+ENDEMBED;
+
+// Returning a single row
+
+childrec testSQLiteRow() := EMBED(SqLite3 : file('test.db'))
+  SELECT * from tbl1 LIMIT 1;
+ENDEMBED;
+
+// Returning scalars
+
+string testSQLiteString() := EMBED(SqLite3 : file('test.db'))
+  SELECT max(name) from tbl1;
+ENDEMBED;
+
+dataset(childrec) testSQLiteStringParam(string filter) := EMBED(SqLite3 : file('test.db'))
+  SELECT * from tbl1 where name = :filter;
+ENDEMBED;
+
+integer testSQLiteInt() := EMBED(SqLite3 : file('test.db'))
+  SELECT max(value) from tbl1;
+ENDEMBED;
+
+boolean testSQLiteBool() := EMBED(SqLite3 : file('test.db'))
+  SELECT max(boolval) from tbl1;
+ENDEMBED;
+
+real8 testSQLiteReal8() := EMBED(SqLite3 : file('test.db'))
+  SELECT max(r8) from tbl1;
+ENDEMBED;
+
+real4 testSQLiteReal4() := EMBED(SqLite3 : file('test.db'))
+  SELECT max(r4) from tbl1;
+ENDEMBED;
+
+data testSQLiteData() := EMBED(SqLite3 : file('test.db'))
+  SELECT max(d) from tbl1;
+ENDEMBED;
+
+UTF8 testSQLiteUtf8() := EMBED(SqLite3 : file('test.db'))
+  SELECT max(u1) from tbl1;
+ENDEMBED;
+
+UNICODE testSQLiteUnicode() := EMBED(SqLite3 : file('test.db'))
+  SELECT max(u2) from tbl1;
+ENDEMBED;
+
+// Run the tests
+
+sequential (
+  drop(),
+  create(),
+  initialize(),
+  initializeNulls(),
+  testSQLiteParms('name2', 1, true, 1.2, 3.4, D'aa55aa55', 23.45, U'Straßaße', U'Straßßßße'),
+  OUTPUT(testSQLiteDS()),
+  OUTPUT(testSQLiteRow().name),
+  OUTPUT(testSQLiteString()),
+  OUTPUT(testSQLiteStringParam(testSQLiteString())),
+  OUTPUT(testSQLiteInt()),
+  OUTPUT(testSQLiteBool()),
+  OUTPUT(testSQLiteReal8()),
+  OUTPUT(testSQLiteReal4()),
+  OUTPUT(testSQLiteData()),
+  OUTPUT(testSQLiteUtf8()),
+  OUTPUT(testSQLiteUnicode())
+);

plugins/CMakeLists.txt

@@ -28,3 +28,4 @@ add_subdirectory (v8embed)
 add_subdirectory (pyembed)
 add_subdirectory (javaembed)
 add_subdirectory (Rembed)
+add_subdirectory (cassandra)

plugins/Rembed/Rembed.cpp

@@ -854,14 +854,26 @@ public:
         vval.assign(cval, cval+len);
         R[name] = vval;
     }
+    virtual void bindFloatParam(const char *name, float val)
+    {
+        R[name] = val;
+    }
     virtual void bindRealParam(const char *name, double val)
     {
         R[name] = val;
     }
+    virtual void bindSignedSizeParam(const char *name, int size, __int64 val)
+    {
+        R[name] = (long int) val;
+    }
     virtual void bindSignedParam(const char *name, __int64 val)
     {
         R[name] = (long int) val;
     }
+    virtual void bindUnsignedSizeParam(const char *name, int size, unsigned __int64 val)
+    {
+        R[name] = (long int) val;
+    }
     virtual void bindUnsignedParam(const char *name, unsigned __int64 val)
     {
         R[name] = (unsigned long int) val;
@@ -1068,7 +1080,7 @@ private:
 class REmbedContext: public CInterfaceOf<IEmbedContext>
 {
 public:
-    virtual IEmbedFunctionContext *createFunctionContext(bool isImport, const char *options)
+    virtual IEmbedFunctionContext *createFunctionContext(unsigned flags, const char *options)
     {
         return new REmbedFunctionContext(*queryGlobalState()->R, options);
     }

plugins/cassandra/CMakeLists.txt

@@ -0,0 +1,91 @@
+################################################################################
+#    HPCC SYSTEMS software Copyright (C) 2014 HPCC Systems.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+################################################################################
+
+
+# Component: cassandraembed
+
+#####################################################
+# Description:
+# ------------
+#    Cmake Input File for cassandraembed
+#####################################################
+
+
+project( cassandraembed )
+
+if (USE_CASSANDRA)
+  ADD_PLUGIN(cassandraembed PACKAGES OPTION MAKE_CASSANDRAEMBED)
+  if ( MAKE_CASSANDRAEMBED )
+
+    # Build libuv, required by the cassandra driver but not available on all distros
+    add_custom_command(OUTPUT ${PROJECT_BINARY_DIR}/libuv.so
+                       COMMAND make builddir_name=${PROJECT_BINARY_DIR}
+                       COMMAND mv ${PROJECT_BINARY_DIR}/libuv.so ${PROJECT_BINARY_DIR}/libuv.so.0.10
+                       COMMAND ln -s ${PROJECT_BINARY_DIR}/libuv.so.0.10 ${PROJECT_BINARY_DIR}/libuv.so
+                       WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/libuv)
+    add_custom_target ( libuv ALL DEPENDS ${PROJECT_BINARY_DIR}/libuv.so )
+
+    set(LIBUV_LIBRARY ${PROJECT_BINARY_DIR}/libuv.so)
+    set(LIBUV_INCLUDE_DIR ${PROJECT_SOURCE_DIR}/libuv/include/)
+
+    # Build the Cassandra cpp driver, only presently available as source
+    option(CASS_INSTALL_HEADER "Install header file" OFF)
+    option(CASS_BUILD_STATIC "Build static library" OFF)
+    option(CASS_BUILD_EXAMPLES "Build examples" OFF)
+    add_subdirectory (cpp-driver ${PROJECT_BINARY_DIR}/cassandra)
+    add_dependencies( cassandra libuv )
+
+    set(SRCS cassandraembed.cpp)
+
+    include_directories (
+         ./../../system/include
+         ./../../rtl/eclrtl
+         ./../../roxie/roxiemem
+         ./../../rtl/include
+         ./../../rtl/nbcd
+         ./../../common/deftype
+         ./../../system/jlib
+
+         ./cpp-driver/include
+       )
+
+    ADD_DEFINITIONS( -D_USRDLL -DCASSANDRAEMBED_EXPORTS)
+
+    HPCC_ADD_LIBRARY( cassandraembed SHARED ${SRCS} )
+    if (${CMAKE_VERSION} VERSION_LESS "2.8.9")
+      message("WARNING: Cannot set NO_SONAME. shlibdeps will give warnings when package is installed")
+    else()
+      set_target_properties( cassandraembed PROPERTIES NO_SONAME 1 )
+    endif()
+
+    install ( TARGETS cassandraembed DESTINATION plugins COMPONENT Runtime )
+    # until such time as the cassandra cpp driver and libuv are available as standard in all distros we want to support,
+    # include them in our rpm
+    # Note that the cassandra driver CMake file already includes the relevant install commands
+
+    install ( FILES ${PROJECT_BINARY_DIR}/libuv.so.0.10 DESTINATION ${LIB_DIR} COMPONENT Runtime )
+
+    target_link_libraries ( cassandraembed
+        cassandra
+        eclrtl
+        roxiemem
+        jlib
+        )
+  endif()
+endif()
+
+# Even if not making the Cassandra plugin, we want to install the header
+install ( FILES ${CMAKE_CURRENT_SOURCE_DIR}/cassandra.ecllib DESTINATION plugins COMPONENT Runtime)

testing/ecl/sqlite.ecl

@@ -1,6 +1,6 @@
 /*##############################################################################
 
-    HPCC SYSTEMS software Copyright (C) 2013 HPCC Systems.
+    HPCC SYSTEMS software Copyright (C) 2014 HPCC Systems.
 
     Licensed under the Apache License, Version 2.0 (the "License");
     you may not use this file except in compliance with the License.
@@ -15,24 +15,11 @@
     limitations under the License.
 ############################################################################## */
 
-IMPORT SqLite3;
-
-childrec := RECORD
-   string name => unsigned value;
+EXPORT Language := SERVICE : plugin('cassandraembed')
+  boolean getEmbedContext():cpp,pure,namespace='cassandraembed',entrypoint='getEmbedContext',prototype='IEmbedContext* getEmbedContext()';
+  boolean syntaxCheck(const varstring src):cpp,pure,namespace='cassandraembed',entrypoint='syntaxCheck';
 END;
-
-dataset(childrec) testSqLite(unsigned lim) := EMBED(SqLite3 : file('test.db'))
-  SELECT * from tbl1 where two = :lim;
-ENDEMBED;
-
-dataset(childrec) testSqLite2(string v) := EMBED(SqLite3 : file('test.db'))
-  SELECT * from tbl1 where one = :v;
-ENDEMBED;
-
-unsigned testSqLite3(string v) := EMBED(SqLite3 : file('test.db'))
-  SELECT count(*) from tbl1 where one = :v;
-ENDEMBED;
-
-output(testSqLite(20));
-output(testSqLite2('hello!'));
-output(testSqLite3('hello!'));
+EXPORT getEmbedContext := Language.getEmbedContext;
+EXPORT syntaxCheck := Language.syntaxCheck;
+EXPORT boolean supportsImport := false;
+EXPORT boolean supportsScript := true;

plugins/cassandra/cpp-driver

@@ -0,0 +1 @@
+Subproject commit d828ec92db0efea72f57716777ab41f0bb9efa1f

plugins/cassandra/libuv

@@ -0,0 +1 @@
+Subproject commit 71b71289871e31b83a60ad9e278c511b8f2a4a95

plugins/javaembed/javaembed.cpp

@@ -1789,6 +1789,25 @@ public:
         v.l = javaData;
         addArg(v);
     }
+    virtual void bindFloatParam(const char *name, float val)
+    {
+        // Could argue that the size should match...
+        jvalue v;
+        switch(*argsig)
+        {
+        case 'D':
+            v.d = val;
+            break;
+        case 'F':
+            v.f = val;
+            break;
+        default:
+            typeError("REAL");
+            break;
+        }
+        argsig++;
+        addArg(v);
+    }
     virtual void bindRealParam(const char *name, double val)
     {
         jvalue v;
@@ -1807,6 +1826,10 @@ public:
         argsig++;
         addArg(v);
     }
+    virtual void bindSignedSizeParam(const char *name, int size, __int64 val)
+    {
+        bindSignedParam(name, val);
+    }
     virtual void bindSignedParam(const char *name, __int64 val)
     {
         jvalue v;
@@ -1831,6 +1854,10 @@ public:
         argsig++;
         addArg(v);
     }
+    virtual void bindUnsignedSizeParam(const char *name, int size, unsigned __int64 val)
+    {
+        bindUnsignedParam(name, val);
+    }
     virtual void bindUnsignedParam(const char *name, unsigned __int64 val)
     {
         throw MakeStringException(MSGAUD_user, 0, "javaembed: Unsigned parameters not supported"); // Java doesn't support unsigned
@@ -2262,9 +2289,9 @@ static JNIEnv *queryJNIEnv()
 class JavaEmbedContext : public CInterfaceOf<IEmbedContext>
 {
 public:
-    virtual IEmbedFunctionContext *createFunctionContext(bool isImport, const char *options)
+    virtual IEmbedFunctionContext *createFunctionContext(unsigned flags, const char *options)
     {
-        assertex(isImport);
+        assertex(flags & EFimport);
         return new JavaEmbedImportContext(queryContext(), options);
     }
 };

plugins/mysql/mysqlembed.cpp

@@ -26,6 +26,7 @@
 #include "eclrtl_imp.hpp"
 #include "rtlds_imp.hpp"
 #include "rtlfield_imp.hpp"
+#include "rtlembed.hpp"
 #include "roxiemem.hpp"
 #include "nbcd.hpp"
 
@@ -343,7 +344,10 @@ static bool isInteger(enum_field_types type)
 static bool getBooleanResult(const RtlFieldInfo *field, const MYSQL_BIND &bound)
 {
     if (*bound.is_null)
-        return false;
+    {
+        NullFieldProcessor p(field);
+        return p.boolResult;
+    }
     if (!isInteger(bound.buffer_type))
         typeError("boolean", field);
     return rtlReadUInt(bound.buffer, *bound.length) != 0;
@@ -353,8 +357,8 @@ static void getDataResult(const RtlFieldInfo *field, const MYSQL_BIND &bound, si
 {
     if (*bound.is_null)
     {
-        result = NULL;
-        chars = 0;
+        NullFieldProcessor p(field);
+        rtlStrToDataX(chars, result, p.resultChars, p.stringResult);
         return;
     }
     if (bound.buffer_type == MYSQL_TYPE_TINY_BLOB ||
@@ -375,7 +379,10 @@ static unsigned __int64 getUnsignedResult(const RtlFieldInfo *field, const MYSQL
 static double getRealResult(const RtlFieldInfo *field, const MYSQL_BIND &bound)
 {
     if (*bound.is_null)
-        return 0;
+    {
+        NullFieldProcessor p(field);
+        return p.doubleResult;
+    }
     if (isInteger(bound.buffer_type))
     {
         if (bound.is_unsigned)
@@ -394,7 +401,10 @@ static double getRealResult(const RtlFieldInfo *field, const MYSQL_BIND &bound)
 static __int64 getSignedResult(const RtlFieldInfo *field, const MYSQL_BIND &bound)
 {
     if (*bound.is_null)
-        return 0;
+    {
+        NullFieldProcessor p(field);
+        return p.intResult;
+    }
     if (isInteger(bound.buffer_type))
     {
         if (bound.is_unsigned)
@@ -409,7 +419,10 @@ static __int64 getSignedResult(const RtlFieldInfo *field, const MYSQL_BIND &boun
 static unsigned __int64 getUnsignedResult(const RtlFieldInfo *field, const MYSQL_BIND &bound)
 {
     if (*bound.is_null)
-        return 0;
+    {
+        NullFieldProcessor p(field);
+        return p.uintResult;
+    }
     if (!isInteger(bound.buffer_type))
         typeError("integer", field);
     if (bound.is_unsigned)
@@ -422,8 +435,8 @@ static void getStringResult(const RtlFieldInfo *field, const MYSQL_BIND &bound,
 {
     if (*bound.is_null)
     {
-        result = NULL;
-        chars = 0;
+        NullFieldProcessor p(field);
+        rtlStrToStrX(chars, result, p.resultChars, p.stringResult);
         return;
     }
     if (bound.buffer_type != MYSQL_TYPE_STRING && bound.buffer_type != MYSQL_TYPE_VAR_STRING)
@@ -438,8 +451,8 @@ static void getUTF8Result(const RtlFieldInfo *field, const MYSQL_BIND &bound, si
 {
     if (*bound.is_null)
     {
-        result = NULL;
-        chars = 0;
+        NullFieldProcessor p(field);
+        rtlUtf8ToUtf8X(chars, result, p.resultChars, p.stringResult);
         return;
     }
     if (bound.buffer_type != MYSQL_TYPE_STRING && bound.buffer_type != MYSQL_TYPE_VAR_STRING)
@@ -454,8 +467,8 @@ static void getUnicodeResult(const RtlFieldInfo *field, const MYSQL_BIND &bound,
 {
     if (*bound.is_null)
     {
-        result = NULL;
-        chars = 0;
+        NullFieldProcessor p(field);
+        rtlUnicodeToUnicodeX(chars, result, p.resultChars, p.unicodeResult);
         return;
     }
     if (bound.buffer_type != MYSQL_TYPE_STRING && bound.buffer_type != MYSQL_TYPE_VAR_STRING)
@@ -470,18 +483,19 @@ static void getDecimalResult(const RtlFieldInfo *field, const MYSQL_BIND &bound,
 {
     if (*bound.is_null)
     {
-        value.setInt(0);
+        NullFieldProcessor p(field);
+        value.set(p.decimalResult);
         return;
     }
     size32_t chars;
     rtlDataAttr result;
     mysqlembed::getStringResult(field, bound, chars, result.refstr());
+    value.setString(chars, result.getstr());
     if (field)
     {
         RtlDecimalTypeInfo *dtype = (RtlDecimalTypeInfo *) field->type;
         value.setPrecision(dtype->getDecimalDigits(), dtype->getDecimalPrecision());
     }
-    value.setString(chars, result.getstr());
 }
 
 static void createBindBuffer(MYSQL_BIND & bindInfo, enum_field_types sqlType, unsigned size)
@@ -1004,17 +1018,30 @@ public:
         bindInfo.buffer_length = bytes;
         bindInfo.length = &bindInfo.buffer_length;
     }
+    virtual void bindFloatParam(const char *name, float val)
+    {
+        MYSQL_BIND &bindInfo = findParameter(name, MYSQL_TYPE_FLOAT, sizeof(val));
+        * (float *) bindInfo.buffer = val;
+    }
     virtual void bindRealParam(const char *name, double val)
     {
         MYSQL_BIND &bindInfo = findParameter(name, MYSQL_TYPE_DOUBLE, sizeof(val));
         * (double *) bindInfo.buffer = val;
     }
+    virtual void bindSignedSizeParam(const char *name, int size, __int64 val)
+    {
+        bindSignedParam(name, val);
+    }
     virtual void bindSignedParam(const char *name, __int64 val)
     {
         MYSQL_BIND &bindInfo = findParameter(name, MYSQL_TYPE_LONGLONG, sizeof(val));
         * (__int64 *) bindInfo.buffer = val;
         bindInfo.is_unsigned = false;
     }
+    virtual void bindUnsignedSizeParam(const char *name, int size, unsigned __int64 val)
+    {
+        bindUnsignedParam(name, val);
+    }
     virtual void bindUnsignedParam(const char *name, unsigned __int64 val)
     {
         MYSQL_BIND &bindInfo = findParameter(name, MYSQL_TYPE_LONGLONG, sizeof(val));
@@ -1064,8 +1091,9 @@ public:
     {
         throwUnexpected();
     }
-    virtual void compileEmbeddedScript(size32_t len, const char *script)
+    virtual void compileEmbeddedScript(size32_t chars, const char *script)
     {
+        size32_t len = rtlUtf8Size(chars, script);
         Owned<MySQLStatement> stmt  = new MySQLStatement(mysql_stmt_init(*conn));
         if (!*stmt)
             fail("failed to create statement");
@@ -1092,7 +1120,7 @@ protected:
     {
         if (!stmtInfo->hasResult() || stmtInfo->queryResultBindings().numColumns() != 1)
             typeError("scalar", NULL);
-        lazyExecute();
+        lazyExecute(); // MORE this seems wrong to me  - or at least needs to check not already executed
         if (!stmtInfo->next())
             typeError("scalar", NULL);
         return stmtInfo->queryResultBindings().queryColumn(0);
@@ -1118,9 +1146,9 @@ protected:
 class MySQLEmbedContext : public CInterfaceOf<IEmbedContext>
 {
 public:
-    virtual IEmbedFunctionContext *createFunctionContext(bool isImport, const char *options)
+    virtual IEmbedFunctionContext *createFunctionContext(unsigned flags, const char *options)
     {
-        if (isImport)
+        if (flags & EFimport)
             UNSUPPORTED("IMPORT");
         else
             return new MySQLEmbedFunctionContext(options);

plugins/pyembed/pyembed.cpp

@@ -1204,14 +1204,26 @@ public:
     {
         addArg(name, PyByteArray_FromStringAndSize((const char *) val, len));
     }
+    virtual void bindFloatParam(const char *name, float val)
+    {
+        addArg(name, PyFloat_FromDouble((double) val));
+    }
     virtual void bindRealParam(const char *name, double val)
     {
         addArg(name, PyFloat_FromDouble(val));
     }
+    virtual void bindSignedSizeParam(const char *name, int size, __int64 val)
+    {
+        addArg(name, PyLong_FromLongLong(val));
+    }
     virtual void bindSignedParam(const char *name, __int64 val)
     {
         addArg(name, PyLong_FromLongLong(val));
     }
+    virtual void bindUnsignedSizeParam(const char *name, int size, unsigned __int64 val)
+    {
+        addArg(name, PyLong_FromUnsignedLongLong(val));
+    }
     virtual void bindUnsignedParam(const char *name, unsigned __int64 val)
     {
         addArg(name, PyLong_FromUnsignedLongLong(val));
@@ -1435,7 +1447,7 @@ private:
 class Python27EmbedContext : public CInterfaceOf<IEmbedContext>
 {
 public:
-    virtual IEmbedFunctionContext *createFunctionContext(bool isImport, const char *options)
+    virtual IEmbedFunctionContext *createFunctionContext(unsigned flags, const char *options)
     {
         if (!threadContext)
         {
@@ -1444,7 +1456,7 @@ public:
             threadContext = new PythonThreadContext;
             threadHookChain = addThreadTermFunc(releaseContext);
         }
-        if (isImport)
+        if (flags & EFimport)
             return new Python27EmbedImportContext(threadContext, options);
         else
             return new Python27EmbedScriptContext(threadContext, options);

plugins/sqlite3/sqlite3.cpp

@@ -26,6 +26,7 @@
 #include "eclrtl_imp.hpp"
 #include "rtlds_imp.hpp"
 #include "rtlfield_imp.hpp"
+#include "rtlembed.hpp"
 #include "nbcd.hpp"
 
 #ifdef _WIN32
@@ -87,12 +88,6 @@ public:
 
 // Conversions from SqLite3 values to ECL data
 
-static void checkSqliteError(int rc)
-{
-    if (rc != SQLITE_OK)
-        rtlFail(rc, "SqLite3 error");
-}
-
 static void typeError(const char *expected, const RtlFieldInfo *field) __attribute__((noreturn));
 
 static void typeError(const char *expected, const RtlFieldInfo *field)
@@ -103,9 +98,19 @@ static void typeError(const char *expected, const RtlFieldInfo *field)
     rtlFail(0, msg.str());
 }
 
+static inline bool isNull(sqlite3_value *val)
+{
+    return sqlite3_value_type(val) == SQLITE_NULL;
+}
+
 static bool getBooleanResult(const RtlFieldInfo *field, sqlite3_value *val)
 {
     assertex(val);
+    if (isNull(val))
+    {
+        NullFieldProcessor p(field);
+        return p.boolResult;
+    }
     if (sqlite3_value_type(val) != SQLITE_INTEGER)
         typeError("boolean", field);
     return sqlite3_value_int64(val) != 0;
@@ -114,7 +119,13 @@ static bool getBooleanResult(const RtlFieldInfo *field, sqlite3_value *val)
 static void getDataResult(const RtlFieldInfo *field, sqlite3_value *val, size32_t &chars, void * &result)
 {
     assertex(val);
-    if (sqlite3_value_type(val) != SQLITE_BLOB)
+    if (isNull(val))
+    {
+        NullFieldProcessor p(field);
+        rtlStrToDataX(chars, result, p.resultChars, p.stringResult);
+        return;
+    }
+    if (sqlite3_value_type(val) != SQLITE_BLOB && sqlite3_value_type(val) != SQLITE_TEXT)
         typeError("blob", field);
     const void *blob = sqlite3_value_blob(val);
     int bytes = sqlite3_value_bytes(val);
@@ -124,6 +135,11 @@ static void getDataResult(const RtlFieldInfo *field, sqlite3_value *val, size32_
 static double getRealResult(const RtlFieldInfo *field, sqlite3_value *val)
 {
     assertex(val);
+    if (isNull(val))
+    {
+        NullFieldProcessor p(field);
+        return p.doubleResult;
+    }
     if (sqlite3_value_type(val) != SQLITE_FLOAT)
         typeError("real", field);
     return sqlite3_value_double(val);
@@ -132,6 +148,11 @@ static double getRealResult(const RtlFieldInfo *field, sqlite3_value *val)
 static __int64 getSignedResult(const RtlFieldInfo *field, sqlite3_value *val)
 {
     assertex(val);
+    if (isNull(val))
+    {
+        NullFieldProcessor p(field);
+        return p.intResult;
+    }
     if (sqlite3_value_type(val) != SQLITE_INTEGER)
         typeError("integer", field);
     return sqlite3_value_int64(val);
@@ -140,6 +161,11 @@ static __int64 getSignedResult(const RtlFieldInfo *field, sqlite3_value *val)
 static unsigned __int64 getUnsignedResult(const RtlFieldInfo *field, sqlite3_value *val)
 {
     assertex(val);
+    if (isNull(val))
+    {
+        NullFieldProcessor p(field);
+        return p.uintResult;
+    }
     if (sqlite3_value_type(val) != SQLITE_INTEGER)
         typeError("integer", field);
     return (unsigned __int64) sqlite3_value_int64(val);
@@ -148,6 +174,12 @@ static unsigned __int64 getUnsignedResult(const RtlFieldInfo *field, sqlite3_val
 static void getStringResult(const RtlFieldInfo *field, sqlite3_value *val, size32_t &chars, char * &result)
 {
     assertex(val);
+    if (isNull(val))
+    {
+        NullFieldProcessor p(field);
+        rtlStrToStrX(chars, result, p.resultChars, p.stringResult);
+        return;
+    }
     if (sqlite3_value_type(val) != SQLITE_TEXT)
         typeError("string", field);
     const char *text = (const char *) sqlite3_value_text(val);
@@ -159,6 +191,12 @@ static void getStringResult(const RtlFieldInfo *field, sqlite3_value *val, size3
 static void getUTF8Result(const RtlFieldInfo *field, sqlite3_value *val, size32_t &chars, char * &result)
 {
     assertex(val);
+    if (isNull(val))
+    {
+        NullFieldProcessor p(field);
+        rtlUtf8ToUtf8X(chars, result, p.resultChars, p.stringResult);
+        return;
+    }
     if (sqlite3_value_type(val) != SQLITE_TEXT)
         typeError("string", field);
     const char *text = (const char *) sqlite3_value_text(val);
@@ -170,10 +208,16 @@ static void getUTF8Result(const RtlFieldInfo *field, sqlite3_value *val, size32_
 static void getUnicodeResult(const RtlFieldInfo *field, sqlite3_value *val, size32_t &chars, UChar * &result)
 {
     assertex(val);
+    if (isNull(val))
+    {
+        NullFieldProcessor p(field);
+        rtlUnicodeToUnicodeX(chars, result, p.resultChars, p.unicodeResult);
+        return;
+    }
     if (sqlite3_value_type(val) != SQLITE_TEXT)
         typeError("string", field);
     const UChar *text = (const UChar *) sqlite3_value_text16(val);
-    int bytes = sqlite3_value_bytes(val);
+    int bytes = sqlite3_value_bytes16(val);
     unsigned numchars = bytes / sizeof(UChar);
     rtlUnicodeToUnicodeX(chars, result, numchars, text);
 }
@@ -319,7 +363,8 @@ protected:
 class SqLite3EmbedFunctionContext : public CInterfaceOf<IEmbedFunctionContext>
 {
 public:
-    SqLite3EmbedFunctionContext(const char *options) : db(NULL)
+    SqLite3EmbedFunctionContext(unsigned _flags, const char *options)
+    : flags(_flags), db(NULL)
     {
         const char *dbname = NULL;
         StringArray opts;
@@ -347,6 +392,15 @@ public:
             sqlite3_close(db);
     }
 
+    void checkSqliteError(int rc)
+    {
+        if (rc != SQLITE_OK)
+        {
+            VStringBuffer msg("sqlite: error %d - %s", rc, sqlite3_errmsg(db));
+            rtlFail(rc, msg.str());
+        }
+    }
+
     virtual bool getBooleanResult()
     {
         return sqlite3embed::getBooleanResult(NULL, getScalarResult());
@@ -427,14 +481,26 @@ public:
     {
         checkSqliteError(sqlite3_bind_blob(stmt, findParameter(name), val, len, SQLITE_TRANSIENT));
     }
+    virtual void bindFloatParam(const char *name, float val)
+    {
+        checkSqliteError(sqlite3_bind_double(stmt, findParameter(name), (double) val));
+    }
     virtual void bindRealParam(const char *name, double val)
     {
         checkSqliteError(sqlite3_bind_double(stmt, findParameter(name), val));
     }
+    virtual void bindSignedSizeParam(const char *name, int size, __int64 val)
+    {
+        bindSignedParam(name, val);
+    }
     virtual void bindSignedParam(const char *name, __int64 val)
     {
         checkSqliteError(sqlite3_bind_int64(stmt, findParameter(name), val));
     }
+    virtual void bindUnsignedSizeParam(const char *name, int size, unsigned __int64 val)
+    {
+        bindUnsignedParam(name, val);
+    }
     virtual void bindUnsignedParam(const char *name, unsigned __int64 val)
     {
         checkSqliteError(sqlite3_bind_int64(stmt, findParameter(name), val));
@@ -469,8 +535,9 @@ public:
     {
         throwUnexpected();
     }
-    virtual void compileEmbeddedScript(size32_t len, const char *script)
+    virtual void compileEmbeddedScript(size32_t chars, const char *script)
     {
+        size32_t len = rtlUtf8Size(chars, script);
         int rc = sqlite3_prepare_v2(db, script, len, stmt.ref(), NULL);
         checkSqliteError(rc);
     }
@@ -479,6 +546,12 @@ public:
         assertex(stmt);
         int rc = sqlite3_reset(stmt);
         checkSqliteError(rc);
+        if (flags & EFnoreturn)
+        {
+            rc = sqlite3_step(stmt);
+            if (rc != SQLITE_DONE)
+                checkSqliteError(rc);
+        }
     }
 protected:
     sqlite3_value *getScalarResult()
@@ -501,17 +574,18 @@ protected:
     }
     OwnedStatement stmt;
     sqlite3 *db;
+    unsigned flags;
 };
 
 class SqLite3EmbedContext : public CInterfaceOf<IEmbedContext>
 {
 public:
-    virtual IEmbedFunctionContext *createFunctionContext(bool isImport, const char *options)
+    virtual IEmbedFunctionContext *createFunctionContext(unsigned flags, const char *options)
     {
-        if (isImport)
+        if (flags & EFimport)
             UNSUPPORTED("IMPORT");
         else
-            return new SqLite3EmbedFunctionContext(options);
+            return new SqLite3EmbedFunctionContext(flags, options);
     }
 };
 

plugins/v8embed/v8embed.cpp

@@ -59,6 +59,13 @@ extern "C" EXPORT bool getECLPluginDefinition(ECLPluginDefinitionBlock *pb)
     return true;
 }
 
+static void UNSUPPORTED(const char *feature) __attribute__((noreturn));
+
+static void UNSUPPORTED(const char *feature)
+{
+    throw MakeStringException(-1, "UNSUPPORTED feature: %s not supported in v8embed plugin", feature);
+}
+
 static void typeError(const char *expected, const RtlFieldInfo *field) __attribute__((noreturn));
 
 static void typeError(const char *expected, const RtlFieldInfo *field)
@@ -470,17 +477,30 @@ public:
         }
         context->Global()->Set(v8::String::New(name), array);
     }
+    virtual void bindFloatParam(const char *name, float val)
+    {
+        v8::HandleScope handle_scope;
+        context->Global()->Set(v8::String::New(name), v8::Number::New(val));
+    }
     virtual void bindRealParam(const char *name, double val)
     {
         v8::HandleScope handle_scope;
         context->Global()->Set(v8::String::New(name), v8::Number::New(val));
     }
+    virtual void bindSignedSizeParam(const char *name, int size, __int64 val)
+    {
+        bindSignedParam(name, val);
+    }
     virtual void bindSignedParam(const char *name, __int64 val)
     {
         // MORE - might need to check does not overflow 32 bits? Or store as a real?
         v8::HandleScope handle_scope;
         context->Global()->Set(v8::String::New(name), v8::Integer::New(val));
     }
+    virtual void bindUnsignedSizeParam(const char *name, int size, unsigned __int64 val)
+    {
+        bindUnsignedParam(name, val);
+    }
     virtual void bindUnsignedParam(const char *name, unsigned __int64 val)
     {
         // MORE - might need to check does not overflow 32 bits
@@ -911,9 +931,10 @@ public:
     V8JavascriptEmbedContext()
     {
     }
-    virtual IEmbedFunctionContext *createFunctionContext(bool isImport, const char *options)
+    virtual IEmbedFunctionContext *createFunctionContext(unsigned flags, const char *options)
     {
-        assertex(!isImport);
+        if (flags & EFimport)
+            UNSUPPORTED("IMPORT");
         if (!theFunctionContext)
         {
             theFunctionContext = new V8JavascriptEmbedFunctionContext;

rtl/eclrtl/eclrtl.hpp

@@ -811,11 +811,17 @@ interface IEmbedFunctionContext : extends IInterface
     virtual size32_t getTransformResult(ARowBuilder & builder) = 0;
     virtual void bindRowParam(const char *name, IOutputMetaData & metaVal, byte *val) = 0;
     virtual void bindDatasetParam(const char *name, IOutputMetaData & metaVal, IRowStream * val) = 0;
+
+    virtual void bindFloatParam(const char *name, float val) = 0;
+    virtual void bindSignedSizeParam(const char *name, int size, __int64 val) = 0;
+    virtual void bindUnsignedSizeParam(const char *name, int size, unsigned __int64 val) = 0;
 };
 
+enum EmbedFlags { EFembed = 1, EFimport = 2, EFnoreturn = 4, EFnoparams = 8 }; // For createFunctionContext flags
+
 interface IEmbedContext : extends IInterface
 {
-    virtual IEmbedFunctionContext *createFunctionContext(bool isImport, const char *options) = 0;
+    virtual IEmbedFunctionContext *createFunctionContext(unsigned flags, const char *options) = 0;
     // MORE - add syntax checked here!
 };
 

rtl/eclrtl/rtlembed.hpp

@@ -0,0 +1,141 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2014 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#ifndef rtlfield_hpp
+#define rtlfield_hpp
+
+// NOTE - not included from generated code (see rtlfield_imp.hpp)
+
+#include "eclrtl.hpp"
+#include "nbcd.hpp"
+
+class NullFieldProcessor : public CInterfaceOf<IFieldProcessor>
+{
+public:
+    NullFieldProcessor(const RtlFieldInfo * field)
+    : intResult(0), uintResult(0), boolResult(false), doubleResult(0),
+      unicodeResult(NULL), stringResult(NULL), resultChars(0), fieldExpected(field)
+    {
+        if (field && field->initializer)
+        {
+            field->process(field->initializer, field->initializer, *this);
+            assertex(fieldExpected==NULL);
+        }
+    }
+    virtual void processString(unsigned len, const char *value, const RtlFieldInfo * field)
+    {
+        checkExpected(field);
+        stringResult = value;
+        resultChars = len;
+    }
+    virtual void processBool(bool value, const RtlFieldInfo * field)
+    {
+        checkExpected(field);
+        boolResult = value;
+    }
+    virtual void processData(unsigned len, const void *value, const RtlFieldInfo * field)
+    {
+        checkExpected(field);
+        stringResult = (const char *) value;
+        resultChars = len;
+    }
+    virtual void processInt(__int64 value, const RtlFieldInfo * field)
+    {
+        checkExpected(field);
+        intResult = value;
+    }
+    virtual void processUInt(unsigned __int64 value, const RtlFieldInfo * field)
+    {
+        checkExpected(field);
+        uintResult = value;
+    }
+    virtual void processReal(double value, const RtlFieldInfo * field)
+    {
+        checkExpected(field);
+        doubleResult = value;
+    }
+    virtual void processDecimal(const void *value, unsigned digits, unsigned precision, const RtlFieldInfo * field)
+    {
+        checkExpected(field);
+        decimalResult.setDecimal(digits, precision, value);
+    }
+    virtual void processUDecimal(const void *value, unsigned digits, unsigned precision, const RtlFieldInfo * field)
+    {
+        checkExpected(field);
+        decimalResult.setDecimal(digits, precision, value);
+    }
+    virtual void processUnicode(unsigned chars, const UChar *value, const RtlFieldInfo * field)
+    {
+        checkExpected(field);
+        unicodeResult = value;
+        resultChars = chars;
+    }
+    virtual void processQString(unsigned len, const char *value, const RtlFieldInfo * field)
+    {
+        checkExpected(field);
+        stringResult = value;
+        resultChars = len;
+    }
+    virtual void processUtf8(unsigned chars, const char *value, const RtlFieldInfo * field)
+    {
+        checkExpected(field);
+        stringResult = value;
+        resultChars = chars;
+    }
+
+    virtual bool processBeginSet(const RtlFieldInfo * field, unsigned numElements, bool isAll, const byte *data)
+    {
+        UNIMPLEMENTED;
+    }
+    virtual bool processBeginDataset(const RtlFieldInfo * field, unsigned numRows)
+    {
+        throwUnexpected();
+    }
+    virtual bool processBeginRow(const RtlFieldInfo * field)
+    {
+        throwUnexpected();
+    }
+    virtual void processEndSet(const RtlFieldInfo * field)
+    {
+        throwUnexpected();
+    }
+    virtual void processEndDataset(const RtlFieldInfo * field)
+    {
+        throwUnexpected();
+    }
+    virtual void processEndRow(const RtlFieldInfo * field)
+    {
+    }
+    __int64 intResult;
+    __uint64 uintResult;
+    bool boolResult;
+    double doubleResult;
+    Decimal decimalResult;
+    const char *stringResult;
+    const UChar *unicodeResult;
+    unsigned resultChars;
+protected:
+    const RtlFieldInfo * fieldExpected;
+    void checkExpected(const RtlFieldInfo * field)
+    {
+        assertex(field==fieldExpected);
+        fieldExpected = NULL;
+    }
+
+};
+
+#endif

rtl/eclrtl/rtlfield.cpp

@@ -1446,8 +1446,8 @@ size32_t RtlUnimplementedTypeInfo::toXML(const byte * self, const byte * selfrow
 
 //-------------------------------------------------------------------------------------------------------------------
 
-RtlFieldStrInfo::RtlFieldStrInfo(const char * _name, const char * _xpath, const RtlTypeInfo * _type) 
-: RtlFieldInfo(rtlCreateFieldNameAtom(_name), _xpath, _type) 
+RtlFieldStrInfo::RtlFieldStrInfo(const char * _name, const char * _xpath, const RtlTypeInfo * _type, const char *_initializer)
+: RtlFieldInfo(rtlCreateFieldNameAtom(_name), _xpath, _type, _initializer)
 {
 }
 

rtl/eclrtl/rtlfield_imp.hpp

@@ -315,7 +315,7 @@ public:
 
 struct ECLRTL_API RtlFieldStrInfo : public RtlFieldInfo
 {
-    RtlFieldStrInfo(const char * _name, const char * _xpath, const RtlTypeInfo * _type);
+    RtlFieldStrInfo(const char * _name, const char * _xpath, const RtlTypeInfo * _type, const char * _initializer = NULL);
 };
 
 

rtl/include/eclhelper.hpp

@@ -39,8 +39,8 @@ if the supplied pointer was not from the roxiemem heap. Usually an OwnedRoxieStr
 
 //Should be incremented whenever the virtuals in the context or a helper are changed, so
 //that a work unit can't be rerun.  Try as hard as possible to retain compatibility.
-#define ACTIVITY_INTERFACE_VERSION      155
-#define MIN_ACTIVITY_INTERFACE_VERSION  155             //minimum value that is compatible with current interface - without using selectInterface
+#define ACTIVITY_INTERFACE_VERSION      156
+#define MIN_ACTIVITY_INTERFACE_VERSION  156             //minimum value that is compatible with current interface - without using selectInterface
 
 typedef unsigned char byte;
 
@@ -381,11 +381,13 @@ public:
 //Core struct used for representing meta for a field.
 struct RtlFieldInfo
 {
-    inline RtlFieldInfo(IAtom * _name, const char * _xpath, const RtlTypeInfo * _type) : name(_name), xpath(_xpath), type(_type) {}
+    inline RtlFieldInfo(IAtom * _name, const char * _xpath, const RtlTypeInfo * _type, const char *_initializer = NULL)
+    : name(_name), xpath(_xpath), type(_type), initializer((const byte *) _initializer) {}
 
     IAtom * name;
     const char * xpath;
     const RtlTypeInfo * type;
+    const byte *initializer;
 
     inline bool isFixedSize() const 
     { 

testing/regress/ecl/cassandra-simple.ecl

@@ -0,0 +1,210 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2014 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+IMPORT cassandra;
+
+/*
+ This example illustrates various calls to embdded Cassandra CQL code
+ */
+
+// This is the record structure in ECL that will correspond to the rows in the Cassabdra dataset
+// Note that the default values specified in the fields will be used when a NULL value is being
+// returned from Cassandra
+
+childrec := RECORD
+   string name,
+   integer4 value { default(99999) },
+   boolean boolval { default(true) },
+   real8 r8 {default(99.99)},
+   real4 r4 {default(999.99)},
+   DATA d {default (D'999999')},
+   DECIMAL10_2 ddd {default(9.99)},
+   UTF8 u1 {default(U'9999 ß')},
+   UNICODE8 u2 {default(U'9999 ßßßß')}
+END;
+
+// Some data we will use to initialize the Cassandra table
+
+init := DATASET([{'name1', 1, true, 1.2, 3.4, D'aa55aa55', 1234567.89, U'Straße', U'Straße'},
+                 {'name2', 2, false, 5.6, 7.8, D'00', -1234567.89, U'là', U'là'}], childrec);
+
+init2 := ROW({'name4' , 3, true, 9.10, 11.12, D'aa55aa55', 987.65, U'Baße', U'Baße'}, childrec);
+
+// Set up the Cassandra database
+// Note that we can execute multiple statements in a single embed, provided that there are
+// no parameters and no result type
+
+createks() := EMBED(cassandra : user('rchapman'))
+  CREATE KEYSPACE IF NOT EXISTS test WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '3' } ;
+ENDEMBED;
+
+createTables() := EMBED(cassandra : user('rchapman'),keyspace('test'))
+  DROP TABLE IF EXISTS tbl1;
+
+  CREATE TABLE tbl1 ( name VARCHAR, value INT, boolval boolean , r8 DOUBLE, r4 FLOAT, d BLOB, ddd VARCHAR, u1 VARCHAR, u2 VARCHAR, PRIMARY KEY (name) );
+  CREATE INDEX tbl1_value  ON tbl1 (value);
+  CREATE INDEX tbl1_boolval  ON tbl1 (boolval);
+  INSERT INTO tbl1 (name, u1) values ('nulls', 'ß');  // Creates some null fields. Also note that query is utf8
+ENDEMBED;
+
+// Initialize the Cassandra table, passing in the ECL dataset to provide the rows
+// Note the batch option to control how cassandra inserts are batched
+// If not supplied, each insert is executed individually - because Cassandra
+// has restrictions about what can be done in a batch, we can't default to using batch
+// unless told to...
+
+initialize(dataset(childrec) values) := EMBED(cassandra : user('rchapman'),keyspace('test'),batch('unlogged'))
+  INSERT INTO tbl1 (name, value, boolval, r8, r4,d,ddd,u1,u2) values (?,?,?,?,?,?,?,?,?);
+ENDEMBED;
+
+initialize2(row(childrec) values) := EMBED(cassandra : user('rchapman'),keyspace('test'))
+  INSERT INTO tbl1 (name, value, boolval, r8, r4,d,ddd,u1,u2) values (?,?,?,?,?,?,?,?,?);
+ENDEMBED;
+
+// Returning a dataset
+
+dataset(childrec) testCassandraDS() := EMBED(cassandra : user('rchapman'),keyspace('test'))
+  SELECT name, value, boolval, r8, r4,d,ddd,u1,u2 from tbl1;
+ENDEMBED;
+
+// Returning a single row
+
+childrec testCassandraRow() := EMBED(cassandra : user('rchapman'),keyspace('test'))
+  SELECT name, value, boolval, r8, r4,d,ddd,u1,u2 from tbl1 LIMIT 1;
+ENDEMBED;
+
+// Passing in parameters
+
+testCassandraParms(
+   string name,
+   integer4 value,
+   boolean boolval,
+   real8 r8,
+   real4 r4,
+   DATA d,
+//   DECIMAL10_2 ddd,
+   UTF8 u1,
+   UNICODE8 u2) := EMBED(cassandra : user('rchapman'),keyspace('test'))
+  INSERT INTO tbl1 (name, value, boolval, r8, r4,d,ddd,u1,u2) values (?,?,?,?,?,?,'8.76543',?,?);
+ENDEMBED;
+
+// Returning scalars
+
+string testCassandraString() := EMBED(cassandra : user('rchapman'),keyspace('test'))
+  SELECT name from tbl1 LIMIT 1;
+ENDEMBED;
+
+dataset(childrec) testCassandraStringParam(string filter) := EMBED(cassandra : user('rchapman'),keyspace('test'))
+  SELECT name, value, boolval, r8, r4,d,ddd,u1,u2 from tbl1 where name = ?;
+ENDEMBED;
+
+integer testCassandraInt() := EMBED(cassandra : user('rchapman'),keyspace('test'))
+  SELECT value from tbl1 LIMIT 1;
+ENDEMBED;
+
+boolean testCassandraBool() := EMBED(cassandra : user('rchapman'),keyspace('test'))
+  SELECT boolval from tbl1 WHERE name='name1';
+ENDEMBED;
+
+real8 testCassandraReal8() := EMBED(cassandra : user('rchapman'),keyspace('test'))
+  SELECT r8 from tbl1 WHERE name='name1';
+ENDEMBED;
+
+real4 testCassandraReal4() := EMBED(cassandra : user('rchapman'),keyspace('test'))
+  SELECT r4 from tbl1 WHERE name='name1';
+ENDEMBED;
+
+data testCassandraData() := EMBED(cassandra : user('rchapman'),keyspace('test'))
+  SELECT d from tbl1 WHERE name='name1';
+ENDEMBED;
+
+UTF8 testCassandraUtf8() := EMBED(cassandra : user('rchapman'),keyspace('test'))
+  SELECT u1 from tbl1 WHERE name='name1';
+ENDEMBED;
+
+UNICODE testCassandraUnicode() := EMBED(cassandra : user('rchapman'),keyspace('test'))
+  SELECT u2 from tbl1 WHERE name='name1';
+ENDEMBED;
+
+// Coding a TRANSFORM to call Cassandra - this ends up acting a little like a join
+
+stringrec := RECORD
+   string name
+END;
+
+TRANSFORM(childrec) t(stringrec L) := EMBED(cassandra : user('rchapman'),keyspace('test'))
+  SELECT name, value, boolval, r8, r4,d,ddd,u1,u2 from tbl1 where name = ?;
+ENDEMBED;
+
+init3 := DATASET([{'name1'},
+                  {'name2'}], stringrec);
+
+testCassandraTransform := PROJECT(init3, t(LEFT));
+
+// Passing in AND returning a dataset - this also ends up acting a bit like a keyed join...
+
+stringrec extractName(childrec l) := TRANSFORM
+  SELF := l;
+END;
+
+dataset(childrec) testCassandraDSParam(dataset(stringrec) inrecs) := EMBED(cassandra : user('rchapman'),keyspace('test'))
+  SELECT name, value, boolval, r8, r4,d,ddd,u1,u2 from tbl1 where name = ?;
+ENDEMBED;
+
+// Testing performance of batch inserts
+
+s1 :=DATASET(25000, TRANSFORM(childrec, SELF.name := 'name'+COUNTER,
+                                         SELF.value:=COUNTER,
+                                         SELF.boolval:=COUNTER % 2 =1,
+                                         SELF.r8:=COUNTER*1.00001,
+                                         SELF.r4:=COUNTER*1.001,
+                                         SELF:=[]));
+
+testCassandraBulk := initialize(s1);
+
+// Check that 25000 got inserted
+
+integer testCassandraCount() := EMBED(cassandra : user('rchapman'),keyspace('test'))
+  SELECT COUNT(*) from tbl1;
+ENDEMBED;
+
+
+// Execute the tests
+
+sequential (
+  createks(),
+  createTables(),
+  initialize(init),
+
+  testCassandraParms('name3', 1, true, 1.2, 3.4, D'aa55aa55', U'Straße', U'Straße'),
+  initialize2(init2),
+  OUTPUT(SORT(testCassandraDS(), name)),
+  OUTPUT(testCassandraRow().name),
+  OUTPUT(testCassandraString()),
+  OUTPUT(testCassandraStringParam(testCassandraString())),
+  OUTPUT(testCassandraInt()),
+  OUTPUT(testCassandraBool()),
+  OUTPUT(testCassandraReal8()),
+  OUTPUT(testCassandraReal4()),
+  OUTPUT(testCassandraData()),
+  OUTPUT(testCassandraUtf8()),
+  OUTPUT(testCassandraUnicode()),
+  OUTPUT(testCassandraTransform()),
+  OUTPUT(testCassandraDSParam(PROJECT(init, extractName(LEFT)))),
+  testCassandraBulk,
+  OUTPUT(testCassandraCount())
+);

testing/regress/ecl/key/cassandra-simple.xml

@@ -0,0 +1,46 @@
+<Dataset name='Result 1'>
+ <Row><name>name1</name><value>1</value><boolval>true</boolval><r8>1.2</r8><r4>3.400000095367432</r4><d>6161353561613535</d><ddd>1234567.89</ddd><u1>Straße</u1><u2>Straße  </u2></Row>
+ <Row><name>name2</name><value>2</value><boolval>false</boolval><r8>5.6</r8><r4>7.800000190734863</r4><d>3030</d><ddd>-1234567.89</ddd><u1>là</u1><u2>là      </u2></Row>
+ <Row><name>name3</name><value>1</value><boolval>true</boolval><r8>1.2</r8><r4>3.400000095367432</r4><d>6161353561613535</d><ddd>8.76</ddd><u1>Straße</u1><u2>Straße  </u2></Row>
+ <Row><name>name4</name><value>3</value><boolval>true</boolval><r8>9.1</r8><r4>11.11999988555908</r4><d>6161353561613535</d><ddd>987.65</ddd><u1>Baße</u1><u2>Baße    </u2></Row>
+ <Row><name>nulls</name><value>99999</value><boolval>true</boolval><r8>99.98999999999999</r8><r4>999.989990234375</r4><d>393939393939</d><ddd>9.99</ddd><u1>ß</u1><u2>9999 ßßß</u2></Row>
+</Dataset>
+<Dataset name='Result 2'>
+ <Row><Result_2>name1</Result_2></Row>
+</Dataset>
+<Dataset name='Result 3'>
+ <Row><Result_3>name1</Result_3></Row>
+</Dataset>
+<Dataset name='Result 4'>
+ <Row><name>name1</name><value>1</value><boolval>true</boolval><r8>1.2</r8><r4>3.400000095367432</r4><d>6161353561613535</d><ddd>1234567.89</ddd><u1>Straße</u1><u2>Straße  </u2></Row>
+</Dataset>
+<Dataset name='Result 5'>
+ <Row><Result_5>1</Result_5></Row>
+</Dataset>
+<Dataset name='Result 6'>
+ <Row><Result_6>true</Result_6></Row>
+</Dataset>
+<Dataset name='Result 7'>
+ <Row><Result_7>1.2</Result_7></Row>
+</Dataset>
+<Dataset name='Result 8'>
+ <Row><Result_8>3.400000095367432</Result_8></Row>
+</Dataset>
+<Dataset name='Result 9'>
+ <Row><Result_9>6161353561613535</Result_9></Row>
+</Dataset>
+<Dataset name='Result 10'>
+ <Row><Result_10>Straße</Result_10></Row>
+</Dataset>
+<Dataset name='Result 11'>
+ <Row><Result_11>Straße  </Result_11></Row>
+</Dataset>
+<Dataset name='Result 12'>
+ <Row><name>name1</name><value>1</value><boolval>true</boolval><r8>1.2</r8><r4>3.400000095367432</r4><d>6161353561613535</d><ddd>1234567.89</ddd><u1>Straße</u1><u2>Straße  </u2></Row>
+ <Row><name>name2</name><value>2</value><boolval>false</boolval><r8>5.6</r8><r4>7.800000190734863</r4><d>3030</d><ddd>-1234567.89</ddd><u1>là</u1><u2>là      </u2></Row>
+</Dataset>
+<Dataset name='Result 13'>
+</Dataset>
+<Dataset name='Result 14'>
+ <Row><Result_14>25001</Result_14></Row>
+</Dataset>

testing/regress/ecl/key/sqlite.xml

@@ -0,0 +1,35 @@
+<Dataset name='Result 1'>
+ <Row><name>name1</name><value>1</value><boolval>true</boolval><r8>1.2</r8><r4>3.400000095367432</r4><d>6161353561613535</d><ddd>1234567.89</ddd><u1>Straße</u1><u2>Straße  </u2></Row>
+ <Row><name>nulls</name><value>99999</value><boolval>true</boolval><r8>99.98999999999999</r8><r4>999.989990234375</r4><d>393939393939</d><ddd>0</ddd><u1>9999 ß</u1><u2>9999 ßßß</u2></Row>
+ <Row><name>name2</name><value>1</value><boolval>true</boolval><r8>1.2</r8><r4>3.400000095367432</r4><d>6161353561613535</d><ddd>23.45</ddd><u1>Straßaße</u1><u2>Straßßßß</u2></Row>
+</Dataset>
+<Dataset name='Result 2'>
+ <Row><Result_2></Result_2></Row>
+</Dataset>
+<Dataset name='Result 3'>
+ <Row><Result_3>nulls</Result_3></Row>
+</Dataset>
+<Dataset name='Result 4'>
+ <Row><name>nulls</name><value>99999</value><boolval>true</boolval><r8>99.98999999999999</r8><r4>999.989990234375</r4><d>393939393939</d><ddd>0</ddd><u1>9999 ß</u1><u2>9999 ßßß</u2></Row>
+</Dataset>
+<Dataset name='Result 5'>
+ <Row><Result_5>1</Result_5></Row>
+</Dataset>
+<Dataset name='Result 6'>
+ <Row><Result_6>true</Result_6></Row>
+</Dataset>
+<Dataset name='Result 7'>
+ <Row><Result_7>1.2</Result_7></Row>
+</Dataset>
+<Dataset name='Result 8'>
+ <Row><Result_8>3.400000095367432</Result_8></Row>
+</Dataset>
+<Dataset name='Result 9'>
+ <Row><Result_9>6161353561613535</Result_9></Row>
+</Dataset>
+<Dataset name='Result 10'>
+ <Row><Result_10>Straße</Result_10></Row>
+</Dataset>
+<Dataset name='Result 11'>
+ <Row><Result_11>Straßßßß</Result_11></Row>
+</Dataset>

testing/regress/ecl/mysqlembed.ecl

@@ -19,14 +19,14 @@ IMPORT mysql;
 
 childrec := RECORD
    string name,
-   integer value,
-   boolean boolval,
-   real8 r8,
-   real4 r4,
-   DATA d,
-   DECIMAL10_2 ddd,
-   UTF8 u1,
-   UNICODE8 u2
+   integer4 value { default(99999) },
+   boolean boolval { default(true) },
+   real8 r8 {default(99.99)},
+   real4 r4 {default(999.99)},
+   DATA d {default (D'999999')},
+   DECIMAL10_2 ddd {default(9.99)},
+   UTF8 u1 {default(U'9999 ß')},
+   UNICODE8 u2 {default(U'9999 ßßßß')}
 END;
 
 stringrec := RECORD
@@ -56,6 +56,10 @@ initializeNulls() := EMBED(mysql : user('rchapman'),database('test'))
   INSERT INTO tbl1 (name) values ('nulls');
 ENDEMBED;
 
+initializeUtf8() := EMBED(mysql : user('rchapman'),database('test'))
+  INSERT INTO tbl1 values ('utf8test', 1, 1, 1.2, 3.4, 'aa55aa55', 1234567.89, 'Straße', 'Straße');
+ENDEMBED;
+
 dataset(childrec) testMySQLDS() := EMBED(mysql : user('rchapman'),database('test'))
   SELECT * from tbl1;
 ENDEMBED;
@@ -121,6 +125,7 @@ sequential (
   create(),
   initialize(init),
   initializeNulls(),
+  initializeUtf8(),
   OUTPUT(testMySQLDS()),
   OUTPUT(testMySQLRow().name),
   OUTPUT(testMySQLParms('name1', 1, true, 1.2, 3.4, D'aa55aa55', U'Straße', U'Straße')),

testing/regress/ecl/sqlite.ecl

@@ -0,0 +1,129 @@
+IMPORT SqLite3;
+
+/*
+ This example illustrates various calls to embdded SQLite code
+ */
+
+// This is the record structure in ECL that will correspond to the rows in the SQLite dataset
+// Note that the default values specified in the fields will be used when a NULL value is being
+// returned from MySQL
+
+childrec := RECORD
+   string name,
+   integer4 value { default(99999) },
+   boolean boolval { default(true) },
+   real8 r8 {default(99.99)},
+   real4 r4 {default(999.99)},
+   DATA d {default (D'999999')},
+   DECIMAL10_2 ddd {default(9.99)},
+   UTF8 u1 {default(U'9999 ß')},
+   UNICODE8 u2 {default(U'9999 ßßßß')}
+END;
+
+// Set up the SQLite database
+
+drop() := EMBED(SqLite3 : file('test.db'))
+  DROP TABLE IF EXISTS tbl1;
+ENDEMBED;
+
+create() := EMBED(SqLite3 : file('test.db'))
+  CREATE TABLE tbl1 ( name VARCHAR(20), value INT, boolval TINYINT, r8 DOUBLE, r4 FLOAT, d BLOB, ddd DECIMAL(10,2), u1 VARCHAR(10), u2 VARCHAR(10) );
+ENDEMBED;
+
+// Add a row
+
+initialize() := EMBED(SqLite3 : file('test.db'))
+  INSERT INTO tbl1 values ('name1', 1, 1, 1.2, 3.4, 'aa55aa55', 1234567.89, 'Straße', 'Straße');
+ENDEMBED;
+
+// Add an additional row containing NULL values, to test that they are handled properly when read in ECL
+
+initializeNulls() := EMBED(SqLite3 : file('test.db'))
+  INSERT INTO tbl1 (name) values ('nulls');
+ENDEMBED;
+
+// Test various types of parameters
+
+testSqLiteParms(
+   string name,
+   integer4 value,
+   boolean boolval,
+   real8 r8,
+   real4 r4,
+   DATA d,
+   REAL8 ddd, // Decimal parms not supported.
+   UTF8 u1,
+   UNICODE8 u2) := EMBED(SqLite3 : file('test.db'))
+  INSERT INTO tbl1 (name, value, boolval, r8, r4,d,ddd,u1,u2) values (:name, :value, :boolval, :r8, :r4,:d,:ddd,:u1,:u2);
+ENDEMBED;
+
+// Returning a dataset
+
+dataset(childrec) testSQLiteDS() := EMBED(SqLite3 : file('test.db'))
+  SELECT * from tbl1;
+ENDEMBED;
+
+// Returning a single row
+
+childrec testSQLiteRow() := EMBED(SqLite3 : file('test.db'))
+  SELECT * from tbl1 LIMIT 1;
+ENDEMBED;
+
+// Returning scalars
+
+string testSQLiteString() := EMBED(SqLite3 : file('test.db'))
+  SELECT max(name) from tbl1;
+ENDEMBED;
+
+dataset(childrec) testSQLiteStringParam(string filter) := EMBED(SqLite3 : file('test.db'))
+  SELECT * from tbl1 where name = :filter;
+ENDEMBED;
+
+integer testSQLiteInt() := EMBED(SqLite3 : file('test.db'))
+  SELECT max(value) from tbl1;
+ENDEMBED;
+
+boolean testSQLiteBool() := EMBED(SqLite3 : file('test.db'))
+  SELECT max(boolval) from tbl1;
+ENDEMBED;
+
+real8 testSQLiteReal8() := EMBED(SqLite3 : file('test.db'))
+  SELECT max(r8) from tbl1;
+ENDEMBED;
+
+real4 testSQLiteReal4() := EMBED(SqLite3 : file('test.db'))
+  SELECT max(r4) from tbl1;
+ENDEMBED;
+
+data testSQLiteData() := EMBED(SqLite3 : file('test.db'))
+  SELECT max(d) from tbl1;
+ENDEMBED;
+
+UTF8 testSQLiteUtf8() := EMBED(SqLite3 : file('test.db'))
+  SELECT max(u1) from tbl1;
+ENDEMBED;
+
+UNICODE testSQLiteUnicode() := EMBED(SqLite3 : file('test.db'))
+  SELECT max(u2) from tbl1;
+ENDEMBED;
+
+// Run the tests
+
+sequential (
+  drop(),
+  create(),
+  initialize(),
+  initializeNulls(),
+  testSQLiteParms('name2', 1, true, 1.2, 3.4, D'aa55aa55', 23.45, U'Straßaße', U'Straßßßße'),
+  OUTPUT(testSQLiteDS()),
+  OUTPUT(testSQLiteRow().name),
+  OUTPUT(testSQLiteString()),
+  OUTPUT(testSQLiteStringParam(testSQLiteString())),
+  OUTPUT(testSQLiteInt()),
+  OUTPUT(testSQLiteBool()),
+  OUTPUT(testSQLiteReal8()),
+  OUTPUT(testSQLiteReal4()),
+  OUTPUT(testSQLiteData()),
+  OUTPUT(testSQLiteUtf8()),
+  OUTPUT(testSQLiteUnicode())
+);