Merge pull request #3915 from richardkchapman/embedding

HPCC-8720 Add support for SET OF simpletype to embed plugins

Reviewed-by: Gavin Halliday <ghalliday@hpccsystems.com>

cmake_modules/FindJNI.cmake

@@ -0,0 +1,310 @@
+# - Find JNI java libraries.
+# This module finds if Java is installed and determines where the
+# include files and libraries are. It also determines what the name of
+# the library is. This code sets the following variables:
+#
+#  JNI_INCLUDE_DIRS      = the include dirs to use
+#  JNI_LIBRARIES         = the libraries to use
+#  JNI_FOUND             = TRUE if JNI headers and libraries were found.
+#  JAVA_AWT_LIBRARY      = the path to the jawt library
+#  JAVA_JVM_LIBRARY      = the path to the jvm library
+#  JAVA_INCLUDE_PATH     = the include path to jni.h
+#  JAVA_INCLUDE_PATH2    = the include path to jni_md.h
+#  JAVA_AWT_INCLUDE_PATH = the include path to jawt.h
+#
+
+# Derived from the version included with CMake 2.8.9, with fixes for
+# the location that openjdk installs to on Ubuntu 12.10
+#=============================================================================
+#CMake - Cross Platform Makefile Generator
+#Copyright 2000-2011 Kitware, Inc., Insight Software Consortium
+#All rights reserved.
+#
+#Redistribution and use in source and binary forms, with or without
+#modification, are permitted provided that the following conditions
+#are met:
+#
+#* Redistributions of source code must retain the above copyright
+#  notice, this list of conditions and the following disclaimer.
+#
+#* Redistributions in binary form must reproduce the above copyright
+#  notice, this list of conditions and the following disclaimer in the
+#  documentation and/or other materials provided with the distribution.
+#
+#* Neither the names of Kitware, Inc., the Insight Software Consortium,
+#  nor the names of their contributors may be used to endorse or promote
+#  products derived from this software without specific prior written
+#  permission.
+#
+#THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+#"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+#LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+#A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+#HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+#SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+#LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+#DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+#THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+#(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+#OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+#------------------------------------------------------------------------------
+#
+#The above copyright and license notice applies to distributions of
+#CMake in source and binary form.  Some source files contain additional
+#notices of original copyright by their contributors; see each source
+#for details.  Third-party software packages supplied with CMake under
+#compatible licenses provide their own copyright notices documented in
+#corresponding subdirectories.
+#
+#------------------------------------------------------------------------------
+#
+#CMake was initially developed by Kitware with the following sponsorship:
+#
+# * National Library of Medicine at the National Institutes of Health
+#   as part of the Insight Segmentation and Registration Toolkit (ITK).
+#
+# * US National Labs (Los Alamos, Livermore, Sandia) ASC Parallel
+#   Visualization Initiative.
+#
+# * National Alliance for Medical Image Computing (NAMIC) is funded by the
+#   National Institutes of Health through the NIH Roadmap for Medical Research,
+#   Grant U54 EB005149.
+#
+# * Kitware, Inc.
+#=============================================================================
+
+# Expand {libarch} occurences to java_libarch subdirectory(-ies) and set ${_var}
+MACRO(java_append_library_directories _var)
+    # Determine java arch-specific library subdir
+    # Mostly based on openjdk/jdk/make/common/shared/Platform.gmk as of openjdk
+    # 1.6.0_18 + icedtea patches. However, it would be much better to base the
+    # guess on the first part of the GNU config.guess platform triplet.
+    IF(CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64")
+        SET(_java_libarch "amd64")
+    ELSEIF(CMAKE_SYSTEM_PROCESSOR MATCHES "^i.86$")
+        SET(_java_libarch "i386")
+    ELSEIF(CMAKE_SYSTEM_PROCESSOR MATCHES "^alpha")
+        SET(_java_libarch "alpha")
+    ELSEIF(CMAKE_SYSTEM_PROCESSOR MATCHES "^arm")
+        # Subdir is "arm" for both big-endian (arm) and little-endian (armel).
+        SET(_java_libarch "arm")
+    ELSEIF(CMAKE_SYSTEM_PROCESSOR MATCHES "^mips")
+        # mips* machines are bi-endian mostly so processor does not tell
+        # endianess of the underlying system.
+        SET(_java_libarch "${CMAKE_SYSTEM_PROCESSOR}" "mips" "mipsel" "mipseb")
+    ELSEIF(CMAKE_SYSTEM_PROCESSOR MATCHES "^(powerpc|ppc)64")
+        SET(_java_libarch "ppc64")
+    ELSEIF(CMAKE_SYSTEM_PROCESSOR MATCHES "^(powerpc|ppc)")
+        SET(_java_libarch "ppc")
+    ELSEIF(CMAKE_SYSTEM_PROCESSOR MATCHES "^sparc")
+        # Both flavours can run on the same processor
+        SET(_java_libarch "${CMAKE_SYSTEM_PROCESSOR}" "sparc" "sparcv9")
+    ELSEIF(CMAKE_SYSTEM_PROCESSOR MATCHES "^(parisc|hppa)")
+        SET(_java_libarch "parisc" "parisc64")
+    ELSEIF(CMAKE_SYSTEM_PROCESSOR MATCHES "^s390")
+        # s390 binaries can run on s390x machines
+        SET(_java_libarch "${CMAKE_SYSTEM_PROCESSOR}" "s390" "s390x")
+    ELSEIF(CMAKE_SYSTEM_PROCESSOR MATCHES "^sh")
+        SET(_java_libarch "sh")
+    ELSE(CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64")
+        SET(_java_libarch "${CMAKE_SYSTEM_PROCESSOR}")
+    ENDIF(CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64")
+
+    # Append default list architectures if CMAKE_SYSTEM_PROCESSOR was empty or
+    # system is non-Linux (where the code above has not been well tested)
+    IF(NOT _java_libarch OR NOT (CMAKE_SYSTEM_NAME MATCHES "Linux"))
+        LIST(APPEND _java_libarch "i386" "amd64" "ppc")
+    ENDIF(NOT _java_libarch OR NOT (CMAKE_SYSTEM_NAME MATCHES "Linux"))
+
+    # Sometimes ${CMAKE_SYSTEM_PROCESSOR} is added to the list to prefer
+    # current value to a hardcoded list. Remove possible duplicates.
+    LIST(REMOVE_DUPLICATES _java_libarch)
+
+    FOREACH(_path ${ARGN})
+        IF(_path MATCHES "{libarch}")
+            FOREACH(_libarch ${_java_libarch})
+                STRING(REPLACE "{libarch}" "${_libarch}" _newpath "${_path}")
+                LIST(APPEND ${_var} "${_newpath}")
+            ENDFOREACH(_libarch)
+        ELSE(_path MATCHES "{libarch}")
+            LIST(APPEND ${_var} "${_path}")
+        ENDIF(_path MATCHES "{libarch}")
+    ENDFOREACH(_path)
+ENDMACRO(java_append_library_directories)
+
+GET_FILENAME_COMPONENT(java_install_version
+  "[HKEY_LOCAL_MACHINE\\SOFTWARE\\JavaSoft\\Java Development Kit;CurrentVersion]" NAME)
+
+SET(JAVA_AWT_LIBRARY_DIRECTORIES
+  "[HKEY_LOCAL_MACHINE\\SOFTWARE\\JavaSoft\\Java Development Kit\\1.4;JavaHome]/lib"
+  "[HKEY_LOCAL_MACHINE\\SOFTWARE\\JavaSoft\\Java Development Kit\\1.3;JavaHome]/lib"
+  "[HKEY_LOCAL_MACHINE\\SOFTWARE\\JavaSoft\\Java Development Kit\\${java_install_version};JavaHome]/lib"
+  )
+
+FILE(TO_CMAKE_PATH "$ENV{JAVA_HOME}" _JAVA_HOME)
+
+JAVA_APPEND_LIBRARY_DIRECTORIES(JAVA_AWT_LIBRARY_DIRECTORIES
+  ${_JAVA_HOME}/jre/lib/{libarch}
+  ${_JAVA_HOME}/jre/lib
+  ${_JAVA_HOME}/lib
+  ${_JAVA_HOME}
+  /usr/lib
+  /usr/local/lib
+  /usr/lib/jvm/java/lib
+  /usr/lib/java/jre/lib/{libarch}
+  /usr/lib/jvm/jre/lib/{libarch}
+  /usr/local/lib/java/jre/lib/{libarch}
+  /usr/local/share/java/jre/lib/{libarch}
+  /usr/lib/j2sdk1.4-sun/jre/lib/{libarch}
+  /usr/lib/j2sdk1.5-sun/jre/lib/{libarch}
+  /opt/sun-jdk-1.5.0.04/jre/lib/{libarch}
+  /usr/lib/jvm/java-6-sun/jre/lib/{libarch}
+  /usr/lib/jvm/java-1.5.0-sun/jre/lib/{libarch}
+  /usr/lib/jvm/java-6-sun-1.6.0.00/jre/lib/{libarch}       # can this one be removed according to #8821 ? Alex
+  /usr/lib/jvm/java-6-openjdk-{libarch}/jre/lib/{libarch}  # Ubuntu 12.10 location
+  /usr/lib/jvm/java-6-openjdk/jre/lib/{libarch}
+  /usr/lib/jvm/java-1.6.0-openjdk-1.6.0.0/jre/lib/{libarch}        # fedora
+  # Debian specific paths for default JVM
+  /usr/lib/jvm/default-java/jre/lib/{libarch}
+  /usr/lib/jvm/default-java/jre/lib
+  /usr/lib/jvm/default-java/lib
+  )
+
+SET(JAVA_JVM_LIBRARY_DIRECTORIES)
+FOREACH(dir ${JAVA_AWT_LIBRARY_DIRECTORIES})
+  SET(JAVA_JVM_LIBRARY_DIRECTORIES
+    ${JAVA_JVM_LIBRARY_DIRECTORIES}
+    "${dir}"
+    "${dir}/client"
+    "${dir}/server"
+    )
+ENDFOREACH(dir)
+
+
+SET(JAVA_AWT_INCLUDE_DIRECTORIES
+  "[HKEY_LOCAL_MACHINE\\SOFTWARE\\JavaSoft\\Java Development Kit\\1.4;JavaHome]/include"
+  "[HKEY_LOCAL_MACHINE\\SOFTWARE\\JavaSoft\\Java Development Kit\\1.3;JavaHome]/include"
+  "[HKEY_LOCAL_MACHINE\\SOFTWARE\\JavaSoft\\Java Development Kit\\${java_install_version};JavaHome]/include"
+  ${_JAVA_HOME}/include
+  /usr/include
+  /usr/local/include
+  /usr/lib/java/include
+  /usr/local/lib/java/include
+  /usr/lib/jvm/java/include
+  /usr/lib/jvm/java-6-sun/include
+  /usr/lib/jvm/java-1.5.0-sun/include
+  /usr/lib/jvm/java-6-sun-1.6.0.00/include       # can this one be removed according to #8821 ? Alex
+  /usr/lib/jvm/java-6-openjdk-amd64/jre/lib/{libarch}  # Ubuntu 12.10 location
+  /usr/lib/jvm/java-6-openjdk/include
+  /usr/local/share/java/include
+  /usr/lib/j2sdk1.4-sun/include
+  /usr/lib/j2sdk1.5-sun/include
+  /opt/sun-jdk-1.5.0.04/include
+  # Debian specific path for default JVM
+  /usr/lib/jvm/default-java/include
+  )
+
+FOREACH(JAVA_PROG "${JAVA_RUNTIME}" "${JAVA_COMPILE}" "${JAVA_ARCHIVE}")
+  GET_FILENAME_COMPONENT(jpath "${JAVA_PROG}" PATH)
+  FOREACH(JAVA_INC_PATH ../include ../java/include ../share/java/include)
+    IF(EXISTS ${jpath}/${JAVA_INC_PATH})
+      SET(JAVA_AWT_INCLUDE_DIRECTORIES ${JAVA_AWT_INCLUDE_DIRECTORIES} "${jpath}/${JAVA_INC_PATH}")
+    ENDIF(EXISTS ${jpath}/${JAVA_INC_PATH})
+  ENDFOREACH(JAVA_INC_PATH)
+  FOREACH(JAVA_LIB_PATH
+    ../lib ../jre/lib ../jre/lib/i386
+    ../java/lib ../java/jre/lib ../java/jre/lib/i386
+    ../share/java/lib ../share/java/jre/lib ../share/java/jre/lib/i386)
+    IF(EXISTS ${jpath}/${JAVA_LIB_PATH})
+      SET(JAVA_AWT_LIBRARY_DIRECTORIES ${JAVA_AWT_LIBRARY_DIRECTORIES} "${jpath}/${JAVA_LIB_PATH}")
+    ENDIF(EXISTS ${jpath}/${JAVA_LIB_PATH})
+  ENDFOREACH(JAVA_LIB_PATH)
+ENDFOREACH(JAVA_PROG)
+
+IF(APPLE)
+  IF(EXISTS ~/Library/Frameworks/JavaVM.framework)
+    SET(JAVA_HAVE_FRAMEWORK 1)
+  ENDIF(EXISTS ~/Library/Frameworks/JavaVM.framework)
+  IF(EXISTS /Library/Frameworks/JavaVM.framework)
+    SET(JAVA_HAVE_FRAMEWORK 1)
+  ENDIF(EXISTS /Library/Frameworks/JavaVM.framework)
+  IF(EXISTS /System/Library/Frameworks/JavaVM.framework)
+    SET(JAVA_HAVE_FRAMEWORK 1)
+  ENDIF(EXISTS /System/Library/Frameworks/JavaVM.framework)
+
+  IF(JAVA_HAVE_FRAMEWORK)
+    IF(NOT JAVA_AWT_LIBRARY)
+      SET (JAVA_AWT_LIBRARY "-framework JavaVM" CACHE FILEPATH "Java Frameworks" FORCE)
+    ENDIF(NOT JAVA_AWT_LIBRARY)
+
+    IF(NOT JAVA_JVM_LIBRARY)
+      SET (JAVA_JVM_LIBRARY "-framework JavaVM" CACHE FILEPATH "Java Frameworks" FORCE)
+    ENDIF(NOT JAVA_JVM_LIBRARY)
+
+    IF(NOT JAVA_AWT_INCLUDE_PATH)
+      IF(EXISTS /System/Library/Frameworks/JavaVM.framework/Headers/jawt.h)
+        SET (JAVA_AWT_INCLUDE_PATH "/System/Library/Frameworks/JavaVM.framework/Headers" CACHE FILEPATH "jawt.h location" FORCE)
+      ENDIF(EXISTS /System/Library/Frameworks/JavaVM.framework/Headers/jawt.h)
+    ENDIF(NOT JAVA_AWT_INCLUDE_PATH)
+
+    # If using "-framework JavaVM", prefer its headers *before* the others in
+    # JAVA_AWT_INCLUDE_DIRECTORIES... (*prepend* to the list here)
+    #
+    SET(JAVA_AWT_INCLUDE_DIRECTORIES
+      ~/Library/Frameworks/JavaVM.framework/Headers
+      /Library/Frameworks/JavaVM.framework/Headers
+      /System/Library/Frameworks/JavaVM.framework/Headers
+      ${JAVA_AWT_INCLUDE_DIRECTORIES}
+      )
+  ENDIF(JAVA_HAVE_FRAMEWORK)
+ELSE(APPLE)
+  FIND_LIBRARY(JAVA_AWT_LIBRARY jawt
+    PATHS ${JAVA_AWT_LIBRARY_DIRECTORIES}
+  )
+  FIND_LIBRARY(JAVA_JVM_LIBRARY NAMES jvm JavaVM
+    PATHS ${JAVA_JVM_LIBRARY_DIRECTORIES}
+  )
+ENDIF(APPLE)
+
+# add in the include path
+FIND_PATH(JAVA_INCLUDE_PATH jni.h
+  ${JAVA_AWT_INCLUDE_DIRECTORIES}
+)
+
+FIND_PATH(JAVA_INCLUDE_PATH2 jni_md.h
+  ${JAVA_INCLUDE_PATH}
+  ${JAVA_INCLUDE_PATH}/win32
+  ${JAVA_INCLUDE_PATH}/linux
+  ${JAVA_INCLUDE_PATH}/freebsd
+  ${JAVA_INCLUDE_PATH}/solaris
+  ${JAVA_INCLUDE_PATH}/hp-ux
+  ${JAVA_INCLUDE_PATH}/alpha
+)
+
+FIND_PATH(JAVA_AWT_INCLUDE_PATH jawt.h
+  ${JAVA_INCLUDE_PATH}
+)
+
+#INCLUDE(${CMAKE_CURRENT_LIST_DIR}/FindPackageHandleStandardArgs.cmake)
+FIND_PACKAGE_HANDLE_STANDARD_ARGS(JNI  DEFAULT_MSG  JAVA_AWT_LIBRARY JAVA_JVM_LIBRARY
+                                                    JAVA_INCLUDE_PATH  JAVA_INCLUDE_PATH2 JAVA_AWT_INCLUDE_PATH)
+
+MARK_AS_ADVANCED(
+  JAVA_AWT_LIBRARY
+  JAVA_JVM_LIBRARY
+  JAVA_AWT_INCLUDE_PATH
+  JAVA_INCLUDE_PATH
+  JAVA_INCLUDE_PATH2
+)
+
+SET(JNI_LIBRARIES
+  ${JAVA_AWT_LIBRARY}
+  ${JAVA_JVM_LIBRARY}
+)
+
+SET(JNI_INCLUDE_DIRS
+  ${JAVA_INCLUDE_PATH}
+  ${JAVA_INCLUDE_PATH2}
+  ${JAVA_AWT_INCLUDE_PATH}
+)

ecl/hql/hqlatoms.cpp

@@ -52,6 +52,7 @@ _ATOM bestAtom;
 _ATOM bindBooleanParamAtom;
 _ATOM bindDataParamAtom;
 _ATOM bindRealParamAtom;
+_ATOM bindSetParamAtom;
 _ATOM bindSignedParamAtom;
 _ATOM bindStringParamAtom;
 _ATOM bindVStringParamAtom;
@@ -149,6 +150,7 @@ _ATOM getEmbedContextAtom;
 _ATOM getBooleanResultAtom;
 _ATOM getDataResultAtom;
 _ATOM getRealResultAtom;
+_ATOM getSetResultAtom;
 _ATOM getSignedResultAtom;
 _ATOM getStringResultAtom;
 _ATOM getUnicodeResultAtom;
@@ -457,6 +459,7 @@ MODULE_INIT(INIT_PRIORITY_HQLATOM)
     MAKEATOM(bindBooleanParam);
     MAKEATOM(bindDataParam);
     MAKEATOM(bindRealParam);
+    MAKEATOM(bindSetParam);
     MAKEATOM(bindSignedParam);
     MAKEATOM(bindStringParam);
     MAKEATOM(bindVStringParam);
@@ -553,6 +556,7 @@ MODULE_INIT(INIT_PRIORITY_HQLATOM)
     MAKEATOM(getBooleanResult);
     MAKEATOM(getDataResult);
     MAKEATOM(getRealResult);
+    MAKEATOM(getSetResult);
     MAKEATOM(getSignedResult);
     MAKEATOM(getStringResult);
     MAKEATOM(getUnicodeResult);

ecl/hql/hqlatoms.hpp

@@ -55,6 +55,7 @@ extern HQL_API _ATOM bestAtom;
 extern HQL_API _ATOM bindBooleanParamAtom;
 extern HQL_API _ATOM bindDataParamAtom;
 extern HQL_API _ATOM bindRealParamAtom;
+extern HQL_API _ATOM bindSetParamAtom;
 extern HQL_API _ATOM bindSignedParamAtom;
 extern HQL_API _ATOM bindStringParamAtom;
 extern HQL_API _ATOM bindVStringParamAtom;
@@ -151,6 +152,7 @@ extern HQL_API _ATOM getEmbedContextAtom;
 extern HQL_API _ATOM getBooleanResultAtom;
 extern HQL_API _ATOM getDataResultAtom;
 extern HQL_API _ATOM getRealResultAtom;
+extern HQL_API _ATOM getSetResultAtom;
 extern HQL_API _ATOM getSignedResultAtom;
 extern HQL_API _ATOM getStringResultAtom;
 extern HQL_API _ATOM getUnicodeResultAtom;

ecl/hql/hqlgram2.cpp

@@ -849,7 +849,7 @@ IHqlExpression * HqlGram::processEmbedBody(const attribute & errpos, IHqlExpress
         if (!matchesBoolean(checkSupport, true))
             reportError(ERR_PluginNoScripting, errpos, "Module %s does not support %s", moduleName->getAtomNamePtr(), isImport ? "import" : "script");
         OwnedHqlExpr syntaxCheckFunc = pluginScope->lookupSymbol(syntaxCheckAtom, LSFpublic, lookupCtx);
-        if (syntaxCheckFunc && !importAtom)
+        if (syntaxCheckFunc && !isImport)
         {
             // MORE - create an expression that calls it, and const fold it, I guess....
         }
@@ -9996,6 +9996,7 @@ static void getTokenText(StringBuffer & msg, int token)
     case ECLCRC: msg.append("ECLCRC"); break;
     case ELSE: msg.append("ELSE"); break;
     case ELSEIF: msg.append("ELSEIF"); break;
+    case EMBED: msg.append("EMBED"); break;
     case EMBEDDED: msg.append("EMBEDDED"); break;
     case _EMPTY_: msg.append("_EMPTY_"); break;
     case ENCODING: msg.append("ENCODING"); break;
@@ -10003,6 +10004,7 @@ static void getTokenText(StringBuffer & msg, int token)
     case ENCRYPTED: msg.append("ENCRYPTED"); break;
     case END: msg.append("END"); break;
     case ENDCPP: msg.append("ENDCPP"); break;
+    case ENDEMBED: msg.append("ENDEMBED"); break;
     case ENTH: msg.append("ENTH"); break;
     case ENUM: msg.append("ENUM"); break;
     case TOK_ERROR: msg.append("ERROR"); break;

ecl/hqlcpp/hqlcpp.cpp

@@ -11320,8 +11320,8 @@ static IHqlExpression * replaceInlineParameters(IHqlExpression * funcdef, IHqlEx
     ForEachChild(i, formals)
     {
         IHqlExpression * param = formals->queryChild(i);
-        OwnedHqlExpr formal = createActualFromFormal(param);
-        simpleTransformer.setMapping(param, formal);
+        OwnedHqlExpr actual = createActualFromFormal(param);
+        simpleTransformer.setMapping(param, actual);
     }
 
     return simpleTransformer.transformRoot(expr);
@@ -11498,6 +11498,17 @@ void HqlCppTranslator::buildScriptFunctionDefinition(BuildCtx &funcctx, IHqlExpr
         case type_data:
             bindFunc = bindDataParamAtom;
             break;
+        case type_set:
+        {
+            bindFunc = bindSetParamAtom;
+            ITypeInfo *childType = paramType->queryChildType();
+            type_t typeCode = childType->getTypeCode();
+            if (childType->isInteger() && !childType->isSigned())
+                typeCode = type_unsigned;
+            args.append(*createIntConstant(typeCode));
+            args.append(*createIntConstant(childType->getSize()));
+            break;
+        }
         default:
             StringBuffer typeText;
             getFriendlyTypeStr(paramType, typeText);
@@ -11508,6 +11519,8 @@ void HqlCppTranslator::buildScriptFunctionDefinition(BuildCtx &funcctx, IHqlExpr
     }
     funcctx.addQuoted("__ctx->callFunction();");
     _ATOM returnFunc;
+    HqlExprArray retargs;
+    retargs.append(*LINK(ctxVar));
     switch (returnType->getTypeCode())
     {
     case type_int:
@@ -11532,13 +11545,23 @@ void HqlCppTranslator::buildScriptFunctionDefinition(BuildCtx &funcctx, IHqlExpr
     case type_data:
         returnFunc = getDataResultAtom;
         break;
+    case type_set:
+    {
+        returnFunc = getSetResultAtom;
+        ITypeInfo *childType = returnType->queryChildType();
+        type_t typeCode = childType->getTypeCode();
+        if (childType->isInteger() && !childType->isSigned())
+            typeCode = type_unsigned;
+        retargs.append(*createIntConstant(typeCode));
+        retargs.append(*createIntConstant(returnType->queryChildType()->getSize()));
+        break;
+    }
     default:
         StringBuffer typeText;
         getFriendlyTypeStr(returnType, typeText);
         throwError1(HQLERR_EmbeddedTypeNotSupported_X, typeText.str());
     }
-    noargs.append(*LINK(ctxVar));
-    OwnedHqlExpr call = bindFunctionCall(returnFunc, noargs);
+    OwnedHqlExpr call = bindFunctionCall(returnFunc, retargs);
     doBuildUserFunctionReturn(funcctx, returnType, call);
 }
 

ecl/hqlcpp/hqlcppsys.ecl

@@ -815,6 +815,8 @@ const char * cppSystemText[]  = {
     "   bindUTF8Param(const varstring name, const utf8 val) : method,entrypoint='bindUTF8Param';",
     "   bindUnicodeParam(const varstring name, const unicode val) : method,entrypoint='bindUnicodeParam';",
 
+    "   bindSetParam(const varstring name, integer4 typeCode, unsigned4 elemSize, const set of any val) : method,entrypoint='bindSetParam';",
+
     "   boolean getBooleanResult() : method,entrypoint='getBooleanResult';",
     "   data getDataResult() : method,entrypoint='getDataResult';",
     "   real getRealResult() : method,entrypoint='getRealResult';",
@@ -824,6 +826,8 @@ const char * cppSystemText[]  = {
     "   utf8 getUTF8Result() : method,entrypoint='getUTF8Result';",
     "   unicode getUnicodeResult() : method,entrypoint='getUnicodeResult';",
 
+    "   set of any getSetResult(integer4 typeCode, unsigned4 elemSize) : method,entrypoint='getSetResult';",
+
     "   compileEmbeddedScript(const utf8 script) : method,entrypoint='compileEmbeddedScript';",
     "   import(const utf8 script) : method,entrypoint='importFunction';",
     "   END;",

plugins/Rembed/CMakeLists.txt

@@ -39,6 +39,7 @@ include_directories (
          ./../../system/include
          ./../../rtl/eclrtl
          ./../../rtl/include
+         ./../../common/deftype
          ./../../system/jlib
     )
 

plugins/Rembed/README

@@ -0,0 +1,33 @@
+################################################################################
+#    HPCC SYSTEMS software Copyright (C) 2013 HPCC Systems.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+################################################################################
+
+Building the Rembed plugin
+--------------------------
+
+Building the plugin to embed R in ECL code is not enabled by default - this is
+because it depends on a component (RInside) that can't be installed via standard
+package-management utilities on most distros.
+
+To install the prequisites for building R support, use the following (Ubuntu 12.04)
+or the equivalent for your distro.
+
+sudo apt-get install r-base r-cran-rcpp
+wget http://cran.r-project.org/src/contrib/RInside_0.2.10.tar.gz
+sudo R CMD INSTALL RInside_0.2.10.tar.gz
+
+Then you can enable building the Rembed plugin using
+
+cmake <path_to_source> -DMAKE_REMBED=1

plugins/Rembed/Rembed.cpp

@@ -17,10 +17,12 @@
 
 #include "platform.h"
 #include "RInside.h"
-#include "eclrtl.hpp"
 #include "jexcept.hpp"
 #include "jthread.hpp"
 #include "hqlplugins.hpp"
+#include "deftype.hpp"
+#include "eclrtl.hpp"
+#include "eclrtl_imp.hpp"
 
 #ifdef _WIN32
 #define EXPORT __declspec(dllexport)
@@ -131,7 +133,7 @@ public:
     }
     virtual void getDataResult(size32_t &__len, void * &__result)
     {
-        std::vector<byte> vval = ::Rcpp::as<std::vector<byte> >(result);;
+        std::vector<byte> vval = ::Rcpp::as<std::vector<byte> >(result);
         rtlStrToDataX(__len, __result, vval.size(), vval.data());
     }
     virtual double getRealResult()
@@ -159,6 +161,126 @@ public:
     {
         throw MakeStringException(MSGAUD_user, 0, "Rembed: %s: Unicode/UTF8 results not supported", func.c_str());
     }
+    virtual void getSetResult(bool & __isAllResult, size32_t & __resultBytes, void * & __result, int _elemType, size32_t elemSize)
+    {
+        type_t elemType = (type_t) _elemType;
+        __isAllResult = false;
+        switch(elemType)
+        {
+
+#define FETCH_ARRAY(type) \
+{  \
+    std::vector<type> vval = ::Rcpp::as< std::vector<type> >(result); \
+    rtlStrToDataX(__resultBytes, __result, vval.size()*elemSize, (const void *) vval.data()); \
+}
+
+        case type_boolean:
+        {
+            std::vector<bool> vval = ::Rcpp::as< std::vector<bool> >(result);
+            size32_t size = vval.size();
+            // Vector of bool is odd, and can't be retrieved via data()
+            // Instead we need to iterate, I guess
+            rtlDataAttr out(size);
+            bool *outData = (bool *) out.getdata();
+            for (std::vector<bool>::iterator iter = vval.begin(); iter < vval.end(); iter++)
+            {
+                *outData++ = *iter;
+            }
+            __resultBytes = size;
+            __result = out.detachdata();
+            break;
+        }
+        case type_int:
+            /* if (elemSize == sizeof(signed char))  // rcpp does not seem to support...
+                FETCH_ARRAY(signed char)
+            else */ if (elemSize == sizeof(short))
+                FETCH_ARRAY(short)
+            else if (elemSize == sizeof(int))
+                FETCH_ARRAY(int)
+            else if (elemSize == sizeof(long))    // __int64 / long long does not work...
+                FETCH_ARRAY(long)
+            else
+                rtlFail(0, "Rembed: Unsupported result type");
+            break;
+        case type_unsigned:
+            if (elemSize == sizeof(byte))
+                FETCH_ARRAY(byte)
+            else if (elemSize == sizeof(unsigned short))
+                FETCH_ARRAY(unsigned short)
+            else if (elemSize == sizeof(unsigned int))
+                FETCH_ARRAY(unsigned int)
+            else if (elemSize == sizeof(unsigned long))    // __int64 / long long does not work...
+                FETCH_ARRAY(unsigned long)
+            else
+                rtlFail(0, "Rembed: Unsupported result type");
+            break;
+        case type_real:
+            if (elemSize == sizeof(float))
+                FETCH_ARRAY(float)
+            else if (elemSize == sizeof(double))
+                FETCH_ARRAY(double)
+            else
+                rtlFail(0, "Rembed: Unsupported result type");
+            break;
+        case type_string:
+        case type_varstring:
+        {
+            std::vector<std::string> vval = ::Rcpp::as< std::vector<std::string> >(result);
+            size32_t numResults = vval.size();
+            rtlRowBuilder out;
+            byte *outData = NULL;
+            size32_t outBytes = 0;
+            if (elemSize != UNKNOWN_LENGTH)
+            {
+                outBytes = numResults * elemSize;  // MORE - check for overflow?
+                out.ensureAvailable(outBytes);
+                outData = out.getbytes();
+            }
+            for (std::vector<std::string>::iterator iter = vval.begin(); iter < vval.end(); iter++)
+            {
+                size32_t lenBytes = (*iter).size();
+                const char *text = (*iter).data();
+                if (elemType == type_string)
+                {
+                    if (elemSize == UNKNOWN_LENGTH)
+                    {
+                        out.ensureAvailable(outBytes + lenBytes + sizeof(size32_t));
+                        outData = out.getbytes() + outBytes;
+                        * (size32_t *) outData = lenBytes;
+                        rtlStrToStr(lenBytes, outData+sizeof(size32_t), lenBytes, text);
+                        outBytes += lenBytes + sizeof(size32_t);
+                    }
+                    else
+                    {
+                        rtlStrToStr(elemSize, outData, lenBytes, text);
+                        outData += elemSize;
+                    }
+                }
+                else
+                {
+                    if (elemSize == UNKNOWN_LENGTH)
+                    {
+                        out.ensureAvailable(outBytes + lenBytes + 1);
+                        outData = out.getbytes() + outBytes;
+                        rtlStrToVStr(0, outData, lenBytes, text);
+                        outBytes += lenBytes + 1;
+                    }
+                    else
+                    {
+                        rtlStrToVStr(elemSize, outData, lenBytes, text);  // Fixed size null terminated strings... weird.
+                        outData += elemSize;
+                    }
+                }
+            }
+            __resultBytes = outBytes;
+            __result = out.detachdata();
+            break;
+        }
+        default:
+            rtlFail(0, "REmbed: Unsupported result type");
+            break;
+        }
+    }
 
     virtual void bindBooleanParam(const char *name, bool val)
     {
@@ -194,11 +316,98 @@ public:
     }
     virtual void bindUTF8Param(const char *name, size32_t chars, const char *val)
     {
-        UNIMPLEMENTED;
+        rtlFail(0, "Rembed: Unsupported parameter type UTF8");
     }
     virtual void bindUnicodeParam(const char *name, size32_t chars, const UChar *val)
     {
-        UNIMPLEMENTED;
+        rtlFail(0, "Rembed: Unsupported parameter type UNICODE");
+    }
+
+    virtual void bindSetParam(const char *name, int _elemType, size32_t elemSize, bool isAll, size32_t totalBytes, void *setData)
+    {
+        if (isAll)
+            rtlFail(0, "Rembed: Unsupported parameter type ALL");
+        type_t elemType = (type_t) _elemType;
+        int numElems = totalBytes / elemSize;
+        switch(elemType)
+        {
+
+#define BIND_ARRAY(type) \
+{  \
+    std::vector<type> vval; \
+    const type *start = (const type *) setData; \
+    vval.assign(start, start+numElems); \
+    R[name] = vval; \
+}
+
+        case type_boolean:
+            BIND_ARRAY(bool)
+            break;
+        case type_int:
+            /* if (elemSize == sizeof(signed char))  // No binding exists in rcpp
+                BIND_ARRAY(signed char)
+            else */ if (elemSize == sizeof(short))
+                BIND_ARRAY(short)
+            else if (elemSize == sizeof(int))
+                BIND_ARRAY(int)
+            else if (elemSize == sizeof(long))    // __int64 / long long does not work...
+                BIND_ARRAY(long)
+            else
+                rtlFail(0, "Rembed: Unsupported parameter type");
+            break;
+        case type_unsigned:
+            if (elemSize == sizeof(unsigned char))
+                BIND_ARRAY(unsigned char)
+            else if (elemSize == sizeof(unsigned short))
+                BIND_ARRAY(unsigned short)
+            else if (elemSize == sizeof(unsigned int))
+                BIND_ARRAY(unsigned int)
+            else if (elemSize == sizeof(unsigned long))    // __int64 / long long does not work...
+                BIND_ARRAY(unsigned long)
+            else
+                rtlFail(0, "Rembed: Unsupported parameter type");
+            break;
+        case type_real:
+            if (elemSize == sizeof(float))
+                BIND_ARRAY(float)
+            else if (elemSize == sizeof(double))
+                BIND_ARRAY(double)
+            else
+                rtlFail(0, "Rembed: Unsupported parameter type");
+            break;
+        case type_string:
+        case type_varstring:
+        {
+            std::vector<std::string> vval;
+            const byte *inData = (const byte *) setData;
+            const byte *endData = inData + totalBytes;
+            while (inData < endData)
+            {
+                int thisSize;
+                if (elemSize == UNKNOWN_LENGTH)
+                {
+                    if (elemType==type_varstring)
+                        thisSize = strlen((const char *) inData) + 1;
+                    else
+                    {
+                        thisSize = * (size32_t *) inData;
+                        inData += sizeof(size32_t);
+                    }
+                }
+                else
+                    thisSize = elemSize;
+                std::string s((const char *) inData, thisSize);
+                vval.push_back(s);
+                inData += thisSize;
+                numElems++;
+            }
+            R[name] = vval;
+            break;
+        }
+        default:
+            rtlFail(0, "REmbed: Unsupported parameter type");
+            break;
+        }
     }
 
     virtual void importFunction(size32_t lenChars, const char *utf)

plugins/javaembed/CMakeLists.txt

@@ -39,6 +39,7 @@ include_directories (
          ./../../system/include
          ./../../rtl/eclrtl
          ./../../rtl/include
+         ./../../common/deftype
          ./../../system/jlib
     )
 

plugins/javaembed/javaembed.cpp

@@ -17,10 +17,12 @@
 
 #include "platform.h"
 #include <jni.h>
-#include "eclrtl.hpp"
 #include "jexcept.hpp"
 #include "jthread.hpp"
 #include "hqlplugins.hpp"
+#include "deftype.hpp"
+#include "eclrtl.hpp"
+#include "eclrtl_imp.hpp"
 
 #ifdef _WIN32
 #define EXPORT __declspec(dllexport)
@@ -119,6 +121,12 @@ MODULE_EXIT()
     globalState = NULL;
 }
 
+static void checkType(type_t javatype, size32_t javasize, type_t ecltype, size32_t eclsize)
+{
+    if (javatype != ecltype || javasize != eclsize)
+        throw MakeStringException(0, "javaembed: Type mismatch"); // MORE - could provide some details!
+}
+
 // There is a singleton JavaThreadContext per thread. This allows us to
 // ensure that we can make repeated calls to a Java function efficiently.
 
@@ -150,7 +158,9 @@ public:
             jmethodID throwableToString = JNIenv->GetMethodID(throwableClass, "toString", "()Ljava/lang/String;");
             jstring cause = (jstring) JNIenv->CallObjectMethod(exception, throwableToString);
             const char *text = JNIenv->GetStringUTFChars(cause, 0);
-            throw MakeStringException(MSGAUD_user, 0, "javaembed: In method %s: %s", prevtext.get(), text);
+            VStringBuffer message("javaembed: In method %s: %s", prevtext.get(), text);
+            JNIenv->ReleaseStringUTFChars(cause, text);
+            rtlFail(0, message.str());
         }
     }
 
@@ -291,7 +301,7 @@ public:
             break;
         }
         default:
-            throwUnexpected();
+            throw MakeStringException(MSGAUD_user, 0, "javaembed: Type mismatch on result");
         }
     }
     inline void getUTF8Result(jvalue &result, size32_t &__chars, char * &__result)
@@ -311,7 +321,7 @@ public:
             break;
         }
         default:
-            throwUnexpected();
+            throw MakeStringException(MSGAUD_user, 0, "javaembed: Type mismatch on result");
         }
     }
     inline void getUnicodeResult(jvalue &result, size32_t &__chars, UChar * &__result)
@@ -332,13 +342,146 @@ public:
             break;
         }
         default:
-            throwUnexpected();
+            throw MakeStringException(MSGAUD_user, 0, "javaembed: Type mismatch on result");
         }
     }
+    inline void getSetResult(jvalue &result, bool & __isAllResult, size32_t & __resultBytes, void * & __result, int _elemType, size32_t elemSize)
+    {
+        if (returnType.get()[0] != '[')
+            throw MakeStringException(MSGAUD_user, 0, "javaembed: Type mismatch on result (array expected)");
+        type_t elemType = (type_t) _elemType;
+        jarray array = (jarray) result.l;
+        int numResults = JNIenv->GetArrayLength(array);
+        rtlRowBuilder out;
+        byte *outData = NULL;
+        size32_t outBytes = 0;
+        if (elemSize != UNKNOWN_LENGTH)
+        {
+            out.ensureAvailable(numResults * elemSize); // MORE - check for overflow?
+            outData = out.getbytes();
+        }
+        switch(returnType.get()[1])
+        {
+        case 'Z':
+            checkType(type_boolean, sizeof(jboolean), elemType, elemSize);
+            JNIenv->GetBooleanArrayRegion((jbooleanArray) array, 0, numResults, (jboolean *) outData);
+            break;
+        case 'B':
+            checkType(type_int, sizeof(jbyte), elemType, elemSize);
+            JNIenv->GetByteArrayRegion((jbyteArray) array, 0, numResults, (jbyte *) outData);
+            break;
+        case 'C':
+            // we COULD map to a set of string1, but is there any point?
+            throw MakeStringException(0, "javaembed: Return type mismatch (char[] not supported)");
+            break;
+        case 'S':
+            checkType(type_int, sizeof(jshort), elemType, elemSize);
+            JNIenv->GetShortArrayRegion((jshortArray) array, 0, numResults, (jshort *) outData);
+            break;
+        case 'I':
+            checkType(type_int, sizeof(jint), elemType, elemSize);
+            JNIenv->GetIntArrayRegion((jintArray) array, 0, numResults, (jint *) outData);
+            break;
+        case 'J':
+            checkType(type_int, sizeof(jlong), elemType, elemSize);
+            JNIenv->GetLongArrayRegion((jlongArray) array, 0, numResults, (jlong *) outData);
+            break;
+        case 'F':
+            checkType(type_real, sizeof(jfloat), elemType, elemSize);
+            JNIenv->GetFloatArrayRegion((jfloatArray) array, 0, numResults, (jfloat *) outData);
+            break;
+        case 'D':
+            checkType(type_real, sizeof(jdouble), elemType, elemSize);
+            JNIenv->GetDoubleArrayRegion((jdoubleArray) array, 0, numResults, (jdouble *) outData);
+            break;
+        case 'L':
+            if (strcmp(returnType, "[Ljava/lang/String;") == 0)
+            {
+                for (int i = 0; i < numResults; i++)
+                {
+                    jstring elem = (jstring) JNIenv->GetObjectArrayElement((jobjectArray) array, i);
+                    size_t lenBytes = JNIenv->GetStringUTFLength(elem);  // in bytes
+                    const char *text =  JNIenv->GetStringUTFChars(elem, NULL);
+
+                    switch (elemType)
+                    {
+                    case type_string:
+                        if (elemSize == UNKNOWN_LENGTH)
+                        {
+                            out.ensureAvailable(outBytes + lenBytes + sizeof(size32_t));
+                            outData = out.getbytes() + outBytes;
+                            * (size32_t *) outData = lenBytes;
+                            rtlStrToStr(lenBytes, outData+sizeof(size32_t), lenBytes, text);
+                            outBytes += lenBytes + sizeof(size32_t);
+                        }
+                        else
+                            rtlStrToStr(elemSize, outData, lenBytes, text);
+                        break;
+                    case type_varstring:
+                        if (elemSize == UNKNOWN_LENGTH)
+                        {
+                            out.ensureAvailable(outBytes + lenBytes + 1);
+                            outData = out.getbytes() + outBytes;
+                            rtlStrToVStr(0, outData, lenBytes, text);
+                            outBytes += lenBytes + 1;
+                        }
+                        else
+                            rtlStrToVStr(elemSize, outData, lenBytes, text);  // Fixed size null terminated strings... weird.
+                        break;
+                    case type_utf8:
+                    case type_unicode:
+                    {
+                        size32_t numchars = rtlUtf8Length(lenBytes, text);
+                        if (elemType == type_utf8)
+                        {
+                            assertex (elemSize == UNKNOWN_LENGTH);
+                            out.ensureAvailable(outBytes + lenBytes + sizeof(size32_t));
+                            outData = out.getbytes() + outBytes;
+                            * (size32_t *) outData = numchars;
+                            rtlStrToStr(lenBytes, outData+sizeof(size32_t), lenBytes, text);
+                            outBytes += lenBytes + sizeof(size32_t);
+                        }
+                        else
+                        {
+                            if (elemSize == UNKNOWN_LENGTH)
+                            {
+                                // You can't assume that number of chars in utf8 matches number in unicode16 ...
+                                size32_t numchars16;
+                                rtlDataAttr unicode16;
+                                rtlUtf8ToUnicodeX(numchars16, unicode16.refustr(), numchars, text);
+                                out.ensureAvailable(outBytes + numchars16*sizeof(UChar) + sizeof(size32_t));
+                                outData = out.getbytes() + outBytes;
+                                * (size32_t *) outData = numchars16;
+                                rtlUnicodeToUnicode(numchars16, (UChar *) (outData+sizeof(size32_t)), numchars16, unicode16.getustr());
+                                outBytes += numchars16*sizeof(UChar) + sizeof(size32_t);
+                            }
+                            else
+                                rtlUtf8ToUnicode(elemSize / sizeof(UChar), (UChar *) outData, numchars, text);
+                        }
+                        break;
+                    }
+                    default:
+                        JNIenv->ReleaseStringUTFChars(elem, text);
+                        throw MakeStringException(0, "javaembed: Return type mismatch (ECL string type expected)");
+                    }
+                    JNIenv->ReleaseStringUTFChars(elem, text);
+                    if (elemSize != UNKNOWN_LENGTH)
+                        outData += elemSize;
+                }
+            }
+            else
+                throw MakeStringException(0, "javaembed: Return type mismatch (%s[] not supported)", returnType.get()+2);
+            break;
+        }
+        __isAllResult = false;
+        __resultBytes = elemSize == UNKNOWN_LENGTH ? outBytes : elemSize * numResults;
+        __result = out.detachdata();
+    }
     inline const char *querySignature()
     {
         return argsig.get();
     }
+
 private:
     StringAttr returnType;
     StringAttr argsig;
@@ -395,7 +538,10 @@ public:
     {
         sharedCtx->getUnicodeResult(result, __chars, __result);
     }
-
+    virtual void getSetResult(bool & __isAllResult, size32_t & __resultBytes, void * & __result, int elemType, size32_t elemSize)
+    {
+        sharedCtx->getSetResult(result, __isAllResult, __resultBytes, __result, elemType, elemSize);
+    }
 
     virtual void bindBooleanParam(const char *name, bool val)
     {
@@ -544,6 +690,161 @@ public:
         }
         addArg(v);
     }
+    virtual void bindSetParam(const char *name, int _elemType, size32_t elemSize, bool isAll, size32_t totalBytes, void *setData)
+    {
+        jvalue v;
+        if (*argsig != '[')
+            typeError("SET");
+        argsig++;
+        type_t elemType = (type_t) _elemType;
+        int numElems = totalBytes / elemSize;
+        switch(*argsig)
+        {
+        case 'Z':
+            checkType(type_boolean, sizeof(jboolean), elemType, elemSize);
+            v.l = sharedCtx->JNIenv->NewBooleanArray(numElems);
+            sharedCtx->JNIenv->SetBooleanArrayRegion((jbooleanArray) v.l, 0, numElems, (jboolean *) setData);
+            break;
+        case 'B':
+            checkType(type_int, sizeof(jbyte), elemType, elemSize);
+            v.l = sharedCtx->JNIenv->NewByteArray(numElems);
+            sharedCtx->JNIenv->SetByteArrayRegion((jbyteArray) v.l, 0, numElems, (jbyte *) setData);
+            break;
+        case 'C':
+            // we COULD map to a set of string1, but is there any point?
+            typeError("");
+            break;
+        case 'S':
+            checkType(type_int, sizeof(jshort), elemType, elemSize);
+            v.l = sharedCtx->JNIenv->NewShortArray(numElems);
+            sharedCtx->JNIenv->SetShortArrayRegion((jshortArray) v.l, 0, numElems, (jshort *) setData);
+            break;
+        case 'I':
+            checkType(type_int, sizeof(jint), elemType, elemSize);
+            v.l = sharedCtx->JNIenv->NewIntArray(numElems);
+            sharedCtx->JNIenv->SetIntArrayRegion((jintArray) v.l, 0, numElems, (jint *) setData);
+            break;
+        case 'J':
+            checkType(type_int, sizeof(jlong), elemType, elemSize);
+            v.l = sharedCtx->JNIenv->NewLongArray(numElems);
+            sharedCtx->JNIenv->SetLongArrayRegion((jlongArray) v.l, 0, numElems, (jlong *) setData);
+            break;
+        case 'F':
+            checkType(type_real, sizeof(jfloat), elemType, elemSize);
+            v.l = sharedCtx->JNIenv->NewFloatArray(numElems);
+            sharedCtx->JNIenv->SetFloatArrayRegion((jfloatArray) v.l, 0, numElems, (jfloat *) setData);
+            break;
+        case 'D':
+            checkType(type_real, sizeof(jdouble), elemType, elemSize);
+            v.l = sharedCtx->JNIenv->NewDoubleArray(numElems);
+            sharedCtx->JNIenv->SetDoubleArrayRegion((jdoubleArray) v.l, 0, numElems, (jdouble *) setData);
+            break;
+        case 'L':
+            if (strncmp(argsig, "Ljava/lang/String;", 18) == 0)
+            {
+                argsig += 17;  // Yes, 17, because we increment again at the end of the case
+                const byte *inData = (const byte *) setData;
+                const byte *endData = inData + totalBytes;
+                if (elemSize == UNKNOWN_LENGTH)
+                {
+                    numElems = 0;
+                    // Will need 2 passes to work out how many elements there are in the set :(
+                    while (inData < endData)
+                    {
+                        int thisSize;
+                        switch (elemType)
+                        {
+                        case type_varstring:
+                            thisSize = strlen((const char *) inData) + 1;
+                            break;
+                        case type_string:
+                            thisSize = * (size32_t *) inData + sizeof(size32_t);
+                            break;
+                        case type_unicode:
+                            thisSize = (* (size32_t *) inData) * sizeof(UChar) + sizeof(size32_t);
+                            break;
+                        case type_utf8:
+                            thisSize = rtlUtf8Size(* (size32_t *) inData, inData + sizeof(size32_t)) + sizeof(size32_t);;
+                            break;
+                        default:
+                            typeError("STRING");
+                        }
+                        inData += thisSize;
+                        numElems++;
+                    }
+                    inData = (const byte *) setData;
+                }
+                int idx = 0;
+                v.l = sharedCtx->JNIenv->NewObjectArray(numElems, sharedCtx->JNIenv->FindClass("java/lang/String"), NULL);
+                while (inData < endData)
+                {
+                    jstring thisElem;
+                    size32_t thisSize = elemSize;
+                    switch (elemType)
+                    {
+                    case type_varstring:
+                    {
+                        size32_t numChars = strlen((const char *) inData);
+                        unsigned unicodeChars;
+                        rtlDataAttr unicode;
+                        rtlStrToUnicodeX(unicodeChars, unicode.refustr(), numChars, (const char *) inData);
+                        thisElem = sharedCtx->JNIenv->NewString(unicode.getustr(), unicodeChars);
+                        if (elemSize == UNKNOWN_LENGTH)
+                            thisSize = numChars + 1;
+                        break;
+                    }
+                    case type_string:
+                    {
+                        if (elemSize == UNKNOWN_LENGTH)
+                        {
+                            thisSize = * (size32_t *) inData;
+                            inData += sizeof(size32_t);
+                        }
+                        unsigned unicodeChars;
+                        rtlDataAttr unicode;
+                        rtlStrToUnicodeX(unicodeChars, unicode.refustr(), thisSize, (const char *) inData);
+                        thisElem = sharedCtx->JNIenv->NewString(unicode.getustr(), unicodeChars);
+                        break;
+                    }
+                    case type_unicode:
+                    {
+                        if (elemSize == UNKNOWN_LENGTH)
+                        {
+                            thisSize = (* (size32_t *) inData) * sizeof(UChar); // NOTE - it's in chars...
+                            inData += sizeof(size32_t);
+                        }
+                        thisElem = sharedCtx->JNIenv->NewString((const UChar *) inData, thisSize / sizeof(UChar));
+                        //checkJPythonError();
+                        break;
+                    }
+                    case type_utf8:
+                    {
+                        assertex (elemSize == UNKNOWN_LENGTH);
+                        size32_t numChars = * (size32_t *) inData;
+                        inData += sizeof(size32_t);
+                        unsigned unicodeChars;
+                        rtlDataAttr unicode;
+                        rtlUtf8ToUnicodeX(unicodeChars, unicode.refustr(), numChars, (const char *) inData);
+                        thisElem = sharedCtx->JNIenv->NewString(unicode.getustr(), unicodeChars);
+                        thisSize = rtlUtf8Size(numChars, inData);
+                        break;
+                    }
+                    default:
+                        typeError("STRING");
+                    }
+                    sharedCtx->checkException();
+                    inData += thisSize;
+                    sharedCtx->JNIenv->SetObjectArrayElement((jobjectArray) v.l, idx, thisElem);
+                    idx++;
+                }
+            }
+            else
+                typeError("");
+            break;
+        }
+        argsig++;
+        addArg(v);
+    }
 
     virtual void importFunction(size32_t lenChars, const char *utf)
     {
@@ -565,6 +866,7 @@ protected:
     JavaThreadContext *sharedCtx;
     jvalue result;
 private:
+
     void typeError(const char *ECLtype)
     {
         const char *javaType;
@@ -579,6 +881,7 @@ private:
         case 'J': javaType = "long"; break;
         case 'F': javaType = "float"; break;
         case 'D': javaType = "double"; break;
+        case '[': javaType = "array"; break;
         case 'L':
             {
                 javaType = argsig+1;

plugins/pyembed/CMakeLists.txt

@@ -41,6 +41,7 @@ include_directories (
          ./../../system/include
          ./../../rtl/eclrtl
          ./../../rtl/include
+         ./../../common/deftype
          ./../../system/jlib
     )
 
@@ -57,7 +58,6 @@ else()
 endif()
 
 install ( TARGETS pyembed DESTINATION plugins )
-
 if (debug_python)
   target_link_libraries ( pyembed ${PYTHON_DEBUG_LIBRARY} )
 else()

plugins/pyembed/pyembed.cpp

@@ -17,10 +17,12 @@
 
 #include "platform.h"
 #include "Python.h"
-#include "eclrtl.hpp"
 #include "jexcept.hpp"
 #include "jthread.hpp"
 #include "hqlplugins.hpp"
+#include "deftype.hpp"
+#include "eclrtl.hpp"
+#include "eclrtl_imp.hpp"
 
 #ifdef _WIN32
 #define EXPORT __declspec(dllexport)
@@ -149,7 +151,7 @@ public:
             // Name should be in the form module.function
             const char *funcname = strrchr(text, '.');
             if (!funcname)
-                rtlFail(0, "Expected module.function");
+                rtlFail(0, "pyembed: Expected module.function");
             StringBuffer modname(funcname-text, text);
             funcname++;  // skip the '.'
             // If the modname is preceded by a path, add it to the python path before importing
@@ -172,7 +174,7 @@ public:
             script.set(PyDict_GetItemString(dict, funcname));
             checkPythonError();
             if (!script || !PyCallable_Check(script))
-                rtlFail(0, "Object is not callable");
+                rtlFail(0, "pyembed: Object is not callable");
             prevtext.set(text);
         }
         return script.getLink();
@@ -282,22 +284,22 @@ public:
 
     virtual bool getBooleanResult()
     {
-        assertex(result);
+        assertex(result && result != Py_None);
         if (!PyBool_Check(result))
-            throw MakeStringException(MSGAUD_user, 0, "pyembed: Type mismatch on result");
+            rtlFail(0, "pyembed: Type mismatch on result - value is not BOOLEAN ");
         return result == Py_True;
     }
     virtual void getDataResult(size32_t &__chars, void * &__result)
     {
         assertex(result && result != Py_None);
         if (!PyByteArray_Check(result))
-            throw MakeStringException(MSGAUD_user, 0, "pyembed: Type mismatch on result");
+            rtlFail(0, "pyembed: Type mismatch on result - value is not a bytearray");
         rtlStrToDataX(__chars, __result, PyByteArray_Size(result), PyByteArray_AsString(result));
     }
     virtual double getRealResult()
     {
         assertex(result && result != Py_None);
-        return (__int64) PyFloat_AsDouble(result);
+        return PyFloat_AsDouble(result);
     }
     virtual __int64 getSignedResult()
     {
@@ -320,7 +322,7 @@ public:
             rtlStrToStrX(__chars, __result, lenBytes, text);
         }
         else
-            rtlFail(0, "Python type mismatch - return value was not a string");
+            rtlFail(0, "pyembed: type mismatch - return value was not a string");
     }
     virtual void getUTF8Result(size32_t &__chars, char * &__result)
     {
@@ -336,7 +338,7 @@ public:
             rtlUtf8ToUtf8X(__chars, __result, numchars, text);
         }
         else
-            rtlFail(0, "Python type mismatch - return value was not a unicode string");
+            rtlFail(0, "pyembed: type mismatch - return value was not a unicode string");
     }
     virtual void getUnicodeResult(size32_t &__chars, UChar * &__result)
     {
@@ -352,69 +354,193 @@ public:
             rtlUtf8ToUnicodeX(__chars, __result, numchars, text);
         }
         else
-            rtlFail(0, "Python type mismatch - return value was not a unicode string");
+            rtlFail(0, "pyembed: type mismatch - return value was not a unicode string");
     }
-
-
-protected:
-    PythonThreadContext *sharedCtx;
-    OwnedPyObject locals;
-    OwnedPyObject globals;
-    OwnedPyObject result;
-    OwnedPyObject script;
-};
-
-class Python27EmbedScriptContext : public Python27EmbedContextBase
-{
-public:
-    Python27EmbedScriptContext(PythonThreadContext *_sharedCtx, const char *options)
-    : Python27EmbedContextBase(_sharedCtx)
-    {
-    }
-    ~Python27EmbedScriptContext()
+    virtual void getSetResult(bool & __isAllResult, size32_t & __resultBytes, void * & __result, int elemType, size32_t elemSize)
     {
+        assertex(result && result != Py_None);
+        if (!PyList_Check(result))
+            rtlFail(0, "pyembed: type mismatch - return value was not a list");
+        Py_ssize_t numResults = PyList_Size(result);
+        rtlRowBuilder out;
+        byte *outData = NULL;
+        size32_t outBytes = 0;
+        if (elemSize != UNKNOWN_LENGTH)
+        {
+            out.ensureAvailable(numResults * elemSize); // MORE - check for overflow?
+            outData = out.getbytes();
+        }
+        for (int i = 0; i < numResults; i++)
+        {
+            PyObject *elem = PyList_GetItem(result, i); // note - borrowed reference
+            switch ((type_t) elemType)
+            {
+            case type_int:
+                rtlWriteInt(outData, PyLong_AsLongLong(elem), elemSize);
+                break;
+            case type_unsigned:
+                rtlWriteInt(outData, PyLong_AsUnsignedLongLong(elem), elemSize);
+                break;
+            case type_real:
+                if (!PyFloat_Check(elem))
+                    rtlFail(0, "pyembed: type mismatch - return value in list was not a REAL");
+                if (elemSize == sizeof(double))
+                    * (double *) outData = (double) PyFloat_AsDouble(elem);
+                else
+                {
+                    assertex(elemSize == sizeof(float));
+                    * (float *) outData = (float) PyFloat_AsDouble(elem);
+                }
+                break;
+            case type_boolean:
+                assertex(elemSize == sizeof(bool));
+                if (!PyBool_Check(elem))
+                    rtlFail(0, "pyembed: type mismatch - return value in list was not a BOOLEAN");
+                * (bool *) outData = (result == Py_True);
+                break;
+            case type_string:
+            case type_varstring:
+            {
+                if (!PyString_Check(elem))
+                    rtlFail(0, "pyembed: type mismatch - return value in list was not a STRING");
+                const char * text =  PyString_AsString(elem);
+                checkPythonError();
+                size_t lenBytes = PyString_Size(elem);
+                if (elemSize == UNKNOWN_LENGTH)
+                {
+                    if (elemType == type_string)
+                    {
+                        out.ensureAvailable(outBytes + lenBytes + sizeof(size32_t));
+                        outData = out.getbytes() + outBytes;
+                        * (size32_t *) outData = lenBytes;
+                        rtlStrToStr(lenBytes, outData+sizeof(size32_t), lenBytes, text);
+                        outBytes += lenBytes + sizeof(size32_t);
+                    }
+                    else
+                    {
+                        out.ensureAvailable(outBytes + lenBytes + 1);
+                        outData = out.getbytes() + outBytes;
+                        rtlStrToVStr(0, outData, lenBytes, text);
+                        outBytes += lenBytes + 1;
+                    }
+                }
+                else
+                {
+                    if (elemType == type_string)
+                        rtlStrToStr(elemSize, outData, lenBytes, text);
+                    else
+                        rtlStrToVStr(elemSize, outData, lenBytes, text);  // Fixed size null terminated strings... weird.
+                }
+                break;
+            }
+            case type_unicode:
+            case type_utf8:
+            {
+                if (!PyUnicode_Check(elem))
+                    rtlFail(0, "pyembed: type mismatch - return value in list was not a unicode STRING");
+                OwnedPyObject utf8 = PyUnicode_AsUTF8String(elem);
+                checkPythonError();
+                size_t lenBytes = PyString_Size(utf8);
+                const char * text =  PyString_AsString(utf8);
+                checkPythonError();
+                size32_t numchars = rtlUtf8Length(lenBytes, text);
+                if (elemType == type_utf8)
+                {
+                    assertex (elemSize == UNKNOWN_LENGTH);
+                    out.ensureAvailable(outBytes + lenBytes + sizeof(size32_t));
+                    outData = out.getbytes() + outBytes;
+                    * (size32_t *) outData = numchars;
+                    rtlStrToStr(lenBytes, outData+sizeof(size32_t), lenBytes, text);
+                    outBytes += lenBytes + sizeof(size32_t);
+                }
+                else
+                {
+                    if (elemSize == UNKNOWN_LENGTH)
+                    {
+                        out.ensureAvailable(outBytes + numchars*sizeof(UChar) + sizeof(size32_t));
+                        outData = out.getbytes() + outBytes;
+                        // You can't assume that number of chars in utf8 matches number in unicode16 ...
+                        size32_t numchars16;
+                        rtlDataAttr unicode16;
+                        rtlUtf8ToUnicodeX(numchars16, unicode16.refustr(), numchars, text);
+                        * (size32_t *) outData = numchars16;
+                        rtlUnicodeToUnicode(numchars16, (UChar *) (outData+sizeof(size32_t)), numchars16, unicode16.getustr());
+                        outBytes += numchars16*sizeof(UChar) + sizeof(size32_t);
+                    }
+                    else
+                        rtlUtf8ToUnicode(elemSize / sizeof(UChar), (UChar *) outData, numchars, text);
+                }
+                break;
+            }
+            case type_data:
+            {
+                if (!PyByteArray_Check(elem))
+                    rtlFail(0, "pyembed: type mismatch - return value in list was not a bytearray");
+                size_t lenBytes = PyByteArray_Size(elem);  // Could check does not overflow size32_t
+                const char *data = PyByteArray_AsString(elem);
+                if (elemSize == UNKNOWN_LENGTH)
+                {
+                    out.ensureAvailable(outBytes + lenBytes + sizeof(size32_t));
+                    outData = out.getbytes() + outBytes;
+                    * (size32_t *) outData = lenBytes;
+                    rtlStrToData(lenBytes, outData+sizeof(size32_t), lenBytes, data);
+                    outBytes += lenBytes + sizeof(size32_t);
+                }
+                else
+                    rtlStrToData(elemSize, outData, lenBytes, data);
+                break;
+            }
+            default:
+                rtlFail(0, "pyembed: type mismatch - unsupported return type");
+            }
+            checkPythonError();
+            if (elemSize != UNKNOWN_LENGTH)
+            {
+                outData += elemSize;
+                outBytes += elemSize;
+            }
+        }
+        __isAllResult = false;
+        __resultBytes = outBytes;
+        __result = out.detachdata();
     }
+
     virtual void bindBooleanParam(const char *name, bool val)
     {
-        OwnedPyObject vval = PyBool_FromLong(val ? 1 : 0);
-        PyDict_SetItemString(locals, name, vval);
+        addArg(name, PyBool_FromLong(val ? 1 : 0));
     }
     virtual void bindDataParam(const char *name, size32_t len, const void *val)
     {
-        OwnedPyObject vval = PyByteArray_FromStringAndSize((const char *) val, len);
-        PyDict_SetItemString(locals, name, vval);
+        addArg(name, PyByteArray_FromStringAndSize((const char *) val, len));
     }
     virtual void bindRealParam(const char *name, double val)
     {
-        OwnedPyObject vval = PyFloat_FromDouble(val);
-        PyDict_SetItemString(locals, name, vval);
+        addArg(name, PyFloat_FromDouble(val));
     }
     virtual void bindSignedParam(const char *name, __int64 val)
     {
-        OwnedPyObject vval = PyLong_FromLongLong(val);
-        PyDict_SetItemString(locals, name, vval);
+        addArg(name, PyLong_FromLongLong(val));
     }
     virtual void bindUnsignedParam(const char *name, unsigned __int64 val)
     {
-        OwnedPyObject vval = PyLong_FromUnsignedLongLong(val);
-        PyDict_SetItemString(locals, name, vval);
+        addArg(name, PyLong_FromUnsignedLongLong(val));
     }
     virtual void bindStringParam(const char *name, size32_t len, const char *val)
     {
-        OwnedPyObject vval = PyString_FromStringAndSize(val, len);
-        PyDict_SetItemString(locals, name, vval);
+        addArg(name, PyString_FromStringAndSize(val, len));
     }
     virtual void bindVStringParam(const char *name, const char *val)
     {
-        OwnedPyObject vval = PyString_FromString(val);
-        PyDict_SetItemString(locals, name, vval);
+        addArg(name, PyString_FromString(val));
     }
     virtual void bindUTF8Param(const char *name, size32_t chars, const char *val)
     {
         size32_t sizeBytes = rtlUtf8Size(chars, val);
-        OwnedPyObject vval = PyUnicode_FromStringAndSize(val, sizeBytes);   // NOTE - requires size in bytes not chars
-        PyDict_SetItemString(locals, name, vval);
+        PyObject *vval = PyUnicode_FromStringAndSize(val, sizeBytes);   // NOTE - requires size in bytes not chars
+        checkPythonError();
+        addArg(name, vval);
     }
+
     virtual void bindUnicodeParam(const char *name, size32_t chars, const UChar *val)
     {
         // You don't really know what size Py_UNICODE is (varies from system to system), so go via utf8
@@ -422,11 +548,117 @@ public:
         char *unicode;
         rtlUnicodeToUtf8X(unicodeChars, unicode, chars, val);
         size32_t sizeBytes = rtlUtf8Size(unicodeChars, unicode);
-        OwnedPyObject vval = PyUnicode_FromStringAndSize(unicode, sizeBytes);   // NOTE - requires size in bytes not chars
+        PyObject *vval = PyUnicode_FromStringAndSize(unicode, sizeBytes);   // NOTE - requires size in bytes not chars
         checkPythonError();
-        PyDict_SetItemString(locals, name, vval);
+        addArg(name, vval);
         rtlFree(unicode);
     }
+    virtual void bindSetParam(const char *name, int elemType, size32_t elemSize, bool isAll, size32_t totalBytes, void *setData)
+    {
+        if (isAll)
+            rtlFail(0, "pyembed: Cannot pass ALL");
+        type_t typecode = (type_t) elemType;
+        const byte *inData = (const byte *) setData;
+        const byte *endData = inData + totalBytes;
+        OwnedPyObject vval = PyList_New(0);
+        while (inData < endData)
+        {
+            OwnedPyObject thisElem;
+            size32_t thisSize = elemSize;
+            switch (typecode)
+            {
+            case type_int:
+                thisElem.setown(PyLong_FromLongLong(rtlReadInt(inData, elemSize)));
+                break;
+            case type_unsigned:
+                thisElem.setown(PyLong_FromUnsignedLongLong(rtlReadUInt(inData, elemSize)));
+                break;
+            case type_varstring:
+            {
+                size32_t numChars = strlen((const char *) inData);
+                thisElem.setown(PyString_FromStringAndSize((const char *) inData, numChars));
+                if (elemSize == UNKNOWN_LENGTH)
+                    thisSize = numChars + 1;
+                break;
+            }
+            case type_string:
+                if (elemSize == UNKNOWN_LENGTH)
+                {
+                    thisSize = * (size32_t *) inData;
+                    inData += sizeof(size32_t);
+                }
+                thisElem.setown(PyString_FromStringAndSize((const char *) inData, thisSize));
+                break;
+            case type_real:
+                if (elemSize == sizeof(double))
+                    thisElem.setown(PyFloat_FromDouble(* (double *) inData));
+                else
+                    thisElem.setown(PyFloat_FromDouble(* (float *) inData));
+                break;
+            case type_boolean:
+                assertex(elemSize == sizeof(bool));
+                thisElem.setown(PyBool_FromLong(*(bool*)inData ? 1 : 0));
+                break;
+            case type_unicode:
+            {
+                if (elemSize == UNKNOWN_LENGTH)
+                {
+                    thisSize = (* (size32_t *) inData) * sizeof(UChar); // NOTE - it's in chars...
+                    inData += sizeof(size32_t);
+                }
+                unsigned unicodeChars;
+                rtlDataAttr unicode;
+                rtlUnicodeToUtf8X(unicodeChars, unicode.refstr(), thisSize / sizeof(UChar), (const UChar *) inData);
+                size32_t sizeBytes = rtlUtf8Size(unicodeChars, unicode.getstr());
+                thisElem.setown(PyUnicode_FromStringAndSize(unicode.getstr(), sizeBytes));   // NOTE - requires size in bytes not chars
+                checkPythonError();
+                break;
+            }
+            case type_utf8:
+            {
+                assertex (elemSize == UNKNOWN_LENGTH);
+                size32_t numChars = * (size32_t *) inData;
+                inData += sizeof(size32_t);
+                thisSize = rtlUtf8Size(numChars, inData);
+                thisElem.setown(PyUnicode_FromStringAndSize((const char *) inData, thisSize));   // NOTE - requires size in bytes not chars
+                break;
+            }
+            case type_data:
+                if (elemSize == UNKNOWN_LENGTH)
+                {
+                    thisSize = * (size32_t *) inData;
+                    inData += sizeof(size32_t);
+                }
+                thisElem.setown(PyByteArray_FromStringAndSize((const char *) inData, thisSize));
+                break;
+            }
+            checkPythonError();
+            inData += thisSize;
+            PyList_Append(vval, thisElem);
+        }
+        addArg(name, vval.getLink());
+    }
+
+protected:
+    virtual void addArg(const char *name, PyObject *arg) = 0;
+
+    PythonThreadContext *sharedCtx;
+    OwnedPyObject locals;
+    OwnedPyObject globals;
+    OwnedPyObject result;
+    OwnedPyObject script;
+};
+
+class Python27EmbedScriptContext : public Python27EmbedContextBase
+{
+public:
+    Python27EmbedScriptContext(PythonThreadContext *_sharedCtx, const char *options)
+    : Python27EmbedContextBase(_sharedCtx)
+    {
+    }
+    ~Python27EmbedScriptContext()
+    {
+    }
 
     virtual void importFunction(size32_t lenChars, const char *text)
     {
@@ -446,6 +678,14 @@ public:
         if (!result || result == Py_None)
             result.set(PyDict_GetItemString(globals, "__result__"));
     }
+protected:
+    virtual void addArg(const char *name, PyObject *arg)
+    {
+        assertex(arg);
+        PyDict_SetItemString(locals, name, arg);
+        Py_DECREF(arg);
+        checkPythonError();
+    }
 };
 
 class Python27EmbedImportContext : public Python27EmbedContextBase
@@ -459,51 +699,6 @@ public:
     ~Python27EmbedImportContext()
     {
     }
-    virtual void bindBooleanParam(const char *name, bool val)
-    {
-        addArg(PyBool_FromLong(val ? 1 : 0));
-    }
-    virtual void bindDataParam(const char *name, size32_t len, const void *val)
-    {
-        addArg(PyByteArray_FromStringAndSize((const char *) val, len));
-    }
-    virtual void bindRealParam(const char *name, double val)
-    {
-        addArg(PyFloat_FromDouble(val));
-    }
-    virtual void bindSignedParam(const char *name, __int64 val)
-    {
-        addArg(PyLong_FromLongLong(val));
-    }
-    virtual void bindUnsignedParam(const char *name, unsigned __int64 val)
-    {
-        addArg(PyLong_FromUnsignedLongLong(val));
-    }
-    virtual void bindStringParam(const char *name, size32_t len, const char *val)
-    {
-        addArg(PyString_FromStringAndSize(val, len));
-    }
-    virtual void bindVStringParam(const char *name, const char *val)
-    {
-        addArg(PyString_FromString(val));
-    }
-    virtual void bindUTF8Param(const char *name, size32_t chars, const char *val)
-    {
-        size32_t sizeBytes = rtlUtf8Size(chars, val);
-        addArg(PyUnicode_FromStringAndSize(val, sizeBytes));   // NOTE - requires size in bytes not chars
-    }
-    virtual void bindUnicodeParam(const char *name, size32_t chars, const UChar *val)
-    {
-        // You don't really know what size Py_UNICODE is (varies from system to system), so go via utf8
-        unsigned unicodeChars;
-        char *unicode;
-        rtlUnicodeToUtf8X(unicodeChars, unicode, chars, val);
-        size32_t sizeBytes = rtlUtf8Size(unicodeChars, unicode);
-        PyObject *vval = PyUnicode_FromStringAndSize(unicode, sizeBytes);   // NOTE - requires size in bytes not chars
-        checkPythonError();
-        addArg(vval);
-        rtlFree(unicode);
-    }
 
     virtual void importFunction(size32_t lenChars, const char *utf)
     {
@@ -519,7 +714,7 @@ public:
         checkPythonError();
     }
 private:
-    void addArg(PyObject *arg)
+    virtual void addArg(const char *name, PyObject *arg)
     {
         if (argcount)
             _PyTuple_Resize(args.ref(), argcount+1);

plugins/v8embed/CMakeLists.txt

@@ -40,6 +40,7 @@ include_directories (
          ./../../system/include
          ./../../rtl/eclrtl
          ./../../rtl/include
+         ./../../common/deftype
          ./../../system/jlib
     )
 

plugins/v8embed/v8embed.cpp

@@ -17,10 +17,12 @@
 
 #include "platform.h"
 #include "v8.h"
-#include "eclrtl.hpp"
 #include "jexcept.hpp"
 #include "jthread.hpp"
 #include "hqlplugins.hpp"
+#include "deftype.hpp"
+#include "eclrtl.hpp"
+#include "eclrtl_imp.hpp"
 
 #ifdef _WIN32
 #define EXPORT __declspec(dllexport)
@@ -119,10 +121,9 @@ public:
     virtual void bindStringParam(const char *name, size32_t len, const char *val)
     {
         size32_t utfCharCount;
-        char *utfText;
-        rtlStrToUtf8X(utfCharCount, utfText, len, val);
-        bindUTF8Param(name, utfCharCount, utfText);
-        rtlFree(utfText);
+        rtlDataAttr utfText;
+        rtlStrToUtf8X(utfCharCount, utfText.refstr(), len, val);
+        bindUTF8Param(name, utfCharCount, utfText.getstr());
     }
     virtual void bindVStringParam(const char *name, const char *val)
     {
@@ -138,6 +139,122 @@ public:
         v8::HandleScope handle_scope;
         context->Global()->Set(v8::String::New(name), v8::String::New(val, chars));
     }
+    virtual void bindSetParam(const char *name, int elemType, size32_t elemSize, bool isAll, size32_t totalBytes, void *setData)
+    {
+        if (isAll)
+            rtlFail(0, "v8embed: Cannot pass ALL");
+        v8::HandleScope handle_scope;
+        type_t typecode = (type_t) elemType;
+        const byte *inData = (const byte *) setData;
+        const byte *endData = inData + totalBytes;
+        int numElems;
+        if (elemSize == UNKNOWN_LENGTH)
+        {
+            numElems = 0;
+            // Will need 2 passes to work out how many elements there are in the set :(
+            while (inData < endData)
+            {
+                int thisSize;
+                switch (elemType)
+                {
+                case type_varstring:
+                    thisSize = strlen((const char *) inData) + 1;
+                    break;
+                case type_string:
+                    thisSize = * (size32_t *) inData + sizeof(size32_t);
+                    break;
+                case type_unicode:
+                    thisSize = (* (size32_t *) inData) * sizeof(UChar) + sizeof(size32_t);
+                    break;
+                case type_utf8:
+                    thisSize = rtlUtf8Size(* (size32_t *) inData, inData + sizeof(size32_t)) + sizeof(size32_t);
+                    break;
+                default:
+                    rtlFail(0, "v8embed: Unsupported parameter type");
+                    break;
+                }
+                inData += thisSize;
+                numElems++;
+            }
+            inData = (const byte *) setData;
+        }
+        else
+            numElems = totalBytes / elemSize;
+        v8::Local<v8::Array> array = v8::Array::New(numElems);
+        v8::Handle<v8::Value> thisItem;
+        size32_t thisSize = elemSize;
+        for (int idx = 0; idx < numElems; idx++)
+        {
+            switch (typecode)
+            {
+            case type_int:
+                thisItem = v8::Integer::New(rtlReadInt(inData, elemSize));
+                break;
+            case type_unsigned:
+                thisItem = v8::Integer::NewFromUnsigned(rtlReadUInt(inData, elemSize));
+                break;
+            case type_varstring:
+            {
+                size32_t numChars = strlen((const char *) inData);
+                size32_t utfCharCount;
+                rtlDataAttr utfText;
+                rtlStrToUtf8X(utfCharCount, utfText.refstr(), numChars, (const char *) inData);
+                thisItem = v8::String::New(utfText.getstr(), rtlUtf8Size(utfCharCount, utfText.getstr()));
+                if (elemSize == UNKNOWN_LENGTH)
+                    thisSize = numChars + 1;
+                break;
+            }
+            case type_string:
+            {
+                if (elemSize == UNKNOWN_LENGTH)
+                {
+                    thisSize = * (size32_t *) inData;
+                    inData += sizeof(size32_t);
+                }
+                size32_t utfCharCount;
+                rtlDataAttr utfText;
+                rtlStrToUtf8X(utfCharCount, utfText.refstr(), thisSize, (const char *) inData);
+                thisItem = v8::String::New(utfText.getstr(), rtlUtf8Size(utfCharCount, utfText.getstr()));
+                break;
+            }
+            case type_real:
+                if (elemSize == sizeof(double))
+                    thisItem = v8::Number::New(* (double *) inData);
+                else
+                    thisItem = v8::Number::New(* (float *) inData);
+                break;
+            case type_boolean:
+                assertex(elemSize == sizeof(bool));
+                thisItem = v8::Boolean::New(* (bool *) inData);
+                break;
+            case type_unicode:
+            {
+                if (elemSize == UNKNOWN_LENGTH)
+                {
+                    thisSize = (* (size32_t *) inData) * sizeof(UChar); // NOTE - it's in chars...
+                    inData += sizeof(size32_t);
+                }
+                thisItem = v8::String::New((const UChar *) inData, thisSize/sizeof(UChar));
+                break;
+            }
+            case type_utf8:
+            {
+                assertex (elemSize == UNKNOWN_LENGTH);
+                size32_t numChars = * (size32_t *) inData;
+                inData += sizeof(size32_t);
+                thisSize = rtlUtf8Size(numChars, inData);
+                thisItem = v8::String::New((const char *) inData, thisSize);
+                break;
+            }
+            default:
+                rtlFail(0, "v8embed: Unsupported parameter type");
+                break;
+            }
+            inData += thisSize;
+            array->Set(v8::Number::New(idx), thisItem);
+        }
+        context->Global()->Set(v8::String::New(name), array);
+    }
 
     virtual bool getBooleanResult()
     {
@@ -199,6 +316,133 @@ public:
         unsigned numchars = rtlUtf8Length(utf8.length(), *utf8);
         rtlUtf8ToUnicodeX(__chars, __result, numchars, *utf8);
     }
+    virtual void getSetResult(bool & __isAllResult, size32_t & __resultBytes, void * & __result, int elemType, size32_t elemSize)
+    {
+        assertex (!result.IsEmpty());
+        if (!result->IsArray())
+            rtlFail(0, "v8embed: type mismatch - return value was not an array");
+        v8::HandleScope handle_scope;
+        v8::Handle<v8::Array> array = v8::Handle<v8::Array>::Cast(result);
+        size_t numResults = array->Length();
+        rtlRowBuilder out;
+        byte *outData = NULL;
+        size32_t outBytes = 0;
+        if (elemSize != UNKNOWN_LENGTH)
+        {
+            out.ensureAvailable(numResults * elemSize); // MORE - check for overflow?
+            outData = out.getbytes();
+        }
+        for (int i = 0; i < numResults; i++)
+        {
+            v8::Local<v8::Value> elem = array->Get(i);
+            if (elem.IsEmpty())
+                rtlFail(0, "v8embed: type mismatch - empty value in returned array");
+            switch ((type_t) elemType)
+            {
+            case type_int:
+                rtlWriteInt(outData, v8::Integer::Cast(*elem)->Value(), elemSize);
+                break;
+            case type_unsigned:
+                rtlWriteInt(outData, v8::Integer::Cast(*elem)->Value(), elemSize);
+                break;
+            case type_real:
+                if (elemSize == sizeof(double))
+                    * (double *) outData = (double) v8::Number::Cast(*elem)->Value();
+                else
+                {
+                    assertex(elemSize == sizeof(float));
+                    * (float *) outData = (float) v8::Number::Cast(*elem)->Value();
+                }
+                break;
+            case type_boolean:
+                assertex(elemSize == sizeof(bool));
+                * (bool *) outData = elem->BooleanValue();
+                break;
+            case type_string:
+            case type_varstring:
+            {
+                if (!elem->IsString())
+                    rtlFail(0, "v8embed: type mismatch - return value in list was not a STRING");
+                v8::String::AsciiValue ascii(elem);
+                const char * text =  *ascii;
+                size_t lenBytes = ascii.length();
+                if (elemSize == UNKNOWN_LENGTH)
+                {
+                    if (elemType == type_string)
+                    {
+                        out.ensureAvailable(outBytes + lenBytes + sizeof(size32_t));
+                        outData = out.getbytes() + outBytes;
+                        * (size32_t *) outData = lenBytes;
+                        rtlStrToStr(lenBytes, outData+sizeof(size32_t), lenBytes, text);
+                        outBytes += lenBytes + sizeof(size32_t);
+                    }
+                    else
+                    {
+                        out.ensureAvailable(outBytes + lenBytes + 1);
+                        outData = out.getbytes() + outBytes;
+                        rtlStrToVStr(0, outData, lenBytes, text);
+                        outBytes += lenBytes + 1;
+                    }
+                }
+                else
+                {
+                    if (elemType == type_string)
+                        rtlStrToStr(elemSize, outData, lenBytes, text);
+                    else
+                        rtlStrToVStr(elemSize, outData, lenBytes, text);  // Fixed size null terminated strings... weird.
+                }
+                break;
+            }
+            case type_unicode:
+            case type_utf8:
+            {
+                if (!elem->IsString())
+                    rtlFail(0, "v8embed: type mismatch - return value in list was not a STRING");
+                v8::String::Utf8Value utf8(elem);
+                size_t lenBytes = utf8.length();
+                const char * text =  *utf8;
+                size32_t numchars = rtlUtf8Length(lenBytes, text);
+                if (elemType == type_utf8)
+                {
+                    assertex (elemSize == UNKNOWN_LENGTH);
+                    out.ensureAvailable(outBytes + lenBytes + sizeof(size32_t));
+                    outData = out.getbytes() + outBytes;
+                    * (size32_t *) outData = numchars;
+                    rtlStrToStr(lenBytes, outData+sizeof(size32_t), lenBytes, text);
+                    outBytes += lenBytes + sizeof(size32_t);
+                }
+                else
+                {
+                    if (elemSize == UNKNOWN_LENGTH)
+                    {
+                        out.ensureAvailable(outBytes + numchars*sizeof(UChar) + sizeof(size32_t));
+                        outData = out.getbytes() + outBytes;
+                        // You can't assume that number of chars in utf8 matches number in unicode16 ...
+                        size32_t numchars16;
+                        rtlDataAttr unicode16;
+                        rtlUtf8ToUnicodeX(numchars16, unicode16.refustr(), numchars, text);
+                        * (size32_t *) outData = numchars16;
+                        rtlUnicodeToUnicode(numchars16, (UChar *) (outData+sizeof(size32_t)), numchars16, unicode16.getustr());
+                        outBytes += numchars16*sizeof(UChar) + sizeof(size32_t);
+                    }
+                    else
+                        rtlUtf8ToUnicode(elemSize / sizeof(UChar), (UChar *) outData, numchars, text);
+                }
+                break;
+            }
+            default:
+                rtlFail(0, "v8embed: type mismatch - unsupported return type");
+            }
+            if (elemSize != UNKNOWN_LENGTH)
+            {
+                outData += elemSize;
+                outBytes += elemSize;
+            }
+        }
+        __isAllResult = false;
+        __resultBytes = outBytes;
+        __result = out.detachdata();
+    }
 
     virtual void compileEmbeddedScript(size32_t lenChars, const char *utf)
     {

rtl/eclrtl/eclrtl.hpp

@@ -423,6 +423,7 @@ ECLRTL_API void rtlWriteInt6(void * data, unsigned __int64 value);
 ECLRTL_API void rtlWriteInt7(void * data, unsigned __int64 value);
 inline void rtlWriteInt8(void * data, unsigned value) { *(unsigned __int64 *)data = value; }
 inline void rtlWriteSize32t(void * data, unsigned value) { *(size32_t *)data = value; }
+ECLRTL_API void rtlWriteInt(void * self, __int64 val, unsigned length);
 
 inline int rtlReadSwapInt1(const void * data) { return *(signed char *)data; }
 ECLRTL_API int rtlReadSwapInt2(const void * data);
@@ -742,6 +743,8 @@ interface IEmbedFunctionContext : extends IInterface
     virtual void bindUTF8Param(const char *name, size32_t chars, const char *val) = 0;
     virtual void bindUnicodeParam(const char *name, size32_t chars, const UChar *val) = 0;
 
+    virtual void bindSetParam(const char *name, int elemType, size32_t elemSize, bool isAll, size32_t totalBytes, void *setData) = 0;
+
     virtual bool getBooleanResult() = 0;
     virtual void getDataResult(size32_t &len, void * &result) = 0;
     virtual double getRealResult() = 0;
@@ -750,6 +753,7 @@ interface IEmbedFunctionContext : extends IInterface
     virtual void getStringResult(size32_t &len, char * &result) = 0;
     virtual void getUTF8Result(size32_t &chars, char * &result) = 0;
     virtual void getUnicodeResult(size32_t &chars, UChar * &result) = 0;
+    virtual void getSetResult(bool & __isAllResult, size32_t & __resultBytes, void * & __result, int elemType, size32_t elemSize) = 0;
 
     virtual void importFunction(size32_t len, const char *function) = 0;
     virtual void compileEmbeddedScript(size32_t len, const char *script) = 0;

rtl/eclrtl/rtlint.cpp

@@ -561,4 +561,22 @@ __int64 rtlReadSwapInt(const void * self, unsigned length)
     return 0;
 }
 
+void rtlWriteInt(void * self, __int64 val, unsigned length)
+{
+    switch (length)
+    {
+    case 1: rtlWriteInt1(self, val); break;
+    case 2: rtlWriteInt2(self, val); break;
+    case 3: rtlWriteInt3(self, val); break;
+    case 4: rtlWriteInt4(self, val); break;
+    case 5: rtlWriteInt5(self, val); break;
+    case 6: rtlWriteInt6(self, val); break;
+    case 7: rtlWriteInt7(self, val); break;
+    case 8: rtlWriteInt8(self, val); break;
+    default:
+        rtlFailUnexpected();
+        break;
+    }
+}
+
 #endif

testing/ecl/JavaCat.java

@@ -66,4 +66,17 @@ public class JavaCat
   {
     return a + b;
   }
+
+  public static int testArrays(boolean[] b, short[] s, int[] i, double[] d)
+  {
+    return b.length + s.length + i.length + d.length;
+  }
+
+  public static String[] testStringArray(String[] in)
+  {
+    String t = in[0];
+    in[0] = in[1];
+    in[1] = t;
+    return in;
+  }
 }

testing/ecl/embed.ecl

@@ -1,2 +1,13 @@
 integer c(integer val) := BEGINC++ return val-3; ENDC++;
 c(10);
+
+set of any s(set of any d) := BEGINC++
+// extern void user2(bool & __isAllResult,size32_t & __lenResult,void * & __result,bool isAllD,size32_t  lenD,void * d) {
+{
+   rtlSetToSetX(__isAllResult, __lenResult, __result, isAllD, lenD, d);
+}
+ENDC++;
+set of integer s1 := s([1,2,3]);
+set of real s2 := s([1.0,2.0,3.0]);
+s1;
+s2;

testing/ecl/embedR.ecl

@@ -13,9 +13,83 @@ val[1] = val[2];
 val;
 ENDEMBED;
 
+set of integer testSet(set of integer val) := EMBED(R)
+t = val [1];
+val[1] = val[2];
+val[2] = t;
+val;
+ENDEMBED;
+
+set of unsigned2 testSet0(set of unsigned2 val) := EMBED(R)
+sort(val);
+ENDEMBED;
+
+set of string testSet2(set of string val) := EMBED(R)
+t = val [1];
+val[1] = val[2];
+val[2] = t;
+val;
+ENDEMBED;
+
+set of string testSet3(set of string8 val) := EMBED(R)
+t = val [1];
+val[1] = val[2];
+val[2] = t;
+val;
+ENDEMBED;
+
+set of varstring testSet4(set of varstring val) := EMBED(R)
+t = val [1];
+val[1] = val[2];
+val[2] = t;
+val;
+ENDEMBED;
+
+set of varstring8 testSet5(set of varstring8 val) := EMBED(R)
+t = val [1];
+val[1] = val[2];
+val[2] = t;
+val;
+ENDEMBED;
+
+set of boolean testSet6(set of boolean val) := EMBED(R)
+t = val [1];
+val[1] = val[2];
+val[2] = t;
+val;
+ENDEMBED;
+
+set of real4 testSet7(set of real4 val) := EMBED(R)
+t = val [1];
+val[1] = val[2];
+val[2] = t;
+val;
+ENDEMBED;
+
+set of real8 testSet8(set of real8 val) := EMBED(R)
+t = val [1];
+val[1] = val[2];
+val[2] = t;
+val;
+ENDEMBED;
+
+set of integer2 testSet9(set of integer2 val) := EMBED(R)
+sort(val);
+ENDEMBED;
+
 add1(10);
 cat('Hello', 'World');
 testData(D'ab');
+testSet([1,2,3]);
+testSet0([30000,40000,50000]);
+testSet2(['one','two','three']);
+testSet3(['uno','dos','tre']);
+testSet4(['un','deux','trois']);
+testSet5(['ein','zwei','drei']);
+testSet6([false,true,false,true]);
+testSet7([1.1,2.2,3.3]);
+testSet8([1.2,2.3,3.4]);
+testSet9([-111,0,113]);
 
 s1 :=DATASET(250000, TRANSFORM({ integer a }, SELF.a := add1(COUNTER)));
 s2 :=DATASET(250000, TRANSFORM({ integer a }, SELF.a := add1(COUNTER/2)));

testing/ecl/embedjava.ecl

@@ -10,6 +10,14 @@ integer testThrow(integer p) := IMPORT(java, 'JavaCat.testThrow:(I)I');
 string addChar(string c) := IMPORT(java, 'JavaCat.addChar:(C)C');
 string cat(string s1, string s2) := IMPORT(java, 'JavaCat.cat:(Ljava/lang/String;Ljava/lang/String;)Ljava/lang/String;');
 data testData(data indata) := IMPORT(java, 'JavaCat.testData:([B)[B');
+integer testArrays(set of boolean b, set of integer2 s, set of integer4 i, set of real8 d) := IMPORT(java, 'JavaCat.testArrays:([Z[S[I[D)I');
+set of string testStringArray1(set of string s) := IMPORT(java, 'JavaCat.testStringArray:([Ljava/lang/String;)[Ljava/lang/String;');
+set of varstring testStringArray2(set of varstring s) := IMPORT(java, 'JavaCat.testStringArray:([Ljava/lang/String;)[Ljava/lang/String;');
+set of string8 testStringArray3(set of string8 s) := IMPORT(java, 'JavaCat.testStringArray:([Ljava/lang/String;)[Ljava/lang/String;');
+set of varstring8 testStringArray4(set of varstring8 s) := IMPORT(java, 'JavaCat.testStringArray:([Ljava/lang/String;)[Ljava/lang/String;');
+set of utf8 testStringArray5(set of utf8 s) := IMPORT(java, 'JavaCat.testStringArray:([Ljava/lang/String;)[Ljava/lang/String;');
+set of unicode8 testStringArray6(set of unicode8 s) := IMPORT(java, 'JavaCat.testStringArray:([Ljava/lang/String;)[Ljava/lang/String;');
+set of unicode testStringArray7(set of unicode s) := IMPORT(java, 'JavaCat.testStringArray:([Ljava/lang/String;)[Ljava/lang/String;');
 
 add1(10);
 add2('Hello');
@@ -30,6 +38,15 @@ end;
 
 catch(d(testThrow(a) = a), onfail(t));
 testData(d'aa');
+testArrays([true],[2,3],[4,5,6,7],[8.0,9.0]);
+testArrays([],[],[],[]);
+testStringArray1(['one', 'two', 'three']);
+testStringArray2(['one', 'two', 'three']);
+testStringArray3(['one', 'two', 'three']);
+testStringArray4(['one', 'two', 'three']);
+testStringArray5(['one', 'two', 'three']);
+testStringArray6(['one', 'two', 'three']);
+testStringArray7(['one', 'two', 'three']);
 
 s1 :=DATASET(250000, TRANSFORM({ integer a }, SELF.a := add1(COUNTER)));
 s2 :=DATASET(250000, TRANSFORM({ integer a }, SELF.a := add1(COUNTER/2)));

testing/ecl/embedjs.ecl

@@ -10,6 +10,75 @@ unicode add5(unicode val) := EMBED(javascript, U' val+\' at Oh là là Straße\'
 
 integer testThrow(integer val) := EMBED(javascript) throw new Error("Error from JavaScript"); ENDEMBED;
 data testData(data val) := EMBED(javascript) val[0] = val[0] + 1; val; ENDEMBED;
+set of integer testSet(set of integer val) := EMBED(javascript)
+t = val [1];
+val[1] = val[2];
+val[2] = t;
+val;
+ENDEMBED;
+
+set of unsigned2 testSet0(set of unsigned2 val) := EMBED(javascript)
+t = val [1];
+val[1] = val[2];
+val[2] = t;
+val;
+ENDEMBED;
+
+set of string testSet2(set of string val) := EMBED(javascript)
+t = val [1];
+val[1] = val[2];
+val[2] = t;
+val;
+ENDEMBED;
+
+set of string testSet3(set of string8 val) := EMBED(javascript)
+t = val [1];
+val[1] = val[2];
+val[2] = t;
+val;
+ENDEMBED;
+
+set of varstring testSet4(set of varstring val) := EMBED(javascript)
+t = val [1];
+val[1] = val[2];
+val[2] = t;
+val;
+ENDEMBED;
+
+set of varstring8 testSet5(set of varstring8 val) := EMBED(javascript)
+t = val [1];
+val[1] = val[2];
+val[2] = t;
+val;
+ENDEMBED;
+
+set of boolean testSet6(set of boolean val) := EMBED(javascript)
+t = val [1];
+val[1] = val[2];
+val[2] = t;
+val;
+ENDEMBED;
+
+set of real4 testSet7(set of real4 val) := EMBED(javascript)
+t = val [1];
+val[1] = val[2];
+val[2] = t;
+val;
+ENDEMBED;
+
+set of real8 testSet8(set of real8 val) := EMBED(javascript)
+t = val [1];
+val[1] = val[2];
+val[2] = t;
+val;
+ENDEMBED;
+
+set of integer2 testSet9(set of integer2 val) := EMBED(javascript)
+t = val [1];
+val[1] = val[2];
+val[2] = t;
+val;
+ENDEMBED;
 
 add1(10);
 add2('Hello');
@@ -30,6 +99,16 @@ end;
 
 catch(d(testThrow(a) = a), onfail(t));
 testdata(D'aa');
+testSet([1,2,3]);
+testSet0([30000,40000,50000]);
+testSet2(['one','two','three']);
+testSet3(['uno','dos','tre']);
+testSet4(['un','deux','trois']);
+testSet5(['ein','zwei','drei']);
+testSet6([false,true,false,true]);
+testSet7([1.1,2.2,3.3]);
+testSet8([1.2,2.3,3.4]);
+testSet9([-111,0,113]);
 
 s1 :=DATASET(250000, TRANSFORM({ integer a }, SELF.a := add1(COUNTER)));
 s2 :=DATASET(250000, TRANSFORM({ integer a }, SELF.a := add1(COUNTER/2)));

testing/ecl/embedpy.ecl

@@ -39,6 +39,42 @@ val[0] = val[0] + 1
 return val
 ENDEMBED;
 
+set of integer testSet(set of integer val) := EMBED(Python)
+return sorted(val)
+ENDEMBED;
+
+set of string testSet2(set of string val) := EMBED(Python)
+return sorted(val)
+ENDEMBED;
+
+set of string testSet3(set of string8 val) := EMBED(Python)
+return sorted(val)
+ENDEMBED;
+
+set of utf8 testSet4(set of utf8 val) := EMBED(Python)
+return sorted(val)
+ENDEMBED;
+
+set of varstring testSet5(set of varstring val) := EMBED(Python)
+return sorted(val)
+ENDEMBED;
+
+set of varstring8 testSet6(set of varstring8 val) := EMBED(Python)
+return sorted(val)
+ENDEMBED;
+
+set of unicode testSet7(set of unicode val) := EMBED(Python)
+return sorted(val)
+ENDEMBED;
+
+set of unicode8 testSet8(set of unicode8 val) := EMBED(Python)
+return sorted(val)
+ENDEMBED;
+
+set of data testSet9(set of data val) := EMBED(Python)
+return val
+ENDEMBED;
+
 add1(10);
 add2('Hello');
 add3('World');
@@ -60,6 +96,15 @@ end;
 
 catch(d(testThrow(a) = a), onfail(t));
 testData(D'aa');
+testSet([1,3,2]);
+testSet2(['red','green','yellow']);
+testSet3(['one','two','three']);
+testSet4([U'Oh', U'là', U'Straße']);
+testSet5(['Un','Deux','Trois']);
+testSet6(['Uno','Dos','Tre']);
+testSet7([U'On', U'der', U'Straße']);
+testSet8([U'Aus', U'zum', U'Straße']);
+testSet9([D'Aus', D'zum', D'Strade']);
 
 s1 :=DATASET(250000, TRANSFORM({ integer a }, SELF.a := add1(COUNTER)));
 s2 :=DATASET(250000, TRANSFORM({ integer a }, SELF.a := add1(COUNTER/2)));

testing/ecl/key/embed.xml

@@ -1,3 +1,9 @@
 <Dataset name='Result 1'>
  <Row><Result_1>7</Result_1></Row>
 </Dataset>
+<Dataset name='Result 2'>
+ <Row><Result_2><Item>1</Item><Item>2</Item><Item>3</Item></Result_2></Row>
+</Dataset>
+<Dataset name='Result 3'>
+ <Row><Result_3><Item>1.0</Item><Item>2.0</Item><Item>3.0</Item></Result_3></Row>
+</Dataset>

testing/ecl/key/embedR.xml

@@ -8,8 +8,35 @@
  <Row><Result_3>6262</Result_3></Row>
 </Dataset>
 <Dataset name='Result 4'>
- <Row><Result_4>46875625000</Result_4></Row>
+ <Row><Result_4><Item>2</Item><Item>1</Item><Item>3</Item></Result_4></Row>
 </Dataset>
 <Dataset name='Result 5'>
- <Row><Result_5>46875625000</Result_5></Row>
+ <Row><Result_5><Item>two</Item><Item>one</Item><Item>three</Item></Result_5></Row>
+</Dataset>
+<Dataset name='Result 6'>
+ <Row><Result_6><Item>dos     </Item><Item>uno     </Item><Item>tre     </Item></Result_6></Row>
+</Dataset>
+<Dataset name='Result 7'>
+ <Row><Result_7><Item>deux</Item><Item>un</Item><Item>trois</Item></Result_7></Row>
+</Dataset>
+<Dataset name='Result 8'>
+ <Row><Result_8><Item>zwei</Item><Item>ein</Item><Item>drei</Item></Result_8></Row>
+</Dataset>
+<Dataset name='Result 9'>
+ <Row><Result_9><Item>true</Item><Item>false</Item><Item>false</Item><Item>true</Item></Result_9></Row>
+</Dataset>
+<Dataset name='Result 10'>
+ <Row><Result_10><Item>2.200000047683716</Item><Item>1.100000023841858</Item><Item>3.299999952316284</Item></Result_10></Row>
+</Dataset>
+<Dataset name='Result 11'>
+ <Row><Result_11><Item>2.3</Item><Item>1.2</Item><Item>3.4</Item></Result_11></Row>
+</Dataset>
+<Dataset name='Result 12'>
+ <Row><Result_12><Item>12</Item><Item>11</Item><Item>13</Item></Result_12></Row>
+</Dataset>
+<Dataset name='Result 13'>
+ <Row><Result_13>46875625000</Result_13></Row>
+</Dataset>
+<Dataset name='Result 14'>
+ <Row><Result_14>46875625000</Result_14></Row>
 </Dataset>

testing/ecl/key/embedjava.xml

@@ -26,14 +26,41 @@
  <Row><Result_9>6261</Result_9></Row>
 </Dataset>
 <Dataset name='Result 10'>
- <Row><Result_10>46875625000</Result_10></Row>
+ <Row><Result_10>9</Result_10></Row>
 </Dataset>
 <Dataset name='Result 11'>
- <Row><Result_11>328126500000</Result_11></Row>
+ <Row><Result_11>0</Result_11></Row>
 </Dataset>
 <Dataset name='Result 12'>
- <Row><Result_12>46875625000</Result_12></Row>
+ <Row><Result_12><Item>two</Item><Item>one</Item><Item>three</Item></Result_12></Row>
 </Dataset>
 <Dataset name='Result 13'>
- <Row><Result_13>328126500000</Result_13></Row>
+ <Row><Result_13><Item>two</Item><Item>one</Item><Item>three</Item></Result_13></Row>
+</Dataset>
+<Dataset name='Result 14'>
+ <Row><Result_14><Item>two     </Item><Item>one     </Item><Item>three   </Item></Result_14></Row>
+</Dataset>
+<Dataset name='Result 15'>
+ <Row><Result_15><Item>two</Item><Item>one</Item><Item>three</Item></Result_15></Row>
+</Dataset>
+<Dataset name='Result 16'>
+ <Row><Result_16><Item>two</Item><Item>one</Item><Item>three</Item></Result_16></Row>
+</Dataset>
+<Dataset name='Result 17'>
+ <Row><Result_17><Item>two     </Item><Item>one     </Item><Item>three   </Item></Result_17></Row>
+</Dataset>
+<Dataset name='Result 18'>
+ <Row><Result_18><Item>two</Item><Item>one</Item><Item>three</Item></Result_18></Row>
+</Dataset>
+<Dataset name='Result 19'>
+ <Row><Result_19>46875625000</Result_19></Row>
+</Dataset>
+<Dataset name='Result 20'>
+ <Row><Result_20>328126500000</Result_20></Row>
+</Dataset>
+<Dataset name='Result 21'>
+ <Row><Result_21>46875625000</Result_21></Row>
+</Dataset>
+<Dataset name='Result 22'>
+ <Row><Result_22>328126500000</Result_22></Row>
 </Dataset>

testing/ecl/key/embedjs.xml

@@ -26,14 +26,44 @@
  <Row><Result_9>6261</Result_9></Row>
 </Dataset>
 <Dataset name='Result 10'>
- <Row><Result_10>46875625000</Result_10></Row>
+ <Row><Result_10><Item>1</Item><Item>3</Item><Item>2</Item></Result_10></Row>
 </Dataset>
 <Dataset name='Result 11'>
- <Row><Result_11>328126500000</Result_11></Row>
+ <Row><Result_11><Item>30000</Item><Item>50000</Item><Item>40000</Item></Result_11></Row>
 </Dataset>
 <Dataset name='Result 12'>
- <Row><Result_12>46875625000</Result_12></Row>
+ <Row><Result_12><Item>one</Item><Item>three</Item><Item>two</Item></Result_12></Row>
 </Dataset>
 <Dataset name='Result 13'>
- <Row><Result_13>328126500000</Result_13></Row>
+ <Row><Result_13><Item>uno     </Item><Item>tre     </Item><Item>dos     </Item></Result_13></Row>
+</Dataset>
+<Dataset name='Result 14'>
+ <Row><Result_14><Item>un</Item><Item>trois</Item><Item>deux</Item></Result_14></Row>
+</Dataset>
+<Dataset name='Result 15'>
+ <Row><Result_15><Item>ein</Item><Item>drei</Item><Item>zwei</Item></Result_15></Row>
+</Dataset>
+<Dataset name='Result 16'>
+ <Row><Result_16><Item>false</Item><Item>false</Item><Item>true</Item><Item>true</Item></Result_16></Row>
+</Dataset>
+<Dataset name='Result 17'>
+ <Row><Result_17><Item>1.100000023841858</Item><Item>3.299999952316284</Item><Item>2.200000047683716</Item></Result_17></Row>
+</Dataset>
+<Dataset name='Result 18'>
+ <Row><Result_18><Item>1.2</Item><Item>3.4</Item><Item>2.3</Item></Result_18></Row>
+</Dataset>
+<Dataset name='Result 19'>
+ <Row><Result_19><Item>-111</Item><Item>113</Item><Item>0</Item></Result_19></Row>
+</Dataset>
+<Dataset name='Result 20'>
+ <Row><Result_20>46875625000</Result_20></Row>
+</Dataset>
+<Dataset name='Result 21'>
+ <Row><Result_21>328126500000</Result_21></Row>
+</Dataset>
+<Dataset name='Result 22'>
+ <Row><Result_22>46875625000</Result_22></Row>
+</Dataset>
+<Dataset name='Result 23'>
+ <Row><Result_23>328126500000</Result_23></Row>
 </Dataset>

testing/ecl/key/embedpy.xml

@@ -32,14 +32,41 @@
  <Row><Result_11>6261</Result_11></Row>
 </Dataset>
 <Dataset name='Result 12'>
- <Row><Result_12>46875625000</Result_12></Row>
+ <Row><Result_12><Item>1</Item><Item>2</Item><Item>3</Item></Result_12></Row>
 </Dataset>
 <Dataset name='Result 13'>
- <Row><Result_13>328126500000</Result_13></Row>
+ <Row><Result_13><Item>green</Item><Item>red</Item><Item>yellow</Item></Result_13></Row>
 </Dataset>
 <Dataset name='Result 14'>
- <Row><Result_14>46875625000</Result_14></Row>
+ <Row><Result_14><Item>one     </Item><Item>three   </Item><Item>two     </Item></Result_14></Row>
 </Dataset>
 <Dataset name='Result 15'>
- <Row><Result_15>328126500000</Result_15></Row>
+ <Row><Result_15><Item>Oh</Item><Item>Straße</Item><Item>là</Item></Result_15></Row>
+</Dataset>
+<Dataset name='Result 16'>
+ <Row><Result_16><Item>Deux</Item><Item>Trois</Item><Item>Un</Item></Result_16></Row>
+</Dataset>
+<Dataset name='Result 17'>
+ <Row><Result_17><Item>Dos</Item><Item>Tre</Item><Item>Uno</Item></Result_17></Row>
+</Dataset>
+<Dataset name='Result 18'>
+ <Row><Result_18><Item>On</Item><Item>Straße</Item><Item>der</Item></Result_18></Row>
+</Dataset>
+<Dataset name='Result 19'>
+ <Row><Result_19><Item>Aus     </Item><Item>Straße  </Item><Item>zum     </Item></Result_19></Row>
+</Dataset>
+<Dataset name='Result 20'>
+ <Row><Result_20><Item>417573</Item><Item>7A756D</Item><Item>537472616465</Item></Result_20></Row>
+</Dataset>
+<Dataset name='Result 21'>
+ <Row><Result_21>46875625000</Result_21></Row>
+</Dataset>
+<Dataset name='Result 22'>
+ <Row><Result_22>328126500000</Result_22></Row>
+</Dataset>
+<Dataset name='Result 23'>
+ <Row><Result_23>46875625000</Result_23></Row>
+</Dataset>
+<Dataset name='Result 24'>
+ <Row><Result_24>328126500000</Result_24></Row>
 </Dataset>