Merge pull request #3845 from richardkchapman/embedding

HPCC-8030 JavaScript/Python language support in ECL

Reviewed-by: Gavin Halliday <ghalliday@hpccsystems.com>

cmake_modules/FindR.cmake

@@ -0,0 +1,53 @@
+################################################################################
+#    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+################################################################################
+
+# - Try to find the R library
+# Once done this will define
+#
+#  R_FOUND - system has the R library
+#  R_INCLUDE_DIR - the R include directory(s)
+#  R_LIBRARIES - The libraries needed to use R
+
+IF (NOT R_FOUND)
+  IF (WIN32)
+    SET (R_lib "libR")
+    SET (Rcpp_lib "libRcpp")
+    SET (RInside_lib "libRInside")
+  ELSE()
+    SET (R_lib "R")
+    SET (Rcpp_lib "Rcpp")
+    SET (RInside_lib "RInside")
+  ENDIF()
+
+  FIND_PATH(R_INCLUDE_DIR R.h PATHS /usr/lib /usr/share /usr/local/lib /usr/local/lib64 PATH_SUFFIXES R/include)
+  FIND_PATH(RCPP_INCLUDE_DIR Rcpp.h PATHS /usr/lib /usr/share /usr/local/lib /usr/local/lib64 PATH_SUFFIXES R/site-library/Rcpp/include/)
+  FIND_PATH(RINSIDE_INCLUDE_DIR RInside.h PATHS /usr/lib /usr/share /usr/local/lib /usr/local/lib64 PATH_SUFFIXES R/site-library/RInside/include)
+
+  FIND_LIBRARY (R_LIBRARY NAMES ${R_lib}  PATHS /usr/lib /usr/share /usr/local/lib /usr/local/lib64 PATH_SUFFIXES R/lib)
+  FIND_LIBRARY (RCPP_LIBRARY NAMES ${Rcpp_lib} PATHS /usr/lib /usr/share /usr/local/lib /usr/local/lib64 PATH_SUFFIXES R/site-library/Rcpp/lib/)
+  FIND_LIBRARY (RINSIDE_LIBRARY NAMES ${RInside_lib} PATHS /usr/lib /usr/share /usr/local/lib /usr/local/lib64 PATH_SUFFIXES R/site-library/RInside/lib/)
+
+  SET (R_INCLUDE_DIRS ${R_INCLUDE_DIR} ${RINSIDE_INCLUDE_DIR} ${RCPP_INCLUDE_DIR})
+  SET (R_LIBRARIES ${R_LIBRARY} ${RINSIDE_LIBRARY} ${RCPP_LIBRARY})
+
+  include(FindPackageHandleStandardArgs)
+  find_package_handle_standard_args(R DEFAULT_MSG
+    R_LIBRARIES
+    R_INCLUDE_DIRS
+  )
+
+  MARK_AS_ADVANCED(R_INCLUDE_DIRS R_LIBRARIES)
+ENDIF()

cmake_modules/FindV8.cmake

@@ -0,0 +1,41 @@
+################################################################################
+#    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+################################################################################
+
+# - Try to find the V8 JavaScript library
+# Once done this will define
+#
+#  V8_FOUND - system has the b8 javascript library
+#  V8_INCLUDE_DIR - the V8 include directory
+#  V8_LIBRARIES - The libraries needed to use V8
+
+IF (NOT V8_FOUND)
+  IF (WIN32)
+    SET (v8_lib "libv8")
+  ELSE()
+    SET (v8_lib "v8")
+  ENDIF()
+
+  FIND_PATH (V8_INCLUDE_DIR NAMES v8.h)
+  FIND_LIBRARY (V8_LIBRARIES NAMES ${v8_lib})
+
+  include(FindPackageHandleStandardArgs)
+  find_package_handle_standard_args(v8 DEFAULT_MSG
+    V8_LIBRARIES
+    V8_INCLUDE_DIR
+  )
+
+  MARK_AS_ADVANCED(V8_INCLUDE_DIR V8_LIBRARIES)
+ENDIF()

cmake_modules/commonSetup.cmake

@@ -71,6 +71,11 @@ IF ("${COMMONSETUP_DONE}" STREQUAL "")
   option(USE_RESOURCE "Use resource download in ECLWatch" OFF)
   option(GENERATE_COVERAGE_INFO "Generate coverage info for gcov" OFF)
 
+  option(MAKE_PYEMBED "Make the plugin for Python embedding" ON)
+  option(MAKE_V8EMBED "Make the plugin for V8 JavaScript embedding" ON)
+  option(MAKE_JAVAEMBED "Make the plugin for Java embedding" ON)
+  option(MAKE_REMBED "Make the plugin for R embedding" OFF)
+
   if ( USE_XALAN AND USE_LIBXSLT )
       set(USE_XALAN OFF)
   endif()

dali/base/daclient.cpp

@@ -34,7 +34,6 @@
 extern bool registerClientProcess(ICommunicator *comm, IGroup *& retcoven,unsigned timeout,DaliClientRole role);
 extern void stopClientProcess();
 
-static bool restoreSEH=false;
 static bool daliClientIsActive = false;
 static INode * daliClientLoggingParent = 0;
 
@@ -89,13 +88,10 @@ IDaliClient_Exception *createClientException(DaliClientError err, const char *ms
 
 
 
-bool initClientProcess(IGroup *servergrp, DaliClientRole role, unsigned short mpport, const char *clientVersion, const char *minServerVersion, unsigned timeout, bool enableSEH)
+bool initClientProcess(IGroup *servergrp, DaliClientRole role, unsigned short mpport, const char *clientVersion, const char *minServerVersion, unsigned timeout)
 {
     assertex(servergrp);
     daliClientIsActive = true;
-    restoreSEH = enableSEH;
-    if (enableSEH)
-      EnableSEHtoExceptionMapping();
     startMPServer(mpport);
     Owned<ICommunicator> comm(createCommunicator(servergrp,true));
     IGroup * covengrp;
@@ -134,10 +130,6 @@ void closedownClientProcess()
     stopClientProcess();
     closeCoven();
     stopMPServer();
-    if (restoreSEH) {
-        DisableSEHtoExceptionMapping();
-        restoreSEH = false;
-    }
     daliClientIsActive = false;
 }
 

dali/base/daclient.hpp

@@ -26,7 +26,7 @@
 #include "mpcomm.hpp"
 #include "dasds.hpp"
 
-extern da_decl bool initClientProcess(IGroup *servergrp, DaliClientRole role, unsigned short mpport=0, const char *clientVersion=NULL, const char *minServerVersion=NULL, unsigned timeout=MP_WAIT_FOREVER,bool enableSEH=true); 
+extern da_decl bool initClientProcess(IGroup *servergrp, DaliClientRole role, unsigned short mpport=0, const char *clientVersion=NULL, const char *minServerVersion=NULL, unsigned timeout=MP_WAIT_FOREVER);
 extern da_decl bool reinitClientProcess(IGroup *servergrp, DaliClientRole role, const char *clientVersion=NULL, const char *minServerVersion=NULL, unsigned timeout=MP_WAIT_FOREVER); 
 extern da_decl void closedownClientProcess();
 extern da_decl bool daliClientActive();

ecl/eclagent/eclagent.cpp

@@ -3248,7 +3248,7 @@ extern int HTHOR_API eclagent_main(int argc, const char *argv[], StringBuffer *
             {
                 MTIME_SECTION(timer, "SDS_Initialize");
                 Owned<IGroup> serverGroup = createIGroup(daliServers.str(), DALI_SERVER_PORT);
-                initClientProcess(serverGroup, DCR_EclAgent, 0, NULL, NULL, MP_WAIT_FOREVER, false);
+                initClientProcess(serverGroup, DCR_EclAgent, 0, NULL, NULL, MP_WAIT_FOREVER);
             }
 #ifdef MONITOR_ECLAGENT_STATUS  
             serverstatus = new CSDSServerStatus("ECLagent");
@@ -3409,6 +3409,7 @@ void usage(const char * exeName)
 
 int STARTQUERY_API start_query(int argc, const char *argv[])
 {
+    EnableSEHtoExceptionMapping();
     InitModuleObjects();
 
     for (int idx = 1; idx < argc; idx++)

ecl/eclagent/eclagentmain.cpp

@@ -71,6 +71,7 @@ int main(int argc, const char *argv[])
     }
     else
     {
+        EnableSEHtoExceptionMapping();
         InitModuleObjects();
         int ret = 0;
         try

ecl/hql/hqlatoms.cpp

@@ -49,6 +49,15 @@ _ATOM backupAtom;
 _ATOM bcdAtom;
 _ATOM beforeAtom;
 _ATOM bestAtom;
+_ATOM bindBooleanParamAtom;
+_ATOM bindDataParamAtom;
+_ATOM bindRealParamAtom;
+_ATOM bindSignedParamAtom;
+_ATOM bindStringParamAtom;
+_ATOM bindVStringParamAtom;
+_ATOM bindUnicodeParamAtom;
+_ATOM bindUnsignedParamAtom;
+_ATOM bindUtf8ParamAtom;
 _ATOM bitmapAtom;
 _ATOM blobAtom;
 _ATOM cAtom;
@@ -60,6 +69,7 @@ _ATOM choosenAtom;
 _ATOM clusterAtom;
 _ATOM _colocal_Atom;
 _ATOM commonAtom;
+_ATOM compileEmbeddedScriptAtom;
 _ATOM _complexKeyed_Atom;
 _ATOM compressedAtom;
 _ATOM __compressed__Atom;
@@ -70,6 +80,7 @@ _ATOM contextSensitiveAtom;
 _ATOM costAtom;
 _ATOM countAtom;
 _ATOM _countProject_Atom;
+_ATOM cppAtom;
 _ATOM _cppBody_Atom;
 _ATOM csvAtom;
 _ATOM ctxmethodAtom;
@@ -134,6 +145,15 @@ _ATOM _function_Atom;
 _ATOM globalContextAtom;
 _ATOM gctxmethodAtom;
 _ATOM getAtom;
+_ATOM getEmbedContextAtom;
+_ATOM getBooleanResultAtom;
+_ATOM getDataResultAtom;
+_ATOM getRealResultAtom;
+_ATOM getSignedResultAtom;
+_ATOM getStringResultAtom;
+_ATOM getUnicodeResultAtom;
+_ATOM getUnsignedResultAtom;
+_ATOM getUTF8ResultAtom;
 _ATOM globalAtom;
 _ATOM graphAtom;
 _ATOM groupAtom;
@@ -170,6 +190,7 @@ _ATOM jobTempAtom;
 _ATOM keepAtom;
 _ATOM keyedAtom;
 _ATOM labeledAtom;
+_ATOM languageAtom;
 _ATOM lastAtom;
 _ATOM leftAtom;
 _ATOM leftonlyAtom;
@@ -318,7 +339,9 @@ _ATOM skipAtom;
 _ATOM singleAtom;
 _ATOM snapshotAtom;
 _ATOM soapActionAtom;
+_ATOM syntaxCheckAtom;
 _ATOM httpHeaderAtom;
+_ATOM prototypeAtom;
 _ATOM proxyAddressAtom;
 _ATOM sort_AllAtom;
 _ATOM sort_KeyedAtom;
@@ -332,6 +355,8 @@ _ATOM storedAtom;
 _ATOM streamedAtom;
 _ATOM _streaming_Atom;
 _ATOM successAtom;
+_ATOM supportsImportAtom;
+_ATOM supportsScriptAtom;
 _ATOM sysAtom;
 _ATOM tempAtom;
 _ATOM templateAtom;
@@ -429,6 +454,15 @@ MODULE_INIT(INIT_PRIORITY_HQLATOM)
     MAKEATOM(bcd);
     MAKEATOM(before);
     MAKEATOM(best);
+    MAKEATOM(bindBooleanParam);
+    MAKEATOM(bindDataParam);
+    MAKEATOM(bindRealParam);
+    MAKEATOM(bindSignedParam);
+    MAKEATOM(bindStringParam);
+    MAKEATOM(bindVStringParam);
+    MAKEATOM(bindUnicodeParam);
+    MAKEATOM(bindUnsignedParam);
+    MAKEATOM(bindUtf8Param);
     MAKEATOM(bitmap);
     MAKEATOM(blob);
     MAKEATOM(c);
@@ -440,6 +474,7 @@ MODULE_INIT(INIT_PRIORITY_HQLATOM)
     MAKEATOM(cluster);
     MAKESYSATOM(colocal);
     MAKEATOM(common);
+    MAKEATOM(compileEmbeddedScript);
     MAKESYSATOM(complexKeyed);
     MAKEATOM(compressed);
     MAKEATOM(__compressed__);
@@ -450,6 +485,7 @@ MODULE_INIT(INIT_PRIORITY_HQLATOM)
     MAKEATOM(cost);
     MAKEATOM(count);
     MAKESYSATOM(countProject);
+    MAKEATOM(cpp);
     MAKESYSATOM(cppBody);
     MAKEATOM(csv);
     MAKEATOM(ctxmethod);
@@ -513,6 +549,15 @@ MODULE_INIT(INIT_PRIORITY_HQLATOM)
     MAKESYSATOM(function);
     MAKEATOM(gctxmethod);
     MAKEATOM(get);
+    MAKEATOM(getEmbedContext);
+    MAKEATOM(getBooleanResult);
+    MAKEATOM(getDataResult);
+    MAKEATOM(getRealResult);
+    MAKEATOM(getSignedResult);
+    MAKEATOM(getStringResult);
+    MAKEATOM(getUnicodeResult);
+    MAKEATOM(getUnsignedResult);
+    MAKEATOM(getUTF8Result);
     MAKEATOM(global);
     MAKEATOM(globalContext);
     MAKEATOM(graph);
@@ -551,6 +596,7 @@ MODULE_INIT(INIT_PRIORITY_HQLATOM)
     MAKEATOM(keep);
     MAKEATOM(keyed);
     MAKEATOM(labeled);
+    MAKEATOM(language);
     MAKEATOM(last);
     MAKEATOM(left);
     leftonlyAtom = createLowerCaseAtom("left only");
@@ -643,6 +689,7 @@ MODULE_INIT(INIT_PRIORITY_HQLATOM)
     MAKEATOM(preload);
     MAKEATOM(priority);
     MAKEATOM(private);
+    MAKEATOM(prototype);
     MAKEATOM(proxyAddress);
     MAKEATOM(pseudoentrypoint);
     MAKEATOM(pull);
@@ -699,6 +746,7 @@ MODULE_INIT(INIT_PRIORITY_HQLATOM)
     MAKEATOM(skip);
     MAKEATOM(snapshot);
     MAKEATOM(soapAction);
+    MAKEATOM(syntaxCheck);
     sort_AllAtom = createLowerCaseAtom("SORT ALL");
     sort_KeyedAtom = createLowerCaseAtom("SORT KEYED");
     MAKEATOM(sorted);
@@ -711,6 +759,8 @@ MODULE_INIT(INIT_PRIORITY_HQLATOM)
     MAKEATOM(streamed);
     MAKESYSATOM(streaming);
     MAKEATOM(success);
+    MAKEATOM(supportsImport);
+    MAKEATOM(supportsScript);
     MAKEATOM(sys);
     MAKEATOM(temp);
     MAKEATOM(template);

ecl/hql/hqlatoms.hpp

@@ -52,6 +52,15 @@ extern HQL_API _ATOM backupAtom;
 extern HQL_API _ATOM bcdAtom;
 extern HQL_API _ATOM beforeAtom;
 extern HQL_API _ATOM bestAtom;
+extern HQL_API _ATOM bindBooleanParamAtom;
+extern HQL_API _ATOM bindDataParamAtom;
+extern HQL_API _ATOM bindRealParamAtom;
+extern HQL_API _ATOM bindSignedParamAtom;
+extern HQL_API _ATOM bindStringParamAtom;
+extern HQL_API _ATOM bindVStringParamAtom;
+extern HQL_API _ATOM bindUnicodeParamAtom;
+extern HQL_API _ATOM bindUnsignedParamAtom;
+extern HQL_API _ATOM bindUtf8ParamAtom;
 extern HQL_API _ATOM bitmapAtom;
 extern HQL_API _ATOM blobAtom;
 extern HQL_API _ATOM cAtom;
@@ -63,6 +72,7 @@ extern HQL_API _ATOM choosenAtom;
 extern HQL_API _ATOM clusterAtom;
 extern HQL_API _ATOM _colocal_Atom;
 extern HQL_API _ATOM commonAtom;
+extern HQL_API _ATOM compileEmbeddedScriptAtom;
 extern HQL_API _ATOM _complexKeyed_Atom;
 extern HQL_API _ATOM compressedAtom;
 extern HQL_API _ATOM __compressed__Atom;
@@ -73,6 +83,7 @@ extern HQL_API _ATOM contextSensitiveAtom;
 extern HQL_API _ATOM costAtom;
 extern HQL_API _ATOM countAtom;
 extern HQL_API _ATOM _countProject_Atom;
+extern HQL_API _ATOM cppAtom;
 extern HQL_API _ATOM _cppBody_Atom;
 extern HQL_API _ATOM csvAtom;
 extern HQL_API _ATOM ctxmethodAtom;
@@ -136,6 +147,15 @@ extern HQL_API _ATOM fullouterAtom;
 extern HQL_API _ATOM _function_Atom;
 extern HQL_API _ATOM gctxmethodAtom;
 extern HQL_API _ATOM getAtom;
+extern HQL_API _ATOM getEmbedContextAtom;
+extern HQL_API _ATOM getBooleanResultAtom;
+extern HQL_API _ATOM getDataResultAtom;
+extern HQL_API _ATOM getRealResultAtom;
+extern HQL_API _ATOM getSignedResultAtom;
+extern HQL_API _ATOM getStringResultAtom;
+extern HQL_API _ATOM getUnicodeResultAtom;
+extern HQL_API _ATOM getUnsignedResultAtom;
+extern HQL_API _ATOM getUTF8ResultAtom;
 extern HQL_API _ATOM globalAtom;
 extern HQL_API _ATOM globalContextAtom;
 extern HQL_API _ATOM graphAtom;
@@ -174,6 +194,7 @@ extern HQL_API _ATOM jobTempAtom;
 extern HQL_API _ATOM keepAtom;
 extern HQL_API _ATOM keyedAtom;
 extern HQL_API _ATOM labeledAtom;
+extern HQL_API _ATOM languageAtom;
 extern HQL_API _ATOM lastAtom;
 extern HQL_API _ATOM leftAtom;
 extern HQL_API _ATOM leftonlyAtom;
@@ -322,7 +343,9 @@ extern HQL_API _ATOM skewAtom;
 extern HQL_API _ATOM skipAtom;
 extern HQL_API _ATOM snapshotAtom;
 extern HQL_API _ATOM soapActionAtom;
+extern HQL_API _ATOM syntaxCheckAtom;
 extern HQL_API _ATOM httpHeaderAtom;
+extern HQL_API _ATOM prototypeAtom;
 extern HQL_API _ATOM proxyAddressAtom;
 extern HQL_API _ATOM sort_AllAtom;
 extern HQL_API _ATOM sort_KeyedAtom;
@@ -336,6 +359,8 @@ extern HQL_API _ATOM storedAtom;
 extern HQL_API _ATOM streamedAtom;
 extern HQL_API _ATOM _streaming_Atom;
 extern HQL_API _ATOM successAtom;
+extern HQL_API _ATOM supportsImportAtom;
+extern HQL_API _ATOM supportsScriptAtom;
 extern HQL_API _ATOM sysAtom;
 extern HQL_API _ATOM tempAtom;
 extern HQL_API _ATOM templateAtom;

ecl/hql/hqlattr.cpp

@@ -406,7 +406,7 @@ unsigned getOperatorMetaFlags(node_operator op)
     case no_null:
     case no_globalscope:
     case no_nothor:
-    case no_cppbody:
+    case no_embedbody:
     case no_alias_scope:
     case no_evalonce:
     case no_forcelocal:

ecl/hql/hqlerrors.hpp

@@ -418,6 +418,7 @@
 #define HQLERR_CannotSubmitMacroX   2388
 #define HQLERR_CannotBeGrouped      2389
 #define HQLERR_CannotAccessShared   2390
+#define ERR_PluginNoScripting       2391
 
 #define ERR_ASSERTION_FAILS         100000
 

ecl/hql/hqlexpr.cpp

@@ -1303,7 +1303,7 @@ const char *getOpString(node_operator op)
     case no_id2blob: return "no_id2blob";
     case no_blob2id: return "no_blob2id";
     case no_anon: return "no_anon";
-    case no_cppbody: return "no_cppbody";
+    case no_embedbody: return "no_embedbody";
     case no_sortpartition: return "no_sortpartition";
     case no_define: return "DEFINE";
     case no_globalscope: return "GLOBAL";
@@ -1562,7 +1562,7 @@ bool checkConstant(node_operator op)
     case no_priority:
     case no_event:
     case no_independent:
-    case no_cppbody:
+    case no_embedbody:
     case no_translated:
     case no_assertkeyed:            // not sure about this - might be better to implement in the constant folder
     case no_assertstepped:
@@ -1785,7 +1785,7 @@ childDatasetType getChildDatasetType(IHqlExpression * expr)
     case no_externalcall:                       // None in the sense it is generally used for.
     case no_alias:
     case no_id2blob:
-    case no_cppbody:
+    case no_embedbody:
     case no_datasetfromrow:
     case no_datasetfromdictionary:
     case no_createrow:
@@ -2197,7 +2197,7 @@ inline unsigned doGetNumChildTables(IHqlExpression * dataset)
     case no_externalcall:                       // None in the sense it is generally used for.
     case no_alias:
     case no_id2blob:
-    case no_cppbody:
+    case no_embedbody:
     case no_datasetfromrow:
     case no_datasetfromdictionary:
     case no_param:
@@ -2488,7 +2488,7 @@ bool definesColumnList(IHqlExpression * dataset)
     case no_newsoapcall_ds:
     case no_alias:
     case no_id2blob:
-    case no_cppbody:
+    case no_embedbody:
     case no_externalcall:
     case no_projectrow:
     case no_datasetfromrow:
@@ -3469,7 +3469,7 @@ void CHqlExpression::updateFlagsAfterOperands()
             {
                 infoFlags2 |= HEF2containsCall;
                 IHqlExpression * bodycode = body->queryChild(0);
-                if (bodycode->getOperator() == no_cppbody)
+                if (bodycode->getOperator() == no_embedbody)
                 {
                     if (bodycode->queryProperty(actionAtom))
                         infoFlags |= HEFvolatile;
@@ -3486,7 +3486,7 @@ void CHqlExpression::updateFlagsAfterOperands()
                 infoFlags |= HEFvolatile;
             break;
         }
-    case no_cppbody:
+    case no_embedbody:
         {
             if (queryProperty(actionAtom))
                 infoFlags |= HEFvolatile;
@@ -4527,7 +4527,7 @@ IHqlExpression *CHqlExpressionWithType::clone(HqlExprArray &newkids)
     {
     case no_outofline:
         return createWrapper(op, newkids);
-    case no_cppbody:
+    case no_embedbody:
         {
             if (queryType()->getTypeCode() == type_transform)
             {
@@ -10993,7 +10993,7 @@ IHqlExpression *createDataset(node_operator op, HqlExprArray & parms)
     case no_xmlproject:
     case no_temptable:
     case no_id2blob:
-    case no_cppbody:
+    case no_embedbody:
         type.setown(makeTableType(makeRowType(createRecordType(&parms.item(1))), NULL, NULL, NULL));
         if (queryProperty(_linkCounted_Atom, parms))
             type.setown(setLinkCountedAttr(type, true));
@@ -11901,7 +11901,7 @@ extern IHqlExpression *createRow(node_operator op, HqlExprArray & args)
                 type = makeRowType(LINK(fieldType));
             break;
         }
-    case no_cppbody:
+    case no_embedbody:
     case no_id2blob:
     case no_temprow:
     case no_projectrow:         // arg(1) is actually a transform

ecl/hql/hqlexpr.hpp

@@ -595,7 +595,7 @@ enum _node_operator {
         no_blob2id,
         no_anon,
         no_projectrow,
-        no_cppbody,
+        no_embedbody,
         no_sortpartition,
         no_define,
         no_globalscope,

ecl/hql/hqlfold.cpp

@@ -5417,7 +5417,7 @@ HqlConstantPercolator * CExprFolderTransformer::gatherConstants(IHqlExpression *
     case no_libraryinput:
     case no_translated:
     case no_id2blob:
-    case no_cppbody:
+    case no_embedbody:
     case no_pipe:
     case no_keyindex:
     case no_newkeyindex:

ecl/hql/hqlgram.hpp

@@ -542,7 +542,7 @@ public:
     IHqlExpression * processAlienType(const attribute & errpos);
     IHqlExpression * processIndexBuild(attribute & indexAttr, attribute * recordAttr, attribute * payloadAttr, attribute & filenameAttr, attribute & flagsAttr);
     IHqlExpression * processCompoundFunction(attribute & result, bool outOfLine);
-    IHqlExpression * processCppBody(const attribute & errpos, IHqlExpression * cpp);
+    IHqlExpression * processEmbedBody(const attribute & errpos, IHqlExpression * embedText, IHqlExpression * language, IHqlExpression *attribs);
     void processEnum(attribute & idAttr, IHqlExpression * value);
     void processError(bool full);
     void processLoadXML(attribute & a1, attribute * a2);
@@ -998,6 +998,7 @@ class HqlLex
         HqlLex(HqlGram *gram, IFileContents * _text, IXmlScope *xmlScope, IHqlExpression *macroExpr);
         ~HqlLex();   
 
+        void enterEmbeddedMode();
         static int doyyFlex(YYSTYPE & returnToken, yyscan_t yyscanner, HqlLex * lexer, bool lookup, const short * activeState);
         static int lookupIdentifierToken(YYSTYPE & returnToken, HqlLex * lexer, bool lookup, const short * activeState, const char * tokenText);
 
@@ -1078,6 +1079,7 @@ class HqlLex
         void init(IFileContents * _text);
 
     private:
+        static void doEnterEmbeddedMode(yyscan_t yyscanner);
         void declareXmlSymbol(const YYSTYPE & errpos, const char *name);
         StringBuffer &lookupXmlSymbol(const YYSTYPE & errpos, const char *name, StringBuffer &value);
         void setXmlSymbol(const YYSTYPE & errpos, const char *name, const char *value, bool append);

ecl/hql/hqlgram.y

@@ -175,6 +175,7 @@ static void eclsyntaxerror(HqlGram * parser, const char * s, short yystate, int
   ECLCRC
   ELSE
   ELSEIF
+  EMBED
   EMBEDDED
   _EMPTY_
   ENCODING
@@ -182,6 +183,7 @@ static void eclsyntaxerror(HqlGram * parser, const char * s, short yystate, int
   ENCRYPTED
   END
   ENDCPP
+  ENDEMBED
   ENTH
   ENUM
   TOK_ERROR
@@ -910,7 +912,7 @@ goodObject
     | transform
     | complexType
     | macro
-    | cppBodyText
+    | embedBody
     | eventObject
     | compoundAttribute
     | abstractModule
@@ -1001,10 +1003,40 @@ macro
                         }
     ;
 
-cppBodyText
+embedBody
     : CPPBODY           {
-                            OwnedHqlExpr cpp = $1.getExpr();
-                            $$.setExpr(parser->processCppBody($1, cpp), $1);
+                            OwnedHqlExpr embeddedCppText = $1.getExpr();
+                            $$.setExpr(parser->processEmbedBody($1, embeddedCppText, NULL, NULL), $1);
+                        }
+    | embedPrefix CPPBODY
+                        {
+                            OwnedHqlExpr language = $1.getExpr();
+                            OwnedHqlExpr embedText = $2.getExpr();
+                            $$.setExpr(parser->processEmbedBody($2, embedText, language, NULL), $1);
+                        }
+    | EMBED '(' abstractModule ',' expression ')'
+                        {
+                            parser->normalizeExpression($5, type_stringorunicode, true);
+                            OwnedHqlExpr language = $3.getExpr();
+                            OwnedHqlExpr embedText = $5.getExpr();
+                            $$.setExpr(parser->processEmbedBody($5, embedText, language, NULL), $1);
+                        }
+    | IMPORT '(' abstractModule ',' expression attribs ')'
+                        {
+                            parser->normalizeExpression($5, type_stringorunicode, true);
+                            OwnedHqlExpr language = $3.getExpr();
+                            OwnedHqlExpr funcname = $5.getExpr();
+                            OwnedHqlExpr attribs = createComma(createAttribute(importAtom), $6.getExpr());
+                            $$.setExpr(parser->processEmbedBody($6, funcname, language, attribs), $1);
+                        }
+    
+    ;
+
+embedPrefix
+    : EMBED '(' abstractModule ')'
+                        {
+                            parser->getLexer()->enterEmbeddedMode();
+                            $$.inherit($3);
                         }
     ;
 

ecl/hql/hqlgram2.cpp

@@ -831,11 +831,32 @@ IHqlExpression * HqlGram::convertToOutOfLineFunction(const ECLlocation & errpos,
     return LINK(expr);
 }
 
-IHqlExpression * HqlGram::processCppBody(const attribute & errpos, IHqlExpression * cpp)
+IHqlExpression * HqlGram::processEmbedBody(const attribute & errpos, IHqlExpression * embedText, IHqlExpression * language, IHqlExpression *attribs)
 {
     HqlExprArray args;
-    cpp->unwindList(args, no_comma);
-
+    embedText->unwindList(args, no_comma);
+    if (language)
+    {
+        IHqlScope *pluginScope = language->queryScope();
+        OwnedHqlExpr getEmbedContextFunc = pluginScope->lookupSymbol(getEmbedContextAtom, LSFpublic, lookupCtx);
+        _ATOM moduleName = language->queryName();
+        if (!moduleName)
+            moduleName = unnamedAtom;
+        if (!getEmbedContextFunc)
+            reportError(ERR_PluginNoScripting, errpos, "Module %s does not export getEmbedContext() function", moduleName->getAtomNamePtr());
+        bool isImport = queryPropertyInList(importAtom, attribs) != NULL;
+        OwnedHqlExpr checkSupport = pluginScope->lookupSymbol(isImport ? supportsImportAtom : supportsScriptAtom, LSFpublic, lookupCtx);
+        if (!matchesBoolean(checkSupport, true))
+            reportError(ERR_PluginNoScripting, errpos, "Module %s does not support %s", moduleName->getAtomNamePtr(), isImport ? "import" : "script");
+        OwnedHqlExpr syntaxCheckFunc = pluginScope->lookupSymbol(syntaxCheckAtom, LSFpublic, lookupCtx);
+        if (syntaxCheckFunc && !importAtom)
+        {
+            // MORE - create an expression that calls it, and const fold it, I guess....
+        }
+        args.append(*createExprAttribute(languageAtom, getEmbedContextFunc.getClear()));
+    }
+    if (attribs)
+        attribs->unwindList(args, no_comma);
     Linked<ITypeInfo> type = current_type;
     if (!type)
         type.setown(makeVoidType());
@@ -853,21 +874,21 @@ IHqlExpression * HqlGram::processCppBody(const attribute & errpos, IHqlExpressio
         {
         case type_row:
         case type_record:
-            result.setown(createRow(no_cppbody, args));
+            result.setown(createRow(no_embedbody, args));
             break;
         case type_table:
         case type_groupedtable:
-            result.setown(createDataset(no_cppbody, args));
+            result.setown(createDataset(no_embedbody, args));
             break;
         case type_transform:
-            result.setown(createValue(no_cppbody, makeTransformType(LINK(record->queryType())), args));
+            result.setown(createValue(no_embedbody, makeTransformType(LINK(record->queryType())), args));
             break;
         default:
             throwUnexpected();
         }
     }
     else
-        result.setown(createValue(no_cppbody, LINK(type), args));
+        result.setown(createValue(no_embedbody, LINK(type), args));
 
     result.setown(createLocationAnnotation(result.getClear(), errpos.pos));
 

ecl/hql/hqlir.cpp

@@ -513,7 +513,7 @@ const char * getOperatorIRText(node_operator op)
     EXPAND_CASE(no,blob2id);
     EXPAND_CASE(no,anon);
     EXPAND_CASE(no,projectrow);
-    EXPAND_CASE(no,cppbody);
+    EXPAND_CASE(no,embedbody);
     EXPAND_CASE(no,sortpartition);
     EXPAND_CASE(no,define);
     EXPAND_CASE(no,globalscope);
@@ -2124,14 +2124,14 @@ extern HQL_API void getIRText(StringArray & target, unsigned options, IHqlExpres
 static StringBuffer staticDebuggingStringBuffer;
 extern HQL_API const char * getIRText(IHqlExpression * expr)
 {
-    StringBufferIRBuilder output(staticDebuggingStringBuffer, defaultDumpOptions);
+    StringBufferIRBuilder output(staticDebuggingStringBuffer.clear(), defaultDumpOptions);
     playIR(output, expr, NULL, NULL);
     return staticDebuggingStringBuffer.str();
 }
 
 extern HQL_API const char * getIRText(ITypeInfo * type)
 {
-    StringBufferIRBuilder output(staticDebuggingStringBuffer, defaultDumpOptions);
+    StringBufferIRBuilder output(staticDebuggingStringBuffer.clear(), defaultDumpOptions);
     playIR(output, NULL, NULL, type);
     return staticDebuggingStringBuffer.str();
 }

ecl/hql/hqllex.l

@@ -673,6 +673,7 @@ ECLCRC              { RETURNSYM(ECLCRC); }
 ELSE                { RETURNSYM(ELSE); }
 ELSIF               { RETURNSYM(ELSEIF); }
 ELSEIF              { RETURNSYM(ELSEIF); }
+EMBED               { RETURNSYM(EMBED); }
 EMBEDDED            { RETURNSYM(EMBEDDED); }
 _EMPTY_             { RETURNSYM(_EMPTY_); }
 ENCODING            { RETURNSYM(ENCODING); }
@@ -1375,6 +1376,7 @@ OUT                 { RETURNSYM(TOK_OUT); }
 CONST               { RETURNSYM(TOK_CONST); }
 ENDMACRO            { setupdatepos; return(ENDMACRO); }
 ENDC\+\+            { setupdatepos; return(ENDCPP); }
+ENDEMBED            { setupdatepos; return(ENDEMBED); }
 ENCRYPTED           {
                         setupdatepos; 
                         if (!lookup || !LOOKUPSYM(ENCRYPTED) || lexer->macroGathering)
@@ -1458,18 +1460,21 @@ FUNCTIONMACRO|MACRO {
                         BEGIN(CPP);
                         lexer->inCpp = true;
                     }
-<CPP>[^\n]*"ENDC++"[^\n]*  { 
+<CPP>[^\n]*("ENDC++"|"ENDEMBED")[^\n]*  {
                         lexer->inCpp = false;
                         int endpos = lexer->yyPosition;
                         //skip to the position of ENDC++ on the line (case insensitive)
-                        while (memicmp(lexer->yyBuffer+endpos, "ENDC++", 6) != 0)
+                        while (memicmp(lexer->yyBuffer+endpos, "ENDC++", 6) != 0 && memicmp(lexer->yyBuffer+endpos, "ENDEMBED", 8) != 0)
                             endpos++;
-                        const int lastpos = endpos+6;
+                        const int lastpos = endpos + (tolower(lexer->yyBuffer[endpos+3])=='c' ? 6 : 8);
                             
                         updatepos1; 
                         BEGIN(0);
-                        int beginpos = returnToken.pos.position;
-                        int startpos = beginpos + 8;
+                        int startpos = returnToken.pos.position;
+                        if (endpos-startpos >= 8 && memicmp(lexer->yyBuffer+startpos, "BEGINC++", 8)==0)
+                            startpos += 8;
+                        else
+                            startpos += 1;  // Skip the ) of EMBED(xxx)
 
                         // keep the orginal format info (like blanks, newlines)
                         while (endpos != startpos && (lexer->yyBuffer[endpos-1] == 13 || lexer->yyBuffer[endpos-1] == 10))
@@ -1483,7 +1488,7 @@ FUNCTIONMACRO|MACRO {
                         yyless(CUR_TOKEN_LENGTH - delta);
                         lexer->yyPosition -= delta;
                         lexer->yyColumn -= delta;
-                        OwnedHqlExpr cppText = createConstant(createStringValue(lexer->yyBuffer+startpos, len));
+                        OwnedHqlExpr cppText = createConstant(createUnicodeValue(lexer->yyBuffer+startpos, len, "", true, false));
                         OwnedHqlExpr annotated = createLocationAnnotation(cppText.getClear(), returnToken.pos);
                         OwnedHqlExpr options = extractCppBodyAttrs(len, lexer->yyBuffer+startpos);
                         returnToken.setExpr(createComma(annotated.getClear(), options.getClear()));
@@ -1941,3 +1946,8 @@ FUNCTIONMACRO|MACRO {
 
 .                   { setupdatepos; return (CUR_TOKEN_TEXT[0]); }
 %%
+void HqlLex::doEnterEmbeddedMode(yyscan_t yyscanner)
+{
+   struct yyguts_t * yyg = (struct yyguts_t*)yyscanner;
+   BEGIN(CPP);
+}

ecl/hql/hqlparse.cpp

@@ -2051,6 +2051,12 @@ unsigned HqlLex::getTypeSize(unsigned lengthTypeName)
     return UNKNOWN_LENGTH;
 }
 
+void HqlLex::enterEmbeddedMode()
+{
+    doEnterEmbeddedMode(scanner);
+    inCpp = true;
+}
+
 int HqlLex::yyLex(YYSTYPE & returnToken, bool lookup, const short * activeState)
 {
     loop
@@ -2102,7 +2108,7 @@ int HqlLex::yyLex(YYSTYPE & returnToken, bool lookup, const short * activeState)
             if (inComment)
                 reportError(returnToken, ERR_COMMENT_UNENDED,"Comment is not terminated");
             else if (inCpp)
-                reportError(returnToken, ERR_COMMENT_UNENDED,"BEGINC++ is not terminated");
+                reportError(returnToken, ERR_COMMENT_UNENDED,"BEGINC++ or EMBED is not terminated");
             if (hashendDepths.ordinality())
             {
                 StringBuffer msg("Unexpected EOF: ");

ecl/hqlcpp/hqlcerrors.hpp

@@ -206,6 +206,7 @@
 #define HQLERR_SkipInsideCreateRow              4184
 #define HQLERR_KeyedJoinNoRightIndex_X          4185
 #define HQLERR_ScalarOutputWithinApply          4186
+#define HQLERR_EmbeddedTypeNotSupported_X       4187
 
 //Warnings....
 #define HQLWRN_PersistDataNotLikely             4500
@@ -484,7 +485,7 @@
 #define HQLERR_SkipInsideCreateRow_Text         "SKIP inside a ROW(<transform>) not supported.  It is only allowed in a DATASET transform."
 #define HQLERR_ScalarOutputWithinApply_Text     "A scalar output within an APPLY is undefined and may fail.  Use OUTPUT(dataset,EXTEND) instead."
 #define HQLERR_KeyedJoinNoRightIndex_X_Text     "Right dataset (%s) for a keyed join isn't a key"
-
+#define HQLERR_EmbeddedTypeNotSupported_X_Text  "Type %s not supported for embedded/external scripts"
 //Warnings.
 #define HQLWRN_CannotRecreateDistribution_Text  "Cannot recreate the distribution for a persistent dataset"
 #define HQLWRN_RecursiveDependendencies_Text    "Recursive filename dependency"

ecl/hqlcpp/hqlcpp.cpp

@@ -3269,8 +3269,8 @@ void HqlCppTranslator::buildExpr(BuildCtx & ctx, IHqlExpression * expr, CHqlBoun
     case no_blob2id:
         doBuildExprBlobToId(ctx, expr, tgt);
         return;
-    case no_cppbody:
-        doBuildExprCppBody(ctx, expr, &tgt);
+    case no_embedbody:
+        doBuildExprEmbedBody(ctx, expr, &tgt);
         return;
     case no_null:
         tgt.length.setown(getSizetConstant(0));
@@ -3598,8 +3598,8 @@ void HqlCppTranslator::buildStmt(BuildCtx & _ctx, IHqlExpression * expr)
     case no_assert:
         doBuildStmtAssert(ctx, expr);
         return;
-    case no_cppbody:
-        doBuildExprCppBody(ctx, expr, NULL);
+    case no_embedbody:
+        doBuildExprEmbedBody(ctx, expr, NULL);
         return;
     case no_setworkflow_cond:
         {
@@ -7236,14 +7236,19 @@ void HqlCppTranslator::processCppBodyDirectives(IHqlExpression * expr)
     }
 }
 
-void HqlCppTranslator::doBuildExprCppBody(BuildCtx & ctx, IHqlExpression * expr, CHqlBoundExpr * tgt)
+void HqlCppTranslator::doBuildExprEmbedBody(BuildCtx & ctx, IHqlExpression * expr, CHqlBoundExpr * tgt)
 {
     if (!allowEmbeddedCpp())
         throwError(HQLERR_EmbeddedCppNotAllowed);
 
     processCppBodyDirectives(expr);
+    IHqlExpression *languageAttr = expr->queryProperty(languageAtom);
+    if (languageAttr)
+    {
+        UNIMPLEMENTED;  // It's not clear if this can ever happen - perhaps a parameterless function that used EMBED ?
+    }
     StringBuffer text;
-    expr->queryChild(0)->queryValue()->getStringValue(text);
+    expr->queryChild(0)->queryValue()->getUTF8Value(text);
     text.setLength(cleanupEmbeddedCpp(text.length(), (char*)text.str()));
     OwnedHqlExpr quoted = createQuoted(text.str(), expr->getType());
 
@@ -11250,11 +11255,60 @@ void HqlCppTranslator::expandFunctionPrototype(BuildCtx & ctx, IHqlExpression *
     if (expandFunctionPrototype(s, funcdef))
     {
         s.append(";");
-        ctx.addQuoted(s);
+        IHqlExpression *body = funcdef->queryChild(0);
+        IHqlExpression *namespaceAttr = body->queryProperty(namespaceAtom);
+        if (namespaceAttr)
+        {
+            StringBuffer ns;
+            getStringValue(ns, namespaceAttr->queryChild(0));
+            ns.insert(0, "namespace ").appendf(" { %s }", s.str());
+            ctx.addQuoted(ns);
+        }
+        else
+            ctx.addQuoted(s);
     }
 }
 
 //Replace no_param with whatever they will have been bound to
+static IHqlExpression *createActualFromFormal(IHqlExpression *param)
+{
+    StringBuffer paramNameText, temp;
+    ITypeInfo *paramType = param->queryType();
+    CHqlBoundExpr bound;
+
+    //Case is significant if these parameters are use for BEGINC++ sections
+    _ATOM paramName = param->queryName();
+    paramNameText.clear().append(paramName).toLowerCase();
+
+    Linked<ITypeInfo> type = paramType;
+    switch (paramType->getTypeCode())
+    {
+    case type_set:
+        {
+            appendCapital(temp.clear().append("isAll"), paramNameText);
+            bound.isAll.setown(createVariable(temp.str(), makeBoolType()));
+        }
+        //fall through
+    case type_string:
+    case type_qstring:
+    case type_data:
+    case type_unicode:
+    case type_utf8:
+    case type_dictionary:
+    case type_table:
+    case type_groupedtable:
+        if (paramType->getSize() == UNKNOWN_LENGTH)
+        {
+            appendCapital(temp.clear().append("len"), paramNameText);
+            bound.length.setown(createVariable(temp.str(), LINK(sizetType)));
+        }
+        type.setown(makeReferenceModifier(LINK(type)));
+        break;
+    }
+    bound.expr.setown(createVariable(paramNameText.str(), LINK(type)));
+    return bound.getTranslatedExpr();
+}
+
 static IHqlExpression * replaceInlineParameters(IHqlExpression * funcdef, IHqlExpression * expr)
 {
     IHqlExpression * body = funcdef->queryChild(0);
@@ -11266,41 +11320,8 @@ static IHqlExpression * replaceInlineParameters(IHqlExpression * funcdef, IHqlEx
     ForEachChild(i, formals)
     {
         IHqlExpression * param = formals->queryChild(i);
-        ITypeInfo *paramType = param->queryType();
-        CHqlBoundExpr bound;
-        
-        //Case is significant if these parameters are use for BEGINC++ sections
-        _ATOM paramName = param->queryName();
-        paramNameText.clear().append(paramName).toLowerCase();
-
-        Linked<ITypeInfo> type = paramType;
-        switch (paramType->getTypeCode())
-        {
-        case type_set:
-            {
-                appendCapital(temp.clear().append("isAll"), paramNameText);
-                bound.isAll.setown(createVariable(temp.str(), makeBoolType()));
-            }
-            //fall through
-        case type_string:
-        case type_qstring:
-        case type_data:
-        case type_unicode:
-        case type_utf8:
-        case type_dictionary:
-        case type_table:
-        case type_groupedtable:
-            if (paramType->getSize() == UNKNOWN_LENGTH)
-            {
-                appendCapital(temp.clear().append("len"), paramNameText);
-                bound.length.setown(createVariable(temp.str(), LINK(sizetType)));
-            }
-            type.setown(makeReferenceModifier(LINK(type)));
-            break;
-        } 
-        bound.expr.setown(createVariable(paramNameText.str(), LINK(type)));
-        OwnedHqlExpr replacement = bound.getTranslatedExpr();
-        simpleTransformer.setMapping(param, replacement);
+        OwnedHqlExpr formal = createActualFromFormal(param);
+        simpleTransformer.setMapping(param, formal);
     }
 
     return simpleTransformer.transformRoot(expr);
@@ -11342,87 +11363,222 @@ void HqlCppTranslator::doBuildUserFunctionReturn(BuildCtx & ctx, ITypeInfo * typ
     }
 }
 
+void HqlCppTranslator::buildCppFunctionDefinition(BuildCtx &funcctx, IHqlExpression * bodyCode, const char *proto)
+{
+    processCppBodyDirectives(bodyCode);
+    IHqlExpression * location = queryLocation(bodyCode);
+    const char * locationFilename = location ? location->querySourcePath()->str() : NULL;
+    unsigned startLine = location ? location->getStartLine() : 0;
+    IHqlExpression * cppBody = bodyCode->queryChild(0);
+    if (cppBody->getOperator() == no_record)
+        cppBody = bodyCode->queryChild(1);
+
+    StringBuffer text;
+    cppBody->queryValue()->getUTF8Value(text);
+    //remove #option, and remove /r so we don't end up with mixed format end of lines.
+    text.setLength(cleanupEmbeddedCpp(text.length(), (char*)text.str()));
+
+    const char * start = text.str();
+    loop
+    {
+        char next = *start;
+        if (next == '\n')
+            startLine++;
+        else if (next != '\r')
+            break;
+        start++;
+    }
+    const char * body = start;
+    const char * cppSeparatorText = "#body";
+    const char * separator = strstr(body, cppSeparatorText);
+    if (separator)
+    {
+        text.setCharAt(separator-text.str(), 0);
+        if (location)
+            funcctx.addLine(locationFilename, startLine);
+        funcctx.addQuoted(body);
+        if (location)
+            funcctx.addLine();
+
+        body = separator + strlen(cppSeparatorText);
+        if (*body == '\r') body++;
+        if (*body == '\n') body++;
+        startLine += memcount(body-start, start, '\n');
+    }
+
+    funcctx.addQuotedCompound(proto);
+    if (location)
+        funcctx.addLine(locationFilename, startLine);
+    funcctx.addQuoted(body);
+    if (location)
+        funcctx.addLine();
+}
+
+void HqlCppTranslator::buildScriptFunctionDefinition(BuildCtx &funcctx, IHqlExpression * funcdef, const char *proto)
+{
+    ITypeInfo * returnType = funcdef->queryType()->queryChildType();
+    IHqlExpression * outofline = funcdef->queryChild(0);
+    assertex(outofline->getOperator() == no_outofline);
+    IHqlExpression * bodyCode = outofline->queryChild(0);
+    IHqlExpression *language = queryPropertyChild(bodyCode, languageAtom, 0);
+    bool isImport = bodyCode->hasProperty(importAtom);
+
+    funcctx.addQuotedCompound(proto);
+
+    HqlExprArray noargs;
+    OwnedHqlExpr getPlugin = bindFunctionCall(language, noargs);
+    OwnedHqlExpr pluginPtr = createQuoted("Owned<IEmbedContext> __plugin", makeBoolType());  // Not really bool - at some point ECL may support without this aliasing...
+    buildAssignToTemp(funcctx, pluginPtr, getPlugin);
+    StringBuffer createParam;
+    createParam.append("Owned<IEmbedFunctionContext> __ctx = __plugin->createFunctionContext(");
+    createParam.append(isImport ? "true" : "false");
+    StringBuffer attrParam;
+    ForEachChild(idx, bodyCode)
+    {
+        IHqlExpression *child = bodyCode->queryChild(idx);
+        if (child->isAttribute() && child->queryName() != languageAtom && child->queryName() != importAtom)
+        {
+            attrParam.append(",");
+            attrParam.append(child->queryName());
+            StringBuffer attrValue;
+            if (getStringValue(attrValue, child->queryChild(0)).length())
+            {
+                attrParam.append('=');
+                appendStringAsCPP(attrParam, attrValue.length(), attrValue.str(), true);
+            }
+        }
+    }
+    if (attrParam.length())
+        createParam.append(",\"").append(attrParam.str()+1).append('"');
+    else
+        createParam.append(",NULL");
+    createParam.append(");");
+    funcctx.addQuoted(createParam);
+    OwnedHqlExpr ctxVar = createVariable("__ctx", makeBoolType());
+
+    HqlExprArray scriptArgs;
+    scriptArgs.append(*LINK(ctxVar));
+    scriptArgs.append(*LINK(bodyCode->queryChild(0)));
+    buildFunctionCall(funcctx, isImport ? importAtom : compileEmbeddedScriptAtom, scriptArgs);
+    IHqlExpression *formals = funcdef->queryChild(1);
+    ForEachChild(i, formals)
+    {
+        HqlExprArray args;
+        args.append(*LINK(ctxVar));
+        IHqlExpression * param = formals->queryChild(i);
+        ITypeInfo *paramType = param->queryType();
+        _ATOM paramName = param->queryName();
+        StringBuffer paramNameText;
+        paramNameText.append(paramName).toLowerCase();
+        args.append(*createConstant(paramNameText));
+        _ATOM bindFunc;
+        switch (paramType->getTypeCode())
+        {
+        case type_int:
+            bindFunc = paramType->isSigned() ? bindSignedParamAtom : bindUnsignedParamAtom;
+            break;
+        case type_varstring:
+            bindFunc = bindVStringParamAtom;
+            break;
+        case type_string:
+            bindFunc = bindStringParamAtom;
+            break;
+        case type_real:
+            bindFunc = bindRealParamAtom;
+            break;
+        case type_boolean:
+            bindFunc = bindBooleanParamAtom;
+            break;
+        case type_utf8:
+            bindFunc = bindUtf8ParamAtom;
+            break;
+        case type_unicode:
+            bindFunc = bindUnicodeParamAtom;
+            break;
+        case type_data:
+            bindFunc = bindDataParamAtom;
+            break;
+        default:
+            StringBuffer typeText;
+            getFriendlyTypeStr(paramType, typeText);
+            throwError1(HQLERR_EmbeddedTypeNotSupported_X, typeText.str());
+        }
+        args.append(*createActualFromFormal(param));
+        buildFunctionCall(funcctx, bindFunc, args);
+    }
+    funcctx.addQuoted("__ctx->callFunction();");
+    _ATOM returnFunc;
+    switch (returnType->getTypeCode())
+    {
+    case type_int:
+        returnFunc = returnType->isSigned() ? getSignedResultAtom : getUnsignedResultAtom;
+        break;
+    case type_varstring:
+    case type_string:
+        returnFunc = getStringResultAtom;
+        break;
+    case type_real:
+        returnFunc = getRealResultAtom;
+        break;
+    case type_boolean:
+        returnFunc = getBooleanResultAtom;
+        break;
+    case type_unicode:
+        returnFunc = getUnicodeResultAtom;
+        break;
+    case type_utf8:
+        returnFunc = getUTF8ResultAtom;
+        break;
+    case type_data:
+        returnFunc = getDataResultAtom;
+        break;
+    default:
+        StringBuffer typeText;
+        getFriendlyTypeStr(returnType, typeText);
+        throwError1(HQLERR_EmbeddedTypeNotSupported_X, typeText.str());
+    }
+    noargs.append(*LINK(ctxVar));
+    OwnedHqlExpr call = bindFunctionCall(returnFunc, noargs);
+    doBuildUserFunctionReturn(funcctx, returnType, call);
+}
 
 void HqlCppTranslator::buildFunctionDefinition(IHqlExpression * funcdef)
 {
     IHqlExpression * outofline = funcdef->queryChild(0);
-    ITypeInfo * returnType = funcdef->queryType()->queryChildType();
     assertex(outofline->getOperator() == no_outofline);
     IHqlExpression * bodyCode = outofline->queryChild(0);
 
-    StringBuffer s;
+    StringBuffer proto;
     BuildCtx funcctx(*code, helperAtom);
     if (options.spanMultipleCpp)
     {
-        const bool inChildActivity = true;  // assume the worse
+        const bool inChildActivity = true;  // assume the worst
         OwnedHqlExpr pass = getSizetConstant(cppIndexNextActivity(inChildActivity));
         funcctx.addGroupPass(pass);
     }
-    expandFunctionPrototype(s, funcdef);
+    expandFunctionPrototype(proto, funcdef);
 
-    if (bodyCode->getOperator() == no_cppbody)
+    if (bodyCode->getOperator() == no_embedbody)
     {
         if (!allowEmbeddedCpp())
             throwError(HQLERR_EmbeddedCppNotAllowed);
 
-        processCppBodyDirectives(bodyCode);
-        IHqlExpression * location = queryLocation(bodyCode);
-        const char * locationFilename = location ? location->querySourcePath()->str() : NULL;
-        unsigned startLine = location ? location->getStartLine() : 0;
-        IHqlExpression * cppBody = bodyCode->queryChild(0);
-        if (cppBody->getOperator() == no_record)
-            cppBody = bodyCode->queryChild(1);
-
-        StringBuffer text;
-        cppBody->queryValue()->getStringValue(text);
-        //remove #option, and remove /r so we don't end up with mixed format end of lines.
-        text.setLength(cleanupEmbeddedCpp(text.length(), (char*)text.str()));
-
-        const char * start = text.str();
-        loop
-        {
-            char next = *start;
-            if (next == '\n')
-                startLine++;
-            else if (next != '\r')
-                break;
-            start++;
-        }
-
-        const char * body = start;
-        const char * cppSeparatorText = "#body";
-        const char * separator = strstr(body, cppSeparatorText);
-        if (separator)
-        {
-            text.setCharAt(separator-text.str(), 0);
-            if (location)
-                funcctx.addLine(locationFilename, startLine);
-            funcctx.addQuoted(body);
-            if (location)
-                funcctx.addLine();
-
-            body = separator + strlen(cppSeparatorText);
-            if (*body == '\r') body++;
-            if (*body == '\n') body++;
-            startLine += memcount(body-start, start, '\n');
-        }
-
-        funcctx.addQuotedCompound(s);
-        if (location)
-            funcctx.addLine(locationFilename, startLine);
-        funcctx.addQuoted(body);
-        if (location)
-            funcctx.addLine();
+        IHqlExpression *languageAttr = bodyCode->queryProperty(languageAtom);
+        if (languageAttr)
+            buildScriptFunctionDefinition(funcctx, funcdef, proto);
+        else
+            buildCppFunctionDefinition(funcctx, bodyCode, proto);
     }
     else
     {
-        funcctx.addQuotedCompound(s);
+        funcctx.addQuotedCompound(proto);
         //MORE: Need to work out how to handle functions that require the context.
         //Need to create a class instead.
         assertex(!outofline->hasProperty(contextAtom));
 
         OwnedHqlExpr newCode = replaceInlineParameters(funcdef, bodyCode);
         newCode.setown(foldHqlExpression(newCode));
+        ITypeInfo * returnType = funcdef->queryType()->queryChildType();
         doBuildUserFunctionReturn(funcctx, returnType, newCode);
     }
 }

ecl/hqlcpp/hqlcpp.ipp

@@ -1112,6 +1112,8 @@ public:
 
     bool expandFunctionPrototype(StringBuffer & s, IHqlExpression * funcdef);
     void expandFunctionPrototype(BuildCtx & ctx, IHqlExpression * funcdef);
+    void buildCppFunctionDefinition(BuildCtx &funcctx, IHqlExpression * bodycode, const char *proto);
+    void buildScriptFunctionDefinition(BuildCtx &funcctx, IHqlExpression * bodycode, const char *proto);
     void buildFunctionDefinition(IHqlExpression * funcdef);
     void assignAndCast(BuildCtx & ctx, const CHqlBoundTarget & target, CHqlBoundExpr & expr);
     void assignCastUnknownLength(BuildCtx & ctx, const CHqlBoundTarget & target, CHqlBoundExpr & pure);
@@ -1269,8 +1271,8 @@ public:
     void doBuildExprCountDict(BuildCtx & ctx, IHqlExpression * expr, CHqlBoundExpr & tgt);
     void doBuildExprCount(BuildCtx & ctx, IHqlExpression * expr, CHqlBoundExpr & tgt);
     void doBuildExprCounter(BuildCtx & ctx, IHqlExpression * expr, CHqlBoundExpr & tgt);
-    void doBuildExprCppBody(BuildCtx & ctx, IHqlExpression * expr, CHqlBoundExpr * tgt);
     void doBuildExprDivide(BuildCtx & ctx, IHqlExpression * expr, CHqlBoundExpr & tgt);
+    void doBuildExprEmbedBody(BuildCtx & ctx, IHqlExpression * expr, CHqlBoundExpr * tgt);
     void doBuildExprEvaluate(BuildCtx & ctx, IHqlExpression * expr, CHqlBoundExpr & tgt);
     void doBuildExprExists(BuildCtx & ctx, IHqlExpression * expr, CHqlBoundExpr & tgt);
     void doBuildExprFailCode(BuildCtx & ctx, IHqlExpression * expr, CHqlBoundExpr & tgt);

ecl/hqlcpp/hqlcppsys.ecl

@@ -804,6 +804,28 @@ const char * cppSystemText[]  = {
     "   _linkcounted_ row(dummyRecord) dictionaryLookup(boolean meta, _linkcounted_ dictionary dict, row key, _linkcounted_ row defaultrow) : eclrtl,include,pure,entrypoint='rtlDictionaryLookup';",
     "    boolean dictionaryLookupExists(boolean meta, _linkcounted_ dictionary dict, row key) : eclrtl,include,pure,entrypoint='rtlDictionaryLookupExists';",
 
+    // Marshalling parameters to external languages
+    "   bindBooleanParam(const varstring name, boolean val) : method,entrypoint='bindBooleanParam';",
+    "   bindDataParam(const varstring name, data val) : method,entrypoint='bindDataParam';",
+    "   bindRealParam(const varstring name, real val) : method,entrypoint='bindRealParam';",
+    "   bindSignedParam(const varstring name, integer val) : method,entrypoint='bindSignedParam';",
+    "   bindUnsignedParam(const varstring name, unsigned val) : method,entrypoint='bindUnsignedParam';",
+    "   bindStringParam(const varstring name, const string val) : method,entrypoint='bindStringParam';",
+    "   bindVStringParam(const varstring name, const varstring val) : method,entrypoint='bindVStringParam';",
+    "   bindUTF8Param(const varstring name, const utf8 val) : method,entrypoint='bindUTF8Param';",
+    "   bindUnicodeParam(const varstring name, const unicode val) : method,entrypoint='bindUnicodeParam';",
+
+    "   boolean getBooleanResult() : method,entrypoint='getBooleanResult';",
+    "   data getDataResult() : method,entrypoint='getDataResult';",
+    "   real getRealResult() : method,entrypoint='getRealResult';",
+    "   integer getSignedResult() : method,entrypoint='getSignedResult';",
+    "   string getStringResult() : method,entrypoint='getStringResult';",
+    "   unsigned getUnsignedResult() : method,entrypoint='getUnsignedResult';",
+    "   utf8 getUTF8Result() : method,entrypoint='getUTF8Result';",
+    "   unicode getUnicodeResult() : method,entrypoint='getUnicodeResult';",
+
+    "   compileEmbeddedScript(const utf8 script) : method,entrypoint='compileEmbeddedScript';",
+    "   import(const utf8 script) : method,entrypoint='importFunction';",
     "   END;",
     NULL };
 

ecl/hqlcpp/hqlcse.cpp

@@ -576,7 +576,7 @@ bool CseSpotter::checkPotentialCSE(IHqlExpression * expr, CseSpotterInfo * extra
     case no_soapcall:
     case no_newsoapcall:
     case no_id2blob:
-    case no_cppbody:
+    case no_embedbody:
     case no_rows:
         return false;
 

ecl/hqlcpp/hqlttcpp.cpp

@@ -5713,7 +5713,7 @@ IHqlExpression * WorkflowTransformer::transformInternalFunction(IHqlExpression *
     OwnedHqlExpr namedFuncDef = newFuncDef->clone(funcdefArgs);
     inheritDependencies(namedFuncDef);
 
-    if (ecl->getOperator() == no_cppbody)
+    if (ecl->getOperator() == no_embedbody)
         return namedFuncDef.getClear();
 
     WorkflowItem * item = new WorkflowItem(namedFuncDef);

ecl/hqlcpp/hqlwcpp.cpp

@@ -590,7 +590,15 @@ bool HqlCppWriter::generateFunctionPrototype(IHqlExpression * funcdef, const cha
     if (body->hasProperty(includeAtom) || body->hasProperty(ctxmethodAtom) || body->hasProperty(gctxmethodAtom) || body->hasProperty(methodAtom) || body->hasProperty(sysAtom) || body->hasProperty(omethodAtom))
         return false;
 
-    enum { ServiceApi, RtlApi, BcdApi, CApi, LocalApi } api = ServiceApi;
+    IHqlExpression *proto = body->queryProperty(prototypeAtom);
+    if (proto)
+    {
+        StringBuffer s;
+        getStringValue(s, proto->queryChild(0));
+        out.append(s);
+        return true;
+    }
+    enum { ServiceApi, RtlApi, BcdApi, CApi, CppApi, LocalApi } api = ServiceApi;
     bool isVirtual = funcdef->hasProperty(virtualAtom);
     bool isLocal = body->hasProperty(localAtom);
     if (body->hasProperty(eclrtlAtom))
@@ -599,16 +607,18 @@ bool HqlCppWriter::generateFunctionPrototype(IHqlExpression * funcdef, const cha
         api = BcdApi;
     else if (body->hasProperty(cAtom))
         api = CApi;
+    else if (body->hasProperty(cppAtom))
+        api = CppApi;
     else if (isLocal || isVirtual)
         api = LocalApi;
 
     if (isVirtual)
         out.append("virtual");
     else
-        out.append("extern ");
+        out.append("extern");
 
     if ((api == ServiceApi) || api == CApi)
-        out.append("\"C\" ");
+        out.append(" \"C\" ");
 
     switch (api)
     {
@@ -1120,6 +1130,11 @@ StringBuffer & HqlCppWriter::generateExprCpp(IHqlExpression * expr)
                     generateExprCpp(expr->queryChild(firstArg)).append(".");
                     ++firstArg;
                 }
+                if (props->hasProperty(namespaceAtom))
+                {
+                    getProperty(props, namespaceAtom, out);
+                    out.append("::");
+                }
                 getProperty(props, entrypointAtom, out);
                 out.append('(');
                 if (props->hasProperty(contextAtom))

plugins/CMakeLists.txt

@@ -24,3 +24,16 @@ add_subdirectory (workunitservices)
 if ("${BUILD_LEVEL}" STREQUAL "COMMUNITY")
   add_subdirectory (proxies)
 endif ()
+
+if (MAKE_V8EMBED)
+  add_subdirectory (v8embed)
+endif ()
+if (MAKE_PYEMBED)
+  add_subdirectory (pyembed)
+endif ()
+if (MAKE_JAVAEMBED)
+  add_subdirectory (javaembed)
+endif()
+if (MAKE_REMBED)
+  add_subdirectory (Rembed)
+endif()

plugins/Rembed/CMakeLists.txt

@@ -0,0 +1,60 @@
+################################################################################
+#    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+################################################################################
+
+
+# Component: Rembed
+
+#####################################################
+# Description:
+# ------------
+#    Cmake Input File for Rembed
+#####################################################
+
+project( Rembed )
+
+find_package(R)
+if (NOT R_FOUND)
+  message(FATAL_ERROR "MAKE_REMBED requested but R libraries not found")
+endif()
+
+set (    SRCS
+         Rembed.cpp
+    )
+
+include_directories (
+         "${R_INCLUDE_DIRS}"
+         ./../../system/include
+         ./../../rtl/eclrtl
+         ./../../rtl/include
+         ./../../system/jlib
+    )
+
+ADD_DEFINITIONS( -D_USRDLL -DREMBED_EXPORTS )
+
+HPCC_ADD_LIBRARY( Rembed SHARED ${SRCS} )
+if (${CMAKE_VERSION} VERSION_LESS "2.8.9")
+  message("WARNING: Cannot set NO_SONAME. shlibdeps will give warnings when package is installed")
+else()
+  set_target_properties( Rembed PROPERTIES NO_SONAME 1 )
+endif()
+
+install ( TARGETS Rembed DESTINATION plugins )
+
+target_link_libraries ( Rembed
+    ${R_LIBRARIES}
+    eclrtl
+    jlib
+    )

plugins/Rembed/Rembed.cpp

@@ -0,0 +1,243 @@
+/*##############################################################################
+
+ HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ ############################################################################## */
+
+#include "platform.h"
+#include "RInside.h"
+#include "eclrtl.hpp"
+#include "jexcept.hpp"
+#include "jthread.hpp"
+#include "hqlplugins.hpp"
+
+#ifdef _WIN32
+#define EXPORT __declspec(dllexport)
+#else
+#define EXPORT
+#endif
+
+static const char * compatibleVersions[] =
+{ "R Embed Helper 1.0.0", NULL };
+
+static const char *version = "R Embed Helper 1.0.0";
+
+static const char * EclDefinition =
+    "EXPORT Language := SERVICE\n"
+    "  boolean getEmbedContext():cpp,pure,namespace='Rembed',entrypoint='getEmbedContext',prototype='IEmbedContext* getEmbedContext()';\n"
+    "  boolean syntaxCheck(const varstring src):cpp,pure,namespace='Rembed',entrypoint='syntaxCheck';\n"
+    "  unload():cpp,pure,namespace='Rembed',entrypoint='unload';\n"
+    "END;"
+    "EXPORT getEmbedContext := Language.getEmbedContext;"
+    "EXPORT syntaxCheck := Language.syntaxCheck;"
+    "EXPORT boolean supportsImport := false;"
+    "EXPORT boolean supportsScript := true;";
+
+extern "C" EXPORT bool getECLPluginDefinition(ECLPluginDefinitionBlock *pb)
+{
+    if (pb->size == sizeof(ECLPluginDefinitionBlockEx))
+    {
+        ECLPluginDefinitionBlockEx * pbx = (ECLPluginDefinitionBlockEx *) pb;
+        pbx->compatibleVersions = compatibleVersions;
+    }
+    else if (pb->size != sizeof(ECLPluginDefinitionBlock))
+        return false;
+    pb->magicVersion = PLUGIN_VERSION;
+    pb->version = version;
+    pb->moduleName = "R";
+    pb->ECL = EclDefinition;
+    pb->flags = PLUGIN_DLL_MODULE | PLUGIN_MULTIPLE_VERSIONS;
+    pb->description = "R Embed Helper";
+    return true;
+}
+
+namespace Rembed
+{
+
+// Use a global object to ensure that the R instance is initialized only once
+
+static class RGlobalState
+{
+public:
+    RGlobalState()
+    {
+        const char *args[] = {"R", "--slave" };
+        R = new RInside(2, args, true, false, false);
+
+    }
+    ~RGlobalState()
+    {
+        delete R;
+    }
+    RInside *R;
+}* globalState = NULL;
+
+static CriticalSection RCrit;  // R is single threaded - need to own this before making any call to R
+
+static RGlobalState *queryGlobalState()
+{
+    CriticalBlock b(RCrit);
+    if (!globalState)
+        globalState = new RGlobalState;
+    return globalState;
+}
+
+extern void unload()
+{
+    CriticalBlock b(RCrit);
+    if (globalState)
+        delete globalState;
+    globalState = NULL;
+}
+
+MODULE_INIT(INIT_PRIORITY_STANDARD)
+{
+    return true;
+}
+MODULE_EXIT()
+{
+    unload();
+}
+
+// Each call to a R function will use a new REmbedFunctionContext object
+// This takes care of ensuring that the critsec is locked while we are executing R code,
+// and released when we are not
+
+class REmbedFunctionContext: public CInterfaceOf<IEmbedFunctionContext>
+{
+public:
+    REmbedFunctionContext(RInside &_R, const char *options)
+    : R(_R), block(RCrit), result(R_NilValue)
+    {
+    }
+    ~REmbedFunctionContext()
+    {
+    }
+
+    virtual bool getBooleanResult()
+    {
+        return ::Rcpp::as<bool>(result);
+    }
+    virtual void getDataResult(size32_t &__len, void * &__result)
+    {
+        std::vector<byte> vval = ::Rcpp::as<std::vector<byte> >(result);;
+        rtlStrToDataX(__len, __result, vval.size(), vval.data());
+    }
+    virtual double getRealResult()
+    {
+        return ::Rcpp::as<double>(result);
+    }
+    virtual __int64 getSignedResult()
+    {
+        return ::Rcpp::as<long int>(result); // Should really be long long, but RInside does not support that
+    }
+    virtual unsigned __int64 getUnsignedResult()
+    {
+        return ::Rcpp::as<unsigned long int>(result); // Should really be long long, but RInside does not support that
+    }
+    virtual void getStringResult(size32_t &__len, char * &__result)
+    {
+        std::string str = ::Rcpp::as<std::string>(result);
+        rtlStrToStrX(__len, __result, str.length(), str.data());
+    }
+    virtual void getUTF8Result(size32_t &chars, char * &result)
+    {
+        throw MakeStringException(MSGAUD_user, 0, "Rembed: %s: Unicode/UTF8 results not supported", func.c_str());
+    }
+    virtual void getUnicodeResult(size32_t &chars, UChar * &result)
+    {
+        throw MakeStringException(MSGAUD_user, 0, "Rembed: %s: Unicode/UTF8 results not supported", func.c_str());
+    }
+
+    virtual void bindBooleanParam(const char *name, bool val)
+    {
+        R[name] = val;
+    }
+    virtual void bindDataParam(const char *name, size32_t len, const void *val)
+    {
+        std::vector<byte> vval;
+        const byte *cval = (const byte *) val;
+        vval.assign(cval, cval+len);
+        R[name] = vval;
+    }
+    virtual void bindRealParam(const char *name, double val)
+    {
+        R[name] = val;
+    }
+    virtual void bindSignedParam(const char *name, __int64 val)
+    {
+        R[name] = (long int) val;
+    }
+    virtual void bindUnsignedParam(const char *name, unsigned __int64 val)
+    {
+        R[name] = (unsigned long int) val;
+    }
+    virtual void bindStringParam(const char *name, size32_t len, const char *val)
+    {
+        std::string s(val, len);
+        R[name] = s;
+    }
+    virtual void bindVStringParam(const char *name, const char *val)
+    {
+        R[name] = val;
+    }
+    virtual void bindUTF8Param(const char *name, size32_t chars, const char *val)
+    {
+        UNIMPLEMENTED;
+    }
+    virtual void bindUnicodeParam(const char *name, size32_t chars, const UChar *val)
+    {
+        UNIMPLEMENTED;
+    }
+
+    virtual void importFunction(size32_t lenChars, const char *utf)
+    {
+        throwUnexpected();
+    }
+    virtual void compileEmbeddedScript(size32_t lenChars, const char *utf)
+    {
+        func.assign(utf, rtlUtf8Size(lenChars, utf));
+    }
+
+    virtual void callFunction()
+    {
+        result = R.parseEval(func);
+    }
+private:
+    RInside &R;
+    RInside::Proxy result;
+    std::string func;
+    CriticalBlock block;
+};
+
+class REmbedContext: public CInterfaceOf<IEmbedContext>
+{
+public:
+    virtual IEmbedFunctionContext *createFunctionContext(bool isImport, const char *options)
+    {
+        return new REmbedFunctionContext(*queryGlobalState()->R, options);
+    }
+};
+
+extern IEmbedContext* getEmbedContext()
+{
+    return new REmbedContext;
+}
+
+extern bool syntaxCheck(const char *script)
+{
+    return true; // MORE
+}
+
+} // namespace

plugins/javaembed/CMakeLists.txt

@@ -0,0 +1,72 @@
+################################################################################
+#    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+################################################################################
+
+
+# Component: javaembed
+
+#####################################################
+# Description:
+# ------------
+#    Cmake Input File for javaembed
+#####################################################
+
+project( javaembed )
+
+find_package(JNI)
+if (NOT JNI_FOUND)
+  message(FATAL_ERROR "MAKE_JAVAEMBED requested but JNI libraries not found")
+endif()
+
+set (    SRCS
+         javaembed.cpp
+    )
+
+include_directories (
+         ${JNI_INCLUDE_DIRS}
+         ./../../system/include
+         ./../../rtl/eclrtl
+         ./../../rtl/include
+         ./../../system/jlib
+    )
+
+ADD_DEFINITIONS( -D_USRDLL -DJAVAEMBED_EXPORTS )
+
+HPCC_ADD_LIBRARY( javaembed SHARED ${SRCS} )
+if (${CMAKE_VERSION} VERSION_LESS "2.8.9")
+  message("WARNING: Cannot set NO_SONAME. shlibdeps will give warnings when package is installed")
+else()
+  set_target_properties( javaembed PROPERTIES NO_SONAME 1 )
+endif()
+
+install ( TARGETS javaembed DESTINATION plugins )
+
+# We link against jsig so that signals are chained from the jvm
+
+get_filename_component(JAVA_LIBRARY_PATH ${JAVA_AWT_LIBRARY} PATH)
+if (APPLE)
+    set(JSIG_LIBRARY ${JAVA_LIBRARY_PATH}/libjsig.dylib)
+elseif (WIN32)
+    set(JSIG_LIBRARY ${JAVA_LIBRARY_PATH}/jsig.dll)
+else()
+    set(JSIG_LIBRARY ${JAVA_LIBRARY_PATH}/libjsig.so)
+endif()
+
+target_link_libraries ( javaembed
+#    ${JSIG_LIBRARY}
+    ${JNI_LIBRARIES}
+    eclrtl
+    jlib
+    )

plugins/javaembed/javaembed.cpp

@@ -0,0 +1,641 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#include "platform.h"
+#include <jni.h>
+#include "eclrtl.hpp"
+#include "jexcept.hpp"
+#include "jthread.hpp"
+#include "hqlplugins.hpp"
+
+#ifdef _WIN32
+#define EXPORT __declspec(dllexport)
+#else
+#define EXPORT
+#endif
+
+static const char * compatibleVersions[] = {
+    "Java Embed Helper 1.0.0",
+    NULL };
+
+static const char *version = "Java Embed Helper 1.0.0";
+
+static const char * EclDefinition =
+    "EXPORT Language := SERVICE\n"
+    "  boolean getEmbedContext():cpp,pure,namespace='javaembed',entrypoint='getEmbedContext',prototype='IEmbedContext* getEmbedContext()';\n"
+    "END;"
+    "EXPORT getEmbedContext := Language.getEmbedContext;"
+    "EXPORT boolean supportsImport := true;"
+    "EXPORT boolean supportsScript := false;";
+
+extern "C" EXPORT bool getECLPluginDefinition(ECLPluginDefinitionBlock *pb)
+{
+    if (pb->size == sizeof(ECLPluginDefinitionBlockEx))
+    {
+        ECLPluginDefinitionBlockEx * pbx = (ECLPluginDefinitionBlockEx *) pb;
+        pbx->compatibleVersions = compatibleVersions;
+    }
+    else if (pb->size != sizeof(ECLPluginDefinitionBlock))
+        return false;
+    pb->magicVersion = PLUGIN_VERSION;
+    pb->version = version;
+    pb->moduleName = "java";
+    pb->ECL = EclDefinition;
+    pb->flags = PLUGIN_DLL_MODULE | PLUGIN_MULTIPLE_VERSIONS;
+    pb->description = "Java Embed Helper";
+    return true;
+}
+
+namespace javaembed {
+
+// Use a global object to ensure that the Java VM  is initialized once only.
+// We would like to create it lazily for two reasons:
+// 1. So that we only get a JVM if we need one (even if we have loaded the plugin)
+// 2. It's important for the JVM to be initialized AFTER we have set up signal handlers, as it
+//    likes to set its own (in particular, it seems to intercept and ignore some SIGSEGV during the
+//    garbage collection).
+// Unfortunately, it seems that the design of the JNI interface is such that JNI_CreateJavaVM has to be called on the 'main thread'.
+// So we can't achieve 1, and 2 requires that we create via the INIT_MODLE mechanism (rather than just a static object), and that
+// any engines that call InitModuleObjects() or load plugins dynamically do so AFTER setting any signal handlers or calling
+// EnableSEHtoExceptionMapping
+//
+
+static class JavaGlobalState
+{
+public:
+    JavaGlobalState()
+    {
+        JavaVMInitArgs vm_args; /* JDK/JRE 6 VM initialization arguments */
+        JavaVMOption* options = new JavaVMOption[3];
+        const char* origPath = getenv("CLASSPATH");
+        StringBuffer newPath;
+        newPath.append("-Djava.class.path=").append(origPath).append(ENVSEPCHAR).append(".");
+        options[0].optionString = (char *) newPath.str();
+        options[1].optionString = (char *) "-Xcheck:jni";
+        options[2].optionString = (char *) "-verbose:jni";
+        vm_args.version = JNI_VERSION_1_6;
+#ifdef _DEBUG
+        vm_args.nOptions = 1;  // set to 3 if you want the verbose...
+#else
+        vm_args.nOptions = 1;
+#endif
+        vm_args.options = options;
+        vm_args.ignoreUnrecognized = false;
+        /* load and initialize a Java VM, return a JNI interface pointer in env */
+        JNIEnv *env;       /* receives pointer to native method interface */
+        JNI_CreateJavaVM(&javaVM, (void**)&env, &vm_args);
+
+        delete [] options;
+    }
+    ~JavaGlobalState()
+    {
+        // We don't attempt to destroy the Java VM, as it's buggy...
+    }
+    JavaVM *javaVM;       /* denotes a Java VM */
+} *globalState;
+
+MODULE_INIT(INIT_PRIORITY_STANDARD)
+{
+    globalState = new JavaGlobalState;
+    return true;
+}
+MODULE_EXIT()
+{
+    delete globalState;
+    globalState = NULL;
+}
+
+// There is a singleton JavaThreadContext per thread. This allows us to
+// ensure that we can make repeated calls to a Java function efficiently.
+
+class JavaThreadContext
+{
+public:
+    JNIEnv *JNIenv;       /* receives pointer to native method interface */
+public:
+    JavaThreadContext()
+    {
+        jint res = globalState->javaVM->AttachCurrentThread((void **) &JNIenv, NULL);
+        assertex(res >= 0);
+        javaClass = NULL;
+        javaMethodID = NULL;
+    }
+    ~JavaThreadContext()
+    {
+        if (javaClass)
+            JNIenv->DeleteGlobalRef(javaClass);
+    }
+
+    void checkException()
+    {
+        jthrowable exception = JNIenv->ExceptionOccurred();
+        if (exception)
+        {
+            JNIenv->ExceptionClear();
+            jclass throwableClass = JNIenv->FindClass("java/lang/Throwable");
+            jmethodID throwableToString = JNIenv->GetMethodID(throwableClass, "toString", "()Ljava/lang/String;");
+            jstring cause = (jstring) JNIenv->CallObjectMethod(exception, throwableToString);
+            const char *text = JNIenv->GetStringUTFChars(cause, 0);
+            throw MakeStringException(MSGAUD_user, 0, "javaembed: In method %s: %s", prevtext.get(), text);
+        }
+    }
+
+    inline void importFunction(size32_t lenChars, const char *utf)
+    {
+        size32_t bytes = rtlUtf8Size(lenChars, utf);
+        StringBuffer text(bytes, utf);
+        if (!prevtext || strcmp(text, prevtext) != 0)
+        {
+            prevtext.clear();
+            // Name should be in the form class.method:signature
+            const char *funcname = strchr(text, '.');
+            if (!funcname)
+                throw MakeStringException(MSGAUD_user, 0, "javaembed: Invalid import name %s - Expected classname.methodname:signature", text.str());
+            const char *signature = strchr(funcname, ':');
+            if (!signature)
+                throw MakeStringException(MSGAUD_user, 0, "javaembed: Invalid import name %s - Expected classname.methodname:signature", text.str());
+            StringBuffer classname(funcname-text, text);
+            funcname++;  // skip the '.'
+            StringBuffer methodname(signature-funcname, funcname);
+            signature++; // skip the ':'
+            if (javaClass)
+                JNIenv->DeleteGlobalRef(javaClass);
+            javaClass = (jclass) JNIenv->NewGlobalRef(JNIenv->FindClass(classname));
+            if (!javaClass)
+                throw MakeStringException(MSGAUD_user, 0, "javaembed: Failed to resolve class name %s", classname.str());
+            javaMethodID = JNIenv->GetStaticMethodID(javaClass, methodname, signature);
+            if (!javaMethodID)
+                throw MakeStringException(MSGAUD_user, 0, "javaembed: Failed to resolve method name %s with signature %s", methodname.str(), signature);
+            const char *returnSig = strrchr(signature, ')');
+            assertex(returnSig);  // Otherwise how did Java accept it??
+            returnSig++;
+            returnType.set(returnSig);
+            argsig.set(signature);
+            prevtext.set(text);
+        }
+    }
+    inline void callFunction(jvalue &result, const jvalue * args)
+    {
+        JNIenv->ExceptionClear();
+        switch (returnType.get()[0])
+        {
+        case 'C': result.c = JNIenv->CallStaticCharMethodA(javaClass, javaMethodID, args); break;
+        case 'Z': result.z = JNIenv->CallStaticBooleanMethodA(javaClass, javaMethodID, args); break;
+        case 'J': result.j = JNIenv->CallStaticLongMethodA(javaClass, javaMethodID, args); break;
+        case 'F': result.f = JNIenv->CallStaticFloatMethodA(javaClass, javaMethodID, args); break;
+        case 'D': result.d = JNIenv->CallStaticDoubleMethodA(javaClass, javaMethodID, args); break;
+        case 'I': result.i = JNIenv->CallStaticIntMethodA(javaClass, javaMethodID, args); break;
+        case 'S': result.s = JNIenv->CallStaticShortMethodA(javaClass, javaMethodID, args); break;
+        case 'B': result.s = JNIenv->CallStaticByteMethodA(javaClass, javaMethodID, args); break;
+
+        case '[':
+        case 'L': result.l = JNIenv->CallStaticObjectMethodA(javaClass, javaMethodID, args); break;
+
+        default: throwUnexpected();
+        }
+        checkException();
+    }
+    inline __int64 getSignedResult(jvalue & result)
+    {
+        switch (returnType.get()[0])
+        {
+        case 'B': return result.b;
+        case 'S': return result.s;
+        case 'I': return result.i;
+        case 'J': return result.j;
+        case 'L':
+            {
+                // Result should be of class 'Number'
+                jmethodID getVal = JNIenv->GetMethodID(JNIenv->GetObjectClass(result.l), "longValue", "()J");
+                if (!getVal)
+                    throw MakeStringException(MSGAUD_user, 0, "javaembed: Type mismatch on result");
+                return JNIenv->CallLongMethod(result.l, getVal);
+            }
+        default:
+            throw MakeStringException(MSGAUD_user, 0, "javaembed: Type mismatch on result");
+        }
+    }
+    inline double getDoubleResult(jvalue &result)
+    {
+        switch (returnType.get()[0])
+        {
+        case 'D': return result.d;
+        case 'F': return result.f;
+        case 'L':
+            {
+                // Result should be of class 'Number'
+                jmethodID getVal = JNIenv->GetMethodID(JNIenv->GetObjectClass(result.l), "doubleValue", "()D");
+                if (!getVal)
+                    throw MakeStringException(MSGAUD_user, 0, "javaembed: Type mismatch on result");
+                return JNIenv->CallDoubleMethod(result.l, getVal);
+            }
+        default:
+            throw MakeStringException(MSGAUD_user, 0, "javaembed: Type mismatch on result");
+        }
+    }
+    inline bool getBooleanResult(jvalue &result)
+    {
+        switch (returnType.get()[0])
+        {
+        case 'Z': return result.z;
+        case 'L':
+            {
+                // Result should be of class 'Boolean'
+                jmethodID getVal = JNIenv->GetMethodID(JNIenv->GetObjectClass(result.l), "booleanValue", "()Z");
+                if (!getVal)
+                    throw MakeStringException(MSGAUD_user, 0, "javaembed: Type mismatch on result");
+                return JNIenv->CallBooleanMethod(result.l, getVal);
+            }
+        default:
+            throw MakeStringException(MSGAUD_user, 0, "javaembed: Type mismatch on result");
+        }
+    }
+    inline void getDataResult(jvalue &result, size32_t &__len, void * &__result)
+    {
+        if (strcmp(returnType, "[B")!=0)
+            throw MakeStringException(MSGAUD_user, 0, "javaembed: Type mismatch on result");
+        jbyteArray array = (jbyteArray) result.l;
+        __len = JNIenv->GetArrayLength(array);
+        __result = rtlMalloc(__len);
+        JNIenv->GetByteArrayRegion(array, 0, __len, (jbyte *) __result);
+    }
+    inline void getStringResult(jvalue &result, size32_t &__len, char * &__result)
+    {
+        switch (returnType.get()[0])
+        {
+        case 'C': // Single char returned, prototyped as STRING or STRING1 in ECL
+            rtlUnicodeToStrX(__len, __result, 1, &result.c);
+            break;
+        case 'L':
+        {
+            jstring sresult = (jstring) result.l;
+            size_t size = JNIenv->GetStringUTFLength(sresult);  // in bytes
+            const char *text =  JNIenv->GetStringUTFChars(sresult, NULL);
+            size32_t chars = rtlUtf8Length(size, text);
+            rtlUtf8ToStrX(__len, __result, chars, text);
+            JNIenv->ReleaseStringUTFChars(sresult, text);
+            break;
+        }
+        default:
+            throwUnexpected();
+        }
+    }
+    inline void getUTF8Result(jvalue &result, size32_t &__chars, char * &__result)
+    {
+        switch (returnType.get()[0])
+        {
+        case 'C': // Single jchar returned, prototyped as UTF8 in ECL
+            rtlUnicodeToUtf8X(__chars, __result, 1, &result.c);
+            break;
+        case 'L':
+        {
+            jstring sresult = (jstring) result.l;
+            size_t size = JNIenv->GetStringUTFLength(sresult); // Returns length in bytes (not chars)
+            const char * text =  JNIenv->GetStringUTFChars(sresult, NULL);
+            rtlUtf8ToUtf8X(__chars, __result, rtlUtf8Length(size, text), text);
+            JNIenv->ReleaseStringUTFChars(sresult, text);
+            break;
+        }
+        default:
+            throwUnexpected();
+        }
+    }
+    inline void getUnicodeResult(jvalue &result, size32_t &__chars, UChar * &__result)
+    {
+        switch (returnType.get()[0])
+        {
+        case 'C': // Single jchar returned, prototyped as UNICODE or UNICODE1 in ECL
+            rtlUnicodeToUnicodeX(__chars, __result, 1, &result.c);
+            break;
+        case 'L':
+        {
+            jstring sresult = (jstring) result.l;
+            size_t size = JNIenv->GetStringUTFLength(sresult);  // in bytes
+            const char *text =  JNIenv->GetStringUTFChars(sresult, NULL);
+            size32_t chars = rtlUtf8Length(size, text);
+            rtlUtf8ToUnicodeX(__chars, __result, chars, text);
+            JNIenv->ReleaseStringUTFChars(sresult, text);
+            break;
+        }
+        default:
+            throwUnexpected();
+        }
+    }
+    inline const char *querySignature()
+    {
+        return argsig.get();
+    }
+private:
+    StringAttr returnType;
+    StringAttr argsig;
+    StringAttr prevtext;
+    jclass javaClass;
+    jmethodID javaMethodID;
+};
+
+// Each call to a Java function will use a new JavaEmbedScriptContext object
+#define MAX_JNI_ARGS 10
+
+class JavaEmbedImportContext : public CInterfaceOf<IEmbedFunctionContext>
+{
+public:
+    JavaEmbedImportContext(JavaThreadContext *_sharedCtx, const char *options)
+    : sharedCtx(_sharedCtx)
+    {
+        argcount = 0;
+        argsig = NULL;
+    }
+    ~JavaEmbedImportContext()
+    {
+    }
+
+    virtual bool getBooleanResult()
+    {
+        return sharedCtx->getBooleanResult(result);
+    }
+    virtual void getDataResult(size32_t &__len, void * &__result)
+    {
+        sharedCtx->getDataResult(result, __len, __result);
+    }
+    virtual double getRealResult()
+    {
+        return sharedCtx->getDoubleResult(result);
+    }
+    virtual __int64 getSignedResult()
+    {
+        return sharedCtx->getSignedResult(result);
+    }
+    virtual unsigned __int64 getUnsignedResult()
+    {
+        throw MakeStringException(MSGAUD_user, 0, "javaembed: Unsigned results not supported"); // Java doesn't support unsigned
+    }
+    virtual void getStringResult(size32_t &__len, char * &__result)
+    {
+        sharedCtx->getStringResult(result, __len, __result);
+    }
+    virtual void getUTF8Result(size32_t &__chars, char * &__result)
+    {
+        sharedCtx->getUTF8Result(result, __chars, __result);
+    }
+    virtual void getUnicodeResult(size32_t &__chars, UChar * &__result)
+    {
+        sharedCtx->getUnicodeResult(result, __chars, __result);
+    }
+
+
+    virtual void bindBooleanParam(const char *name, bool val)
+    {
+        if (*argsig != 'B')
+            typeError("BOOLEAN");
+        argsig++;
+        jvalue v;
+        v.z = val;
+        addArg(v);
+    }
+    virtual void bindDataParam(const char *name, size32_t len, const void *val)
+    {
+        if (argsig[0] != '[' || argsig[1] != 'B')
+            typeError("DATA");
+        argsig += 2;
+        jvalue v;
+        jbyteArray javaData = sharedCtx->JNIenv->NewByteArray(len);
+        sharedCtx->JNIenv->SetByteArrayRegion(javaData, 0, len, (jbyte *) val);
+        v.l = javaData;
+        addArg(v);
+    }
+    virtual void bindRealParam(const char *name, double val)
+    {
+        jvalue v;
+        switch(*argsig)
+        {
+        case 'D':
+            v.d = val;
+            break;
+        case 'F':
+            v.f = val;
+            break;
+        default:
+            typeError("REAL");
+            break;
+        }
+        argsig++;
+        addArg(v);
+    }
+    virtual void bindSignedParam(const char *name, __int64 val)
+    {
+        jvalue v;
+        switch(*argsig)
+        {
+        case 'I':
+            v.i = val;
+            break;
+        case 'J':
+            v.j = val;
+            break;
+        case 'S':
+            v.s = val;
+            break;
+        case 'B':
+            v.b = val;
+            break;
+        default:
+            typeError("INTEGER");
+            break;
+        }
+        argsig++;
+        addArg(v);
+    }
+    virtual void bindUnsignedParam(const char *name, unsigned __int64 val)
+    {
+        throw MakeStringException(MSGAUD_user, 0, "javaembed: Unsigned parameters not supported"); // Java doesn't support unsigned
+    }
+    virtual void bindStringParam(const char *name, size32_t len, const char *val)
+    {
+        jvalue v;
+        switch(*argsig)
+        {
+        case 'C':
+            rtlStrToUnicode(1, &v.c, len, val);
+            argsig++;
+            break;
+        case 'L':
+            if (strncmp(argsig, "Ljava/lang/String;", 18) == 0)
+            {
+                argsig += 18;
+                unsigned unicodeChars;
+                UChar *unicode;
+                rtlStrToUnicodeX(unicodeChars, unicode, len, val);
+                v.l = sharedCtx->JNIenv->NewString(unicode, unicodeChars);
+                rtlFree(unicode);
+                break;
+            }
+            // fall into ...
+        default:
+            typeError("STRING");
+            break;
+        }
+        addArg(v);
+    }
+    virtual void bindVStringParam(const char *name, const char *val)
+    {
+        bindStringParam(name, strlen(val), val);
+    }
+    virtual void bindUTF8Param(const char *name, size32_t numchars, const char *val)
+    {
+        jvalue v;
+        switch(*argsig)
+        {
+        case 'C':
+            rtlUtf8ToUnicode(1, &v.c, numchars, val);
+            argsig++;
+            break;
+        case 'L':
+            if (strncmp(argsig, "Ljava/lang/String;", 18) == 0)
+            {
+                argsig += 18;
+                unsigned unicodeChars;
+                UChar *unicode;
+                rtlUtf8ToUnicodeX(unicodeChars, unicode, numchars, val);
+                v.l = sharedCtx->JNIenv->NewString(unicode, unicodeChars);
+                rtlFree(unicode);
+                break;
+            }
+            // fall into ...
+        default:
+            typeError("UTF8");
+            break;
+        }
+        addArg(v);
+    }
+    virtual void bindUnicodeParam(const char *name, size32_t numchars, const UChar *val)
+    {
+        jvalue v;
+        switch(*argsig)
+        {
+        case 'C':
+            rtlUnicodeToUnicode(1, &v.c, numchars, val);
+            argsig++;
+            break;
+        case 'L':
+            if (strncmp(argsig, "Ljava/lang/String;", 18) == 0)
+            {
+                argsig += 18;
+                v.l = sharedCtx->JNIenv->NewString(val, numchars);
+                break;
+            }
+            // fall into ...
+        default:
+            typeError("UNICODE");
+            break;
+        }
+        addArg(v);
+    }
+
+    virtual void importFunction(size32_t lenChars, const char *utf)
+    {
+        sharedCtx->importFunction(lenChars, utf);
+        argsig = sharedCtx->querySignature();
+        assertex(*argsig == '(');
+        argsig++;
+    }
+    virtual void callFunction()
+    {
+        sharedCtx->callFunction(result, args);
+    }
+
+    virtual void compileEmbeddedScript(size32_t lenChars, const char *script)
+    {
+        throwUnexpected();  // The java language helper supports only imported functions, not embedding java code in ECL.
+    }
+protected:
+    JavaThreadContext *sharedCtx;
+    jvalue result;
+private:
+    void typeError(const char *ECLtype)
+    {
+        const char *javaType;
+        int javaLen = 0;
+        switch (*argsig)
+        {
+        case 'Z': javaType = "boolean"; break;
+        case 'B': javaType = "byte"; break;
+        case 'C': javaType = "char"; break;
+        case 'S': javaType = "short"; break;
+        case 'I': javaType = "int"; break;
+        case 'J': javaType = "long"; break;
+        case 'F': javaType = "float"; break;
+        case 'D': javaType = "double"; break;
+        case 'L':
+            {
+                javaType = argsig+1;
+                const char *semi = strchr(argsig, ';');
+                if (semi)
+                    javaLen = semi - javaType;
+                break;
+            }
+        case ')':
+            throw MakeStringException(0, "javaembed: Too many ECL parameters passed for Java signature %s", sharedCtx->querySignature());
+        default:
+            throw MakeStringException(0, "javaembed: Unrecognized character %c in java signature %s", *argsig, sharedCtx->querySignature());
+        }
+        if (!javaLen)
+            javaLen = strlen(argsig);
+        throw MakeStringException(0, "javaembed: ECL type %s cannot be passed to Java type %.*s", ECLtype, javaLen, javaType);
+    }
+    void addArg(jvalue &arg)
+    {
+        assertex(argcount < MAX_JNI_ARGS);
+        args[argcount] = arg;
+        argcount++;
+    }
+    jvalue args[MAX_JNI_ARGS];
+    int argcount;
+    const char *argsig;
+};
+
+static __thread JavaThreadContext* threadContext;  // We reuse per thread, for speed
+static __thread ThreadTermFunc threadHookChain;
+
+static void releaseContext()
+{
+    delete threadContext;
+    threadContext = NULL;
+    if (threadHookChain)
+        (*threadHookChain)();
+}
+
+class JavaEmbedContext : public CInterfaceOf<IEmbedContext>
+{
+public:
+    virtual IEmbedFunctionContext *createFunctionContext(bool isImport, const char *options)
+    {
+        if (!threadContext)
+        {
+            threadContext = new JavaThreadContext;
+            threadHookChain = addThreadTermFunc(releaseContext);
+        }
+        assertex(isImport);
+        return new JavaEmbedImportContext(threadContext, options);
+    }
+};
+
+extern IEmbedContext* getEmbedContext()
+{
+    return new JavaEmbedContext;
+}
+
+} // namespace

plugins/pyembed/CMakeLists.txt

@@ -0,0 +1,70 @@
+################################################################################
+#    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+################################################################################
+
+
+# Component: pyembed
+
+#####################################################
+# Description:
+# ------------
+#    Cmake Input File for pyembed
+#####################################################
+
+set ( debug_python Off )   # A lot slower but can assist in debugging...
+
+project( pyembed )
+
+find_package(PythonLibs)
+if ("${PYTHON_INCLUDE_DIR}" STREQUAL "")
+  message(FATAL_ERROR "MAKE_PYEMBED requested but python libraries not found")
+endif()
+
+set (    SRCS
+         pyembed.cpp
+    )
+
+include_directories (
+         "${PYTHON_INCLUDE_DIR}"
+         ./../../system/include
+         ./../../rtl/eclrtl
+         ./../../rtl/include
+         ./../../system/jlib
+    )
+
+ADD_DEFINITIONS( -D_USRDLL -DPYEMBED_EXPORTS )
+if (debug_python)
+  ADD_DEFINITIONS(-DPy_DEBUG)
+endif()
+
+HPCC_ADD_LIBRARY( pyembed SHARED ${SRCS} )
+if (${CMAKE_VERSION} VERSION_LESS "2.8.9")
+  message("WARNING: Cannot set NO_SONAME. shlibdeps will give warnings when package is installed")
+else()
+  set_target_properties( pyembed PROPERTIES NO_SONAME 1 )
+endif()
+
+install ( TARGETS pyembed DESTINATION plugins )
+
+if (debug_python)
+  target_link_libraries ( pyembed ${PYTHON_DEBUG_LIBRARY} )
+else()
+  target_link_libraries ( pyembed ${PYTHON_LIBRARY} )
+endif()
+
+target_link_libraries ( pyembed
+    eclrtl
+    jlib
+    )

plugins/pyembed/pyembed.cpp

@@ -0,0 +1,561 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#include "platform.h"
+#include "Python.h"
+#include "eclrtl.hpp"
+#include "jexcept.hpp"
+#include "jthread.hpp"
+#include "hqlplugins.hpp"
+
+#ifdef _WIN32
+#define EXPORT __declspec(dllexport)
+#else
+#define EXPORT
+#endif
+
+static const char * compatibleVersions[] = {
+    "Python2.7 Embed Helper 1.0.0",
+    NULL };
+
+static const char *version = "Python2.7 Embed Helper 1.0.0";
+
+static const char * EclDefinition =
+    "EXPORT Language := SERVICE\n"
+    "  boolean getEmbedContext():cpp,pure,namespace='pyembed',entrypoint='getEmbedContext',prototype='IEmbedContext* getEmbedContext()';\n"
+    "  boolean syntaxCheck(const varstring src):cpp,pure,namespace='pyembed',entrypoint='syntaxCheck';\n"
+    "END;"
+    "EXPORT getEmbedContext := Language.getEmbedContext;"
+    "EXPORT syntaxCheck := Language.syntaxCheck;"
+    "EXPORT boolean supportsImport := true;"
+    "EXPORT boolean supportsScript := true;";
+
+extern "C" EXPORT bool getECLPluginDefinition(ECLPluginDefinitionBlock *pb)
+{
+    if (pb->size == sizeof(ECLPluginDefinitionBlockEx))
+    {
+        ECLPluginDefinitionBlockEx * pbx = (ECLPluginDefinitionBlockEx *) pb;
+        pbx->compatibleVersions = compatibleVersions;
+    }
+    else if (pb->size != sizeof(ECLPluginDefinitionBlock))
+        return false;
+    pb->magicVersion = PLUGIN_VERSION;
+    pb->version = version;
+    pb->moduleName = "python";
+    pb->ECL = EclDefinition;
+    pb->flags = PLUGIN_DLL_MODULE | PLUGIN_MULTIPLE_VERSIONS;
+    pb->description = "Python2.7 Embed Helper";
+    return true;
+}
+
+namespace pyembed {
+
+// Use class OwnedPyObject for any objects that are not 'borrowed references'
+// so that the appropriate Py_DECREF call is made when the OwnedPyObject goes
+// out of scope, even if the function returns prematurely (such as via an exception).
+// In particular, checkPythonError is a lot easier to call safely if this is used.
+
+class OwnedPyObject
+{
+    PyObject *ptr;
+public:
+    inline OwnedPyObject() : ptr(NULL)     {}
+    inline OwnedPyObject(PyObject *_ptr) : ptr(_ptr) {}
+    inline ~OwnedPyObject()                { if (ptr) Py_DECREF(ptr); }
+    inline PyObject * get() const           { return ptr; }
+    inline PyObject * operator -> () const { return ptr; }
+    inline operator PyObject *() const    { return ptr; }
+    inline void clear()                     { if (ptr) Py_DECREF(ptr); ptr = NULL; }
+    inline void setown(PyObject *_ptr)      { clear(); ptr = _ptr; }
+    inline void set(PyObject *_ptr)         { clear(); ptr = _ptr; if (ptr) Py_INCREF(ptr);}
+    inline PyObject *getLink()              { if (ptr) Py_INCREF(ptr); return ptr;}
+    inline PyObject **ref()                 { return &ptr; }
+};
+
+// call checkPythonError to throw an exception if Python error state is set
+
+static void checkPythonError()
+{
+    PyObject* err = PyErr_Occurred();
+    if (err)
+    {
+        OwnedPyObject pType, pValue, pTraceBack;
+        PyErr_Fetch(pType.ref(), pValue.ref(), pTraceBack.ref());
+        OwnedPyObject valStr = PyObject_Str(pValue);
+        PyErr_Clear();
+        VStringBuffer errMessage("pyembed: %s", PyString_AsString(valStr));
+        rtlFail(0, errMessage.str());
+    }
+}
+
+// The Python Global Interpreter Lock (GIL) won't know about C++-created threads, so we need to
+// call PyGILState_Ensure() and PyGILState_Release at the start and end of every function.
+// Wrapping them in a class like this ensures that the release always happens even if
+// the function exists prematurely
+
+class GILstateWrapper
+{
+    PyGILState_STATE gstate;
+public:
+    GILstateWrapper()
+    {
+        gstate = PyGILState_Ensure();
+    }
+    ~GILstateWrapper()
+    {
+        PyGILState_Release(gstate);
+    }
+};
+
+// There is a singleton PythonThreadContext per thread. This allows us to
+// ensure that we can make repeated calls to a Python function efficiently.
+
+class PythonThreadContext
+{
+public:
+    PyThreadState *threadState;
+public:
+    PythonThreadContext()
+    {
+        threadState = PyEval_SaveThread();
+    }
+    ~PythonThreadContext()
+    {
+        PyEval_RestoreThread(threadState);
+        script.clear();
+    }
+
+    inline PyObject * importFunction(size32_t lenChars, const char *utf)
+    {
+        size32_t bytes = rtlUtf8Size(lenChars, utf);
+        StringBuffer text(bytes, utf);
+        if (!prevtext || strcmp(text, prevtext) != 0)
+        {
+            prevtext.clear();
+            // Name should be in the form module.function
+            const char *funcname = strrchr(text, '.');
+            if (!funcname)
+                rtlFail(0, "Expected module.function");
+            StringBuffer modname(funcname-text, text);
+            funcname++;  // skip the '.'
+            // If the modname is preceded by a path, add it to the python path before importing
+            const char *pathsep = strrchr(modname, PATHSEPCHAR);
+            if (pathsep)
+            {
+                StringBuffer path(pathsep-modname, modname);
+                modname.remove(0, 1+pathsep-modname);
+                PyObject *sys_path = PySys_GetObject((char *) "path");
+                OwnedPyObject new_path = PyString_FromString(path);
+                if (sys_path)
+                {
+                    PyList_Append(sys_path, new_path);
+                    checkPythonError();
+                }
+            }
+            module.setown(PyImport_ImportModule(modname));
+            checkPythonError();
+            PyObject *dict = PyModule_GetDict(module);  // this is a borrowed reference and does not need to be released
+            script.set(PyDict_GetItemString(dict, funcname));
+            checkPythonError();
+            if (!script || !PyCallable_Check(script))
+                rtlFail(0, "Object is not callable");
+            prevtext.set(text);
+        }
+        return script.getLink();
+    }
+
+    inline PyObject *compileEmbeddedScript(size32_t lenChars, const char *utf)
+    {
+        size32_t bytes = rtlUtf8Size(lenChars, utf);
+        StringBuffer text(bytes, utf);
+        if (!prevtext || strcmp(text, prevtext) != 0)
+        {
+            prevtext.clear();
+            // Try compiling as a eval first... if that fails, try as a script.
+            script.setown(Py_CompileString(text, "", Py_eval_input));
+            if (!script)
+            {
+                PyErr_Clear();
+                StringBuffer wrapped;
+                wrapPythonText(wrapped, text);
+                script.setown(Py_CompileString(wrapped, "<embed>", Py_file_input));
+            }
+            checkPythonError();
+            prevtext.set(text);
+        }
+        return script.getLink();
+    }
+private:
+    static StringBuffer &wrapPythonText(StringBuffer &out, const char *in)
+    {
+        out.append("def __user__():\n  ");
+        char c;
+        while ((c = *in++) != '\0')
+        {
+            out.append(c);
+            if (c=='\n')
+                out.append("  ");
+        }
+        out.append("\n__result__ = __user__()\n");
+        return out;
+    }
+    GILstateWrapper GILState;
+    OwnedPyObject module;
+    OwnedPyObject script;
+    StringAttr prevtext;
+};
+
+static __thread PythonThreadContext* threadContext;  // We reuse per thread, for speed
+static __thread ThreadTermFunc threadHookChain;
+
+static void releaseContext()
+{
+    delete threadContext;
+    threadContext = NULL;
+    if (threadHookChain)
+        (*threadHookChain)();
+}
+
+// Use a global object to ensure that the Python interpreter is initialized on main thread
+
+static class Python27GlobalState
+{
+public:
+    Python27GlobalState()
+    {
+        // Initialize the Python Interpreter
+        Py_Initialize();
+        PyEval_InitThreads();
+        tstate = PyEval_SaveThread();
+    }
+    ~Python27GlobalState()
+    {
+        if (threadContext)
+            delete threadContext;   // The one on the main thread won't get picked up by the thread hook mechanism
+        threadContext = NULL;
+        PyEval_RestoreThread(tstate);
+        // Finish the Python Interpreter
+        Py_Finalize();
+    }
+protected:
+    PyThreadState *tstate;
+} globalState;
+
+// Each call to a Python function will use a new Python27EmbedFunctionContext object
+// This takes care of ensuring that the Python GIL is locked while we are executing python code,
+// and released when we are not
+
+class Python27EmbedContextBase : public CInterfaceOf<IEmbedFunctionContext>
+{
+public:
+    Python27EmbedContextBase(PythonThreadContext *_sharedCtx)
+    : sharedCtx(_sharedCtx)
+    {
+        PyEval_RestoreThread(sharedCtx->threadState);
+        locals.setown(PyDict_New());
+        globals.setown(PyDict_New());
+        PyDict_SetItemString(locals, "__builtins__", PyEval_GetBuiltins());  // required for import to work
+    }
+    ~Python27EmbedContextBase()
+    {
+        // We need to clear these before calling savethread, or we won't own the GIL
+        locals.clear();
+        globals.clear();
+        result.clear();
+        script.clear();
+        sharedCtx->threadState = PyEval_SaveThread();
+    }
+
+    virtual bool getBooleanResult()
+    {
+        assertex(result);
+        if (!PyBool_Check(result))
+            throw MakeStringException(MSGAUD_user, 0, "pyembed: Type mismatch on result");
+        return result == Py_True;
+    }
+    virtual void getDataResult(size32_t &__chars, void * &__result)
+    {
+        assertex(result && result != Py_None);
+        if (!PyByteArray_Check(result))
+            throw MakeStringException(MSGAUD_user, 0, "pyembed: Type mismatch on result");
+        rtlStrToDataX(__chars, __result, PyByteArray_Size(result), PyByteArray_AsString(result));
+    }
+    virtual double getRealResult()
+    {
+        assertex(result && result != Py_None);
+        return (__int64) PyFloat_AsDouble(result);
+    }
+    virtual __int64 getSignedResult()
+    {
+        assertex(result && result != Py_None);
+        return (__int64) PyLong_AsLongLong(result);
+    }
+    virtual unsigned __int64 getUnsignedResult()
+    {
+        assertex(result && result != Py_None);
+        return (__int64) PyLong_AsUnsignedLongLong(result);
+    }
+    virtual void getStringResult(size32_t &__chars, char * &__result)
+    {
+        assertex(result && result != Py_None);
+        if (PyString_Check(result))
+        {
+            const char * text =  PyString_AsString(result);
+            checkPythonError();
+            size_t lenBytes = PyString_Size(result);
+            rtlStrToStrX(__chars, __result, lenBytes, text);
+        }
+        else
+            rtlFail(0, "Python type mismatch - return value was not a string");
+    }
+    virtual void getUTF8Result(size32_t &__chars, char * &__result)
+    {
+        assertex(result && result != Py_None);
+        if (PyUnicode_Check(result))
+        {
+            OwnedPyObject utf8 = PyUnicode_AsUTF8String(result);
+            checkPythonError();
+            size_t lenBytes = PyString_Size(utf8);
+            const char * text =  PyString_AsString(utf8);
+            checkPythonError();
+            size32_t numchars = rtlUtf8Length(lenBytes, text);
+            rtlUtf8ToUtf8X(__chars, __result, numchars, text);
+        }
+        else
+            rtlFail(0, "Python type mismatch - return value was not a unicode string");
+    }
+    virtual void getUnicodeResult(size32_t &__chars, UChar * &__result)
+    {
+        assertex(result && result != Py_None);
+        if (PyUnicode_Check(result))
+        {
+            OwnedPyObject utf8 = PyUnicode_AsUTF8String(result);
+            checkPythonError();
+            size_t lenBytes = PyString_Size(utf8);
+            const char * text =  PyString_AsString(utf8);
+            checkPythonError();
+            size32_t numchars = rtlUtf8Length(lenBytes, text);
+            rtlUtf8ToUnicodeX(__chars, __result, numchars, text);
+        }
+        else
+            rtlFail(0, "Python type mismatch - return value was not a unicode string");
+    }
+
+
+protected:
+    PythonThreadContext *sharedCtx;
+    OwnedPyObject locals;
+    OwnedPyObject globals;
+    OwnedPyObject result;
+    OwnedPyObject script;
+};
+
+class Python27EmbedScriptContext : public Python27EmbedContextBase
+{
+public:
+    Python27EmbedScriptContext(PythonThreadContext *_sharedCtx, const char *options)
+    : Python27EmbedContextBase(_sharedCtx)
+    {
+    }
+    ~Python27EmbedScriptContext()
+    {
+    }
+    virtual void bindBooleanParam(const char *name, bool val)
+    {
+        OwnedPyObject vval = PyBool_FromLong(val ? 1 : 0);
+        PyDict_SetItemString(locals, name, vval);
+    }
+    virtual void bindDataParam(const char *name, size32_t len, const void *val)
+    {
+        OwnedPyObject vval = PyByteArray_FromStringAndSize((const char *) val, len);
+        PyDict_SetItemString(locals, name, vval);
+    }
+    virtual void bindRealParam(const char *name, double val)
+    {
+        OwnedPyObject vval = PyFloat_FromDouble(val);
+        PyDict_SetItemString(locals, name, vval);
+    }
+    virtual void bindSignedParam(const char *name, __int64 val)
+    {
+        OwnedPyObject vval = PyLong_FromLongLong(val);
+        PyDict_SetItemString(locals, name, vval);
+    }
+    virtual void bindUnsignedParam(const char *name, unsigned __int64 val)
+    {
+        OwnedPyObject vval = PyLong_FromUnsignedLongLong(val);
+        PyDict_SetItemString(locals, name, vval);
+    }
+    virtual void bindStringParam(const char *name, size32_t len, const char *val)
+    {
+        OwnedPyObject vval = PyString_FromStringAndSize(val, len);
+        PyDict_SetItemString(locals, name, vval);
+    }
+    virtual void bindVStringParam(const char *name, const char *val)
+    {
+        OwnedPyObject vval = PyString_FromString(val);
+        PyDict_SetItemString(locals, name, vval);
+    }
+    virtual void bindUTF8Param(const char *name, size32_t chars, const char *val)
+    {
+        size32_t sizeBytes = rtlUtf8Size(chars, val);
+        OwnedPyObject vval = PyUnicode_FromStringAndSize(val, sizeBytes);   // NOTE - requires size in bytes not chars
+        PyDict_SetItemString(locals, name, vval);
+    }
+    virtual void bindUnicodeParam(const char *name, size32_t chars, const UChar *val)
+    {
+        // You don't really know what size Py_UNICODE is (varies from system to system), so go via utf8
+        unsigned unicodeChars;
+        char *unicode;
+        rtlUnicodeToUtf8X(unicodeChars, unicode, chars, val);
+        size32_t sizeBytes = rtlUtf8Size(unicodeChars, unicode);
+        OwnedPyObject vval = PyUnicode_FromStringAndSize(unicode, sizeBytes);   // NOTE - requires size in bytes not chars
+        checkPythonError();
+        PyDict_SetItemString(locals, name, vval);
+        rtlFree(unicode);
+    }
+
+    virtual void importFunction(size32_t lenChars, const char *text)
+    {
+        throwUnexpected();
+    }
+    virtual void compileEmbeddedScript(size32_t lenChars, const char *utf)
+    {
+        script.setown(sharedCtx->compileEmbeddedScript(lenChars, utf));
+    }
+
+    virtual void callFunction()
+    {
+        result.setown(PyEval_EvalCode((PyCodeObject *) script.get(), locals, globals));
+        checkPythonError();
+        if (!result || result == Py_None)
+            result.set(PyDict_GetItemString(locals, "__result__"));
+        if (!result || result == Py_None)
+            result.set(PyDict_GetItemString(globals, "__result__"));
+    }
+};
+
+class Python27EmbedImportContext : public Python27EmbedContextBase
+{
+public:
+    Python27EmbedImportContext(PythonThreadContext *_sharedCtx, const char *options)
+    : Python27EmbedContextBase(_sharedCtx)
+    {
+        argcount = 0;
+    }
+    ~Python27EmbedImportContext()
+    {
+    }
+    virtual void bindBooleanParam(const char *name, bool val)
+    {
+        addArg(PyBool_FromLong(val ? 1 : 0));
+    }
+    virtual void bindDataParam(const char *name, size32_t len, const void *val)
+    {
+        addArg(PyByteArray_FromStringAndSize((const char *) val, len));
+    }
+    virtual void bindRealParam(const char *name, double val)
+    {
+        addArg(PyFloat_FromDouble(val));
+    }
+    virtual void bindSignedParam(const char *name, __int64 val)
+    {
+        addArg(PyLong_FromLongLong(val));
+    }
+    virtual void bindUnsignedParam(const char *name, unsigned __int64 val)
+    {
+        addArg(PyLong_FromUnsignedLongLong(val));
+    }
+    virtual void bindStringParam(const char *name, size32_t len, const char *val)
+    {
+        addArg(PyString_FromStringAndSize(val, len));
+    }
+    virtual void bindVStringParam(const char *name, const char *val)
+    {
+        addArg(PyString_FromString(val));
+    }
+    virtual void bindUTF8Param(const char *name, size32_t chars, const char *val)
+    {
+        size32_t sizeBytes = rtlUtf8Size(chars, val);
+        addArg(PyUnicode_FromStringAndSize(val, sizeBytes));   // NOTE - requires size in bytes not chars
+    }
+    virtual void bindUnicodeParam(const char *name, size32_t chars, const UChar *val)
+    {
+        // You don't really know what size Py_UNICODE is (varies from system to system), so go via utf8
+        unsigned unicodeChars;
+        char *unicode;
+        rtlUnicodeToUtf8X(unicodeChars, unicode, chars, val);
+        size32_t sizeBytes = rtlUtf8Size(unicodeChars, unicode);
+        PyObject *vval = PyUnicode_FromStringAndSize(unicode, sizeBytes);   // NOTE - requires size in bytes not chars
+        checkPythonError();
+        addArg(vval);
+        rtlFree(unicode);
+    }
+
+    virtual void importFunction(size32_t lenChars, const char *utf)
+    {
+        script.setown(sharedCtx->importFunction(lenChars, utf));
+    }
+    virtual void compileEmbeddedScript(size32_t len, const char *text)
+    {
+        throwUnexpected();
+    }
+    virtual void callFunction()
+    {
+        result.setown(PyObject_CallObject(script, args));
+        checkPythonError();
+    }
+private:
+    void addArg(PyObject *arg)
+    {
+        if (argcount)
+            _PyTuple_Resize(args.ref(), argcount+1);
+        else
+            args.setown(PyTuple_New(1));
+        PyTuple_SET_ITEM((PyTupleObject *) args.get(), argcount++, arg);  // Note - 'steals' the arg reference
+    }
+    int argcount;
+    OwnedPyObject args;
+};
+
+class Python27EmbedContext : public CInterfaceOf<IEmbedContext>
+{
+public:
+    virtual IEmbedFunctionContext *createFunctionContext(bool isImport, const char *options)
+    {
+        if (!threadContext)
+        {
+            threadContext = new PythonThreadContext;
+            threadHookChain = addThreadTermFunc(releaseContext);
+        }
+        if (isImport)
+            return new Python27EmbedImportContext(threadContext, options);
+        else
+            return new Python27EmbedScriptContext(threadContext, options);
+    }
+};
+
+extern IEmbedContext* getEmbedContext()
+{
+    return new Python27EmbedContext;
+}
+
+extern bool syntaxCheck(const char *script)
+{
+    return true; // MORE
+}
+
+} // namespace

plugins/v8embed/CMakeLists.txt

@@ -0,0 +1,61 @@
+################################################################################
+#    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+################################################################################
+
+
+# Component: v8embed
+
+#####################################################
+# Description:
+# ------------
+#    Cmake Input File for v8embed
+#####################################################
+
+project( v8embed )
+
+set (    SRCS
+         v8embed.cpp
+    )
+
+find_package(V8)
+if (NOT V8_FOUND)
+  message(FATAL_ERROR "MAKE_V8EMBED requested but v8 not found")
+endif()
+
+
+include_directories (
+         ${V8_INCLUDE_DIR}
+         ./../../system/include
+         ./../../rtl/eclrtl
+         ./../../rtl/include
+         ./../../system/jlib
+    )
+
+ADD_DEFINITIONS( -D_USRDLL -DV8EMBED_EXPORTS )
+
+HPCC_ADD_LIBRARY( v8embed SHARED ${SRCS} )
+if (${CMAKE_VERSION} VERSION_LESS "2.8.9")
+  message("WARNING: Cannot set NO_SONAME. shlibdeps will give warnings when package is installed")
+else()
+  set_target_properties( v8embed PROPERTIES NO_SONAME 1 )
+endif()
+
+install ( TARGETS v8embed DESTINATION plugins )
+
+target_link_libraries ( v8embed
+    ${V8_LIBRARIES}
+    eclrtl
+    jlib
+    )

plugins/v8embed/v8embed.cpp

@@ -0,0 +1,274 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#include "platform.h"
+#include "v8.h"
+#include "eclrtl.hpp"
+#include "jexcept.hpp"
+#include "jthread.hpp"
+#include "hqlplugins.hpp"
+
+#ifdef _WIN32
+#define EXPORT __declspec(dllexport)
+#else
+#define EXPORT
+#endif
+
+static const char * compatibleVersions[] = {
+    "V8 JavaScript Embed Helper 1.0.0",
+    NULL };
+
+static const char *version = "V8 JavaScript Embed Helper 1.0.0";
+static const char * EclDefinition =
+    "EXPORT Language := SERVICE\n"
+    "  boolean getEmbedContext():cpp,pure,namespace='javascriptLanguageHelper',entrypoint='getEmbedContext',prototype='IEmbedContext* getEmbedContext()';\n"
+    "  boolean syntaxCheck(const varstring src):cpp,pure,namespace='javascriptLanguageHelper',entrypoint='syntaxCheck';\n"
+    "END;"
+    "export getEmbedContext := Language.getEmbedContext;"
+    "export syntaxCheck := Language.syntaxCheck;"
+    "EXPORT boolean supportsImport := false;"
+    "EXPORT boolean supportsScript := true;";
+
+extern "C" EXPORT bool getECLPluginDefinition(ECLPluginDefinitionBlock *pb)
+{
+    if (pb->size == sizeof(ECLPluginDefinitionBlockEx))
+    {
+        ECLPluginDefinitionBlockEx * pbx = (ECLPluginDefinitionBlockEx *) pb;
+        pbx->compatibleVersions = compatibleVersions;
+    }
+    else if (pb->size != sizeof(ECLPluginDefinitionBlock))
+        return false;
+    pb->magicVersion = PLUGIN_VERSION;
+    pb->version = version;
+    pb->moduleName = "javascript";
+    pb->ECL = EclDefinition;
+    pb->flags = PLUGIN_DLL_MODULE | PLUGIN_MULTIPLE_VERSIONS;
+    pb->description = "V8 JavaScript Embed Helper";
+    return true;
+}
+
+namespace javascriptLanguageHelper {
+
+class V8JavascriptEmbedFunctionContext : public CInterfaceOf<IEmbedFunctionContext>
+{
+public:
+    V8JavascriptEmbedFunctionContext()
+    {
+        isolate = v8::Isolate::New();
+        isolate->Enter();
+        context = v8::Context::New();
+        context->Enter();
+    }
+    ~V8JavascriptEmbedFunctionContext()
+    {
+        script.Dispose();
+        result.Dispose();
+        context->Exit();
+        context.Dispose();
+        isolate->Exit();
+        isolate->Dispose();
+    }
+
+    virtual void bindBooleanParam(const char *name, bool val)
+    {
+        v8::HandleScope handle_scope;
+        context->Global()->Set(v8::String::New(name), v8::Boolean::New(val));
+    }
+    virtual void bindDataParam(const char *name, size32_t len, const void *val)
+    {
+        v8::HandleScope handle_scope;
+        v8::Local<v8::Array> array = v8::Array::New(len);
+        const byte *vval = (const byte *) val;
+        for (int i = 0; i < len; i++)
+        {
+            array->Set(v8::Number::New(i), v8::Integer::New(vval[i])); // feels horridly inefficient, but seems to be the expected approach
+        }
+        context->Global()->Set(v8::String::New(name), array);
+    }
+    virtual void bindRealParam(const char *name, double val)
+    {
+        v8::HandleScope handle_scope;
+        context->Global()->Set(v8::String::New(name), v8::Number::New(val));
+    }
+    virtual void bindSignedParam(const char *name, __int64 val)
+    {
+        // MORE - might need to check does not overflow 32 bits? Or store as a real?
+        v8::HandleScope handle_scope;
+        context->Global()->Set(v8::String::New(name), v8::Integer::New(val));
+    }
+    virtual void bindUnsignedParam(const char *name, unsigned __int64 val)
+    {
+        // MORE - might need to check does not overflow 32 bits
+        v8::HandleScope handle_scope;
+        context->Global()->Set(v8::String::New(name), v8::Integer::NewFromUnsigned(val));
+    }
+    virtual void bindStringParam(const char *name, size32_t len, const char *val)
+    {
+        size32_t utfCharCount;
+        char *utfText;
+        rtlStrToUtf8X(utfCharCount, utfText, len, val);
+        bindUTF8Param(name, utfCharCount, utfText);
+        rtlFree(utfText);
+    }
+    virtual void bindVStringParam(const char *name, const char *val)
+    {
+        bindStringParam(name, strlen(val), val);
+    }
+    virtual void bindUTF8Param(const char *name, size32_t chars, const char *val)
+    {
+        v8::HandleScope handle_scope;
+        context->Global()->Set(v8::String::New(name), v8::String::New(val, rtlUtf8Size(chars, val)));
+    }
+    virtual void bindUnicodeParam(const char *name, size32_t chars, const UChar *val)
+    {
+        v8::HandleScope handle_scope;
+        context->Global()->Set(v8::String::New(name), v8::String::New(val, chars));
+    }
+
+    virtual bool getBooleanResult()
+    {
+        assertex (!result.IsEmpty());
+        v8::HandleScope handle_scope;
+        return result->BooleanValue();
+    }
+    virtual void getDataResult(size32_t &__len, void * &__result)
+    {
+        assertex (!result.IsEmpty() && result->IsArray());
+        v8::HandleScope handle_scope;
+        v8::Handle<v8::Array> array = v8::Handle<v8::Array>::Cast(result);
+        __len = array->Length();
+        __result = rtlMalloc(__len);
+        byte *bresult = (byte *) __result;
+        for (size32_t i = 0; i < __len; i++)
+        {
+            bresult[i] = v8::Integer::Cast(*array->Get(i))->Value(); // feels horridly inefficient, but seems to be the expected approach
+        }
+    }
+    virtual double getRealResult()
+    {
+        assertex (!result.IsEmpty());
+        v8::HandleScope handle_scope;
+        return v8::Number::Cast(*result)->Value();
+    }
+    virtual __int64 getSignedResult()
+    {
+        assertex (!result.IsEmpty());
+        v8::HandleScope handle_scope;
+        return v8::Integer::Cast(*result)->Value();
+    }
+    virtual unsigned __int64 getUnsignedResult()
+    {
+        assertex (!result.IsEmpty());
+        v8::HandleScope handle_scope;
+        return v8::Integer::Cast(*result)->Value();
+    }
+    virtual void getStringResult(size32_t &__chars, char * &__result)
+    {
+        assertex (!result.IsEmpty() && result->IsString());
+        v8::HandleScope handle_scope;
+        v8::String::AsciiValue ascii(result);
+        rtlStrToStrX(__chars, __result, ascii.length(), *ascii);
+    }
+    virtual void getUTF8Result(size32_t &__chars, char * &__result)
+    {
+        assertex (!result.IsEmpty() && result->IsString());
+        v8::HandleScope handle_scope;
+        v8::String::Utf8Value utf8(result);
+        unsigned numchars = rtlUtf8Length(utf8.length(), *utf8);
+        rtlUtf8ToUtf8X(__chars, __result, numchars, *utf8);
+    }
+    virtual void getUnicodeResult(size32_t &__chars, UChar * &__result)
+    {
+        assertex (!result.IsEmpty() && result->IsString());
+        v8::HandleScope handle_scope;
+        v8::String::Utf8Value utf8(result);
+        unsigned numchars = rtlUtf8Length(utf8.length(), *utf8);
+        rtlUtf8ToUnicodeX(__chars, __result, numchars, *utf8);
+    }
+
+    virtual void compileEmbeddedScript(size32_t lenChars, const char *utf)
+    {
+        v8::HandleScope handle_scope;
+        v8::Handle<v8::String> source = v8::String::New(utf, rtlUtf8Size(lenChars, utf));
+        v8::Handle<v8::Script> lscript = v8::Script::Compile(source);
+        script = v8::Persistent<v8::Script>::New(lscript);
+    }
+    virtual void importFunction(size32_t lenChars, const char *utf)
+    {
+        UNIMPLEMENTED; // Not sure if meaningful for js
+    }
+    virtual void callFunction()
+    {
+        assertex (!script.IsEmpty());
+        v8::HandleScope handle_scope;
+        v8::TryCatch tryCatch;
+        result = v8::Persistent<v8::Value>::New(script->Run());
+        v8::Handle<v8::Value> exception = tryCatch.Exception();
+        if (!exception.IsEmpty())
+        {
+            v8::String::AsciiValue msg(exception);
+            throw MakeStringException(MSGAUD_user, 0, "v8embed: %s", *msg);
+        }
+    }
+
+protected:
+    v8::Isolate *isolate;
+    v8::Persistent<v8::Context> context;
+    v8::Persistent<v8::Script> script;
+    v8::Persistent<v8::Value> result;
+};
+
+static __thread V8JavascriptEmbedFunctionContext * theFunctionContext;  // We reuse per thread, for speed
+static __thread ThreadTermFunc threadHookChain;
+
+static void releaseContext()
+{
+    ::Release(theFunctionContext);
+    if (threadHookChain)
+        (*threadHookChain)();
+}
+
+class V8JavascriptEmbedContext : public CInterfaceOf<IEmbedContext>
+{
+public:
+    V8JavascriptEmbedContext()
+    {
+    }
+    virtual IEmbedFunctionContext *createFunctionContext(bool isImport, const char *options)
+    {
+        assertex(!isImport);
+        if (!theFunctionContext)
+        {
+            theFunctionContext = new V8JavascriptEmbedFunctionContext;
+            threadHookChain = addThreadTermFunc(releaseContext);
+        }
+        return LINK(theFunctionContext);
+    }
+} theEmbedContext;
+
+
+extern IEmbedContext* getEmbedContext()
+{
+    return LINK(&theEmbedContext);
+}
+
+extern bool syntaxCheck(const char *script)
+{
+    return true; // MORE
+}
+
+} // namespace

roxie/ccd/ccdmain.cpp

@@ -433,6 +433,10 @@ public:
 
 int STARTQUERY_API start_query(int argc, const char *argv[])
 {
+    EnableSEHtoExceptionMapping();
+    setTerminateOnSEH();
+    init_signals();
+    // We need to do the above BEFORE we call InitModuleObjects
     InitModuleObjects();
     getDaliServixPort();
     init_signals();
@@ -492,8 +496,6 @@ int STARTQUERY_API start_query(int argc, const char *argv[])
 #endif
     srand( (unsigned)time( NULL ) );
     ccdChannels = createPTree("Channels");
-    EnableSEHtoExceptionMapping();
-    setTerminateOnSEH();
 
     char currentDirectory[_MAX_DIR];
     if (!getcwd(currentDirectory, sizeof(currentDirectory)))

rtl/eclrtl/eclrtl.hpp

@@ -728,4 +728,38 @@ ECLRTL_API unsigned rtlDelayReturn(unsigned value, unsigned sleepTime);
 
 ECLRTL_API bool rtlGPF();
 
+//-----------------------------------------------------------------------------
+
+interface IEmbedFunctionContext : extends IInterface
+{
+    virtual void bindBooleanParam(const char *name, bool val) = 0;
+    virtual void bindDataParam(const char *name, size32_t len, const void *val) = 0;
+    virtual void bindRealParam(const char *name, double val) = 0;
+    virtual void bindSignedParam(const char *name, __int64 val) = 0;
+    virtual void bindUnsignedParam(const char *name, unsigned __int64 val) = 0;
+    virtual void bindStringParam(const char *name, size32_t len, const char *val) = 0;
+    virtual void bindVStringParam(const char *name, const char *val) = 0;
+    virtual void bindUTF8Param(const char *name, size32_t chars, const char *val) = 0;
+    virtual void bindUnicodeParam(const char *name, size32_t chars, const UChar *val) = 0;
+
+    virtual bool getBooleanResult() = 0;
+    virtual void getDataResult(size32_t &len, void * &result) = 0;
+    virtual double getRealResult() = 0;
+    virtual __int64 getSignedResult() = 0;
+    virtual unsigned __int64 getUnsignedResult() = 0;
+    virtual void getStringResult(size32_t &len, char * &result) = 0;
+    virtual void getUTF8Result(size32_t &chars, char * &result) = 0;
+    virtual void getUnicodeResult(size32_t &chars, UChar * &result) = 0;
+
+    virtual void importFunction(size32_t len, const char *function) = 0;
+    virtual void compileEmbeddedScript(size32_t len, const char *script) = 0;
+    virtual void callFunction() = 0;
+};
+
+interface IEmbedContext : extends IInterface
+{
+    virtual IEmbedFunctionContext *createFunctionContext(bool isImport, const char *options) = 0;
+    // MORE - add syntax checked here!
+};
+
 #endif

system/include/platform.h

@@ -137,6 +137,7 @@ typedef memsize_t rowsize_t;
 #define LoadSucceeded(h)           ((unsigned)h >= 32)
 #define GetSharedObjectError()     GetLastError()
 #define strtok_r(a,b,c)             j_strtok_r(a,b,c)
+#define __thread __declspec(thread)
 
 typedef unsigned __int64 off64_t;
 typedef int socklen_t;

system/jlib/jthread.cpp

@@ -43,9 +43,20 @@
 
 //#define NO_CATCHALL
 
+static __thread ThreadTermFunc threadTerminationHook;
+
+ThreadTermFunc addThreadTermFunc(ThreadTermFunc onTerm)
+{
+    ThreadTermFunc old = threadTerminationHook;
+    threadTerminationHook = onTerm;
+    return old;
+}
+
 PointerArray *exceptionHandlers = NULL;
 MODULE_INIT(INIT_PRIORITY_JTHREAD)
 {
+    if (threadTerminationHook)
+        (*threadTerminationHook)();  // May be too late :(
     exceptionHandlers = new PointerArray();
     return true;
 }
@@ -255,6 +266,8 @@ int Thread::begin()
         handleException(MakeStringException(0, "Unknown exception in Thread %s", getName()));
     }
 #endif
+    if (threadTerminationHook)
+        (*threadTerminationHook)();
 #ifdef _WIN32
 #ifndef _DEBUG
     CloseHandle(hThread);   // leak handle when debugging, 
@@ -543,6 +556,7 @@ void CThreadedPersistent::main()
         try
         {
             owner->main();
+            // Note we do NOT call the thread reset hook here - these threads are expected to be able to preserve state, I think
         }
         catch (IException *e)
         {
@@ -794,6 +808,8 @@ public:
                 handleException(MakeStringException(0, "Unknown exception in Thread from pool %s", parent.poolname.get()));
             }
 #endif
+            if (threadTerminationHook)
+                (*threadTerminationHook)();    // Reset any pre-thread state.
         } while (parent.notifyStopped(this));
         return 0;
     }

system/jlib/jthread.hpp

@@ -52,6 +52,17 @@ extern jlib_decl void disableThreadSEH();
 
 extern jlib_decl unsigned threadLogID();  // for use in logging
 
+// A function registered via addThreadTermFunc will be called when the thread that registered that function
+// terminates. Such a function should call on to the previously registered function (if any) - generally you
+// would expect to store that value in thread-local storage.
+// This can be used to ensure that thread-specific objects can be properly destructed.
+// Note that threadpools also call the thread termination hook when each thread's main function terminates,
+// so the hook function should clear any variables if necessary rather than assuming that they will be cleared
+// at thread startup time.
+
+typedef void (*ThreadTermFunc)();
+extern jlib_decl ThreadTermFunc addThreadTermFunc(ThreadTermFunc onTerm);
+
 class jlib_decl Thread : public CInterface, public IThread
 {
 private:
@@ -176,31 +187,6 @@ public:
     virtual void Do(unsigned idx=0)=0;
 };
 
-// Thread local storage - use MAKETHREADLOCALIINTERFACE macro to get a thread local type
-
-template <class CLASS, class CLASSINIT = CLASS, class MAP = MapBetween<ThreadId, ThreadId, CLASS, CLASSINIT> >
-class ThreadLocalOf : public MAP
-{
-public:
-    CLASS * query()
-    {
-        CLASS * find = threadMap.getValue(GetCurrentThreadId());
-        if(find) return find;
-        threadMap.setValue(GetCurrentThreadId(), CLASSINIT());
-        return threadMap.getValue(GetCurrentThreadId());
-    }
-    operator CLASS & ()
-    {
-        return *query();
-    }
-private:
-    MAP threadMap;
-};
-
-#define MAKETHREADLOCALIINTERFACE(C, CI, NAME)                                                      \
-typedef ThreadLocalOf<C, CI> NAME
-
-
 // ---------------------------------------------------------------------------
 // Thread Pools
 // ---------------------------------------------------------------------------

testing/ecl/JavaCat.java

@@ -0,0 +1,69 @@
+public class JavaCat
+{
+  public static int add1(int a)
+  {
+    return a + 1;
+  }
+  public static String add2(String a)
+  {
+    return a + '1';
+  }
+  public static char addChar(char a)
+  {
+    return ++a;
+  }
+  public static int testThrow(int a) throws Exception
+  {
+    throw new Exception("Exception from Java");
+  }
+
+  public static byte[] testData(byte [] indata)
+  {
+    indata[0]++;
+    return indata;
+  }
+
+  public static int add(int a, int b)
+  {
+    return a + b;
+  }
+  public static long addL(int a, int b)
+  {
+    return a + b;
+  }
+  public static Integer addI(int a, int b)
+  {
+    return a + b;
+  }
+  public static float fadd(float a, float b)
+  {
+    System.out.print("fadd(");
+    System.out.print(a);
+    System.out.print(",");
+    System.out.print(b);
+    System.out.println(")");
+    return a + b;
+  }
+  public static double dadd(double a, double b)
+  {
+    System.out.print("fadd(");
+    System.out.print(a);
+    System.out.print(",");
+    System.out.print(b);
+    System.out.println(")");
+    return a + b;
+  }
+  public static Double daddD(double a, double b)
+  {
+    System.out.print("fadd(");
+    System.out.print(a);
+    System.out.print(",");
+    System.out.print(b);
+    System.out.println(")");
+    return a + b;
+  }
+  public static String cat(String a, String b)
+  {
+    return a + b;
+  }
+}

testing/ecl/embed.ecl

@@ -0,0 +1,2 @@
+integer c(integer val) := BEGINC++ return val-3; ENDC++;
+c(10);

testing/ecl/embedR.ecl

@@ -0,0 +1,26 @@
+IMPORT R;
+
+integer add1(integer val) := EMBED(R)
+val+1
+ENDEMBED;
+
+string cat(varstring what, string who) := EMBED(R)
+paste(what,who)
+ENDEMBED;
+
+data testData(data val) := EMBED(R)
+val[1] = val[2];
+val;
+ENDEMBED;
+
+add1(10);
+cat('Hello', 'World');
+testData(D'ab');
+
+s1 :=DATASET(250000, TRANSFORM({ integer a }, SELF.a := add1(COUNTER)));
+s2 :=DATASET(250000, TRANSFORM({ integer a }, SELF.a := add1(COUNTER/2)));
+SUM(NOFOLD(s1 + s2), a);
+
+s1b :=DATASET(250000, TRANSFORM({ integer a }, SELF.a := COUNTER+1));
+s2b :=DATASET(250000, TRANSFORM({ integer a }, SELF.a := (COUNTER/2)+1));
+SUM(NOFOLD(s1b + s2b), a);

testing/ecl/embedjava.ecl

@@ -0,0 +1,48 @@
+IMPORT java;
+
+integer add1(integer val) := IMPORT(java, 'JavaCat.add1:(I)I');
+string add2(string val) := IMPORT(java, 'JavaCat.add2:(Ljava/lang/String;)Ljava/lang/String;');
+string add3(varstring val) := IMPORT(java, 'JavaCat.add2:(Ljava/lang/String;)Ljava/lang/String;');
+utf8 add4(utf8 val) := IMPORT(java, 'JavaCat.add2:(Ljava/lang/String;)Ljava/lang/String;');
+unicode add5(unicode val) := IMPORT(java, 'JavaCat.add2:(Ljava/lang/String;)Ljava/lang/String;');
+integer testThrow(integer p) := IMPORT(java, 'JavaCat.testThrow:(I)I');
+
+string addChar(string c) := IMPORT(java, 'JavaCat.addChar:(C)C');
+string cat(string s1, string s2) := IMPORT(java, 'JavaCat.cat:(Ljava/lang/String;Ljava/lang/String;)Ljava/lang/String;');
+data testData(data indata) := IMPORT(java, 'JavaCat.testData:([B)[B');
+
+add1(10);
+add2('Hello');
+add3('World');
+add4(U'Leovenaðes');
+add5(U'Стоял');
+addChar('A');
+
+cat('Hello', ' world');
+// Can't catch an expression(only a dataset)
+d := dataset([{ 1, '' }], { integer a, string m} ) : stored('nofold');
+
+d t := transform
+  self.a := FAILCODE;
+  self.m := FAILMESSAGE;
+  self := [];
+end;
+
+catch(d(testThrow(a) = a), onfail(t));
+testData(d'aa');
+
+s1 :=DATASET(250000, TRANSFORM({ integer a }, SELF.a := add1(COUNTER)));
+s2 :=DATASET(250000, TRANSFORM({ integer a }, SELF.a := add1(COUNTER/2)));
+ SUM(NOFOLD(s1 + s2), a);
+
+s1a :=DATASET(250000, TRANSFORM({ integer a }, SELF.a := (integer) add2((STRING)COUNTER)));
+s2a :=DATASET(250000, TRANSFORM({ integer a }, SELF.a := (integer) add3((STRING)(COUNTER/2))));
+ SUM(NOFOLD(s1a + s2a), a);
+
+s1b :=DATASET(250000, TRANSFORM({ integer a }, SELF.a := COUNTER+1));
+s2b :=DATASET(250000, TRANSFORM({ integer a }, SELF.a := (COUNTER/2)+1));
+ SUM(NOFOLD(s1b + s2b), a);
+
+s1c :=DATASET(250000, TRANSFORM({ integer a }, SELF.a := (integer) ((STRING) COUNTER + '1')));
+s2c :=DATASET(250000, TRANSFORM({ integer a }, SELF.a := (integer) ((STRING)(COUNTER/2) + '1')));
+ SUM(NOFOLD(s1c + s2c), a);

testing/ecl/embedjs.ecl

@@ -0,0 +1,48 @@
+IMPORT javascript;
+
+javascript.Language.syntaxcheck('1+2');
+
+integer add1(integer val) := EMBED(javascript) val+1; ENDEMBED;
+string add2(string val) := EMBED(javascript) val+'1'; ENDEMBED;
+string add3(varstring val) := EMBED(javascript) val+'1'; ENDEMBED;
+utf8 add4(utf8 val) := EMBED(javascript) val+'1'; ENDEMBED;
+unicode add5(unicode val) := EMBED(javascript, U' val+\' at Oh là là Straße\';');
+
+integer testThrow(integer val) := EMBED(javascript) throw new Error("Error from JavaScript"); ENDEMBED;
+data testData(data val) := EMBED(javascript) val[0] = val[0] + 1; val; ENDEMBED;
+
+add1(10);
+add2('Hello');
+add3('World');
+add4(U'Oh là là Straße');
+add5(U'Стоял');
+
+add2('Oh là là Straße');  // Passing latin chars - should be untranslated
+
+// Test exception throwing/catching
+d := dataset([{ 1, '' }], { integer a, string m} ) : stored('nofold');
+
+d t := transform
+  self.a := FAILCODE;
+  self.m := FAILMESSAGE;
+  self := [];
+end;
+
+catch(d(testThrow(a) = a), onfail(t));
+testdata(D'aa');
+
+s1 :=DATASET(250000, TRANSFORM({ integer a }, SELF.a := add1(COUNTER)));
+s2 :=DATASET(250000, TRANSFORM({ integer a }, SELF.a := add1(COUNTER/2)));
+SUM(NOFOLD(s1 + s2), a);
+
+s1a :=DATASET(250000, TRANSFORM({ integer a }, SELF.a := (integer) add2((STRING)COUNTER)));
+s2a :=DATASET(250000, TRANSFORM({ integer a }, SELF.a := (integer) add3((STRING)(COUNTER/2))));
+SUM(NOFOLD(s1a + s2a), a);
+
+s1b :=DATASET(250000, TRANSFORM({ integer a }, SELF.a := COUNTER+1));
+s2b :=DATASET(250000, TRANSFORM({ integer a }, SELF.a := (COUNTER/2)+1));
+SUM(NOFOLD(s1b + s2b), a);
+
+s1c :=DATASET(250000, TRANSFORM({ integer a }, SELF.a := (integer) ((STRING) COUNTER + '1')));
+s2c :=DATASET(250000, TRANSFORM({ integer a }, SELF.a := (integer) ((STRING)(COUNTER/2) + '1')));
+SUM(NOFOLD(s1c + s2c), a);

testing/ecl/embedjs2.ecl

@@ -0,0 +1,17 @@
+import Javascript;
+
+string anagram(string word) := EMBED(Javascript)
+
+function anagram(word)
+{
+  if (word == 'cat')
+     return 'act';
+  else
+    return word;
+}
+
+anagram(word)
+ENDEMBED;
+
+anagram('dog');
+anagram('cat');

testing/ecl/embedp2.ecl

@@ -0,0 +1,17 @@
+import python;
+
+string anagram(string word) := EMBED(Python)
+  def anagram(w):
+    if word == 'cat':
+      return 'act'
+    else:
+      return w
+
+  return anagram(word)
+ENDEMBED;
+
+anagram('dog');
+anagram('cat');
+
+
+

testing/ecl/embedpy.ecl

@@ -0,0 +1,78 @@
+IMPORT Python;
+
+Python.Language.syntaxcheck('1+2');
+
+integer add1(integer val) := EMBED(Python)
+val+1
+ENDEMBED;
+
+string add2(string val) := EMBED(Python)
+val+'1'
+ENDEMBED;
+
+string add3(varstring val) := EMBED(Python)
+val+'1'
+ENDEMBED;
+
+utf8 add4(utf8 val) := EMBED(Python)
+val+'1'
+ENDEMBED;
+
+unicode add5(unicode val) := EMBED(Python)
+val+'1'
+ENDEMBED;
+
+utf8 add6(utf8 val) := EMBED(Python)
+return val+'1'
+ENDEMBED;
+
+unicode add7(unicode val) := EMBED(Python)
+return val+'1'
+ENDEMBED;
+
+integer testThrow(integer val) := EMBED(Python)
+raise Exception('Error from Python')
+ENDEMBED;
+
+data testData(data val) := EMBED(Python)
+val[0] = val[0] + 1
+return val
+ENDEMBED;
+
+add1(10);
+add2('Hello');
+add3('World');
+add4(U'Oh là là Straße');
+add5(U'Стоял');
+add6(U'Oh là là Straße');
+add7(U'Стоял');
+
+add2('Oh là là Straße');  // Passing latin chars - should be untranslated
+
+// Can't catch an expression(only a dataset)
+d := dataset([{ 1, '' }], { integer a, string m} ) : stored('nofold');
+
+d t := transform
+  self.a := FAILCODE;
+  self.m := FAILMESSAGE;
+  self := [];
+end;
+
+catch(d(testThrow(a) = a), onfail(t));
+testData(D'aa');
+
+s1 :=DATASET(250000, TRANSFORM({ integer a }, SELF.a := add1(COUNTER)));
+s2 :=DATASET(250000, TRANSFORM({ integer a }, SELF.a := add1(COUNTER/2)));
+ SUM(NOFOLD(s1 + s2), a);
+
+s1a :=DATASET(250000, TRANSFORM({ integer a }, SELF.a := (integer) add2((STRING)COUNTER)));
+s2a :=DATASET(250000, TRANSFORM({ integer a }, SELF.a := (integer) add3((STRING)(COUNTER/2))));
+ SUM(NOFOLD(s1a + s2a), a);
+
+s1b :=DATASET(250000, TRANSFORM({ integer a }, SELF.a := COUNTER+1));
+s2b :=DATASET(250000, TRANSFORM({ integer a }, SELF.a := (COUNTER/2)+1));
+ SUM(NOFOLD(s1b + s2b), a);
+
+s1c :=DATASET(250000, TRANSFORM({ integer a }, SELF.a := (integer) ((STRING) COUNTER + '1')));
+s2c :=DATASET(250000, TRANSFORM({ integer a }, SELF.a := (integer) ((STRING)(COUNTER/2) + '1')));
+ SUM(NOFOLD(s1c + s2c), a);

testing/ecl/javaimport.ecl

@@ -0,0 +1,19 @@
+import java;
+string jcat(string a, string b) := IMPORT(java, 'JavaCat.cat:(Ljava/lang/String;Ljava/lang/String;)Ljava/lang/String;');
+
+integer jadd(integer a, integer b) := IMPORT(java, 'JavaCat.add:(II)I');
+integer jaddl(integer a, integer b) := IMPORT(java, 'JavaCat.addL:(II)J');
+integer jaddi(integer a, integer b) := IMPORT(java, 'JavaCat.addI:(II)Ljava/lang/Integer;');
+
+real jfadd(real4 a, real4 b) := IMPORT(java, 'JavaCat.fadd:(FF)F');
+real jdadd(real a, real b) := IMPORT(java, 'JavaCat.dadd:(DD)D');
+real jdaddD(real a, real b) := IMPORT(java, 'JavaCat.daddD:(DD)Ljava/lang/Double;');
+
+jcat('Hello ', 'world!');
+jadd(1,2);
+jaddl(3,4);
+jaddi(5,6);
+
+jfadd(1,2);
+jdadd(3,4);
+jdaddD(5,6);

testing/ecl/key/embed.xml

@@ -0,0 +1,3 @@
+<Dataset name='Result 1'>
+ <Row><Result_1>7</Result_1></Row>
+</Dataset>

testing/ecl/key/embedR.xml

@@ -0,0 +1,15 @@
+<Dataset name='Result 1'>
+ <Row><Result_1>11</Result_1></Row>
+</Dataset>
+<Dataset name='Result 2'>
+ <Row><Result_2>Hello World</Result_2></Row>
+</Dataset>
+<Dataset name='Result 3'>
+ <Row><Result_3>6262</Result_3></Row>
+</Dataset>
+<Dataset name='Result 4'>
+ <Row><Result_4>46875625000</Result_4></Row>
+</Dataset>
+<Dataset name='Result 5'>
+ <Row><Result_5>46875625000</Result_5></Row>
+</Dataset>

testing/ecl/key/embedjava.xml

@@ -0,0 +1,39 @@
+<Dataset name='Result 1'>
+ <Row><Result_1>11</Result_1></Row>
+</Dataset>
+<Dataset name='Result 2'>
+ <Row><Result_2>Hello1</Result_2></Row>
+</Dataset>
+<Dataset name='Result 3'>
+ <Row><Result_3>World1</Result_3></Row>
+</Dataset>
+<Dataset name='Result 4'>
+ <Row><Result_4>Leovenaðes1</Result_4></Row>
+</Dataset>
+<Dataset name='Result 5'>
+ <Row><Result_5>Стоял1</Result_5></Row>
+</Dataset>
+<Dataset name='Result 6'>
+ <Row><Result_6>B</Result_6></Row>
+</Dataset>
+<Dataset name='Result 7'>
+ <Row><Result_7>Hello world</Result_7></Row>
+</Dataset>
+<Dataset name='Result 8'>
+ <Row><a>0</a><m>javaembed: In method JavaCat.testThrow:(I)I: java.lang.Exception: Exception from Java</m></Row>
+</Dataset>
+<Dataset name='Result 9'>
+ <Row><Result_9>6261</Result_9></Row>
+</Dataset>
+<Dataset name='Result 10'>
+ <Row><Result_10>46875625000</Result_10></Row>
+</Dataset>
+<Dataset name='Result 11'>
+ <Row><Result_11>328126500000</Result_11></Row>
+</Dataset>
+<Dataset name='Result 12'>
+ <Row><Result_12>46875625000</Result_12></Row>
+</Dataset>
+<Dataset name='Result 13'>
+ <Row><Result_13>328126500000</Result_13></Row>
+</Dataset>

testing/ecl/key/embedjs.xml

@@ -0,0 +1,39 @@
+<Dataset name='Result 1'>
+ <Row><Result_1>true</Result_1></Row>
+</Dataset>
+<Dataset name='Result 2'>
+ <Row><Result_2>11</Result_2></Row>
+</Dataset>
+<Dataset name='Result 3'>
+ <Row><Result_3>Hello1</Result_3></Row>
+</Dataset>
+<Dataset name='Result 4'>
+ <Row><Result_4>World1</Result_4></Row>
+</Dataset>
+<Dataset name='Result 5'>
+ <Row><Result_5>Oh là là Straße1</Result_5></Row>
+</Dataset>
+<Dataset name='Result 6'>
+ <Row><Result_6>Стоял at Oh là là Straße</Result_6></Row>
+</Dataset>
+<Dataset name='Result 7'>
+ <Row><Result_7>Oh l&#224; l&#224; Stra&#223;e1</Result_7></Row>
+</Dataset>
+<Dataset name='Result 8'>
+ <Row><a>0</a><m>v8embed: Error: Error from JavaScript</m></Row>
+</Dataset>
+<Dataset name='Result 9'>
+ <Row><Result_9>6261</Result_9></Row>
+</Dataset>
+<Dataset name='Result 10'>
+ <Row><Result_10>46875625000</Result_10></Row>
+</Dataset>
+<Dataset name='Result 11'>
+ <Row><Result_11>328126500000</Result_11></Row>
+</Dataset>
+<Dataset name='Result 12'>
+ <Row><Result_12>46875625000</Result_12></Row>
+</Dataset>
+<Dataset name='Result 13'>
+ <Row><Result_13>328126500000</Result_13></Row>
+</Dataset>

testing/ecl/key/embedjs2.xml

@@ -0,0 +1,6 @@
+<Dataset name='Result 1'>
+ <Row><Result_1>dog</Result_1></Row>
+</Dataset>
+<Dataset name='Result 2'>
+ <Row><Result_2>act</Result_2></Row>
+</Dataset>

testing/ecl/key/embedp2.xml

@@ -0,0 +1,6 @@
+<Dataset name='Result 1'>
+ <Row><Result_1>dog</Result_1></Row>
+</Dataset>
+<Dataset name='Result 2'>
+ <Row><Result_2>act</Result_2></Row>
+</Dataset>

testing/ecl/key/embedpy.xml

@@ -0,0 +1,45 @@
+<Dataset name='Result 1'>
+ <Row><Result_1>true</Result_1></Row>
+</Dataset>
+<Dataset name='Result 2'>
+ <Row><Result_2>11</Result_2></Row>
+</Dataset>
+<Dataset name='Result 3'>
+ <Row><Result_3>Hello1</Result_3></Row>
+</Dataset>
+<Dataset name='Result 4'>
+ <Row><Result_4>World1</Result_4></Row>
+</Dataset>
+<Dataset name='Result 5'>
+ <Row><Result_5>Oh là là Straße1</Result_5></Row>
+</Dataset>
+<Dataset name='Result 6'>
+ <Row><Result_6>Стоял1</Result_6></Row>
+</Dataset>
+<Dataset name='Result 7'>
+ <Row><Result_7>Oh là là Straße1</Result_7></Row>
+</Dataset>
+<Dataset name='Result 8'>
+ <Row><Result_8>Стоял1</Result_8></Row>
+</Dataset>
+<Dataset name='Result 9'>
+ <Row><Result_9>Oh l&#224; l&#224; Stra&#223;e1</Result_9></Row>
+</Dataset>
+<Dataset name='Result 10'>
+ <Row><a>0</a><m>pyembed: Error from Python</m></Row>
+</Dataset>
+<Dataset name='Result 11'>
+ <Row><Result_11>6261</Result_11></Row>
+</Dataset>
+<Dataset name='Result 12'>
+ <Row><Result_12>46875625000</Result_12></Row>
+</Dataset>
+<Dataset name='Result 13'>
+ <Row><Result_13>328126500000</Result_13></Row>
+</Dataset>
+<Dataset name='Result 14'>
+ <Row><Result_14>46875625000</Result_14></Row>
+</Dataset>
+<Dataset name='Result 15'>
+ <Row><Result_15>328126500000</Result_15></Row>
+</Dataset>

testing/ecl/pyimport.ecl

@@ -0,0 +1,3 @@
+import python;
+string pcat(string a, string b) := IMPORT(Python, '/opt/HPCCSystems/examples/python/python_cat.cat');
+pcat('Hello ', 'world!');

testing/ecl/python_cat.py

@@ -0,0 +1,7 @@
+#!/usr/bin/python
+
+def cat(a, b):
+  return a + b
+
+if __name__=="__main__":
+   print cat("Hello", " world")