HPCC-8030 Java/Python/Javascript language support in ECL

Added support in code generator for pluggable external language support.

New syntax: EMBED(module, 'script'), or EMBED(module) script ENDEMBED

A module for embedding Javascript using the v8 engine is provided in the plugin
v8embed - this can be imported using IMPORT javascript;

Additional plugins to support embedding Python and possibly Java will follow.

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

ecl/hql/hqlatoms.cpp

@@ -49,6 +49,10 @@ _ATOM backupAtom;
 _ATOM bcdAtom;
 _ATOM beforeAtom;
 _ATOM bestAtom;
+_ATOM bindRealParamAtom;
+_ATOM bindSignedParamAtom;
+_ATOM bindStringParamAtom;
+_ATOM bindUnsignedParamAtom;
 _ATOM bitmapAtom;
 _ATOM blobAtom;
 _ATOM cAtom;
@@ -60,6 +64,7 @@ _ATOM choosenAtom;
 _ATOM clusterAtom;
 _ATOM _colocal_Atom;
 _ATOM commonAtom;
+_ATOM compileEmbeddedScriptAtom;
 _ATOM _complexKeyed_Atom;
 _ATOM compressedAtom;
 _ATOM __compressed__Atom;
@@ -70,6 +75,7 @@ _ATOM contextSensitiveAtom;
 _ATOM costAtom;
 _ATOM countAtom;
 _ATOM _countProject_Atom;
+_ATOM cppAtom;
 _ATOM _cppBody_Atom;
 _ATOM csvAtom;
 _ATOM ctxmethodAtom;
@@ -134,6 +140,11 @@ _ATOM _function_Atom;
 _ATOM globalContextAtom;
 _ATOM gctxmethodAtom;
 _ATOM getAtom;
+_ATOM getEmbedContextAtom;
+_ATOM getRealResultAtom;
+_ATOM getSignedResultAtom;
+_ATOM getStringResultAtom;
+_ATOM getUnsignedResultAtom;
 _ATOM globalAtom;
 _ATOM graphAtom;
 _ATOM groupAtom;
@@ -170,6 +181,7 @@ _ATOM jobTempAtom;
 _ATOM keepAtom;
 _ATOM keyedAtom;
 _ATOM labeledAtom;
+_ATOM languageAtom;
 _ATOM lastAtom;
 _ATOM leftAtom;
 _ATOM leftonlyAtom;
@@ -318,7 +330,9 @@ _ATOM skipAtom;
 _ATOM singleAtom;
 _ATOM snapshotAtom;
 _ATOM soapActionAtom;
+_ATOM syntaxCheckAtom;
 _ATOM httpHeaderAtom;
+_ATOM prototypeAtom;
 _ATOM proxyAddressAtom;
 _ATOM sort_AllAtom;
 _ATOM sort_KeyedAtom;
@@ -429,6 +443,10 @@ MODULE_INIT(INIT_PRIORITY_HQLATOM)
     MAKEATOM(bcd);
     MAKEATOM(before);
     MAKEATOM(best);
+    MAKEATOM(bindRealParam);
+    MAKEATOM(bindSignedParam);
+    MAKEATOM(bindStringParam);
+    MAKEATOM(bindUnsignedParam);
     MAKEATOM(bitmap);
     MAKEATOM(blob);
     MAKEATOM(c);
@@ -440,6 +458,7 @@ MODULE_INIT(INIT_PRIORITY_HQLATOM)
     MAKEATOM(cluster);
     MAKESYSATOM(colocal);
     MAKEATOM(common);
+    MAKEATOM(compileEmbeddedScript);
     MAKESYSATOM(complexKeyed);
     MAKEATOM(compressed);
     MAKEATOM(__compressed__);
@@ -450,6 +469,7 @@ MODULE_INIT(INIT_PRIORITY_HQLATOM)
     MAKEATOM(cost);
     MAKEATOM(count);
     MAKESYSATOM(countProject);
+    MAKEATOM(cpp);
     MAKESYSATOM(cppBody);
     MAKEATOM(csv);
     MAKEATOM(ctxmethod);
@@ -513,6 +533,11 @@ MODULE_INIT(INIT_PRIORITY_HQLATOM)
     MAKESYSATOM(function);
     MAKEATOM(gctxmethod);
     MAKEATOM(get);
+    MAKEATOM(getEmbedContext);
+    MAKEATOM(getRealResult);
+    MAKEATOM(getSignedResult);
+    MAKEATOM(getStringResult);
+    MAKEATOM(getUnsignedResult);
     MAKEATOM(global);
     MAKEATOM(globalContext);
     MAKEATOM(graph);
@@ -551,6 +576,7 @@ MODULE_INIT(INIT_PRIORITY_HQLATOM)
     MAKEATOM(keep);
     MAKEATOM(keyed);
     MAKEATOM(labeled);
+    MAKEATOM(language);
     MAKEATOM(last);
     MAKEATOM(left);
     leftonlyAtom = createLowerCaseAtom("left only");
@@ -643,6 +669,7 @@ MODULE_INIT(INIT_PRIORITY_HQLATOM)
     MAKEATOM(preload);
     MAKEATOM(priority);
     MAKEATOM(private);
+    MAKEATOM(prototype);
     MAKEATOM(proxyAddress);
     MAKEATOM(pseudoentrypoint);
     MAKEATOM(pull);
@@ -699,6 +726,7 @@ MODULE_INIT(INIT_PRIORITY_HQLATOM)
     MAKEATOM(skip);
     MAKEATOM(snapshot);
     MAKEATOM(soapAction);
+    MAKEATOM(syntaxCheck);
     sort_AllAtom = createLowerCaseAtom("SORT ALL");
     sort_KeyedAtom = createLowerCaseAtom("SORT KEYED");
     MAKEATOM(sorted);

ecl/hql/hqlatoms.hpp

@@ -52,6 +52,10 @@ extern HQL_API _ATOM backupAtom;
 extern HQL_API _ATOM bcdAtom;
 extern HQL_API _ATOM beforeAtom;
 extern HQL_API _ATOM bestAtom;
+extern HQL_API _ATOM bindRealParamAtom;
+extern HQL_API _ATOM bindSignedParamAtom;
+extern HQL_API _ATOM bindStringParamAtom;
+extern HQL_API _ATOM bindUnsignedParamAtom;
 extern HQL_API _ATOM bitmapAtom;
 extern HQL_API _ATOM blobAtom;
 extern HQL_API _ATOM cAtom;
@@ -63,6 +67,7 @@ extern HQL_API _ATOM choosenAtom;
 extern HQL_API _ATOM clusterAtom;
 extern HQL_API _ATOM _colocal_Atom;
 extern HQL_API _ATOM commonAtom;
+extern HQL_API _ATOM compileEmbeddedScriptAtom;
 extern HQL_API _ATOM _complexKeyed_Atom;
 extern HQL_API _ATOM compressedAtom;
 extern HQL_API _ATOM __compressed__Atom;
@@ -73,6 +78,7 @@ extern HQL_API _ATOM contextSensitiveAtom;
 extern HQL_API _ATOM costAtom;
 extern HQL_API _ATOM countAtom;
 extern HQL_API _ATOM _countProject_Atom;
+extern HQL_API _ATOM cppAtom;
 extern HQL_API _ATOM _cppBody_Atom;
 extern HQL_API _ATOM csvAtom;
 extern HQL_API _ATOM ctxmethodAtom;
@@ -136,6 +142,11 @@ extern HQL_API _ATOM fullouterAtom;
 extern HQL_API _ATOM _function_Atom;
 extern HQL_API _ATOM gctxmethodAtom;
 extern HQL_API _ATOM getAtom;
+extern HQL_API _ATOM getEmbedContextAtom;
+extern HQL_API _ATOM getRealResultAtom;
+extern HQL_API _ATOM getSignedResultAtom;
+extern HQL_API _ATOM getStringResultAtom;
+extern HQL_API _ATOM getUnsignedResultAtom;
 extern HQL_API _ATOM globalAtom;
 extern HQL_API _ATOM globalContextAtom;
 extern HQL_API _ATOM graphAtom;
@@ -174,6 +185,7 @@ extern HQL_API _ATOM jobTempAtom;
 extern HQL_API _ATOM keepAtom;
 extern HQL_API _ATOM keyedAtom;
 extern HQL_API _ATOM labeledAtom;
+extern HQL_API _ATOM languageAtom;
 extern HQL_API _ATOM lastAtom;
 extern HQL_API _ATOM leftAtom;
 extern HQL_API _ATOM leftonlyAtom;
@@ -322,7 +334,9 @@ extern HQL_API _ATOM skewAtom;
 extern HQL_API _ATOM skipAtom;
 extern HQL_API _ATOM snapshotAtom;
 extern HQL_API _ATOM soapActionAtom;
+extern HQL_API _ATOM syntaxCheckAtom;
 extern HQL_API _ATOM httpHeaderAtom;
+extern HQL_API _ATOM prototypeAtom;
 extern HQL_API _ATOM proxyAddressAtom;
 extern HQL_API _ATOM sort_AllAtom;
 extern HQL_API _ATOM sort_KeyedAtom;

ecl/hql/hqlerrors.hpp

@@ -418,6 +418,7 @@
 #define HQLERR_CannotSubmitMacroX   2388
 #define HQLERR_CannotBeGrouped      2389
 #define HQLERR_CannotAccessShared   2390
+#define ERR_PluginNoScripting       2391
 
 #define ERR_ASSERTION_FAILS         100000
 

ecl/hql/hqlgram.hpp

@@ -542,7 +542,7 @@ public:
     IHqlExpression * processAlienType(const attribute & errpos);
     IHqlExpression * processIndexBuild(attribute & indexAttr, attribute * recordAttr, attribute * payloadAttr, attribute & filenameAttr, attribute & flagsAttr);
     IHqlExpression * processCompoundFunction(attribute & result, bool outOfLine);
-    IHqlExpression * processCppBody(const attribute & errpos, IHqlExpression * cpp);
+    IHqlExpression * processCppBody(const attribute & errpos, IHqlExpression * cpp, IHqlExpression * language);
     void processEnum(attribute & idAttr, IHqlExpression * value);
     void processError(bool full);
     void processLoadXML(attribute & a1, attribute * a2);
@@ -998,6 +998,7 @@ class HqlLex
         HqlLex(HqlGram *gram, IFileContents * _text, IXmlScope *xmlScope, IHqlExpression *macroExpr);
         ~HqlLex();   
 
+        void enterEmbeddedMode();
         static int doyyFlex(YYSTYPE & returnToken, yyscan_t yyscanner, HqlLex * lexer, bool lookup, const short * activeState);
         static int lookupIdentifierToken(YYSTYPE & returnToken, HqlLex * lexer, bool lookup, const short * activeState, const char * tokenText);
 
@@ -1078,6 +1079,7 @@ class HqlLex
         void init(IFileContents * _text);
 
     private:
+        static void doEnterEmbeddedMode(yyscan_t yyscanner);
         void declareXmlSymbol(const YYSTYPE & errpos, const char *name);
         StringBuffer &lookupXmlSymbol(const YYSTYPE & errpos, const char *name, StringBuffer &value);
         void setXmlSymbol(const YYSTYPE & errpos, const char *name, const char *value, bool append);

ecl/hql/hqlgram.y

@@ -175,6 +175,7 @@ static void eclsyntaxerror(HqlGram * parser, const char * s, short yystate, int
   ECLCRC
   ELSE
   ELSEIF
+  EMBED
   EMBEDDED
   _EMPTY_
   ENCODING
@@ -182,6 +183,7 @@ static void eclsyntaxerror(HqlGram * parser, const char * s, short yystate, int
   ENCRYPTED
   END
   ENDCPP
+  ENDEMBED
   ENTH
   ENUM
   TOK_ERROR
@@ -1004,7 +1006,28 @@ macro
 cppBodyText
     : CPPBODY           {
                             OwnedHqlExpr cpp = $1.getExpr();
-                            $$.setExpr(parser->processCppBody($1, cpp), $1);
+                            $$.setExpr(parser->processCppBody($1, cpp, NULL), $1);
+                        }
+    | embedPrefix CPPBODY
+                        {
+                            OwnedHqlExpr language = $1.getExpr();
+                            OwnedHqlExpr cpp = $2.getExpr();
+                            $$.setExpr(parser->processCppBody($2, cpp, language), $1);
+                        }
+    | EMBED '(' abstractModule ',' expression ')'
+                        {
+                            parser->normalizeExpression($5, type_string, true);
+                            OwnedHqlExpr language = $3.getExpr();
+                            OwnedHqlExpr cpp = $5.getExpr();
+                            $$.setExpr(parser->processCppBody($5, cpp, language), $1);
+                        }
+    ;
+
+embedPrefix
+    : EMBED '(' abstractModule ')'
+                        {
+                            parser->getLexer()->enterEmbeddedMode();
+                            $$.inherit($3);
                         }
     ;
 

ecl/hql/hqlgram2.cpp

@@ -831,11 +831,23 @@ IHqlExpression * HqlGram::convertToOutOfLineFunction(const ECLlocation & errpos,
     return LINK(expr);
 }
 
-IHqlExpression * HqlGram::processCppBody(const attribute & errpos, IHqlExpression * cpp)
+IHqlExpression * HqlGram::processCppBody(const attribute & errpos, IHqlExpression * cpp, IHqlExpression * language)
 {
     HqlExprArray args;
     cpp->unwindList(args, no_comma);
-
+    if (language)
+    {
+        IHqlScope *pluginScope = language->queryScope();
+        OwnedHqlExpr getEmbedContextFunc = pluginScope->lookupSymbol(getEmbedContextAtom, LSFsharedOK, lookupCtx);
+        if (!getEmbedContextFunc)
+            reportError(ERR_PluginNoScripting, errpos, "Module %s does not export getEmbedContext() function", language->queryName()->getAtomNamePtr());
+        OwnedHqlExpr syntaxCheckFunc = pluginScope->lookupSymbol(cppAtom, LSFsharedOK, lookupCtx);
+        if (syntaxCheckFunc)
+        {
+            // MORE - create an expression that calls it, and const fold it, I guess....
+        }
+        args.append(*createAttribute(languageAtom, getEmbedContextFunc.getClear()));
+    }
     Linked<ITypeInfo> type = current_type;
     if (!type)
         type.setown(makeVoidType());

ecl/hql/hqllex.l

@@ -673,6 +673,7 @@ ECLCRC              { RETURNSYM(ECLCRC); }
 ELSE                { RETURNSYM(ELSE); }
 ELSIF               { RETURNSYM(ELSEIF); }
 ELSEIF              { RETURNSYM(ELSEIF); }
+EMBED               { RETURNSYM(EMBED); }
 EMBEDDED            { RETURNSYM(EMBEDDED); }
 _EMPTY_             { RETURNSYM(_EMPTY_); }
 ENCODING            { RETURNSYM(ENCODING); }
@@ -1375,6 +1376,7 @@ OUT                 { RETURNSYM(TOK_OUT); }
 CONST               { RETURNSYM(TOK_CONST); }
 ENDMACRO            { setupdatepos; return(ENDMACRO); }
 ENDC\+\+            { setupdatepos; return(ENDCPP); }
+ENDEMBED            { setupdatepos; return(ENDEMBED); }
 ENCRYPTED           {
                         setupdatepos; 
                         if (!lookup || !LOOKUPSYM(ENCRYPTED) || lexer->macroGathering)
@@ -1458,18 +1460,21 @@ FUNCTIONMACRO|MACRO {
                         BEGIN(CPP);
                         lexer->inCpp = true;
                     }
-<CPP>[^\n]*"ENDC++"[^\n]*  { 
+<CPP>[^\n]*("ENDC++"|"ENDEMBED")[^\n]*  {
                         lexer->inCpp = false;
                         int endpos = lexer->yyPosition;
                         //skip to the position of ENDC++ on the line (case insensitive)
-                        while (memicmp(lexer->yyBuffer+endpos, "ENDC++", 6) != 0)
+                        while (memicmp(lexer->yyBuffer+endpos, "ENDC++", 6) != 0 && memicmp(lexer->yyBuffer+endpos, "ENDEMBED", 8) != 0)
                             endpos++;
-                        const int lastpos = endpos+6;
+                        const int lastpos = endpos + (tolower(lexer->yyBuffer[endpos+3])=='c' ? 6 : 8);
                             
                         updatepos1; 
                         BEGIN(0);
-                        int beginpos = returnToken.pos.position;
-                        int startpos = beginpos + 8;
+                        int startpos = returnToken.pos.position;
+                        if (endpos-startpos >= 8 && memicmp(lexer->yyBuffer+startpos, "BEGINC++", 8)==0)
+                            startpos += 8;
+                        else
+                            startpos += 1;  // Skip the ) of EMBED(xxx)
 
                         // keep the orginal format info (like blanks, newlines)
                         while (endpos != startpos && (lexer->yyBuffer[endpos-1] == 13 || lexer->yyBuffer[endpos-1] == 10))
@@ -1941,3 +1946,8 @@ FUNCTIONMACRO|MACRO {
 
 .                   { setupdatepos; return (CUR_TOKEN_TEXT[0]); }
 %%
+void HqlLex::doEnterEmbeddedMode(yyscan_t yyscanner)
+{
+   struct yyguts_t * yyg = (struct yyguts_t*)yyscanner;
+   BEGIN(CPP);
+}

ecl/hql/hqlparse.cpp

@@ -2051,6 +2051,12 @@ unsigned HqlLex::getTypeSize(unsigned lengthTypeName)
     return UNKNOWN_LENGTH;
 }
 
+void HqlLex::enterEmbeddedMode()
+{
+    doEnterEmbeddedMode(scanner);
+    inCpp = true;
+}
+
 int HqlLex::yyLex(YYSTYPE & returnToken, bool lookup, const short * activeState)
 {
     loop

ecl/hqlcpp/hqlcpp.cpp

@@ -7241,6 +7241,11 @@ void HqlCppTranslator::doBuildExprCppBody(BuildCtx & ctx, IHqlExpression * expr,
         throwError(HQLERR_EmbeddedCppNotAllowed);
 
     processCppBodyDirectives(expr);
+    IHqlExpression *languageAttr = expr->queryProperty(languageAtom);
+    if (languageAtom)
+    {
+        UNIMPLEMENTED;  // It's not clear if this can ever happen - perhaps a parameterless function that used EMBED ?
+    }
     StringBuffer text;
     expr->queryChild(0)->queryValue()->getStringValue(text);
     text.setLength(cleanupEmbeddedCpp(text.length(), (char*)text.str()));
@@ -11249,11 +11254,60 @@ void HqlCppTranslator::expandFunctionPrototype(BuildCtx & ctx, IHqlExpression *
     if (expandFunctionPrototype(s, funcdef))
     {
         s.append(";");
-        ctx.addQuoted(s);
+        IHqlExpression *body = funcdef->queryChild(0);
+        IHqlExpression *namespaceAttr = body->queryProperty(namespaceAtom);
+        if (namespaceAttr)
+        {
+            StringBuffer ns;
+            getStringValue(ns, namespaceAttr->queryChild(0));
+            ns.insert(0, "namespace ").appendf(" { %s }", s.str());
+            ctx.addQuoted(ns);
+        }
+        else
+            ctx.addQuoted(s);
     }
 }
 
 //Replace no_param with whatever they will have been bound to
+static IHqlExpression *createActualFromFormal(IHqlExpression *param)
+{
+    StringBuffer paramNameText, temp;
+    ITypeInfo *paramType = param->queryType();
+    CHqlBoundExpr bound;
+
+    //Case is significant if these parameters are use for BEGINC++ sections
+    _ATOM paramName = param->queryName();
+    paramNameText.clear().append(paramName).toLowerCase();
+
+    Linked<ITypeInfo> type = paramType;
+    switch (paramType->getTypeCode())
+    {
+    case type_set:
+        {
+            appendCapital(temp.clear().append("isAll"), paramNameText);
+            bound.isAll.setown(createVariable(temp.str(), makeBoolType()));
+        }
+        //fall through
+    case type_string:
+    case type_qstring:
+    case type_data:
+    case type_unicode:
+    case type_utf8:
+    case type_dictionary:
+    case type_table:
+    case type_groupedtable:
+        if (paramType->getSize() == UNKNOWN_LENGTH)
+        {
+            appendCapital(temp.clear().append("len"), paramNameText);
+            bound.length.setown(createVariable(temp.str(), LINK(sizetType)));
+        }
+        type.setown(makeReferenceModifier(LINK(type)));
+        break;
+    }
+    bound.expr.setown(createVariable(paramNameText.str(), LINK(type)));
+    return bound.getTranslatedExpr();
+}
+
 static IHqlExpression * replaceInlineParameters(IHqlExpression * funcdef, IHqlExpression * expr)
 {
     IHqlExpression * body = funcdef->queryChild(0);
@@ -11265,41 +11319,7 @@ static IHqlExpression * replaceInlineParameters(IHqlExpression * funcdef, IHqlEx
     ForEachChild(i, formals)
     {
         IHqlExpression * param = formals->queryChild(i);
-        ITypeInfo *paramType = param->queryType();
-        CHqlBoundExpr bound;
-        
-        //Case is significant if these parameters are use for BEGINC++ sections
-        _ATOM paramName = param->queryName();
-        paramNameText.clear().append(paramName).toLowerCase();
-
-        Linked<ITypeInfo> type = paramType;
-        switch (paramType->getTypeCode())
-        {
-        case type_set:
-            {
-                appendCapital(temp.clear().append("isAll"), paramNameText);
-                bound.isAll.setown(createVariable(temp.str(), makeBoolType()));
-            }
-            //fall through
-        case type_string:
-        case type_qstring:
-        case type_data:
-        case type_unicode:
-        case type_utf8:
-        case type_dictionary:
-        case type_table:
-        case type_groupedtable:
-            if (paramType->getSize() == UNKNOWN_LENGTH)
-            {
-                appendCapital(temp.clear().append("len"), paramNameText);
-                bound.length.setown(createVariable(temp.str(), LINK(sizetType)));
-            }
-            type.setown(makeReferenceModifier(LINK(type)));
-            break;
-        } 
-        bound.expr.setown(createVariable(paramNameText.str(), LINK(type)));
-        OwnedHqlExpr replacement = bound.getTranslatedExpr();
-        simpleTransformer.setMapping(param, replacement);
+        simpleTransformer.setMapping(param, createActualFromFormal(param));
     }
 
     return simpleTransformer.transformRoot(expr);
@@ -11341,87 +11361,167 @@ void HqlCppTranslator::doBuildUserFunctionReturn(BuildCtx & ctx, ITypeInfo * typ
     }
 }
 
+void HqlCppTranslator::buildCppFunctionDefinition(BuildCtx &funcctx, IHqlExpression * bodyCode, const char *proto)
+{
+    processCppBodyDirectives(bodyCode);
+    IHqlExpression * location = queryLocation(bodyCode);
+    const char * locationFilename = location ? location->querySourcePath()->str() : NULL;
+    unsigned startLine = location ? location->getStartLine() : 0;
+    IHqlExpression * cppBody = bodyCode->queryChild(0);
+    if (cppBody->getOperator() == no_record)
+        cppBody = bodyCode->queryChild(1);
+
+    StringBuffer text;
+    cppBody->queryValue()->getStringValue(text);
+    //remove #option, and remove /r so we don't end up with mixed format end of lines.
+    text.setLength(cleanupEmbeddedCpp(text.length(), (char*)text.str()));
+
+    const char * start = text.str();
+    loop
+    {
+        char next = *start;
+        if (next == '\n')
+            startLine++;
+        else if (next != '\r')
+            break;
+        start++;
+    }
+    const char * body = start;
+    const char * cppSeparatorText = "#body";
+    const char * separator = strstr(body, cppSeparatorText);
+    if (separator)
+    {
+        text.setCharAt(separator-text.str(), 0);
+        if (location)
+            funcctx.addLine(locationFilename, startLine);
+        funcctx.addQuoted(body);
+        if (location)
+            funcctx.addLine();
+
+        body = separator + strlen(cppSeparatorText);
+        if (*body == '\r') body++;
+        if (*body == '\n') body++;
+        startLine += memcount(body-start, start, '\n');
+    }
+
+    funcctx.addQuotedCompound(proto);
+    if (location)
+        funcctx.addLine(locationFilename, startLine);
+    funcctx.addQuoted(body);
+    if (location)
+        funcctx.addLine();
+}
+
+void HqlCppTranslator::buildScriptFunctionDefinition(BuildCtx &funcctx, IHqlExpression * funcdef, const char *proto)
+{
+    ITypeInfo * returnType = funcdef->queryType()->queryChildType();
+    IHqlExpression * outofline = funcdef->queryChild(0);
+    assertex(outofline->getOperator() == no_outofline);
+    IHqlExpression * bodyCode = outofline->queryChild(0);
+    IHqlExpression *language = queryPropertyChild(bodyCode, languageAtom, 0);
+
+    funcctx.addQuotedCompound(proto);
+
+    HqlExprArray noargs;
+    OwnedHqlExpr getPlugin = bindFunctionCall(language, noargs);
+    OwnedHqlExpr pluginPtr = createQuoted("Owned<IEmbedContext> __plugin", makeBoolType());
+    buildAssignToTemp(funcctx, pluginPtr, getPlugin);
+    funcctx.addQuoted("Owned<IEmbedFunctionContext> __ctx = __plugin->createFunctionContext();");
+    OwnedHqlExpr ctxVar = createVariable("__ctx", makeBoolType());
+
+    HqlExprArray scriptArgs;
+    scriptArgs.append(*LINK(ctxVar));
+    scriptArgs.append(*LINK(bodyCode->queryChild(0)));
+    buildFunctionCall(funcctx, compileEmbeddedScriptAtom, scriptArgs);
+    IHqlExpression *formals = funcdef->queryChild(1);
+    ForEachChild(i, formals)
+    {
+        HqlExprArray args;
+        args.append(*LINK(ctxVar));
+        IHqlExpression * param = formals->queryChild(i);
+        ITypeInfo *paramType = param->queryType();
+        _ATOM paramName = param->queryName();
+        StringBuffer paramNameText;
+        paramNameText.append(paramName).toLowerCase();
+        args.append(*createConstant(paramNameText));
+        _ATOM bindFunc;
+        switch (paramType->getTypeCode())
+        {
+        case type_int:
+            bindFunc = paramType->isSigned() ? bindSignedParamAtom : bindUnsignedParamAtom;
+            break;
+        case type_varstring:
+        case type_string:
+            bindFunc = bindStringParamAtom;
+            break;
+        case type_real:
+            bindFunc = bindRealParamAtom;
+            break;
+        default:
+            UNIMPLEMENTED;
+        }
+        args.append(*createActualFromFormal(param));
+        buildFunctionCall(funcctx, bindFunc, args);
+    }
+    funcctx.addQuoted("__ctx->callFunction();");
+    _ATOM returnFunc;
+    switch (returnType->getTypeCode())
+    {
+    case type_int:
+        returnFunc = returnType->isSigned() ? getSignedResultAtom : getUnsignedResultAtom;
+        break;
+    case type_varstring:
+    case type_string:
+        returnFunc = getStringResultAtom;
+        break;
+    case type_real:
+        returnFunc = getRealResultAtom;
+        break;
+    default:
+        UNIMPLEMENTED;
+    }
+    noargs.append(*LINK(ctxVar));
+    OwnedHqlExpr call = bindFunctionCall(returnFunc, noargs);
+    doBuildUserFunctionReturn(funcctx, returnType, call);
+}
 
 void HqlCppTranslator::buildFunctionDefinition(IHqlExpression * funcdef)
 {
     IHqlExpression * outofline = funcdef->queryChild(0);
-    ITypeInfo * returnType = funcdef->queryType()->queryChildType();
     assertex(outofline->getOperator() == no_outofline);
     IHqlExpression * bodyCode = outofline->queryChild(0);
 
-    StringBuffer s;
+    StringBuffer proto;
     BuildCtx funcctx(*code, helperAtom);
     if (options.spanMultipleCpp)
     {
-        const bool inChildActivity = true;  // assume the worse
+        const bool inChildActivity = true;  // assume the worst
         OwnedHqlExpr pass = getSizetConstant(cppIndexNextActivity(inChildActivity));
         funcctx.addGroupPass(pass);
     }
-    expandFunctionPrototype(s, funcdef);
+    expandFunctionPrototype(proto, funcdef);
 
     if (bodyCode->getOperator() == no_cppbody)
     {
         if (!allowEmbeddedCpp())
             throwError(HQLERR_EmbeddedCppNotAllowed);
 
-        processCppBodyDirectives(bodyCode);
-        IHqlExpression * location = queryLocation(bodyCode);
-        const char * locationFilename = location ? location->querySourcePath()->str() : NULL;
-        unsigned startLine = location ? location->getStartLine() : 0;
-        IHqlExpression * cppBody = bodyCode->queryChild(0);
-        if (cppBody->getOperator() == no_record)
-            cppBody = bodyCode->queryChild(1);
-
-        StringBuffer text;
-        cppBody->queryValue()->getStringValue(text);
-        //remove #option, and remove /r so we don't end up with mixed format end of lines.
-        text.setLength(cleanupEmbeddedCpp(text.length(), (char*)text.str()));
-
-        const char * start = text.str();
-        loop
-        {
-            char next = *start;
-            if (next == '\n')
-                startLine++;
-            else if (next != '\r')
-                break;
-            start++;
-        }
-
-        const char * body = start;
-        const char * cppSeparatorText = "#body";
-        const char * separator = strstr(body, cppSeparatorText);
-        if (separator)
-        {
-            text.setCharAt(separator-text.str(), 0);
-            if (location)
-                funcctx.addLine(locationFilename, startLine);
-            funcctx.addQuoted(body);
-            if (location)
-                funcctx.addLine();
-
-            body = separator + strlen(cppSeparatorText);
-            if (*body == '\r') body++;
-            if (*body == '\n') body++;
-            startLine += memcount(body-start, start, '\n');
-        }
-
-        funcctx.addQuotedCompound(s);
-        if (location)
-            funcctx.addLine(locationFilename, startLine);
-        funcctx.addQuoted(body);
-        if (location)
-            funcctx.addLine();
+        IHqlExpression *languageAttr = bodyCode->queryProperty(languageAtom);
+        if (languageAttr)
+            buildScriptFunctionDefinition(funcctx, funcdef, proto);
+        else
+            buildCppFunctionDefinition(funcctx, bodyCode, proto);
     }
     else
     {
-        funcctx.addQuotedCompound(s);
+        funcctx.addQuotedCompound(proto);
         //MORE: Need to work out how to handle functions that require the context.
         //Need to create a class instead.
         assertex(!outofline->hasProperty(contextAtom));
 
         OwnedHqlExpr newCode = replaceInlineParameters(funcdef, bodyCode);
         newCode.setown(foldHqlExpression(newCode));
+        ITypeInfo * returnType = funcdef->queryType()->queryChildType();
         doBuildUserFunctionReturn(funcctx, returnType, newCode);
     }
 }

ecl/hqlcpp/hqlcpp.ipp

@@ -1108,6 +1108,8 @@ public:
 
     bool expandFunctionPrototype(StringBuffer & s, IHqlExpression * funcdef);
     void expandFunctionPrototype(BuildCtx & ctx, IHqlExpression * funcdef);
+    void buildCppFunctionDefinition(BuildCtx &funcctx, IHqlExpression * bodycode, const char *proto);
+    void buildScriptFunctionDefinition(BuildCtx &funcctx, IHqlExpression * bodycode, const char *proto);
     void buildFunctionDefinition(IHqlExpression * funcdef);
     void assignAndCast(BuildCtx & ctx, const CHqlBoundTarget & target, CHqlBoundExpr & expr);
     void assignCastUnknownLength(BuildCtx & ctx, const CHqlBoundTarget & target, CHqlBoundExpr & pure);

ecl/hqlcpp/hqlcppsys.ecl

@@ -804,6 +804,19 @@ const char * cppSystemText[]  = {
     "   _linkcounted_ row(dummyRecord) dictionaryLookup(boolean meta, _linkcounted_ dictionary dict, row key, _linkcounted_ row defaultrow) : eclrtl,include,pure,entrypoint='rtlDictionaryLookup';",
     "    boolean dictionaryLookupExists(boolean meta, _linkcounted_ dictionary dict, row key) : eclrtl,include,pure,entrypoint='rtlDictionaryLookupExists';",
 
+    // Marshalling parameters to external languages
+    "   bindSignedParam(const varstring name, integer val) : method,entrypoint='bindSignedParam';",
+    "   bindUnsignedParam(const varstring name, unsigned val) : method,entrypoint='bindUnsignedParam';",
+    "   bindStringParam(const varstring name, const string val) : method,entrypoint='bindStringParam';",
+    "   bindVStringParam(const varstring name, const varstring val) : method,entrypoint='bindvStringParam';",  // Not yet used
+    "   bindRealParam(const varstring name, real val) : method,entrypoint='bindRealParam';",
+
+    "   real getRealResult() : method,entrypoint='getRealResult';",
+    "   integer getSignedResult() : method,entrypoint='getSignedResult';",
+    "   string getStringResult() : method,entrypoint='getStringResult';",
+    "   unsigned getUnsignedResult() : method,entrypoint='getUnsignedResult';",
+
+    "   compileEmbeddedScript(const varstring script) : method,entrypoint='compileEmbeddedScript';",
     "   END;",
     NULL };
 

ecl/hqlcpp/hqlwcpp.cpp

@@ -590,7 +590,15 @@ bool HqlCppWriter::generateFunctionPrototype(IHqlExpression * funcdef, const cha
     if (body->hasProperty(includeAtom) || body->hasProperty(ctxmethodAtom) || body->hasProperty(gctxmethodAtom) || body->hasProperty(methodAtom) || body->hasProperty(sysAtom) || body->hasProperty(omethodAtom))
         return false;
 
-    enum { ServiceApi, RtlApi, BcdApi, CApi, LocalApi } api = ServiceApi;
+    IHqlExpression *proto = body->queryProperty(prototypeAtom);
+    if (proto)
+    {
+        StringBuffer s;
+        getStringValue(s, proto->queryChild(0));
+        out.append(s);
+        return true;
+    }
+    enum { ServiceApi, RtlApi, BcdApi, CApi, CppApi, LocalApi } api = ServiceApi;
     bool isVirtual = funcdef->hasProperty(virtualAtom);
     bool isLocal = body->hasProperty(localAtom);
     if (body->hasProperty(eclrtlAtom))
@@ -599,16 +607,18 @@ bool HqlCppWriter::generateFunctionPrototype(IHqlExpression * funcdef, const cha
         api = BcdApi;
     else if (body->hasProperty(cAtom))
         api = CApi;
+    else if (body->hasProperty(cppAtom))
+        api = CppApi;
     else if (isLocal || isVirtual)
         api = LocalApi;
 
     if (isVirtual)
         out.append("virtual");
     else
-        out.append("extern ");
+        out.append("extern");
 
     if ((api == ServiceApi) || api == CApi)
-        out.append("\"C\" ");
+        out.append(" \"C\" ");
 
     switch (api)
     {
@@ -1120,6 +1130,11 @@ StringBuffer & HqlCppWriter::generateExprCpp(IHqlExpression * expr)
                     generateExprCpp(expr->queryChild(firstArg)).append(".");
                     ++firstArg;
                 }
+                if (props->hasProperty(namespaceAtom))
+                {
+                    getProperty(props, namespaceAtom, out);
+                    out.append("::");
+                }
                 getProperty(props, entrypointAtom, out);
                 out.append('(');
                 if (props->hasProperty(contextAtom))

plugins/CMakeLists.txt

@@ -24,3 +24,5 @@ add_subdirectory (workunitservices)
 if ("${BUILD_LEVEL}" STREQUAL "COMMUNITY")
   add_subdirectory (proxies)
 endif ()
+
+add_subdirectory (v8embed)

plugins/v8embed/CMakeLists.txt

@@ -0,0 +1,51 @@
+################################################################################
+#    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+################################################################################
+
+
+# Component: v8embed
+
+#####################################################
+# Description:
+# ------------
+#    Cmake Input File for v8embed
+#####################################################
+
+set ( toolsdir "${HPCC_SOURCE_DIR}/tools" )
+
+
+project( v8embed )
+
+set (    SRCS
+         v8embed.cpp
+    )
+
+include_directories (
+         ./../../system/include
+         ./../../rtl/eclrtl
+         ./../../rtl/include
+         ./../../system/jlib
+    )
+
+ADD_DEFINITIONS( -D_USRDLL -DV8EMBED_EXPORTS )
+
+HPCC_ADD_LIBRARY( v8embed SHARED ${SRCS} )
+install ( TARGETS v8embed DESTINATION plugins )
+
+target_link_libraries ( v8embed
+    v8
+    eclrtl
+    jlib
+    )

plugins/v8embed/v8embed.cpp

@@ -0,0 +1,201 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#include "platform.h"
+#include "v8.h"
+#include "eclrtl.hpp"
+#include "jexcept.hpp"
+#include "jthread.hpp"
+#include "hqlplugins.hpp"
+
+#ifdef _WIN32
+#define EXPORT __declspec(dllexport)
+#else
+#define EXPORT
+#endif
+
+static const char * compatibleVersions[] = {
+    "JAVASCRIPT 1.0.0",
+    NULL };
+
+static const char *version = "V8JavaScriptHelper 1.1.14";
+static const char * EclDefinition =
+    "EXPORT Language := SERVICE\n"
+    "  boolean getEmbedContext():cpp,pure,namespace='javascriptLanguageHelper',entrypoint='getEmbedContext',prototype='IEmbedContext* getEmbedContext()';\n"
+    "  boolean syntaxCheck(const varstring src):cpp,pure,namespace='javascriptLanguageHelper',entrypoint='syntaxCheck';\n"
+    "END;"
+    "export getEmbedContext := Language.getEmbedContext;"
+    "export syntaxCheck := Language.syntaxCheck;";
+
+extern "C" EXPORT bool getECLPluginDefinition(ECLPluginDefinitionBlock *pb)
+{
+    if (pb->size == sizeof(ECLPluginDefinitionBlockEx))
+    {
+        ECLPluginDefinitionBlockEx * pbx = (ECLPluginDefinitionBlockEx *) pb;
+        pbx->compatibleVersions = compatibleVersions;
+    }
+    else if (pb->size != sizeof(ECLPluginDefinitionBlock))
+        return false;
+    pb->magicVersion = PLUGIN_VERSION;
+    pb->version = version;
+    pb->moduleName = "javascript";
+    pb->ECL = EclDefinition;
+    pb->flags = PLUGIN_DLL_MODULE | PLUGIN_MULTIPLE_VERSIONS;
+    pb->description = "V8 Javascript language helper";
+    return true;
+}
+
+namespace javascriptLanguageHelper {
+
+class V8JavascriptEmbedFunctionContext : public CInterfaceOf<IEmbedFunctionContext>
+{
+public:
+    V8JavascriptEmbedFunctionContext()
+    {
+        isolate = v8::Isolate::New();
+        isolate->Enter();
+        context = v8::Context::New();
+        context->Enter();
+    }
+    ~V8JavascriptEmbedFunctionContext()
+    {
+        script.Dispose();
+        result.Dispose();
+        context->Exit();
+        context.Dispose();
+        isolate->Exit();
+        isolate->Dispose();
+    }
+
+    virtual void bindRealParam(const char *name, double val)
+    {
+        v8::HandleScope handle_scope;
+        context->Global()->Set(v8::String::New(name), v8::Number::New(val));
+    }
+    virtual void bindSignedParam(const char *name, __int64 val)
+    {
+        // MORE - might need to check does not overflow 32 bits? Or store as a real?
+        v8::HandleScope handle_scope;
+        context->Global()->Set(v8::String::New(name), v8::Integer::New(val));
+    }
+    virtual void bindUnsignedParam(const char *name, unsigned __int64 val)
+    {
+        // MORE - might need to check does not overflow 32 bits
+        v8::HandleScope handle_scope;
+        context->Global()->Set(v8::String::New(name), v8::Integer::NewFromUnsigned(val));
+    }
+    virtual void bindStringParam(const char *name, size32_t len, const char *val)
+    {
+        v8::HandleScope handle_scope;
+        context->Global()->Set(v8::String::New(name), v8::String::New(val, len));
+    }
+    virtual void bindVStringParam(const char *name, const char *val)
+    {
+        v8::HandleScope handle_scope;
+        context->Global()->Set(v8::String::New(name), v8::String::New(val));
+    }
+
+    virtual double getRealResult()
+    {
+        assertex (!result.IsEmpty());
+        v8::HandleScope handle_scope;
+        return v8::Number::Cast(*result)->Value();
+    }
+    virtual __int64 getSignedResult()
+    {
+        assertex (!result.IsEmpty());
+        v8::HandleScope handle_scope;
+        return v8::Integer::Cast(*result)->Value();
+    }
+    virtual unsigned __int64 getUnsignedResult()
+    {
+        assertex (!result.IsEmpty());
+        v8::HandleScope handle_scope;
+        return v8::Integer::Cast(*result)->Value();
+    }
+    virtual void getStringResult(size32_t &__len, char * &__result)
+    {
+        assertex (!result.IsEmpty());
+        v8::HandleScope handle_scope;   // May not strictly be needed?
+        v8::String::AsciiValue ascii(result);
+        const char *chars= *ascii;
+        __len = strlen(chars);
+        __result = (char *)rtlMalloc(__len);
+        memcpy(__result, chars, __len);
+    }
+
+    virtual void compileEmbeddedScript(const char *text)
+    {
+        v8::HandleScope handle_scope;
+        v8::Handle<v8::String> source = v8::String::New(text);
+        v8::Handle<v8::Script> lscript = v8::Script::Compile(source);
+        script = v8::Persistent<v8::Script>::New(lscript);
+    }
+    virtual void callFunction()
+    {
+        assertex (!script.IsEmpty());
+        v8::HandleScope handle_scope;
+        result = v8::Persistent<v8::Value>::New(script->Run());
+    }
+
+protected:
+    v8::Isolate *isolate;
+    v8::Persistent<v8::Context> context;
+    v8::Persistent<v8::Script> script;
+    v8::Persistent<v8::Value> result;
+};
+
+__thread V8JavascriptEmbedFunctionContext * theFunctionContext;  // We reuse per thread, for speed
+__thread ThreadTermFunc threadHookChain;
+
+static void releaseContext()
+{
+    ::Release(theFunctionContext);
+    if (threadHookChain)
+        (*threadHookChain)();
+}
+
+class V8JavascriptEmbedContext : public CInterfaceOf<IEmbedContext>
+{
+public:
+    V8JavascriptEmbedContext()
+    {
+        Link();  // Deliberately 'leak' in order to avoid freeing this global object
+    }
+    virtual IEmbedFunctionContext *createFunctionContext()
+    {
+        if (!theFunctionContext)
+        {
+            theFunctionContext = new V8JavascriptEmbedFunctionContext;
+            threadHookChain = addThreadTermFunc(releaseContext);
+        }
+        return LINK(theFunctionContext);
+    }
+} theEmbedContext;
+
+
+extern IEmbedContext* getEmbedContext()
+{
+    return LINK(&theEmbedContext);
+}
+
+extern bool syntaxCheck(const char *script)
+{
+    return true; // MORE
+}
+
+} // namespace

rtl/eclrtl/eclrtl.hpp

@@ -729,4 +729,29 @@ ECLRTL_API unsigned rtlDelayReturn(unsigned value, unsigned sleepTime);
 
 ECLRTL_API bool rtlGPF();
 
+//-----------------------------------------------------------------------------
+
+interface IEmbedFunctionContext : extends IInterface
+{
+    virtual void bindRealParam(const char *name, double val) = 0;
+    virtual void bindSignedParam(const char *name, __int64 val) = 0;
+    virtual void bindUnsignedParam(const char *name, unsigned __int64 val) = 0;
+    virtual void bindStringParam(const char *name, size32_t len, const char *val) = 0;
+    virtual void bindVStringParam(const char *name, const char *val) = 0;
+
+    virtual double getRealResult() = 0;
+    virtual __int64 getSignedResult() = 0;
+    virtual unsigned __int64 getUnsignedResult() = 0;
+    virtual void getStringResult(size32_t &len, char * &result) = 0;
+
+    virtual void compileEmbeddedScript(const char *script) = 0;
+    virtual void callFunction() = 0;
+};
+
+interface IEmbedContext : extends IInterface
+{
+    virtual IEmbedFunctionContext *createFunctionContext() = 0;
+    // MORE - add syntax checked here!
+};
+
 #endif

system/include/platform.h

@@ -137,6 +137,7 @@ typedef memsize_t rowsize_t;
 #define LoadSucceeded(h)           ((unsigned)h >= 32)
 #define GetSharedObjectError()     GetLastError()
 #define strtok_r(a,b,c)             j_strtok_r(a,b,c)
+#define __thread __declspec(thread)
 
 typedef unsigned __int64 off64_t;
 typedef int socklen_t;

system/jlib/jthread.cpp

@@ -54,6 +54,15 @@ MODULE_EXIT()
     delete exceptionHandlers;
 }
 
+__thread ThreadTermFunc threadTerminationHook;
+
+ThreadTermFunc addThreadTermFunc(ThreadTermFunc onTerm)
+{
+    ThreadTermFunc old = threadTerminationHook;
+    threadTerminationHook = onTerm;
+    return old;
+}
+
 void addThreadExceptionHandler(IExceptionHandler *handler)
 {
     assertex(exceptionHandlers); // have to ensure MODULE_INIT has appropriate priority.
@@ -95,6 +104,8 @@ void *Thread::_threadmain(void *v)
     t->tidlog = threadLogID();
 #endif
     int ret = t->begin();
+    if (threadTerminationHook)
+        (*threadTerminationHook)();
     char *&threadname = t->cthreadname.threadname;
     if (threadname) {
         memsize_t l=strlen(threadname);

system/jlib/jthread.hpp

@@ -52,6 +52,14 @@ extern jlib_decl void disableThreadSEH();
 
 extern jlib_decl unsigned threadLogID();  // for use in logging
 
+// A function registered via addThreadTermFunc will be called when the thread that registered that function
+// terminates. Such a function should call on to the previously registered function (if any) - generally you
+// would expect to store that value in thread-local storage.
+// This can be used to ensure that thread-specific objects can be properly destructed.
+
+typedef void (*ThreadTermFunc)();
+extern jlib_decl ThreadTermFunc addThreadTermFunc(ThreadTermFunc onTerm);
+
 class jlib_decl Thread : public CInterface, public IThread
 {
 private:
@@ -176,31 +184,6 @@ public:
     virtual void Do(unsigned idx=0)=0;
 };
 
-// Thread local storage - use MAKETHREADLOCALIINTERFACE macro to get a thread local type
-
-template <class CLASS, class CLASSINIT = CLASS, class MAP = MapBetween<ThreadId, ThreadId, CLASS, CLASSINIT> >
-class ThreadLocalOf : public MAP
-{
-public:
-    CLASS * query()
-    {
-        CLASS * find = threadMap.getValue(GetCurrentThreadId());
-        if(find) return find;
-        threadMap.setValue(GetCurrentThreadId(), CLASSINIT());
-        return threadMap.getValue(GetCurrentThreadId());
-    }
-    operator CLASS & ()
-    {
-        return *query();
-    }
-private:
-    MAP threadMap;
-};
-
-#define MAKETHREADLOCALIINTERFACE(C, CI, NAME)                                                      \
-typedef ThreadLocalOf<C, CI> NAME
-
-
 // ---------------------------------------------------------------------------
 // Thread Pools
 // ---------------------------------------------------------------------------

testing/ecl/embed.ecl

@@ -0,0 +1,2 @@
+integer c(integer val) := BEGINC++ return val-3; ENDC++;
+c(10);

testing/ecl/embedjs.ecl

@@ -0,0 +1,27 @@
+IMPORT javascript;
+
+javascript.Language.syntaxcheck('1+2');
+
+integer add1(integer val) := EMBED(javascript) val+1; ENDEMBED;
+string add2(string val) := EMBED(javascript) val+'1'; ENDEMBED;
+string add3(varstring val) := EMBED(javascript) val+'1'; ENDEMBED;
+
+add1(10);
+add2('Hello');
+add3('World');
+
+s1 :=DATASET(250000, TRANSFORM({ integer a }, SELF.a := add1(COUNTER)));
+s2 :=DATASET(250000, TRANSFORM({ integer a }, SELF.a := add1(COUNTER/2)));
+SUM(NOFOLD(s1 & s2), a);
+
+s1a :=DATASET(250000, TRANSFORM({ integer a }, SELF.a := (integer) add2((STRING)COUNTER)));
+s2a :=DATASET(250000, TRANSFORM({ integer a }, SELF.a := (integer) add3((STRING)(COUNTER/2))));
+SUM(NOFOLD(s1a & s2a), a);
+
+s1b :=DATASET(250000, TRANSFORM({ integer a }, SELF.a := COUNTER+1));
+s2b :=DATASET(250000, TRANSFORM({ integer a }, SELF.a := (COUNTER/2)+1));
+SUM(NOFOLD(s1b & s2b), a);
+
+s1c :=DATASET(250000, TRANSFORM({ integer a }, SELF.a := (integer) ((STRING) COUNTER + '1')));
+s2c :=DATASET(250000, TRANSFORM({ integer a }, SELF.a := (integer) ((STRING)(COUNTER/2) + '1')));
+SUM(NOFOLD(s1c & s2c), a);

testing/ecl/embedp.ecl

@@ -0,0 +1,14 @@
+python := module
+  export a := 1;
+end;
+
+integer a(integer val) := EMBED(python) return val+1; ENDEMBED;
+string a2(string val) := EMBED(python) return val+'1'; ENDEMBED;
+string a3(varstring val) := EMBED(python) return val+'1'; ENDEMBED;
+
+integer b(integer val) := EMBED(python, 'return val-2;');
+
+a(10);
+a2('Hello');
+a3('world');
+b(10);

testing/ecl/key/embedjs.xml

@@ -0,0 +1,21 @@
+<Dataset name='Result 1'>
+ <Row><Result_1>11</Result_1></Row>
+</Dataset>
+<Dataset name='Result 2'>
+ <Row><Result_2>Hello1</Result_2></Row>
+</Dataset>
+<Dataset name='Result 3'>
+ <Row><Result_3>World1</Result_3></Row>
+</Dataset>
+<Dataset name='Result 4'>
+ <Row><Result_4>46875625000</Result_4></Row>
+</Dataset>
+<Dataset name='Result 5'>
+ <Row><Result_5>328126500000</Result_5></Row>
+</Dataset>
+<Dataset name='Result 6'>
+ <Row><Result_6>46875625000</Result_6></Row>
+</Dataset>
+<Dataset name='Result 7'>
+ <Row><Result_7>328126500000</Result_7></Row>
+</Dataset>