HPCC-10456 Codegen support for streamed datasets from embedded languages

Add support for row return type, and embedded transform.

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

ecl/hql/hqlatoms.cpp

@@ -23,6 +23,7 @@ IIdAtom * failId;
 IIdAtom * bindBooleanParamId;
 IIdAtom * bindDataParamId;
 IIdAtom * bindRealParamId;
+IIdAtom * bindRowParamId;
 IIdAtom * bindSetParamId;
 IIdAtom * bindSignedParamId;
 IIdAtom * bindStringParamId;
@@ -36,9 +37,11 @@ IIdAtom * getBooleanResultId;
 IIdAtom * getDataResultId;
 IIdAtom * getDatasetResultId;
 IIdAtom * getRealResultId;
+IIdAtom * getRowResultId;
 IIdAtom * getSetResultId;
 IIdAtom * getSignedResultId;
 IIdAtom * getStringResultId;
+IIdAtom * getTransformResultId;
 IIdAtom * getUnicodeResultId;
 IIdAtom * getUnsignedResultId;
 IIdAtom * getUTF8ResultId;
@@ -427,6 +430,7 @@ MODULE_INIT(INIT_PRIORITY_HQLATOM)
     MAKEID(bindBooleanParam);
     MAKEID(bindDataParam);
     MAKEID(bindRealParam);
+    MAKEID(bindRowParam);
     MAKEID(bindSetParam);
     MAKEID(bindSignedParam);
     MAKEID(bindStringParam);
@@ -442,9 +446,11 @@ MODULE_INIT(INIT_PRIORITY_HQLATOM)
     MAKEID(getDataResult);
     MAKEID(getDatasetResult);
     MAKEID(getRealResult);
+    MAKEID(getRowResult);
     MAKEID(getSetResult);
     MAKEID(getSignedResult);
     MAKEID(getStringResult);
+    MAKEID(getTransformResult);
     MAKEID(getUnicodeResult);
     MAKEID(getUnsignedResult);
     MAKEID(getUTF8Result);

ecl/hql/hqlatoms.hpp

@@ -23,6 +23,7 @@ extern HQL_API IIdAtom * atId;
 extern HQL_API IIdAtom * bindBooleanParamId;
 extern HQL_API IIdAtom * bindDataParamId;
 extern HQL_API IIdAtom * bindRealParamId;
+extern HQL_API IIdAtom * bindRowParamId;
 extern HQL_API IIdAtom * bindSetParamId;
 extern HQL_API IIdAtom * bindSignedParamId;
 extern HQL_API IIdAtom * bindStringParamId;
@@ -38,9 +39,11 @@ extern HQL_API IIdAtom * getBooleanResultId;
 extern HQL_API IIdAtom * getDataResultId;
 extern HQL_API IIdAtom * getDatasetResultId;
 extern HQL_API IIdAtom * getRealResultId;
+extern HQL_API IIdAtom * getRowResultId;
 extern HQL_API IIdAtom * getSetResultId;
 extern HQL_API IIdAtom * getSignedResultId;
 extern HQL_API IIdAtom * getStringResultId;
+extern HQL_API IIdAtom * getTransformResultId;
 extern HQL_API IIdAtom * getUnicodeResultId;
 extern HQL_API IIdAtom * getUnsignedResultId;
 extern HQL_API IIdAtom * getUTF8ResultId;

ecl/hql/hqlexpr.cpp

@@ -11588,6 +11588,8 @@ extern IHqlExpression *createRow(node_operator op, HqlExprArray & args)
         {
             IHqlExpression & record = args.item(1);
             type = makeRowType(LINK(record.queryRecordType()));
+            if (queryAttribute(_linkCounted_Atom, args))
+                type = makeAttributeModifier(type, getLinkCountedAttr());
             break;
         }
     case no_typetransfer:

ecl/hql/hqlexpr.hpp

@@ -1862,5 +1862,6 @@ extern HQL_API bool getBoolAttribute(IHqlExpression * expr, IAtom * name, bool d
 extern HQL_API void setLegacyEclSemantics(bool _value);
 extern HQL_API bool queryLegacyEclSemantics();
 void exportSymbols(IPropertyTree* data, IHqlScope * scope, HqlLookupContext & ctx);
+inline bool isCall(node_operator op) { return (op == no_call || op == no_externalcall); }
 
 #endif

ecl/hql/hqlgram2.cpp

@@ -862,22 +862,21 @@ IHqlExpression * HqlGram::processEmbedBody(const attribute & errpos, IHqlExpress
     if (!type)
         type.setown(makeVoidType());
 
-    //All non C++ dataset embedded functions must return streamed datasets
-    if (language && isDatasetType(type))
-    {
-        type.setown(setStreamedAttr(type, true));
-        type.setown(setLinkCountedAttr(type, true));
-        //MORE: Recursively set link counted on the records.
-    }
+    if (type->getTypeCode() == type_record)
+        type.setown(makeRowType(LINK(type)));
 
     IHqlExpression * record = queryOriginalRecord(type);
     OwnedHqlExpr result;
     if (record)
     {
         args.add(*LINK(record),1);
-        if (hasLinkCountedModifier(type))
+        if (hasLinkCountedModifier(type) || language)
+        {
+            //MORE: Recursively set link counted on the records. for language != NULL
             args.append(*getLinkCountedAttr());
-        if (hasStreamedModifier(type))
+        }
+        //All non C++ dataset embedded functions must return streamed datasets
+        if (hasStreamedModifier(type) || (language && isDatasetType(type)))
             args.append(*getStreamedAttr());
         switch (type->getTypeCode())
         {
@@ -902,7 +901,18 @@ IHqlExpression * HqlGram::processEmbedBody(const attribute & errpos, IHqlExpress
     result.setown(createLocationAnnotation(result.getClear(), errpos.pos));
 
     if (queryParametered())
-        return createWrapper(no_outofline, result.getClear(), createAttribute(contextAtom));    // MORE: this needs more thought
+    {
+        HqlExprArray args;
+        args.append(*LINK(result));
+        if (language)
+        {
+            args.append(*createAttribute(contextAtom));
+            if (result->isDatarow())
+                args.append(*createAttribute(allocatorAtom));
+        }
+
+        return createWrapper(no_outofline, result->queryType(), args);
+    }
     return result.getClear();
 }
 

ecl/hqlcpp/hqlcpp.cpp

@@ -3350,7 +3350,8 @@ void HqlCppTranslator::buildReturn(BuildCtx & ctx, IHqlExpression * expr, ITypeI
     expr = queryExpandAliasScope(ctx, expr);
 
     node_operator op = expr->getOperator();
-    if ((retType->getSize() == UNKNOWN_LENGTH) && (retType->getTypeCode() == type_varstring))
+    type_t returntc = retType->getTypeCode();
+    if ((retType->getSize() == UNKNOWN_LENGTH) && (returntc == type_varstring))
     {
         if (hasConstModifier(retType) && (hasConstModifier(exprType) || expr->queryValue()))
         {
@@ -3402,7 +3403,20 @@ void HqlCppTranslator::buildReturn(BuildCtx & ctx, IHqlExpression * expr, ITypeI
             ctx.addReturn(temp);
         }
     }
-    else if ((retType->getTypeCode() == type_boolean) && specialCaseBoolReturn(ctx, expr))
+    else if ((returntc == type_row) && hasLinkCountedModifier(retType))
+    {
+        CHqlBoundTarget result;
+        buildTempExpr(ctx, ctx, result, expr, FormatNatural, false);
+
+        //MORE: There should be a cleaner way of doing this
+        StringBuffer s;
+        result.expr->toString(s);
+        s.append(".getClear()");
+
+        OwnedHqlExpr temp = createQuoted(s.str(), LINK(exprType));
+        ctx.addReturn(temp);
+    }
+    else if ((returntc == type_boolean) && specialCaseBoolReturn(ctx, expr))
     {
         bool successValue = true;
         if (op == no_not)
@@ -4265,6 +4279,14 @@ void HqlCppTranslator::buildTempExpr(BuildCtx & ctx, BuildCtx & declareCtx, CHql
     case type_row:
         {
             Owned<BoundRow> tempRow = declareTempRow(declareCtx, subctx, expr);
+            tempTarget.expr.set(tempRow->queryBound());
+            //MORE: This should be more general - i) to avoid unnecessary temporaries, and ii) to allow rows to be copied by linking.
+            if (isCall(expr->getOperator()))
+            {
+                buildExprAssign(subctx, tempTarget, expr);
+                return;
+            }
+
             IHqlStmt * stmt = subctx.addGroup();
             stmt->setIncomplete(true);
 
@@ -4275,7 +4297,6 @@ void HqlCppTranslator::buildTempExpr(BuildCtx & ctx, BuildCtx & declareCtx, CHql
 
             stmt->setIncomplete(false);
             stmt->mergeScopeWithContainer();
-            tempTarget.expr.set(tempRow->queryBound());
             
             ctx.associate(*tempRow);
             break;
@@ -5785,6 +5806,8 @@ void HqlCppTranslator::doBuildCall(BuildCtx & ctx, const CHqlBoundTarget * tgt,
         {
             if (hasLinkCountedModifier(retType))
             {
+                if (hasNonNullRecord(retType) && getBoolAttribute(external, allocatorAtom, true))
+                    args.append(*createRowAllocator(ctx, ::queryRecord(retType)));
                 //Always assign link counted rows to a temporary (or the target) to ensure the are not leaked.
                 returnMustAssign = true;
                 if (tgt && hasLinkCountedModifier(targetType) && recordTypesMatch(targetType, retType))
@@ -11566,6 +11589,9 @@ void HqlCppTranslator::buildScriptFunctionDefinition(BuildCtx &funcctx, IHqlExpr
         case type_data:
             bindFunc = bindDataParamId;
             break;
+        case type_row:
+            bindFunc = bindRowParamId; // more
+            break;
         case type_set:
         {
             bindFunc = bindSetParamId;
@@ -11625,12 +11651,18 @@ void HqlCppTranslator::buildScriptFunctionDefinition(BuildCtx &funcctx, IHqlExpr
         retargs.append(*createIntConstant(returnType->queryChildType()->getSize()));
         break;
     }
+    case type_row:
+        returnFunc = getRowResultId;
+        newReturnType.set(returnType);
+        break;
     case type_table:
-        {
-            returnFunc = getDatasetResultId;
-            newReturnType.set(returnType);
-            break;
-        }
+        returnFunc = getDatasetResultId;
+        newReturnType.set(returnType);
+        break;
+    case type_transform:
+        returnFunc = getTransformResultId;
+        newReturnType.set(returnType);
+        break;
     default:
         StringBuffer typeText;
         getFriendlyTypeStr(returnType, typeText);
@@ -11655,6 +11687,8 @@ void HqlCppTranslator::buildFunctionDefinition(IHqlExpression * funcdef)
         funcctx.addGroupPass(pass);
     }
     expandFunctionPrototype(proto, funcdef);
+//    BoundRow * row = bindSelf(deserializectx, dataset, "crSelf");
+
 
     if (bodyCode->getOperator() == no_embedbody)
     {

ecl/hqlcpp/hqlcppds.cpp

@@ -127,7 +127,6 @@ IReferenceSelector * HqlCppTranslator::doBuildRowDeserializeRow(BuildCtx & ctx,
     target.expr.set(tempRow->queryBound());
 
     HqlExprArray args;  
-    args.append(*createRowAllocator(ctx, record));
     args.append(*createSerializer(ctx, record, serializeForm, deserializerAtom));
     args.append(*LINK(srcRow));
     Owned<ITypeInfo> resultType = makeReferenceModifier(makeAttributeModifier(makeRowType(record->getType()), getLinkCountedAttr()));
@@ -352,7 +351,6 @@ IReferenceSelector * HqlCppTranslator::doBuildRowFromXML(BuildCtx & ctx, IHqlExp
     }
 
     HqlExprArray args;
-    args.append(*createRowAllocator(ctx, record));
     args.append(*ensureExprType(expr->queryChild(1), utf8Type));
     args.append(*createQuoted(xmlInstanceName, makeBoolType()));
     args.append(*createConstant(expr->hasAttribute(trimAtom)));

ecl/hqlcpp/hqlcppsys.ecl

@@ -421,8 +421,8 @@ const char * cppSystemText[]  = {
     "   unicode deserializeUnicodeX(boolean o) :    eclrtl,include='eclrtl.hpp',library='eclrtl',entrypoint='deserializeUnicodeX';",
     "   utf8 deserializeUtf8X(boolean o) :  eclrtl,include='eclrtl.hpp',library='eclrtl',entrypoint='deserializeUtf8X';",
     "   varunicode deserializeVUnicodeX(boolean o) :    eclrtl,include='eclrtl.hpp',library='eclrtl',entrypoint='deserializeVUnicodeX';",
-    "   _linkcounted_ row(dummyRecord) deserializeRow(boolean _allocator, boolean _deserializer, boolean _in) : eclrtl,include,entrypoint='rtlDeserializeBufferRow';",
-    "   _linkcounted_ row(dummyRecord) createRowFromXml(boolean _allocator, utf8 _text, boolean xmltransformer, boolean _stripWhitespace) : ctxmethod,pure,entrypoint='fromXml';",
+    "   _linkcounted_ row(dummyRecord) deserializeRow(boolean _deserializer, boolean _in) : eclrtl,include,entrypoint='rtlDeserializeBufferRow';",
+    "   _linkcounted_ row(dummyRecord) createRowFromXml(utf8 _text, boolean xmltransformer, boolean _stripWhitespace) : ctxmethod,pure,entrypoint='fromXml';",
 
     "   _linkcounted_ dataset appendRowsToRowset(_array_ dataset _in) : pure,eclrtl,include,entrypoint='appendRowsToRowset';",
 
@@ -740,7 +740,7 @@ const char * cppSystemText[]  = {
     "   reportRowOverflow(unsigned4 curSize, unsigned4 maxRowSize) : eclrtl,pure,include='eclrtl.hpp',library='eclrtl',entrypoint='rtlReportRowOverflow';", 
     "   checkFieldOverflow(unsigned4 curSize, unsigned4 maxRowSize, const varstring _name) : eclrtl,pure,include='eclrtl.hpp',library='eclrtl',entrypoint='rtlCheckFieldOverflow';", 
     "   reportFieldOverflow(unsigned4 curSize, unsigned4 maxRowSize, const varstring _name) : eclrtl,pure,include='eclrtl.hpp',library='eclrtl',entrypoint='rtlReportFieldOverflow';", 
-    "   _linkcounted_ row(dummyRecord) ensureCapacity(unsigned4 curSize, const varstring _fieldName) : omethod,entrypoint='ensureCapacity';", 
+    "   _linkcounted_ row(dummyRecord) ensureCapacity(unsigned4 curSize, const varstring _fieldName) : omethod,entrypoint='ensureCapacity',allocator=false;",
     "   ensureRowAvailable(unsigned4 curSize) : omethod,entrypoint='ensureAvailable';", 
     "   IIndirectMemberVisitor_visitRowset(_linkcounted_ dataset _x) : omethod,entrypoint='visitRowset';", 
     "   IIndirectMemberVisitor_visitRow(row _x) : omethod,entrypoint='visitRow';", 
@@ -753,7 +753,7 @@ const char * cppSystemText[]  = {
     "   linkcounted dictionary getLocalDictionaryResult(unsigned4 id) : method,allocator(false),pure,entrypoint='getDictionaryResult';",
     "   unsigned4 getGraphLoopCounter() : ctxmethod,entrypoint='getGraphLoopCounter';",
 
-    "   _linkcounted_ row(dummyRecord) finalizeRowClear(unsigned4 _size) : omethod,entrypoint='finalizeRowClear';",
+    "   _linkcounted_ row(dummyRecord) finalizeRowClear(unsigned4 _size) : omethod,entrypoint='finalizeRowClear',allocator=false;",
     "   setMethod(boolean _allocator) : omethod,entrypoint='set';",
     "   setownMethod(boolean _allocator) : omethod,entrypoint='setown';",
 
@@ -779,7 +779,7 @@ const char * cppSystemText[]  = {
 
     "   dummyRecord rtlSerializeToBuilder(boolean _serializer, dummyRecord _in) : eclrtl,include,entrypoint='rtlSerializeToBuilder';",
     "   dummyRecord rtlDeserializeToBuilder(boolean _serializer, dummyRecord _in) : eclrtl,include,entrypoint='rtlDeserializeToBuilder';",
-    "   _linkcounted_ row(dummyRecord) rtlDeserializeRow(boolean _allocator, boolean _deserializer, dummyRecord _in) : eclrtl,include,entrypoint='rtlDeserializeRow';",
+    "   _linkcounted_ row(dummyRecord) rtlDeserializeRow(boolean _deserializer, dummyRecord _in) : eclrtl,include,entrypoint='rtlDeserializeRow';",
 
     "   releaseRow(row _x) : include,entrypoint='rtlReleaseRow';",
     "   releaseRowset(_linkcounted_ dataset _x) : include,allocator(false),entrypoint='rtlReleaseRowset';",
@@ -824,13 +824,13 @@ const char * cppSystemText[]  = {
     // Dictionary support
     "    integer8 dictionaryCount(_linkcounted_ dictionary dict) : eclrtl,include,pure,entrypoint='rtlDictionaryCount';",
     "    boolean dictionaryExists(_linkcounted_ dictionary dict) : eclrtl,include,pure,entrypoint='rtlDictionaryExists';",
-    "   _linkcounted_ row(dummyRecord) dictionaryLookup(IHThorHashLookupInfo meta, _linkcounted_ dictionary dict, row key, _linkcounted_ row defaultrow) : eclrtl,include,pure,entrypoint='rtlDictionaryLookup';",
-    "   _linkcounted_ row(dummyRecord) dictionaryLookupString(_linkcounted_ dictionary dict, const string key, _linkcounted_ row defaultrow) : eclrtl,include,pure,entrypoint='rtlDictionaryLookupString';",
-    "   _linkcounted_ row(dummyRecord) dictionaryLookupStringN(_linkcounted_ dictionary dict, const unsigned4 size, const string key, _linkcounted_ row defaultrow) : eclrtl,include,pure,entrypoint='rtlDictionaryLookupStringN';",
-    "   _linkcounted_ row(dummyRecord) dictionaryLookupSigned(_linkcounted_ dictionary dict, const integer key, _linkcounted_ row defaultrow) : eclrtl,include,pure,entrypoint='rtlDictionaryLookupSigned';",
-    "   _linkcounted_ row(dummyRecord) dictionaryLookupUnsigned(_linkcounted_ dictionary dict, const unsigned key, _linkcounted_ row defaultrow) : eclrtl,include,pure,entrypoint='rtlDictionaryLookupUnsigned';",
-    "   _linkcounted_ row(dummyRecord) dictionaryLookupSignedN(_linkcounted_ dictionary dict, const unsigned4 size, const integer key, _linkcounted_ row defaultrow) : eclrtl,include,pure,entrypoint='rtlDictionaryLookupSignedN';",
-    "   _linkcounted_ row(dummyRecord) dictionaryLookupUnsignedN(_linkcounted_ dictionary dict, const unsigned4 size, const unsigned key, _linkcounted_ row defaultrow) : eclrtl,include,pure,entrypoint='rtlDictionaryLookupUnsignedN';",
+    "   _linkcounted_ row(dummyRecord) dictionaryLookup(IHThorHashLookupInfo meta, _linkcounted_ dictionary dict, row key, _linkcounted_ row defaultrow) : eclrtl,include,pure,entrypoint='rtlDictionaryLookup',allocator=false;",
+    "   _linkcounted_ row(dummyRecord) dictionaryLookupString(_linkcounted_ dictionary dict, const string key, _linkcounted_ row defaultrow) : eclrtl,include,pure,entrypoint='rtlDictionaryLookupString',allocator=false;",
+    "   _linkcounted_ row(dummyRecord) dictionaryLookupStringN(_linkcounted_ dictionary dict, const unsigned4 size, const string key, _linkcounted_ row defaultrow) : eclrtl,include,pure,entrypoint='rtlDictionaryLookupStringN',allocator=false;",
+    "   _linkcounted_ row(dummyRecord) dictionaryLookupSigned(_linkcounted_ dictionary dict, const integer key, _linkcounted_ row defaultrow) : eclrtl,include,pure,entrypoint='rtlDictionaryLookupSigned',allocator=false;",
+    "   _linkcounted_ row(dummyRecord) dictionaryLookupUnsigned(_linkcounted_ dictionary dict, const unsigned key, _linkcounted_ row defaultrow) : eclrtl,include,pure,entrypoint='rtlDictionaryLookupUnsigned',allocator=false;",
+    "   _linkcounted_ row(dummyRecord) dictionaryLookupSignedN(_linkcounted_ dictionary dict, const unsigned4 size, const integer key, _linkcounted_ row defaultrow) : eclrtl,include,pure,entrypoint='rtlDictionaryLookupSignedN',allocator=false;",
+    "   _linkcounted_ row(dummyRecord) dictionaryLookupUnsignedN(_linkcounted_ dictionary dict, const unsigned4 size, const unsigned key, _linkcounted_ row defaultrow) : eclrtl,include,pure,entrypoint='rtlDictionaryLookupUnsignedN',allocator=false;",
 
     "   boolean dictionaryLookupExists(boolean meta, _linkcounted_ dictionary dict, row key) : eclrtl,include,pure,entrypoint='rtlDictionaryLookupExists';",
     "   boolean dictionaryLookupExistsString(_linkcounted_ dictionary dict, const string key) : eclrtl,include,pure,entrypoint='rtlDictionaryLookupExistsString';",
@@ -844,6 +844,7 @@ const char * cppSystemText[]  = {
     "   bindBooleanParam(const varstring name, boolean val) : method,entrypoint='bindBooleanParam';",
     "   bindDataParam(const varstring name, data val) : method,entrypoint='bindDataParam';",
     "   bindRealParam(const varstring name, real val) : method,entrypoint='bindRealParam';",
+    "   bindRowParam(const varstring name, _linkcounted_ row row) : method,entrypoint='bindRowParam';",
     "   bindSignedParam(const varstring name, integer val) : method,entrypoint='bindSignedParam';",
     "   bindUnsignedParam(const varstring name, unsigned val) : method,entrypoint='bindUnsignedParam';",
     "   bindStringParam(const varstring name, const string val) : method,entrypoint='bindStringParam';",
@@ -857,6 +858,8 @@ const char * cppSystemText[]  = {
     "   data getDataResult() : method,entrypoint='getDataResult';",
     "   streamed dataset getDatasetResult() : method,entrypoint='getDatasetResult';",
     "   real getRealResult() : method,entrypoint='getRealResult';",
+    "   _linkcounted_ row(dummyRecord) getRowResult() : method,entrypoint='getRowResult';",
+    "   transform(dummyRecord) getTransformResult() : method,entrypoint='getTransformResult';",
     "   integer getSignedResult() : method,entrypoint='getSignedResult';",
     "   string getStringResult() : method,entrypoint='getStringResult';",
     "   unsigned getUnsignedResult() : method,entrypoint='getUnsignedResult';",

ecl/hqlcpp/hqlhtcpp.cpp

@@ -2992,6 +2992,7 @@ void HqlCppTranslator::doBuildFunctionReturn(BuildCtx & ctx, ITypeInfo * type, I
 {
     bool returnByReference = false;
     CHqlBoundTarget target;
+    OwnedHqlExpr returnValue;
 
     switch (type->getTypeCode())
     {
@@ -3008,13 +3009,29 @@ void HqlCppTranslator::doBuildFunctionReturn(BuildCtx & ctx, ITypeInfo * type, I
     case type_dictionary:
     case type_table:
     case type_groupedtable:
-    case type_row:
         if (!hasStreamedModifier(type))
         {
             initBoundStringTarget(target, type, "__lenResult", "__result");
             returnByReference = true;
         }
         break;
+    case type_row:
+        if (!hasLinkCountedModifier(type))
+        {
+            initBoundStringTarget(target, type, "__lenResult", "__result");
+            returnByReference = true;
+        }
+        break;
+    case type_transform:
+        {
+            OwnedHqlExpr dataset = createDataset(no_anon, LINK(::queryRecord(type)));
+            BoundRow * row = bindSelf(ctx, dataset, "__self");
+            target.expr.set(row->querySelector());
+            returnByReference = true;
+            //A transform also returns the size that was generated (which will be bound to a local variable)
+            returnValue.setown(getRecordSize(row->querySelector()));
+            break;
+        }
     case type_set:
         target.isAll.setown(createVariable("__isAllResult", makeBoolType()));
         target.length.setown(createVariable("__lenResult", LINK(sizetType)));
@@ -3024,7 +3041,11 @@ void HqlCppTranslator::doBuildFunctionReturn(BuildCtx & ctx, ITypeInfo * type, I
     }
 
     if (returnByReference)
+    {
         buildExprAssign(ctx, target, value);
+        if (returnValue)
+            buildReturn(ctx, returnValue);
+    }
     else
         buildReturn(ctx, value, type);
 }
@@ -7310,7 +7331,6 @@ void HqlCppTranslator::ensureSerialized(const CHqlBoundTarget & variable, BuildC
                 buildFunctionCall(serializectx, serializeRowId, serializeArgs);
 
 
-                deserializeArgs.append(*createRowAllocator(deserializectx, record));
                 deserializeArgs.append(*createSerializer(deserializectx, record, serializeForm, deserializerAtom));
                 deserializeArgs.append(*createVariable(inBufferName, makeBoolType()));
                 Owned<ITypeInfo> resultType = makeReferenceModifier(makeAttributeModifier(makeRowType(record->getType()), getLinkCountedAttr()));

ecl/hqlcpp/hqlwcpp.cpp

@@ -1024,8 +1024,17 @@ void HqlCppWriter::generateFunctionReturnType(StringBuffer & params, ITypeInfo *
             break;
         }
     case type_row:
-        out.append("void");
-        params.append("byte * __result");
+        if (hasLinkCountedModifier(retType))
+        {
+            out.append("byte *");
+            if (hasNonNullRecord(retType) && getBoolAttribute(attrs, allocatorAtom, true))
+                params.append("IEngineRowAllocator * _resultAllocator");
+        }
+        else
+        {
+            out.append("void");
+            params.append("byte * __result");
+        }
         break;
     default:
         generateType(retType, NULL);

plugins/Rembed/Rembed.cpp

@@ -370,6 +370,14 @@ public:
     {
         UNIMPLEMENTED;
     }
+    virtual byte * getRowResult(IEngineRowAllocator * _resultAllocator)
+    {
+        UNIMPLEMENTED;
+    }
+    virtual size32_t getTransformResult(ARowBuilder & builder)
+    {
+        UNIMPLEMENTED;
+    }
 
     virtual void bindBooleanParam(const char *name, bool val)
     {

plugins/javaembed/javaembed.cpp

@@ -727,6 +727,14 @@ public:
     {
         UNIMPLEMENTED;
     }
+    virtual byte * getRowResult(IEngineRowAllocator * _resultAllocator)
+    {
+        UNIMPLEMENTED;
+    }
+    virtual size32_t getTransformResult(ARowBuilder & builder)
+    {
+        UNIMPLEMENTED;
+    }
     virtual void bindBooleanParam(const char *name, bool val)
     {
         if (*argsig != 'B')

plugins/pyembed/pyembed.cpp

@@ -862,7 +862,14 @@ public:
     {
         return new PythonRowStream(sharedCtx, result, _resultAllocator);
     }
-
+    virtual byte * getRowResult(IEngineRowAllocator * _resultAllocator)
+    {
+        UNIMPLEMENTED;
+    }
+    virtual size32_t getTransformResult(ARowBuilder & builder)
+    {
+        UNIMPLEMENTED;
+    }
     virtual void bindBooleanParam(const char *name, bool val)
     {
         addArg(name, PyBool_FromLong(val ? 1 : 0));

plugins/v8embed/v8embed.cpp

@@ -445,6 +445,14 @@ public:
 //        resultAllocator.set(_resultAllocator);
 //        return LINK(this);
     }
+    virtual byte * getRowResult(IEngineRowAllocator * _resultAllocator)
+    {
+        UNIMPLEMENTED;
+    }
+    virtual size32_t getTransformResult(ARowBuilder & builder)
+    {
+        UNIMPLEMENTED;
+    }
     virtual const void *nextRow()
     {
 //        assertex(resultAllocator);

rtl/eclrtl/eclrtl.hpp

@@ -769,7 +769,7 @@ ECLRTL_API bool rtlGPF();
 //-----------------------------------------------------------------------------
 interface IRowStream;
 struct RtlTypeInfo;
-
+class ARowBuilder;
 interface IEmbedFunctionContext : extends IInterface
 {
     virtual void bindBooleanParam(const char *name, bool val) = 0;
@@ -799,7 +799,8 @@ interface IEmbedFunctionContext : extends IInterface
     virtual void callFunction() = 0;
 
     virtual IRowStream *getDatasetResult(IEngineRowAllocator * _resultAllocator) = 0;
-
+    virtual byte * getRowResult(IEngineRowAllocator * _resultAllocator) = 0;
+    virtual size32_t getTransformResult(ARowBuilder & builder) = 0;
 };
 
 interface IEmbedContext : extends IInterface

testing/ecl/streame2.ecl

@@ -54,7 +54,21 @@ dataset(namerec) testMissingTuple2(unsigned lim) := EMBED(Python)
   return [ ('1'), ('2'), ('3') ]
 ENDEMBED;
 
+// Test returning a row
+childrec testRowReturn(unsigned lim) := EMBED(Python)
+  return ("Hello", lim)
+ENDEMBED;
+
+// Test defining a transform
+transform(childrec) testTransform(unsigned lim) := EMBED(Python)
+  return ("Hello", lim)
+ENDEMBED;
+
+
 //output (testGenerator(10));
-output (testNamedTuple(10));
+//output (testNamedTuple(10));
 //output (testMissingTuple1(10));
-//output (testMissingTuple2(10));
+//output (testMissingTuple2(10));
+
+output(testRowReturn(10));
+//output(row(testTransform(10)));