Merge branch 'candidate-7.2.x'

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

common/deftype/deftype.cpp

@@ -3327,6 +3327,13 @@ ITypeInfo * replaceChildType(ITypeInfo * type, ITypeInfo * newChild)
     case type_rule:
         newType.setown(makeRuleType(LINK(newChild)));
         break;
+    case type_function:
+    {
+        IFunctionTypeExtra * extra = dynamic_cast<IFunctionTypeExtra *>(type);
+        assertex(extra);
+        newType.setown(makeFunctionType(LINK(newChild), LINK(extra->queryParameters()), LINK(extra->queryDefaults()), LINK(extra->queryAttributes())));
+        break;
+    }
     default:
         throwUnexpected();
     }

common/workunit/workunit.cpp

@@ -3760,6 +3760,8 @@ public:
             { return c->getResultByName(name); }
     virtual IConstWUResult * getResultBySequence(unsigned seq) const
             { return c->getResultBySequence(seq); }
+    virtual IConstWUResult * getQueryResultByName(const char * name) const
+            { return c->getQueryResultByName(name); }
     virtual unsigned getResultLimit() const
             { return c->getResultLimit(); }
     virtual IConstWUResultIterator & getResults() const
@@ -4296,6 +4298,8 @@ extern WORKUNIT_API bool isSpecialResultSequence(unsigned sequence)
         return true;
     default:
         assertex(sequence <= INT_MAX);
+        if ((int) sequence >= LibraryBaseSequence)
+            return true;
         return false;
     }
 }
@@ -8938,6 +8942,27 @@ IConstWUResult* CLocalWorkUnit::getResultByName(const char *qname) const
     return NULL;
 }
 
+IConstWUResult* CLocalWorkUnit::getQueryResultByName(const char *qname) const
+{
+    CriticalBlock block(crit);
+    loadResults();
+    ForEachItemIn(idx, results)
+    {
+        IConstWUResult &cur = results.item(idx);
+        if (!isSpecialResultSequence(cur.getResultSequence()))
+        {
+            SCMStringBuffer name;
+            cur.getResultName(name);
+            if (stricmp(name.str(), qname)==0)
+            {
+                cur.Link();
+                return &cur;
+            }
+        }
+    }
+    return NULL;
+}
+
 IConstWUResult* CLocalWorkUnit::getResultBySequence(unsigned seq) const
 {
     CriticalBlock block(crit);

common/workunit/workunit.hpp

@@ -238,6 +238,7 @@ interface IConstWUGraphMetaIterator : extends IScmIterator
 };
 
 
+constexpr int LibraryBaseSequence = 1000000000;
 //! IWUResult
 enum
 {
@@ -1216,6 +1217,8 @@ interface IConstWorkUnit : extends IConstWorkUnitInfo
     virtual bool getRescheduleFlag() const = 0;
     virtual IConstWUResult * getResultByName(const char * name) const = 0;
     virtual IConstWUResult * getResultBySequence(unsigned seq) const = 0;
+    // Like getResultByName, but ignores "special" results or results from libraries
+    virtual IConstWUResult * getQueryResultByName(const char * name) const = 0;
     virtual unsigned getResultLimit() const = 0;
     virtual IConstWUResultIterator & getResults() const = 0;
     virtual IStringVal & getScope(IStringVal & str) const = 0;

common/workunit/workunit.ipp

@@ -290,6 +290,7 @@ public:
     virtual bool getRescheduleFlag() const;
     virtual IConstWUResult * getResultByName(const char * name) const;
     virtual IConstWUResult * getResultBySequence(unsigned seq) const;
+    virtual IConstWUResult * getQueryResultByName(const char *qname) const;
     virtual unsigned getResultLimit() const;
     virtual IConstWUResultIterator & getResults() const;
     virtual IStringVal & getScope(IStringVal & str) const;

ecl/hql/hqlerrors.hpp

@@ -511,6 +511,9 @@
 #define HQLERR_CacheMissingEntry                3153
 #define HQLERR_PotentialAmbiguity               3154
 #define HQLERR_CannotDefineFunctionFunction     3155
+#define HQLERR_NoScalarOutputInLibrary          3156
+#define HQLERR_OnlyExtendOutputInLibrary        3157
+#define HQLERR_UnnamedOutputInLibrary           3158
 
 #define HQLERR_DedupFieldNotFound_Text          "Field removed from dedup could not be found"
 #define HQLERR_CycleWithModuleDefinition_Text   "Module definition contains an illegal cycle/recursive definition %s"
@@ -560,6 +563,9 @@
 #define HQLERR_CacheMissingEntry_Text           "Cannot process cache entry '%s'"
 #define HQLERR_PotentialAmbiguity_Text          "INTERNAL: Mapping introduces potential ambiguity into expression - please report issue"
 #define HQLERR_CannotDefineFunctionFunction_Text "Cannot define a function that returns a function"
+#define HQLERR_NoScalarOutputInLibrary_Text     "Scalar outputs are not supported inside libraries"
+#define HQLERR_OnlyExtendOutputInLibrary_Text   "OUTPUT within a library must use EXTEND"
+#define HQLERR_UnnamedOutputInLibrary_Text      "OUTPUT within a library must be NAMED"
 
 /* parser error */
 #define ERR_PARSER_CANNOTRECOVER    3005  /* The parser can not recover from previous error(s) */

ecl/hqlcpp/hqlcpp.cpp

@@ -6118,7 +6118,8 @@ void HqlCppTranslator::doBuildCall(BuildCtx & ctx, const CHqlBoundTarget * tgt,
             }
             const CHqlBoundTarget * curTarget;
             if (tgt && !tgt->isFixedSize() && 
-                (hasLinkCountedModifier(targetType) == hasLinkCountedModifier(retType)))
+                (hasLinkCountedModifier(targetType) == hasLinkCountedModifier(retType)) &&
+                !hasStreamedModifier(targetType))
             {
                 doneAssign = true;
                 curTarget = tgt;

ecl/hqlcpp/hqlcpp.ipp

@@ -2015,6 +2015,7 @@ public:
     inline StringBuffer & getUniqueId(StringBuffer & target) { return appendUniqueId(target, getUniqueId()); }
     inline unsigned curGraphSequence() const { return activeGraph ? graphSeqNumber : 0; }
     UniqueSequenceCounter & querySpillSequence() { return spillSequence; }
+    unsigned nextLibrarySequence() { return librarySequence++; }
 
 public:
     void traceExpression(const char * title, IHqlExpression * expr, unsigned level=500);
@@ -2049,6 +2050,7 @@ protected:
     ClusterType         targetClusterType;
     bool contextAvailable;
     unsigned maxSequence;
+    unsigned librarySequence = LibraryBaseSequence;
     unsigned            startCursorSet;
     bool                requireTable;
     BuildCtx *          activeGraphCtx;

ecl/hqlcpp/hqlcppds.cpp

@@ -2760,7 +2760,7 @@ void HqlCppTranslator::buildDatasetAssign(BuildCtx & ctx, const CHqlBoundTarget
     case no_translated:
         {
             bool sourceOutOfLine = isArrayRowset(exprType);
-            if (sourceOutOfLine != targetOutOfLine && !hasStreamedModifier(exprType))
+            if (sourceOutOfLine != targetOutOfLine && !hasStreamedModifier(exprType) && !hasStreamedModifier(to))
             {
                 IAtom * serializeFormat = internalAtom; // The format of serialized expressions in memory must match the internal serialization format
                 OwnedITypeInfo serializedSourceType = getSerializedForm(exprType, serializeFormat);
@@ -2810,7 +2810,7 @@ void HqlCppTranslator::buildDatasetAssign(BuildCtx & ctx, const CHqlBoundTarget
                 IIdAtom * func = NULL;
                 if (!isArrayRowset(to))
                 {
-                    if (!isArrayRowset(exprType))
+                    if (!isArrayRowset(exprType) && !hasStreamedModifier(to))
                         func = dataset2DatasetXId;
                 }
                 else if (hasLinkCountedModifier(to))
@@ -2847,9 +2847,17 @@ void HqlCppTranslator::buildDatasetAssign(BuildCtx & ctx, const CHqlBoundTarget
 
                 if (func)
                 {
+                    OwnedHqlExpr function = needFunction(func);
+                    ITypeInfo * funcType = function->queryType();
+                    ITypeInfo * funcDsType = funcType->queryChildType();
+                    ITypeInfo * funcRowType = funcDsType->queryChildType();
+                    ITypeInfo * toRecordType = to->queryChildType()->queryChildType();
+
+                    Owned<ITypeInfo> newRowType = replaceChildType(funcRowType, toRecordType);
+                    Owned<ITypeInfo> newType = replaceChildType(funcDsType, newRowType);
                     HqlExprArray args;
                     args.append(*LINK(expr));
-                    OwnedHqlExpr call = bindFunctionCall(func, args);
+                    OwnedHqlExpr call = bindFunctionCall(func, args, newType);
                     buildExprAssign(ctx, target, call);
                     return;
                 }

ecl/hqlcpp/hqlhtcpp.cpp

@@ -8983,7 +8983,8 @@ ABoundActivity * HqlCppTranslator::doBuildActivityEmbed(BuildCtx & ctx, IHqlExpr
     if (expr->isDataset())
     {
         MemberFunction func(*this, instance->startctx, "virtual IRowStream * createOutput(IThorActivityContext * activityContext) override");
-        buildReturn(func.ctx, newCall);
+        OwnedITypeInfo streamedType = setStreamedAttr(newCall->queryType(), true);
+        buildReturn(func.ctx, newCall, streamedType);
     }
     else
     {

ecl/hqlcpp/hqlttcpp.cpp

@@ -1330,6 +1330,14 @@ SequenceNumberAllocator::SequenceNumberAllocator(HqlCppTranslator & _translator)
     sequence = 0;
 }
 
+unsigned SequenceNumberAllocator::getNextSequence()
+{
+    if (translator.insideLibrary())
+        return translator.nextLibrarySequence();    // library sequence numbers must be unique for multiple embedded libraries
+    else
+        return sequence++;
+}
+
 void SequenceNumberAllocator::nextSequence(HqlExprArray & args, IHqlExpression * name, IAtom * overwriteAction, IHqlExpression * value, bool needAttr, bool * duplicate)
 {
     IHqlExpression * seq = NULL;
@@ -1380,13 +1388,13 @@ void SequenceNumberAllocator::nextSequence(HqlExprArray & args, IHqlExpression *
 
         if (!seq)
         {
-            seq = createConstant(signedType->castFrom(true, (__int64)sequence++));
+            seq = createConstant(signedType->castFrom(true, (__int64)getNextSequence()));
             OwnedHqlExpr saveValue = overwriteAction ? createAttribute(overwriteAction, LINK(seq), LINK(value)) : LINK(seq);
             namedMap.setValue(name, saveValue);
         }
     }
     else
-        seq = createConstant(signedType->castFrom(true, (__int64)sequence++));
+        seq = createConstant(signedType->castFrom(true, (__int64)getNextSequence()));
 
     if (needAttr)
         args.append(*createAttribute(sequenceAtom, seq));
@@ -1519,8 +1527,16 @@ IHqlExpression * SequenceNumberAllocator::attachSequenceNumber(IHqlExpression *
 {
     switch (expr->getOperator())
     {
-    case no_buildindex:
     case no_output:
+        if (translator.insideLibrary())
+        {
+            if (!expr->hasAttribute(extendAtom))
+                throwError(HQLERR_OnlyExtendOutputInLibrary);
+            if (!expr->hasAttribute(namedAtom))
+                throwError(HQLERR_UnnamedOutputInLibrary);
+        }
+        //fall through
+    case no_buildindex:
     case no_apply:
     case no_distribution:
     case no_keydiff:
@@ -1537,6 +1553,9 @@ IHqlExpression * SequenceNumberAllocator::attachSequenceNumber(IHqlExpression *
         break;
     case no_outputscalar:
         {
+            if (translator.insideLibrary())
+                throwError(HQLERR_NoScalarOutputInLibrary);
+
             IHqlExpression * name = queryResultName(expr);
             queryExtra(expr)->setGetsSequence();
             HqlExprArray args;

ecl/hqlcpp/hqlttcpp.ipp

@@ -170,6 +170,7 @@ protected:
     void nextSequence(HqlExprArray & args, IHqlExpression * name, IAtom * overwriteAction, IHqlExpression * value, bool needAttr, bool * duplicate);
     virtual IHqlExpression * doTransformRootExpr(IHqlExpression * expr);
     IHqlExpression * attachSequenceNumber(IHqlExpression * expr);
+    unsigned getNextSequence();
 
 protected:
     HqlCppTranslator & translator; // should really be an error handler - could do with refactoring.

ecl/regress/aaalibrary6a_err.ecl

@@ -0,0 +1,45 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//nohthor
+//nothor
+//nothorlcr
+//publish
+
+#option ('targetService', 'aaaLibrary5');
+#option ('createServiceAlias', true);
+
+namesRecord :=
+            RECORD
+string20        surname;
+string10        forename;
+integer2        age := 25;
+            END;
+
+FilterDatasetInterface(dataset(namesRecord) ds, string search) := interface
+    export dataset(namesRecord) matches;
+    export dataset(namesRecord) others;
+end;
+
+
+filterDatasetLibrary(dataset(namesRecord) ds, string search) := module,library(FilterDatasetInterface)
+    shared f := ds;
+    export matches := when(if (count(f(surname = search)) > 0, f), output('Search for ' + search), success);
+    export others := if (count(f(surname = search)) <= 0, f);
+end;
+
+build(filterDatasetLibrary);

ecl/regress/aaalibrary6b_err.ecl

@@ -0,0 +1,46 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//nohthor
+//nothor
+//nothorlcr
+//publish
+
+#option ('targetService', 'aaaLibrary5');
+#option ('createServiceAlias', true);
+
+namesRecord :=
+            RECORD
+string20        surname;
+string10        forename;
+integer2        age := 25;
+            END;
+
+FilterDatasetInterface(dataset(namesRecord) ds, string search) := interface
+    export dataset(namesRecord) matches;
+    export dataset(namesRecord) others;
+end;
+
+
+filterDatasetLibrary(dataset(namesRecord) ds, string search) := module,library(FilterDatasetInterface)
+    shared f := ds;
+    msg := DATASET([transform({string txt}, SELF.txt := 'Search for ' + search)]);
+    export matches := when(if (count(f(surname = search)) > 0, f), output(msg,named('logging')), success);
+    export others := if (count(f(surname = search)) <= 0, f);
+end;
+
+build(filterDatasetLibrary);

ecl/regress/aaalibrary6c_err.ecl

@@ -0,0 +1,46 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//nohthor
+//nothor
+//nothorlcr
+//publish
+
+#option ('targetService', 'aaaLibrary5');
+#option ('createServiceAlias', true);
+
+namesRecord :=
+            RECORD
+string20        surname;
+string10        forename;
+integer2        age := 25;
+            END;
+
+FilterDatasetInterface(dataset(namesRecord) ds, string search) := interface
+    export dataset(namesRecord) matches;
+    export dataset(namesRecord) others;
+end;
+
+
+filterDatasetLibrary(dataset(namesRecord) ds, string search) := module,library(FilterDatasetInterface)
+    shared f := ds;
+    msg := DATASET([transform({string txt}, SELF.txt := 'Search for ' + search)]);
+    export matches := when(if (count(f(surname = search)) > 0, f), output(msg,EXTEND), success);
+    export others := if (count(f(surname = search)) <= 0, f);
+end;
+
+build(filterDatasetLibrary);

esp/src/eclwatch/DFUQueryWidget.js

@@ -398,7 +398,7 @@ define([
                     request: this.updatedFilter
                 }).then(function (response) {
                     if (lang.exists("DFUQueryResponse", response)) {
-                        if (response.DFUQueryResponse.Warning) {
+                        if (response.DFUQueryResponse.Warning && dojo.byId(context.id).offsetParent !== null) {
                             context.filter.open();
                             context.filter.setFilterMessage(context.i18n.FilesWarning);
                         } else {
@@ -441,7 +441,16 @@ define([
                 });
 
                 this.initWorkunitsGrid();
-                this.checkIfWarning();
+
+                if (!params.searchResults) {
+                    this.checkIfWarning();
+                }
+
+                ESPUtil.MonitorVisibility(this.workunitsTab, function (visibility) {
+                    if (visibility) {
+                        context.checkIfWarning();
+                    }
+                });
 
                 this.filter.on("clear", function (evt) {
                     context.refreshHRef();

roxie/ccd/ccdcontext.cpp

@@ -3074,6 +3074,11 @@ public:
     {
         return this;
     }
+    virtual const IQueryFactory *queryQueryFactory() const override
+    {
+        return factory;
+    }
+
 
     virtual IGlobalCodeContext *queryGlobalCodeContext()
     {

roxie/ccd/ccdcontext.hpp

@@ -87,6 +87,7 @@ interface IRoxieServerContext : extends IInterface
 
     virtual unsigned getXmlFlags() const = 0;
     virtual IConstWorkUnit *queryWorkUnit() const = 0;
+    virtual const IQueryFactory *queryQueryFactory() const = 0;
     virtual bool outputResultsToSocket() const = 0;
 
     virtual IRoxieDaliHelper *checkDaliConnection() = 0;

roxie/ccd/ccdserver.cpp

@@ -6412,7 +6412,7 @@ public:
 
     virtual unsigned __int64 queryTotalCycles() const
     {
-        return input->queryTotalCycles();
+        return input ? input->queryTotalCycles() : 0;
     }
 
     virtual bool gatherConjunctions(ISteppedConjunctionCollector & collector)
@@ -21308,34 +21308,50 @@ class CRoxieServerWorkUnitWriteActivity : public CRoxieServerInternalSinkActivit
     bool isReread;
     bool grouped;
     IRoxieServerContext *serverContext;
+    int sequence;
 
 public:
     CRoxieServerWorkUnitWriteActivity(IRoxieSlaveContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager, bool _isReread, unsigned _numOutputs)
         : CRoxieServerInternalSinkActivity(_ctx, _factory, _probeManager, _numOutputs), helper((IHThorWorkUnitWriteArg &)basehelper), isReread(_isReread)
     {
         grouped = (helper.getFlags() & POFgrouped) != 0;
-        serverContext = NULL;
-    }
-
-    virtual void onCreate(IHThorArg *_colocalParent)
-    {
-        CRoxieServerInternalSinkActivity::onCreate(_colocalParent);
         serverContext = ctx->queryServerContext();
         if (!serverContext)
         {
-            throw MakeStringException(ROXIE_PIPE_ERROR, "Pipe output activity cannot be executed in slave context");
+            throw MakeStringException(ROXIE_PIPE_ERROR, "Workunit output activity cannot be executed in slave context");
+        }
+        sequence = helper.getSequence();
+        if (sequence >= LibraryBaseSequence)
+        {
+            IConstWorkUnit *workunit = serverContext->queryQueryFactory()->queryWorkUnit();
+            assertex(workunit);
+            const char *storedName = helper.queryName();
+            assertex(storedName);
+            Owned<IConstWUResult> queryRes =  workunit->getQueryResultByName(storedName);
+            if (!queryRes)
+                throw makeStringExceptionV(0, "Library cannot write to result %s that does not exist in calling query", storedName);
+            Owned<IConstWUResult> libraryRes =  factory->queryQueryFactory().queryWorkUnit()->getResultBySequence(sequence);
+            if (libraryRes) // Should always be present, but rather than assert, just ignore if not
+            {
+                SCMStringBuffer queryFormat;
+                SCMStringBuffer libraryFormat;
+                queryRes->getResultEclSchema(queryFormat);
+                libraryRes->getResultEclSchema(libraryFormat);
+                if (!streq(queryFormat.str(), libraryFormat.str()))
+                {
+                    DBGLOG("Query format: %s", queryFormat.str());
+                    DBGLOG("Library format: %s", libraryFormat.str());
+                    throw makeStringExceptionV(0, "Library cannot write to result %s: result type in query does not match", storedName);
+                }
+            }
+            sequence = queryRes->getResultSequence();
         }
     }
 
     virtual bool needsAllocator() const { return true; }
 
-    virtual void onExecute() 
+    virtual void onExecute()
     {
-        int sequence = helper.getSequence();
-        const char *storedName = helper.queryName();
-        if (!storedName)
-            storedName = "Dataset";
-
         MemoryBuffer result;
         bool saveInContext = (int) sequence < 0 || isReread;
         if (!meta.queryOriginal()) // this is a bit of a hack - don't know why no meta on an output....
@@ -21463,6 +21479,9 @@ public:
         }
         if (xmlwriter)
             xmlwriter->outputEndArray(DEFAULTXMLROWTAG);
+        const char *storedName = helper.queryName();
+        if (!storedName)
+            storedName = "Dataset";
         if (saveInContext)
             serverContext->appendResultDeserialized(storedName, sequence, builder.getcount(), builder.linkrows(), (helper.getFlags() & POFextend) != 0, LINK(meta.queryOriginal()));
         if (workunit)

testing/regress/ecl/aaalibrary6.ecl

@@ -0,0 +1,46 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//nohthor
+//nothor
+//nothorlcr
+//publish
+
+#option ('targetService', 'aaaLibrary6');
+#option ('createServiceAlias', true);
+
+namesRecord :=
+            RECORD
+string20        surname;
+string10        forename;
+integer2        age := 25;
+            END;
+
+FilterDatasetInterface(dataset(namesRecord) ds, string search) := interface
+    export dataset(namesRecord) matches;
+    export dataset(namesRecord) others;
+end;
+
+
+filterDatasetLibrary(dataset(namesRecord) ds, string search) := module,library(FilterDatasetInterface)
+    shared f := ds;
+    msg := DATASET([transform({string txt}, SELF.txt := 'Search for ' + search)]);
+    export matches := when(if (count(f(surname = search)) > 0, f), output(msg,named('logging'),EXTEND), success);
+    export others := if (count(f(surname = search)) <= 0, f);
+end;
+
+build(filterDatasetLibrary);

testing/regress/ecl/embedactivity5.ecl

@@ -0,0 +1,29 @@
+r := RECORD
+    UNSIGNED value;
+END;
+
+//This function returns the sum of the squares of the inputs from the dataset as a single non streamed row
+
+dataset(r) myDataset(streamed dataset(r) ds) := EMBED(C++ : activity)
+#include <stdio.h>
+#body
+    unsigned __int64 sum = 0;
+    for (;;)
+    {
+        const byte * next = (const byte *)ds->nextRow();
+        if (!next)
+            break;
+        unsigned __int64 value = *(const unsigned __int64 *)next;
+        rtlReleaseRow(next);
+        sum += value * value;
+    }
+
+    __lenResult = sizeof(unsigned __int64);
+    __result = rtlMalloc(__lenResult);
+    *(unsigned __int64 *)__result = sum;
+ENDEMBED;
+
+
+ds1 := DATASET([1,3,4,5,9,10,1,1], r, distributed);
+
+output(TABLE(myDataset(ds1), { unsigned value := SUM(GROUP, value); }));

testing/regress/ecl/embedactivity5b.ecl

@@ -0,0 +1,31 @@
+r := RECORD
+    UNSIGNED value;
+END;
+
+//This function returns the sum of the squares of the inputs from the dataset as a single non streamed row
+
+linkcounted dataset(r) myDataset(streamed dataset(r) ds) := EMBED(C++ : activity)
+#include <stdio.h>
+#body
+    unsigned __int64 sum = 0;
+    for (;;)
+    {
+        const byte * next = (const byte *)ds->nextRow();
+        if (!next)
+            break;
+        unsigned __int64 value = *(const unsigned __int64 *)next;
+        rtlReleaseRow(next);
+        sum += value * value;
+    }
+
+    byte * row = (byte *)_resultAllocator->createRow();
+    *(unsigned __int64 *)row = sum;
+    __countResult = 1;
+    __result = _resultAllocator->createRowset(1);
+    *__result = row;
+ENDEMBED;
+
+
+ds1 := DATASET([1,3,4,5,9,10,1,1], r, distributed);
+
+output(TABLE(myDataset(ds1), { unsigned value := SUM(GROUP, value); }));

testing/regress/ecl/key/embedactivity5.xml

@@ -0,0 +1,3 @@
+<Dataset name='Result 1'>
+ <Row><value>234</value></Row>
+</Dataset>

testing/regress/ecl/key/embedactivity5b.xml

@@ -0,0 +1,3 @@
+<Dataset name='Result 1'>
+ <Row><value>234</value></Row>
+</Dataset>

testing/regress/ecl/key/library6.xml

@@ -0,0 +1,30 @@
+<Dataset name='logging'>
+ <Row><txt>Logging</txt></Row>
+ <Row><txt>Search for Smith</txt></Row>
+ <Row><txt>Search for Halliday</txt></Row>
+ <Row><txt>Search for Halliday</txt></Row>
+</Dataset>
+<Dataset name='MatchSmith'>
+ <Row><surname>Halliday            </surname><forename>Gavin     </forename><age>31</age></Row>
+ <Row><surname>Halliday            </surname><forename>Liz       </forename><age>30</age></Row>
+ <Row><surname>Jones               </surname><forename>John      </forename><age>44</age></Row>
+ <Row><surname>Smith               </surname><forename>George    </forename><age>75</age></Row>
+ <Row><surname>Smith               </surname><forename>Baby      </forename><age>2</age></Row>
+</Dataset>
+<Dataset name='NotHalliday'>
+</Dataset>
+<Dataset name='NotTricky'>
+ <Row><surname>Halliday            </surname><forename>Gavin     </forename><age>31</age></Row>
+ <Row><surname>Halliday            </surname><forename>Liz       </forename><age>30</age></Row>
+ <Row><surname>Jones               </surname><forename>John      </forename><age>44</age></Row>
+ <Row><surname>Smith               </surname><forename>George    </forename><age>75</age></Row>
+ <Row><surname>Smith               </surname><forename>Baby      </forename><age>2</age></Row>
+</Dataset>
+<Dataset name='Result 5'>
+ <Row><matches><Row><surname>Halliday            </surname><forename>Gavin     </forename><age>31</age></Row><Row><surname>Halliday            </surname><forename>Liz       </forename><age>30</age></Row></matches></Row>
+ <Row><matches></matches></Row>
+</Dataset>
+<Dataset name='Result 6'>
+ <Row><others></others></Row>
+ <Row><others><Row><surname>Jones               </surname><forename>John      </forename><age>44</age></Row><Row><surname>Smith               </surname><forename>George    </forename><age>75</age></Row><Row><surname>Smith               </surname><forename>Baby      </forename><age>2</age></Row></others></Row>
+</Dataset>

testing/regress/ecl/library6.ecl

@@ -0,0 +1,69 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//nohthor
+//nothor
+//nothorlcr
+//The library is defined and built in aaalibrary6.ecl
+
+namesRecord :=
+            RECORD
+string20        surname;
+string10        forename;
+integer2        age := 25;
+            END;
+
+FilterDatasetInterface(dataset(namesRecord) ds, string search) := interface
+    export dataset(namesRecord) matches;
+    export dataset(namesRecord) others;
+end;
+
+
+filterDataset(dataset(namesRecord) ds, string search) := library('aaaLibrary6',FilterDatasetInterface(ds,search));
+
+namesTable := dataset([
+        {'Halliday','Gavin',31},
+        {'Halliday','Liz',30},
+        {'Jones','John', 44},
+        {'Smith','George',75},
+        {'Smith','Baby', 2}], namesRecord);
+
+filtered := filterDataset(namesTable, 'Smith');
+filtered2 := filterDataset(namesTable, 'Halliday');
+filtered3 := filterDataset(namesTable, 'Tricky');
+
+addressRecord := RECORD
+unsigned            id;
+dataset(namesRecord)    ds;
+    END;
+
+addressTable := dataset([
+    {1, [{'Halliday','Gavin',31},{'Halliday','Liz',30}]},
+    {2, [{'Jones','John', 44},
+        {'Smith','George',75},
+        {'Smith','Baby', 2}]}
+    ], addressRecord);
+
+logging := DATASET([{'Logging'}], {string txt});
+sequential(
+  output(logging,named('logging'));
+  output(filtered.matches,,named('MatchSmith'));
+  output(filtered2.others,,named('NotHalliday'));
+  output(filtered3.others,,named('NotTricky'));
+  output(addressTable, { dataset matches := filterDataset(ds, 'Halliday').matches });
+  output(addressTable, { dataset others := filterDataset(ds, 'Halliday').others });
+)