HPCC-21886 Support OUTPUT, EXTEND, NAMED within a LIBRARY

Note: Requires that an output with the same name exists within the calling
query, and that the type matches.

Uses special sequence numbers for outputs within libraries.

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

common/workunit/workunit.cpp

@@ -3760,6 +3760,8 @@ public:
             { return c->getResultByName(name); }
     virtual IConstWUResult * getResultBySequence(unsigned seq) const
             { return c->getResultBySequence(seq); }
+    virtual IConstWUResult * getQueryResultByName(const char * name) const
+            { return c->getQueryResultByName(name); }
     virtual unsigned getResultLimit() const
             { return c->getResultLimit(); }
     virtual IConstWUResultIterator & getResults() const
@@ -4296,6 +4298,8 @@ extern WORKUNIT_API bool isSpecialResultSequence(unsigned sequence)
         return true;
     default:
         assertex(sequence <= INT_MAX);
+        if ((int) sequence >= LibraryBaseSequence)
+            return true;
         return false;
     }
 }
@@ -8938,6 +8942,27 @@ IConstWUResult* CLocalWorkUnit::getResultByName(const char *qname) const
     return NULL;
 }
 
+IConstWUResult* CLocalWorkUnit::getQueryResultByName(const char *qname) const
+{
+    CriticalBlock block(crit);
+    loadResults();
+    ForEachItemIn(idx, results)
+    {
+        IConstWUResult &cur = results.item(idx);
+        if (!isSpecialResultSequence(cur.getResultSequence()))
+        {
+            SCMStringBuffer name;
+            cur.getResultName(name);
+            if (stricmp(name.str(), qname)==0)
+            {
+                cur.Link();
+                return &cur;
+            }
+        }
+    }
+    return NULL;
+}
+
 IConstWUResult* CLocalWorkUnit::getResultBySequence(unsigned seq) const
 {
     CriticalBlock block(crit);

common/workunit/workunit.hpp

@@ -238,6 +238,7 @@ interface IConstWUGraphMetaIterator : extends IScmIterator
 };
 
 
+constexpr int LibraryBaseSequence = 1000000000;
 //! IWUResult
 enum
 {
@@ -1216,6 +1217,8 @@ interface IConstWorkUnit : extends IConstWorkUnitInfo
     virtual bool getRescheduleFlag() const = 0;
     virtual IConstWUResult * getResultByName(const char * name) const = 0;
     virtual IConstWUResult * getResultBySequence(unsigned seq) const = 0;
+    // Like getResultByName, but ignores "special" results or results from libraries
+    virtual IConstWUResult * getQueryResultByName(const char * name) const = 0;
     virtual unsigned getResultLimit() const = 0;
     virtual IConstWUResultIterator & getResults() const = 0;
     virtual IStringVal & getScope(IStringVal & str) const = 0;

common/workunit/workunit.ipp

@@ -290,6 +290,7 @@ public:
     virtual bool getRescheduleFlag() const;
     virtual IConstWUResult * getResultByName(const char * name) const;
     virtual IConstWUResult * getResultBySequence(unsigned seq) const;
+    virtual IConstWUResult * getQueryResultByName(const char *qname) const;
     virtual unsigned getResultLimit() const;
     virtual IConstWUResultIterator & getResults() const;
     virtual IStringVal & getScope(IStringVal & str) const;

ecl/hql/hqlerrors.hpp

@@ -511,6 +511,9 @@
 #define HQLERR_CacheMissingEntry                3153
 #define HQLERR_PotentialAmbiguity               3154
 #define HQLERR_CannotDefineFunctionFunction     3155
+#define HQLERR_NoScalarOutputInLibrary          3156
+#define HQLERR_OnlyExtendOutputInLibrary        3157
+#define HQLERR_UnnamedOutputInLibrary           3158
 
 #define HQLERR_DedupFieldNotFound_Text          "Field removed from dedup could not be found"
 #define HQLERR_CycleWithModuleDefinition_Text   "Module definition contains an illegal cycle/recursive definition %s"
@@ -560,6 +563,9 @@
 #define HQLERR_CacheMissingEntry_Text           "Cannot process cache entry '%s'"
 #define HQLERR_PotentialAmbiguity_Text          "INTERNAL: Mapping introduces potential ambiguity into expression - please report issue"
 #define HQLERR_CannotDefineFunctionFunction_Text "Cannot define a function that returns a function"
+#define HQLERR_NoScalarOutputInLibrary_Text     "Scalar outputs are not supported inside libraries"
+#define HQLERR_OnlyExtendOutputInLibrary_Text   "OUTPUT within a library must use EXTEND"
+#define HQLERR_UnnamedOutputInLibrary_Text      "OUTPUT within a library must be NAMED"
 
 /* parser error */
 #define ERR_PARSER_CANNOTRECOVER    3005  /* The parser can not recover from previous error(s) */

ecl/hqlcpp/hqlcpp.ipp

@@ -2015,6 +2015,7 @@ public:
     inline StringBuffer & getUniqueId(StringBuffer & target) { return appendUniqueId(target, getUniqueId()); }
     inline unsigned curGraphSequence() const { return activeGraph ? graphSeqNumber : 0; }
     UniqueSequenceCounter & querySpillSequence() { return spillSequence; }
+    unsigned nextLibrarySequence() { return librarySequence++; }
 
 public:
     void traceExpression(const char * title, IHqlExpression * expr, unsigned level=500);
@@ -2049,6 +2050,7 @@ protected:
     ClusterType         targetClusterType;
     bool contextAvailable;
     unsigned maxSequence;
+    unsigned librarySequence = LibraryBaseSequence;
     unsigned            startCursorSet;
     bool                requireTable;
     BuildCtx *          activeGraphCtx;

ecl/hqlcpp/hqlttcpp.cpp

@@ -1330,6 +1330,14 @@ SequenceNumberAllocator::SequenceNumberAllocator(HqlCppTranslator & _translator)
     sequence = 0;
 }
 
+unsigned SequenceNumberAllocator::getNextSequence()
+{
+    if (translator.insideLibrary())
+        return translator.nextLibrarySequence();    // library sequence numbers must be unique for multiple embedded libraries
+    else
+        return sequence++;
+}
+
 void SequenceNumberAllocator::nextSequence(HqlExprArray & args, IHqlExpression * name, IAtom * overwriteAction, IHqlExpression * value, bool needAttr, bool * duplicate)
 {
     IHqlExpression * seq = NULL;
@@ -1380,13 +1388,13 @@ void SequenceNumberAllocator::nextSequence(HqlExprArray & args, IHqlExpression *
 
         if (!seq)
         {
-            seq = createConstant(signedType->castFrom(true, (__int64)sequence++));
+            seq = createConstant(signedType->castFrom(true, (__int64)getNextSequence()));
             OwnedHqlExpr saveValue = overwriteAction ? createAttribute(overwriteAction, LINK(seq), LINK(value)) : LINK(seq);
             namedMap.setValue(name, saveValue);
         }
     }
     else
-        seq = createConstant(signedType->castFrom(true, (__int64)sequence++));
+        seq = createConstant(signedType->castFrom(true, (__int64)getNextSequence()));
 
     if (needAttr)
         args.append(*createAttribute(sequenceAtom, seq));
@@ -1519,8 +1527,16 @@ IHqlExpression * SequenceNumberAllocator::attachSequenceNumber(IHqlExpression *
 {
     switch (expr->getOperator())
     {
-    case no_buildindex:
     case no_output:
+        if (translator.insideLibrary())
+        {
+            if (!expr->hasAttribute(extendAtom))
+                throwError(HQLERR_OnlyExtendOutputInLibrary);
+            if (!expr->hasAttribute(namedAtom))
+                throwError(HQLERR_UnnamedOutputInLibrary);
+        }
+        //fall through
+    case no_buildindex:
     case no_apply:
     case no_distribution:
     case no_keydiff:
@@ -1537,6 +1553,9 @@ IHqlExpression * SequenceNumberAllocator::attachSequenceNumber(IHqlExpression *
         break;
     case no_outputscalar:
         {
+            if (translator.insideLibrary())
+                throwError(HQLERR_NoScalarOutputInLibrary);
+
             IHqlExpression * name = queryResultName(expr);
             queryExtra(expr)->setGetsSequence();
             HqlExprArray args;

ecl/hqlcpp/hqlttcpp.ipp

@@ -170,6 +170,7 @@ protected:
     void nextSequence(HqlExprArray & args, IHqlExpression * name, IAtom * overwriteAction, IHqlExpression * value, bool needAttr, bool * duplicate);
     virtual IHqlExpression * doTransformRootExpr(IHqlExpression * expr);
     IHqlExpression * attachSequenceNumber(IHqlExpression * expr);
+    unsigned getNextSequence();
 
 protected:
     HqlCppTranslator & translator; // should really be an error handler - could do with refactoring.

ecl/regress/aaalibrary6a_err.ecl

@@ -0,0 +1,45 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//nohthor
+//nothor
+//nothorlcr
+//publish
+
+#option ('targetService', 'aaaLibrary5');
+#option ('createServiceAlias', true);
+
+namesRecord :=
+            RECORD
+string20        surname;
+string10        forename;
+integer2        age := 25;
+            END;
+
+FilterDatasetInterface(dataset(namesRecord) ds, string search) := interface
+    export dataset(namesRecord) matches;
+    export dataset(namesRecord) others;
+end;
+
+
+filterDatasetLibrary(dataset(namesRecord) ds, string search) := module,library(FilterDatasetInterface)
+    shared f := ds;
+    export matches := when(if (count(f(surname = search)) > 0, f), output('Search for ' + search), success);
+    export others := if (count(f(surname = search)) <= 0, f);
+end;
+
+build(filterDatasetLibrary);

ecl/regress/aaalibrary6b_err.ecl

@@ -0,0 +1,46 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//nohthor
+//nothor
+//nothorlcr
+//publish
+
+#option ('targetService', 'aaaLibrary5');
+#option ('createServiceAlias', true);
+
+namesRecord :=
+            RECORD
+string20        surname;
+string10        forename;
+integer2        age := 25;
+            END;
+
+FilterDatasetInterface(dataset(namesRecord) ds, string search) := interface
+    export dataset(namesRecord) matches;
+    export dataset(namesRecord) others;
+end;
+
+
+filterDatasetLibrary(dataset(namesRecord) ds, string search) := module,library(FilterDatasetInterface)
+    shared f := ds;
+    msg := DATASET([transform({string txt}, SELF.txt := 'Search for ' + search)]);
+    export matches := when(if (count(f(surname = search)) > 0, f), output(msg,named('logging')), success);
+    export others := if (count(f(surname = search)) <= 0, f);
+end;
+
+build(filterDatasetLibrary);

ecl/regress/aaalibrary6c_err.ecl

@@ -0,0 +1,46 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//nohthor
+//nothor
+//nothorlcr
+//publish
+
+#option ('targetService', 'aaaLibrary5');
+#option ('createServiceAlias', true);
+
+namesRecord :=
+            RECORD
+string20        surname;
+string10        forename;
+integer2        age := 25;
+            END;
+
+FilterDatasetInterface(dataset(namesRecord) ds, string search) := interface
+    export dataset(namesRecord) matches;
+    export dataset(namesRecord) others;
+end;
+
+
+filterDatasetLibrary(dataset(namesRecord) ds, string search) := module,library(FilterDatasetInterface)
+    shared f := ds;
+    msg := DATASET([transform({string txt}, SELF.txt := 'Search for ' + search)]);
+    export matches := when(if (count(f(surname = search)) > 0, f), output(msg,EXTEND), success);
+    export others := if (count(f(surname = search)) <= 0, f);
+end;
+
+build(filterDatasetLibrary);

roxie/ccd/ccdcontext.cpp

@@ -3074,6 +3074,11 @@ public:
     {
         return this;
     }
+    virtual const IQueryFactory *queryQueryFactory() const override
+    {
+        return factory;
+    }
+
 
     virtual IGlobalCodeContext *queryGlobalCodeContext()
     {

roxie/ccd/ccdcontext.hpp

@@ -87,6 +87,7 @@ interface IRoxieServerContext : extends IInterface
 
     virtual unsigned getXmlFlags() const = 0;
     virtual IConstWorkUnit *queryWorkUnit() const = 0;
+    virtual const IQueryFactory *queryQueryFactory() const = 0;
     virtual bool outputResultsToSocket() const = 0;
 
     virtual IRoxieDaliHelper *checkDaliConnection() = 0;

roxie/ccd/ccdserver.cpp

@@ -6412,7 +6412,7 @@ public:
 
     virtual unsigned __int64 queryTotalCycles() const
     {
-        return input->queryTotalCycles();
+        return input ? input->queryTotalCycles() : 0;
     }
 
     virtual bool gatherConjunctions(ISteppedConjunctionCollector & collector)
@@ -21308,34 +21308,50 @@ class CRoxieServerWorkUnitWriteActivity : public CRoxieServerInternalSinkActivit
     bool isReread;
     bool grouped;
     IRoxieServerContext *serverContext;
+    int sequence;
 
 public:
     CRoxieServerWorkUnitWriteActivity(IRoxieSlaveContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager, bool _isReread, unsigned _numOutputs)
         : CRoxieServerInternalSinkActivity(_ctx, _factory, _probeManager, _numOutputs), helper((IHThorWorkUnitWriteArg &)basehelper), isReread(_isReread)
     {
         grouped = (helper.getFlags() & POFgrouped) != 0;
-        serverContext = NULL;
-    }
-
-    virtual void onCreate(IHThorArg *_colocalParent)
-    {
-        CRoxieServerInternalSinkActivity::onCreate(_colocalParent);
         serverContext = ctx->queryServerContext();
         if (!serverContext)
         {
-            throw MakeStringException(ROXIE_PIPE_ERROR, "Pipe output activity cannot be executed in slave context");
+            throw MakeStringException(ROXIE_PIPE_ERROR, "Workunit output activity cannot be executed in slave context");
+        }
+        sequence = helper.getSequence();
+        if (sequence >= LibraryBaseSequence)
+        {
+            IConstWorkUnit *workunit = serverContext->queryQueryFactory()->queryWorkUnit();
+            assertex(workunit);
+            const char *storedName = helper.queryName();
+            assertex(storedName);
+            Owned<IConstWUResult> queryRes =  workunit->getQueryResultByName(storedName);
+            if (!queryRes)
+                throw makeStringExceptionV(0, "Library cannot write to result %s that does not exist in calling query", storedName);
+            Owned<IConstWUResult> libraryRes =  factory->queryQueryFactory().queryWorkUnit()->getResultBySequence(sequence);
+            if (libraryRes) // Should always be present, but rather than assert, just ignore if not
+            {
+                SCMStringBuffer queryFormat;
+                SCMStringBuffer libraryFormat;
+                queryRes->getResultEclSchema(queryFormat);
+                libraryRes->getResultEclSchema(libraryFormat);
+                if (!streq(queryFormat.str(), libraryFormat.str()))
+                {
+                    DBGLOG("Query format: %s", queryFormat.str());
+                    DBGLOG("Library format: %s", libraryFormat.str());
+                    throw makeStringExceptionV(0, "Library cannot write to result %s: result type in query does not match", storedName);
+                }
+            }
+            sequence = queryRes->getResultSequence();
         }
     }
 
     virtual bool needsAllocator() const { return true; }
 
-    virtual void onExecute() 
+    virtual void onExecute()
     {
-        int sequence = helper.getSequence();
-        const char *storedName = helper.queryName();
-        if (!storedName)
-            storedName = "Dataset";
-
         MemoryBuffer result;
         bool saveInContext = (int) sequence < 0 || isReread;
         if (!meta.queryOriginal()) // this is a bit of a hack - don't know why no meta on an output....
@@ -21463,6 +21479,9 @@ public:
         }
         if (xmlwriter)
             xmlwriter->outputEndArray(DEFAULTXMLROWTAG);
+        const char *storedName = helper.queryName();
+        if (!storedName)
+            storedName = "Dataset";
         if (saveInContext)
             serverContext->appendResultDeserialized(storedName, sequence, builder.getcount(), builder.linkrows(), (helper.getFlags() & POFextend) != 0, LINK(meta.queryOriginal()));
         if (workunit)

testing/regress/ecl/aaalibrary6.ecl

@@ -0,0 +1,46 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//nohthor
+//nothor
+//nothorlcr
+//publish
+
+#option ('targetService', 'aaaLibrary6');
+#option ('createServiceAlias', true);
+
+namesRecord :=
+            RECORD
+string20        surname;
+string10        forename;
+integer2        age := 25;
+            END;
+
+FilterDatasetInterface(dataset(namesRecord) ds, string search) := interface
+    export dataset(namesRecord) matches;
+    export dataset(namesRecord) others;
+end;
+
+
+filterDatasetLibrary(dataset(namesRecord) ds, string search) := module,library(FilterDatasetInterface)
+    shared f := ds;
+    msg := DATASET([transform({string txt}, SELF.txt := 'Search for ' + search)]);
+    export matches := when(if (count(f(surname = search)) > 0, f), output(msg,named('logging'),EXTEND), success);
+    export others := if (count(f(surname = search)) <= 0, f);
+end;
+
+build(filterDatasetLibrary);

testing/regress/ecl/key/library6.xml

@@ -0,0 +1,30 @@
+<Dataset name='logging'>
+ <Row><txt>Logging</txt></Row>
+ <Row><txt>Search for Smith</txt></Row>
+ <Row><txt>Search for Halliday</txt></Row>
+ <Row><txt>Search for Halliday</txt></Row>
+</Dataset>
+<Dataset name='MatchSmith'>
+ <Row><surname>Halliday            </surname><forename>Gavin     </forename><age>31</age></Row>
+ <Row><surname>Halliday            </surname><forename>Liz       </forename><age>30</age></Row>
+ <Row><surname>Jones               </surname><forename>John      </forename><age>44</age></Row>
+ <Row><surname>Smith               </surname><forename>George    </forename><age>75</age></Row>
+ <Row><surname>Smith               </surname><forename>Baby      </forename><age>2</age></Row>
+</Dataset>
+<Dataset name='NotHalliday'>
+</Dataset>
+<Dataset name='NotTricky'>
+ <Row><surname>Halliday            </surname><forename>Gavin     </forename><age>31</age></Row>
+ <Row><surname>Halliday            </surname><forename>Liz       </forename><age>30</age></Row>
+ <Row><surname>Jones               </surname><forename>John      </forename><age>44</age></Row>
+ <Row><surname>Smith               </surname><forename>George    </forename><age>75</age></Row>
+ <Row><surname>Smith               </surname><forename>Baby      </forename><age>2</age></Row>
+</Dataset>
+<Dataset name='Result 5'>
+ <Row><matches><Row><surname>Halliday            </surname><forename>Gavin     </forename><age>31</age></Row><Row><surname>Halliday            </surname><forename>Liz       </forename><age>30</age></Row></matches></Row>
+ <Row><matches></matches></Row>
+</Dataset>
+<Dataset name='Result 6'>
+ <Row><others></others></Row>
+ <Row><others><Row><surname>Jones               </surname><forename>John      </forename><age>44</age></Row><Row><surname>Smith               </surname><forename>George    </forename><age>75</age></Row><Row><surname>Smith               </surname><forename>Baby      </forename><age>2</age></Row></others></Row>
+</Dataset>

testing/regress/ecl/library6.ecl

@@ -0,0 +1,69 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//nohthor
+//nothor
+//nothorlcr
+//The library is defined and built in aaalibrary6.ecl
+
+namesRecord :=
+            RECORD
+string20        surname;
+string10        forename;
+integer2        age := 25;
+            END;
+
+FilterDatasetInterface(dataset(namesRecord) ds, string search) := interface
+    export dataset(namesRecord) matches;
+    export dataset(namesRecord) others;
+end;
+
+
+filterDataset(dataset(namesRecord) ds, string search) := library('aaaLibrary6',FilterDatasetInterface(ds,search));
+
+namesTable := dataset([
+        {'Halliday','Gavin',31},
+        {'Halliday','Liz',30},
+        {'Jones','John', 44},
+        {'Smith','George',75},
+        {'Smith','Baby', 2}], namesRecord);
+
+filtered := filterDataset(namesTable, 'Smith');
+filtered2 := filterDataset(namesTable, 'Halliday');
+filtered3 := filterDataset(namesTable, 'Tricky');
+
+addressRecord := RECORD
+unsigned            id;
+dataset(namesRecord)    ds;
+    END;
+
+addressTable := dataset([
+    {1, [{'Halliday','Gavin',31},{'Halliday','Liz',30}]},
+    {2, [{'Jones','John', 44},
+        {'Smith','George',75},
+        {'Smith','Baby', 2}]}
+    ], addressRecord);
+
+logging := DATASET([{'Logging'}], {string txt});
+sequential(
+  output(logging,named('logging'));
+  output(filtered.matches,,named('MatchSmith'));
+  output(filtered2.others,,named('NotHalliday'));
+  output(filtered3.others,,named('NotTricky'));
+  output(addressTable, { dataset matches := filterDataset(ds, 'Halliday').matches });
+  output(addressTable, { dataset others := filterDataset(ds, 'Halliday').others });
+)