Merge branch 'candidate-7.4.x' into candidate-7.6.x

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

ecl/hql/hqlutil.cpp

@@ -9662,11 +9662,16 @@ static void createMappingAssigns(HqlExprArray & assigns, IHqlExpression * selfSe
 
 IHqlExpression * createMappingTransform(IHqlExpression * selfSelector, IHqlExpression * inSelector, bool replaceMissingWithDefault, IErrorReceiver &err, ECLlocation &location)
 {
+    return createMappingTransform(no_transform, selfSelector, inSelector, replaceMissingWithDefault, err, location);
+}
+
+IHqlExpression * createMappingTransform(node_operator transformOp, IHqlExpression * selfSelector, IHqlExpression * inSelector, bool replaceMissingWithDefault, IErrorReceiver &err, ECLlocation &location)
+{
     HqlExprArray assigns;
     IHqlExpression * selfRecord = selfSelector->queryRecord();
     IHqlExpression * inRecord = inSelector->queryRecord();
     createMappingAssigns(assigns, selfSelector, inSelector, inRecord->querySimpleScope(), selfRecord, replaceMissingWithDefault, err, location);
-    return createValue(no_transform, makeTransformType(selfRecord->getType()), assigns);
+    return createValue(transformOp, makeTransformType(selfRecord->getType()), assigns);
 
 }
 

ecl/hql/hqlutil.hpp

@@ -187,6 +187,7 @@ extern HQL_API IHqlExpression * createNullTransform(IHqlExpression * record);
  * @param location      Location to use when reporting errors
  */
 extern HQL_API IHqlExpression * createMappingTransform(IHqlExpression * selfSelector, IHqlExpression * inSelector, bool replaceMissingWithDefault, IErrorReceiver &err, ECLlocation &location);
+extern HQL_API IHqlExpression * createMappingTransform(node_operator transformOp, IHqlExpression * selfSelector, IHqlExpression * inSelector, bool replaceMissingWithDefault, IErrorReceiver &err, ECLlocation &location);
 
 extern HQL_API IHqlExpression * getFailCode(IHqlExpression * failExpr);
 extern HQL_API IHqlExpression * getFailMessage(IHqlExpression * failExpr, bool nullIfOmitted);

ecl/hqlcpp/hqlhtcpp.cpp

@@ -15619,7 +15619,9 @@ ABoundActivity * HqlCppTranslator::doBuildActivityProject(BuildCtx & ctx, IHqlEx
 
 ABoundActivity * HqlCppTranslator::doBuildActivitySerialize(BuildCtx & ctx, IHqlExpression * expr)
 {
-    IHqlExpression * dataset = expr->queryChild(0);
+    LinkedHqlExpr dataset = expr->queryChild(0);
+    if (dataset->isDatarow())
+        dataset.setown(createDatasetFromRow(LINK(dataset)));
     bool serialize = (expr->getOperator() == no_serialize);
 
     Owned<ABoundActivity> boundDataset = buildCachedActivity(ctx, dataset);

ecl/hqlcpp/hqlttcpp.cpp

@@ -9158,7 +9158,7 @@ IHqlExpression * DFSLayoutTransformer::createTransformed(IHqlExpression * expr)
                     OwnedHqlExpr left = createSelector(no_left, oldTransform, seq);
                     OwnedHqlExpr self = getSelf(dfsLayout);
                     MultiErrorReceiver dummy;
-                    OwnedHqlExpr transformECLtoDFS = createMappingTransform(self, left, true, dummy, where);  // transforms from ECL layout to DFS layout. Don't report missing here!
+                    OwnedHqlExpr transformECLtoDFS = createMappingTransform(no_newtransform, self, left, true, dummy, where);  // transforms from ECL layout to DFS layout. Don't report missing here!
 
                     NewProjectMapper2 mapper;
                     mapper.setMapping(oldTransform);
@@ -9906,6 +9906,17 @@ IHqlExpression * HqlLinkedChildRowTransformer::createTransformedBody(IHqlExpress
             return QuickHqlTransformer::createTransformedBody(expr);
         }
         break;
+    case no_libraryinput:
+        {
+            //no_libraryinput are passed in in their serialized form.  But this transformation adds LCR to all records
+            //of the same type regardless of where they are used.
+            //Ideally it would be context depenedent, but I have no idea how to implement that currently, so for the
+            //moment ensure the no_libraryinput uses the serialized form, but then transform to in memory form.
+            OwnedHqlExpr transformed = QuickHqlTransformer::createTransformedBody(expr);
+            if (expr != transformed)
+                return ensureDeserialized(expr, transformed->queryType(), diskAtom);
+            return transformed.getClear();
+        }
     case no_output:
         //would this be a good idea for output to file?  output to pipe?
         if (false)

esp/src/eclwatch/JSGraphWidget.js

@@ -398,7 +398,7 @@ define([
                 arrayUtil.forEach(this.graphData.edges, function (item, idx) {
                     var source = item.getSource();
                     var target = item.getTarget();
-                    if (!this.option("vhidespills") || !target.isSpill()) {
+                    if (source && target && (!this.option("vhidespills") || !target.isSpill())) {
                         var label = this.format(labelTpl, item);
                         var tooltip = this.format(tooltipTpl, item);
                         var numSlaves = parseInt(item.NumSlaves);
@@ -423,29 +423,31 @@ define([
                             label += "\n(" + nlsHPCC.Spill + ")";
                             weight = 25;
                             strokeDasharray = "5,5,10,5";
-                            while (source.isSpill()) {
+                            while (source && source.isSpill()) {
                                 var inputs = source.getInVertices();
                                 source = inputs[0];
                             }
                         }
-                        if (!merge || !item.__widget) {
-                            item.__widget = new hpccGraph.Edge()
-                                .sourceVertex(source.__widget)
-                                .targetVertex(target.__widget)
-                                .targetMarker("arrow")
-                                .weight(weight)
-                                .strokeDasharray(strokeDasharray)
-                                ;
-                            item.__widget.__hpcc_globalID = item.__hpcc_id;
+                        if (source) {
+                            if (!merge || !item.__widget) {
+                                item.__widget = new hpccGraph.Edge()
+                                    .sourceVertex(source.__widget)
+                                    .targetVertex(target.__widget)
+                                    .targetMarker("arrow")
+                                    .weight(weight)
+                                    .strokeDasharray(strokeDasharray)
+                                    ;
+                                item.__widget.__hpcc_globalID = item.__hpcc_id;
+                            }
+                            item.__widget.text(label);
+                            item.__widget.tooltip(tooltip);
+                            item.__widget.classed({
+                                started: started && !finished && !active,
+                                finished: finished && !active,
+                                active: active
+                            });
+                            edges.push(item.__widget);
                         }
-                        item.__widget.text(label);
-                        item.__widget.tooltip(tooltip);
-                        item.__widget.classed({
-                            started: started && !finished && !active,
-                            finished: finished && !active,
-                            active: active
-                        });
-                        edges.push(item.__widget);
                     }
                 }, this);
                 if (this.option("subgraph")) {

esp/src/src/ESPGraph.ts

@@ -194,7 +194,7 @@ var LocalisedXGMMLWriter = declare([], {
     },
 
     addSemiVisibleEdge: function (edge) {
-        if (!this.m_visibleEdges[edge.__hpcc_id]) {
+        if (edge && !this.m_visibleEdges[edge.__hpcc_id]) {
             this.m_visibleEdges[edge.__hpcc_id] = edge;
         }
     },
@@ -213,17 +213,25 @@ var LocalisedXGMMLWriter = declare([], {
             this.addSemiVisibleVertex(source);
             while (this.noSpills && source.isSpill()) {
                 var inEdges = source.getInEdges();
-                this.addSemiVisibleEdge(inEdges[0]);
-                source = inEdges[0].getSource();
-                this.addSemiVisibleVertex(source);
+                if (inEdges.length) {
+                    this.addSemiVisibleEdge(inEdges[0]);
+                    source = inEdges[0].getSource();
+                    this.addSemiVisibleVertex(source);
+                } else {
+                    break;
+                }
             }
             var target = edge.getTarget();
             this.addSemiVisibleVertex(target);
             while (this.noSpills && target.isSpill()) {
                 var outEdges = target.getOutEdges();
-                this.addSemiVisibleEdge(outEdges[0]);
-                target = outEdges[0].getTarget();
-                this.addSemiVisibleVertex(target);
+                if (outEdges.length) {
+                    this.addSemiVisibleEdge(outEdges[0]);
+                    target = outEdges[0].getTarget();
+                    this.addSemiVisibleVertex(target);
+                } else {
+                    break;
+                }
             }
         }
     },

roxie/ccd/ccdserver.cpp

@@ -13807,12 +13807,13 @@ public:
             rows->stop();
             rows.clear();
         }
-        CRoxieServerMultiInputBaseActivity::stop();
+        CRoxieServerMultiInputActivity::stop();
     }
 
     virtual void reset()
     {
         rows.clear();
+        CRoxieServerMultiInputActivity::reset();
     }
 
     virtual const void * nextRow()

testing/regress/ecl/aaalibrary7.ecl

@@ -0,0 +1,60 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2019 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//nohthor
+//nothor
+//publish
+
+#option ('targetService', 'aaaLibrary7');
+#option ('createServiceAlias', true);
+
+idRecord := RECORD
+    unsigned id;
+END;
+
+childRecord := RECORD
+    unsigned5 childId;
+    dataset(idRecord) ids;
+END;
+
+searchOptions := RECORD
+    unsigned3 mainId;
+    dataset(childRecord) children;
+END;
+
+
+namesRecord :=
+            RECORD
+string20        surname;
+string10        forename;
+integer2        age := 25;
+            END;
+
+FilterDatasetInterface(dataset(namesRecord) ds, searchOptions options) := interface
+    export dataset(namesRecord) matches;
+    export dataset(namesRecord) others;
+end;
+
+
+filterDatasetLibrary(dataset(namesRecord) ds, searchOptions options) := module,library(FilterDatasetInterface)
+    shared f := ds;
+    shared searchAge := options.children[1].ids[1].id;
+    export matches := f(age = searchAge);
+    export others := f(age != searchAge);
+end;
+
+build(filterDatasetLibrary);

testing/regress/ecl/key/library7.xml

@@ -0,0 +1,15 @@
+<Dataset name='logging'>
+ <Row><txt>Logging</txt></Row>
+</Dataset>
+<Dataset name='Liz'>
+ <Row><surname>Halliday            </surname><forename>Liz       </forename><age>30</age></Row>
+</Dataset>
+<Dataset name='Gavin'>
+ <Row><surname>Smith               </surname><forename>George    </forename><age>75</age></Row>
+</Dataset>
+<Dataset name='NotGavin'>
+ <Row><surname>Halliday            </surname><forename>Gavin     </forename><age>31</age></Row>
+ <Row><surname>Halliday            </surname><forename>Liz       </forename><age>30</age></Row>
+ <Row><surname>Jones               </surname><forename>John      </forename><age>44</age></Row>
+ <Row><surname>Smith               </surname><forename>Baby      </forename><age>2</age></Row>
+</Dataset>

testing/regress/ecl/library7.ecl

@@ -0,0 +1,71 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2019 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//nohthor
+//nothor
+//The library is defined and built in aaalibrary6.ecl
+
+idRecord := RECORD
+    unsigned id;
+END;
+
+childRecord := RECORD
+    unsigned5 childId;
+    dataset(idRecord) ids;
+END;
+
+searchOptions := RECORD
+    unsigned3 mainId;
+    dataset(childRecord) children;
+END;
+
+
+namesRecord :=
+            RECORD
+string20        surname;
+string10        forename;
+integer2        age := 25;
+            END;
+
+FilterDatasetInterface(dataset(namesRecord) ds, searchOptions options) := interface
+    export dataset(namesRecord) matches;
+    export dataset(namesRecord) others;
+end;
+
+
+filterDataset(dataset(namesRecord) ds, searchOptions options) := library('aaaLibrary7',FilterDatasetInterface(ds,options));
+
+
+mkOptions(unsigned age) := ROW({0, [{2, [{age}]}]}, searchOptions);
+
+namesTable := dataset([
+        {'Halliday','Gavin',31},
+        {'Halliday','Liz',30},
+        {'Jones','John', 44},
+        {'Smith','George',75},
+        {'Smith','Baby', 2}], namesRecord);
+
+filtered1 := filterDataset(namesTable, mkOptions(30));
+filtered2 := filterDataset(namesTable, mkOptions(75));
+
+logging := DATASET([{'Logging'}], {string txt});
+sequential(
+  output(logging,named('logging'));
+  output(filtered1.matches,,named('Liz'));
+  output(filtered2.matches,,named('Gavin'));
+  output(filtered2.others,,named('NotGavin'));
+)

thorlcr/activities/diskread/thdiskread.cpp

@@ -44,8 +44,31 @@ public:
             Owned<IException> e = MakeActivityWarning(&container, TE_GroupMismatch, "DFS and code generated group info. differs: DFS(%s), CodeGen(%s), using DFS info", isGrouped?"grouped":"ungrouped", codeGenGrouped?"grouped":"ungrouped");
             queryJobChannel().fireException(e);
         }
-        if (RecordTranslationMode::None == getTranslationMode(*this))
+        RecordTranslationMode translationMode = getTranslationMode(*this);
+        if (RecordTranslationMode::None != translationMode)
         {
+            /* Turn off translation if not "flat" or not RecordTranslationMode::AlwaysDisk
+             * NB: translation is turned off in slaves in this case too,
+             * turning off here, ensures format crc's are checked and any mismatch spotted
+             * with sensible error.
+             */
+
+            const char *kind = queryFileKind(file);
+            if (!strisame(kind, "flat") && (RecordTranslationMode::AlwaysDisk != translationMode))
+                translationMode = RecordTranslationMode::None;
+        }
+        if (RecordTranslationMode::None == translationMode)
+        {
+            unsigned dfsCrc;
+            if (file->getFormatCrc(dfsCrc) && helper->getDiskFormatCrc() != dfsCrc)
+            {
+                VStringBuffer fileStr("File: %s", file->queryLogicalName());
+                Owned<IThorException> e = MakeActivityException(this, TE_FormatCrcMismatch, "Automatic translation not available, and layout does not match published layout. %s", fileStr.str());
+                if (!queryContainer().queryJob().getWorkUnitValueInt("skipFileFormatCrcCheck", 0))
+                    throw e.getClear();
+                e->setAction(tea_warning);
+                fireException(e);
+            }
             if (recordSize->isFixedSize()) // fixed size
             {
                 if (0 != fileDesc->queryProperties().getPropInt("@recordSize"))