Merge branch 'candidate-7.8.x' into candidate-7.10.x

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

roxie/ccd/ccdserver.cpp

@@ -16455,7 +16455,8 @@ public:
         outputUsed = new bool[numOutputs];
         for (unsigned i2 = 0; i2 < numOutputs; i2++)
         {
-            outputAdaptors[i2].setParent(this, helper.queryOutputMeta(i2), i2);
+            unsigned actualOutput = extra.outputs.item(i2);
+            outputAdaptors[i2].setParent(this, helper.queryOutputMeta(actualOutput), i2);
             outputUsed[i2] = false;
         }
         started = false;

rtl/eclrtl/rtldynfield.cpp

@@ -995,6 +995,7 @@ enum FieldMatchType {
     match_inifblock   = 0x400,   // matching to a field in an ifblock - may not be present
     match_deblob      = 0x1000,  // source needs fetching from a blob prior to translation
     match_dynamic     = 0x2000,  // source needs fetching from dynamic source (callback)
+    match_filepos     = 0x4000,  // type moving in or out of filepos field - cast required
 };
 
 StringBuffer &describeFlags(StringBuffer &out, FieldMatchType flags)
@@ -1016,6 +1017,7 @@ StringBuffer &describeFlags(StringBuffer &out, FieldMatchType flags)
     if (flags & match_virtual) out.append("|virtual");
     if (flags & match_deblob) out.append("|blob");
     if (flags & match_dynamic) out.append("|dynamic");
+    if (flags & match_filepos) out.append("|filepos");
     assertex(out.length() > origlen);
     return out.remove(origlen, 1);
 }
@@ -1261,6 +1263,7 @@ private:
                         offset += fillSize;
                         break;
                     }
+                    case match_filepos:
                     case match_typecast:
                         offset = translateScalar(builder, offset, field, *type, *sourceType, source);
                         break;
@@ -1581,12 +1584,15 @@ private:
     void createMatchInfo()
     {
         unsigned defaulted = 0;
+        bool destHasNested = destRecInfo.hasNested();
+        bool sourceHasNested = sourceRecInfo.hasNested();
         for (unsigned idx = 0; idx < destRecInfo.getNumFields(); idx++)
         {
             const RtlFieldInfo *field = destRecInfo.queryField(idx);
             const RtlTypeInfo *type = field->type;
             MatchInfo &info = matchInfo[idx];
-            info.matchIdx = sourceRecInfo.getFieldNum(destRecInfo.queryName(idx));
+            const char *name = destRecInfo.queryName(idx);
+            info.matchIdx = sourceRecInfo.getFieldNum(name);
             if (info.matchIdx == (unsigned) -1)
             {
                 const byte * initializer = (const byte *) field->initializer;
@@ -1602,6 +1608,27 @@ private:
                 if ((field->flags & RFTMispayloadfield) == 0)
                     matchFlags |= match_keychange;
                 defaulted++;
+                // If dest field is in a nested record, we need to check that there's no "non-record" field in source matching current nested record name
+                if (name)
+                {
+                    if (destHasNested)
+                    {
+                        const char *ldot = strrchr(name, '.');
+                        if (ldot)
+                        {
+                            StringBuffer recname(ldot-name, name);
+                            if (sourceRecInfo.getFieldNum(recname) != (unsigned) -1)
+                                info.matchType = match_fail;  // No translation from non-record to record
+                        }
+                    }
+                    if (sourceHasNested && sourceRecInfo.queryOriginalField(name))
+                    {
+                        // Similarly if dest field IS not a nested record, but there is a field in source which is.
+                        // Note that we already know there is no matching field called name in the exapanded version of source,
+                        // so any match we find must be a record
+                        info.matchType = match_fail;  // No translation from record to non-record
+                    }
+                }
             }
             else
             {
@@ -1741,6 +1768,9 @@ private:
                         }
                     }
                 }
+                else if ((type->getType()==type_filepos || sourceType->getType()==type_filepos) &&
+                         type->isUnsigned()==sourceType->isUnsigned())
+                    info.matchType = match_filepos;
                 else
                     info.matchType = match_typecast;
                 if (deblob)

rtl/eclrtl/rtlfield.cpp

@@ -3192,6 +3192,45 @@ extern bool ECLRTL_API hasTrailingFileposition(const RtlTypeInfo * type)
     return false;
 }
 
+//Does the field list contain a keyed int?  We are not interested in fields inside child datasets.
+//And keyed fields cannot exist within ifblocks.
+static bool containsKeyedSignedInt(const RtlFieldInfo * const * fields)
+{
+    if (!*fields)
+        return false;
+    while (*fields)
+    {
+        const RtlTypeInfo * type = (*fields)->type;
+        if (type->getType() == type_keyedint)
+        {
+            const RtlTypeInfo * baseType = type->queryChildType();
+            if (baseType->isSigned() && (baseType->getType() == type_int))
+                return true;
+        }
+        else if (type->getType() == type_record)
+        {
+            const RtlRecordTypeInfo * record = static_cast<const RtlRecordTypeInfo *>(type);
+            if (containsKeyedSignedInt(record->fields))
+                return true;
+        }
+        fields++;
+    }
+    return false;
+}
+
+extern bool ECLRTL_API containsKeyedSignedInt(const RtlTypeInfo * type)
+{
+    switch (type->getType())
+    {
+    case type_record:
+    {
+        const RtlRecordTypeInfo * record = static_cast<const RtlRecordTypeInfo *>(type);
+        return containsKeyedSignedInt(record->fields);
+    }
+    }
+    return false;
+}
+
 //-------------------------------------------------------------------------------------------------------------------
 
 size32_t RtlRecordTypeInfo::getMinSize() const

rtl/eclrtl/rtlfield.hpp

@@ -752,6 +752,8 @@ extern int ECLRTL_API compareFields(const RtlFieldInfo * const * cur, const byte
 extern unsigned ECLRTL_API hashFields(const RtlFieldInfo * const * cur, const byte *self, unsigned inhash, bool excludePayload = false);
 extern bool ECLRTL_API hasTrailingFileposition(const RtlFieldInfo * const * fields);
 extern bool ECLRTL_API hasTrailingFileposition(const RtlTypeInfo * type);
+extern bool ECLRTL_API containsKeyedSignedInt(const RtlTypeInfo * type);
+
 extern size32_t translateScalar(ARowBuilder &builder, size32_t offset, const RtlFieldInfo *field, const RtlTypeInfo &destType, const RtlTypeInfo &sourceType, const byte *source);
 
 #endif

rtl/eclrtl/rtlnktest.cpp

@@ -311,6 +311,10 @@ protected:
     {
         RtlStringTypeInfo str1(type_string, 1);
         RtlIntTypeInfo int2(type_int, 2);
+        RtlIntTypeInfo int8(type_int, 8);
+        RtlSwapIntTypeInfo swapint2(type_int, 2);
+        RtlSwapIntTypeInfo swapint8(type_int, 8);
+        RtlKeyedIntTypeInfo keyedint8(type_keyedint, 8, &swapint8);
         RtlStringTypeInfo strx(type_string|RFTMunknownsize, 0);
         RtlUtf8TypeInfo utf8(type_utf8|RFTMunknownsize, 0, nullptr);
         testSerialize(int2, "[123]");
@@ -323,6 +327,23 @@ protected:
         testSerialize(int2, "(123,234),(456,567)");
         testSerialize(int2, "(456,567),(123,234)", "(123,234),(456,567)");
 
+        testSerialize(swapint2, "[123]");
+        testSerialize(swapint2, "(123,234]");
+        testSerialize(swapint2, "[123,234)");
+        testSerialize(swapint2, "(123,234)");
+        testSerialize(swapint2, "(,234)");
+        testSerialize(swapint2, "(128,)");
+        testSerialize(swapint2, "(,)");
+        testSerialize(swapint2, "(123,234),(456,567)");
+        testSerialize(swapint2, "(456,567),(123,234)", "(123,234),(456,567)");
+
+        testSerialize(int8, "[9223372036854775807]");
+        testSerialize(int8, "[-9223372036854775808]");
+        testSerialize(swapint8, "[9223372036854775807]");
+        testSerialize(swapint8, "[-9223372036854775808]");
+        testSerialize(keyedint8, "[10]");
+        testSerialize(keyedint8, "[-10]");
+
         testSerialize(str1, "['A']");
         testSerialize(str1, "[',']");
         testSerialize(str1, "['\\'']");

rtl/eclrtl/rtlrecord.cpp

@@ -587,6 +587,23 @@ const RtlRecord *RtlRecord::queryNested(unsigned fieldId) const
     return nullptr;
 }
 
+bool RtlRecord::hasNested() const
+{
+    return fields != originalFields;
+}
+
+const RtlFieldInfo * RtlRecord::queryOriginalField(const char *fieldName) const
+{
+    // Used when setting up then checking unusual translation scenarios - doesn't need to be lightning-fast
+    for (const RtlFieldInfo * const * finger = originalFields; *finger; finger++)
+    {
+        const char *srcName = (*finger)->name;
+        if (srcName && strieq(fieldName, srcName))
+            return *finger;
+    }
+    return nullptr;
+}
+
 const RtlFieldInfo *RtlRecord::queryOriginalField(unsigned idx) const
 {
     const RtlFieldInfo *field = queryField(idx);

rtl/eclrtl/rtlrecord.hpp

@@ -238,6 +238,9 @@ public:
     unsigned getFieldNum(const char *fieldName) const;
     const RtlRecord *queryNested(unsigned field) const;
     bool excluded(const RtlFieldInfo *field, const byte *row, byte *conditions) const;
+    bool hasNested() const;
+    const RtlFieldInfo * queryOriginalField(const char *fieldName) const;
+
 protected:
     size_t * fixedOffsets;         // fixed portion of the field offsets + 1 extra
     unsigned * whichVariableOffset;// which variable offset should be added to the fixed

testing/regress/ecl/key/library8.xml

@@ -0,0 +1,3 @@
+<Dataset name='Result 1'>
+ <Row><Result_1>60</Result_1></Row>
+</Dataset>

testing/regress/ecl/library8.ecl

@@ -0,0 +1,47 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2020 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//nothor
+
+//Catch corruption when the wrong meta is returned for a library call result
+#option ('checkingHeap', true);
+
+smallRecord := { unsigned id; };
+largeRecord := { unsigned id; string1000 bigid; };
+
+
+FilterDatasetInterface(dataset(smallRecord) ds, unsigned every) := interface
+    export dataset(smallRecord) result1;
+    export dataset(largeRecord) result2;
+end;
+
+
+FilterDatasetLibrary(dataset(smallRecord) ds, unsigned every) := module,library(FilterDatasetInterface)
+    export result1 := ds;
+    export result2 := project(ds((id % every) = 1), transform(largeRecord, SELF.id := LEFT.id; SELF.bigid := (string)LEFT.id));
+end;
+
+
+
+filterDataset(dataset(smallRecord) ds, unsigned every) := library(internal(FilterDatasetLibrary), FilterDatasetInterface(ds, every));
+
+
+recs := DATASET(100, transform(smallRecord, SELF.id := COUNTER));
+
+filtered := filterDataset(recs, 2);
+j := JOIN(recs(id > 40), filtered.result2, LEFT.id = RIGHT.id, left outer);
+output(count(nofold(j)));

thorlcr/activities/indexread/thindexreadslave.cpp

@@ -189,7 +189,8 @@ public:
                     Owned<ITranslator> translator = getTranslators(part);
                     IOutputMetaData *actualFormat = translator ? &translator->queryActualFormat() : expectedFormat;
                     bool tryRemoteStream = actualFormat->queryTypeInfo()->canInterpret() && actualFormat->queryTypeInfo()->canSerialize() &&
-                                           projectedFormat->queryTypeInfo()->canInterpret() && projectedFormat->queryTypeInfo()->canSerialize();
+                                           projectedFormat->queryTypeInfo()->canInterpret() && projectedFormat->queryTypeInfo()->canSerialize() &&
+                                           !containsKeyedSignedInt(actualFormat->queryTypeInfo());
 
                     /* If part can potentially be remotely streamed, 1st check if any part is local,
                      * then try to remote stream, and otherwise failover to legacy remote access