Merge pull request #13102 from ghalliday/issue23028

HPCC-23028 Fix problems projecting all fields from a dataset

Reviewed-By: Jake Smith <jake.smith@lexisnexis.com>
Reviewed-By: Richard Chapman <rchapman@hpccsystems.com>

common/thorhelper/thorcommon.cpp

@@ -1229,7 +1229,7 @@ public:
                         translateBuf.setLength(0);
                         MemoryBufferBuilder rowBuilder(translateBuf, 0);
                         translator->translate(rowBuilder, *fieldCallback, row);
-                        row = reinterpret_cast<const byte *>(translateBuf.toByteArray());
+                        row = rowBuilder.getSelf();
                     }
                     return row;
                 }

ecl/hthor/hthor.cpp

@@ -8769,7 +8769,7 @@ const void *CHThorDiskReadActivity::nextRow()
                         {
                             MemoryBufferBuilder aBuilder(translated, 0);
                             translator->translate(aBuilder, *this, next);
-                            next = reinterpret_cast<const byte *>(translated.toByteArray());
+                            next = aBuilder.getSelf();
                         }
                         if (likely(helper.canMatch(next)))
                             thisSize = helper.transform(outBuilder.ensureRow(), next);

ecl/hthor/hthorkey.cpp

@@ -2489,7 +2489,7 @@ public:
             MemoryBufferBuilder aBuilder(buf, 0);
             FetchVirtualFieldCallback fieldCallback(fetch->pos);
             translator->translate(aBuilder, fieldCallback, rawBuffer);
-            rawBuffer = reinterpret_cast<const byte *>(buf.toByteArray());
+            rawBuffer = aBuilder.getSelf();
         }
 
         CriticalBlock procedure(transformCrit);
@@ -3681,7 +3681,7 @@ public:
             MemoryBufferBuilder aBuilder(buf, 0);
             FetchVirtualFieldCallback fieldCallback(pos);
             translator->translate(aBuilder, fieldCallback, row);
-            row = reinterpret_cast<const byte *>(buf.toByteArray());
+            row = aBuilder.getSelf();
         }
         if(match(fetch->ms, row))
         {

ecl/regress/issue23028.ecl

@@ -0,0 +1,19 @@
+#onwarning (1021, ignore); // Ignore warning that all output values are constant
+
+r1 := RECORD
+   string35 code;
+  END;
+
+r2 := RECORD
+  integer2 id;
+  embedded DATASET(r1) child;
+ END;
+
+mk1(unsigned c) := TRANSFORM(r1, SELF.code := (string)c; SELF := []);
+mk2(unsigned c) := TRANSFORM(r2, SELF.id := c; SELF.child := DATASET((c-1) % 5, mk1(COUNTER+c)), SELF := []);
+
+d := DATASET(20, mk2(counter));
+sequential(
+    output(d,,'~myspill', overwrite);
+    output(DATASET('~myspill', r2, THOR), { one := 1 })
+);

roxie/ccd/ccdactivities.cpp

@@ -3765,7 +3765,7 @@ public:
             translator->translate(aBuilder, fieldCallback, diskRow);
             //  note the swapped parameters - left and right map to input and raw differently for JOIN vs FETCH
             IHThorFetchArg *h = (IHThorFetchArg *) helper;
-            return h->transform(rowBuilder, buf.toByteArray(), inputData, rawpos);
+            return h->transform(rowBuilder, aBuilder.getSelf(), inputData, rawpos);
         }
         else
         {
@@ -4458,7 +4458,7 @@ IMessagePacker *CRoxieKeyedJoinFetchActivity::process()
             MemoryBufferBuilder aBuilder(buf, 0);
             FetchVirtualFieldCallback fieldCallback(headerPtr->fpos);
             translator->translate(aBuilder, fieldCallback, rawRHS);
-            rawRHS = (const byte *) buf.toByteArray();
+            rawRHS = aBuilder.getSelf();
         }
 
         inputData = &headerPtr->rhsdata[0];

roxie/ccd/ccdkey.cpp

@@ -471,7 +471,7 @@ protected:
             buf.setLength(0);
             MemoryBufferBuilder aBuilder(buf, 0);
             translator->translate(aBuilder, *this, row);
-            return reinterpret_cast<const byte *>(buf.toByteArray());
+            return aBuilder.getSelf();
         }
         else
             return row.queryRow();
@@ -1153,7 +1153,7 @@ public:
                     buf.setLength(0);
                     MemoryBufferBuilder aBuilder(buf, 0);
                     translator->translate(aBuilder, unexpectedVirtualFieldCallback, keySearcher->queryRow().queryRow()); // MORE - could pass in partially-resolved RtlRow
-                    return (const byte *) buf.toByteArray();
+                    return aBuilder.getSelf();
                 }
                 else
                     return keySearcher->queryRow().queryRow();

rtl/eclrtl/rtlds_imp.hpp

@@ -678,11 +678,11 @@ class ECLRTL_API MemoryBufferBuilder : public RtlRowBuilderBase
 {
 public:
     MemoryBufferBuilder(MemoryBuffer & _buffer, unsigned _minSize)
-        : buffer(&_buffer), minSize(_minSize)
+        : buffer(&_buffer), minSize(_minSize ? _minSize : 1U)
     {
     }
 
-    MemoryBufferBuilder(unsigned _minSize) : minSize(_minSize)
+    MemoryBufferBuilder(unsigned _minSize) : minSize(_minSize ? _minSize : 1U)
     {
     }
 

system/jhtree/jhtree.cpp

@@ -460,7 +460,7 @@ public:
             buf.setLength(0);
             MemoryBufferBuilder aBuilder(buf, 0);
             layoutSize = layoutTrans->translate(aBuilder, unexpectedFieldCallback, reinterpret_cast<byte const *>(keyCursor->queryKeyBuffer()));
-            return reinterpret_cast<byte const *>(buf.toByteArray());
+            return aBuilder.getSelf();
         }
         else
             return reinterpret_cast<byte const *>(keyCursor->queryKeyBuffer());

testing/regress/ecl/key/projectnone.xml

@@ -0,0 +1,27 @@
+<Dataset name='Result 1'>
+</Dataset>
+<Dataset name='Result 2'>
+ <Row><one>1</one></Row>
+ <Row><one>1</one></Row>
+ <Row><one>1</one></Row>
+ <Row><one>1</one></Row>
+ <Row><one>1</one></Row>
+ <Row><one>1</one></Row>
+ <Row><one>1</one></Row>
+ <Row><one>1</one></Row>
+ <Row><one>1</one></Row>
+ <Row><one>1</one></Row>
+ <Row><one>1</one></Row>
+ <Row><one>1</one></Row>
+ <Row><one>1</one></Row>
+ <Row><one>1</one></Row>
+ <Row><one>1</one></Row>
+ <Row><one>1</one></Row>
+ <Row><one>1</one></Row>
+ <Row><one>1</one></Row>
+ <Row><one>1</one></Row>
+ <Row><one>1</one></Row>
+</Dataset>
+<Dataset name='Result 3'>
+ <Row><Result_3>20</Result_3></Row>
+</Dataset>

testing/regress/ecl/projectnone.ecl

@@ -0,0 +1,40 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2019 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+import $.setup;
+prefix := setup.Files(false, false).QueryFilePrefix;
+
+#onwarning (1021, ignore); // Ignore warning that all output values are constant
+
+r1 := RECORD
+   string35 code;
+  END;
+
+r2 := RECORD
+  integer2 id;
+  embedded DATASET(r1) child;
+ END;
+
+mk1(unsigned c) := TRANSFORM(r1, SELF.code := (string)c; SELF := []);
+mk2(unsigned c) := TRANSFORM(r2, SELF.id := c; SELF.child := DATASET((c-1) % 5, mk1(COUNTER+c)), SELF := []);
+
+d := DATASET(20, mk2(counter));
+sequential(
+    output(d,,prefix+'myspill', overwrite);
+    output(DATASET(prefix+'myspill', r2, THOR), { one := 1 });
+    output(count(nocombine(DATASET(prefix+'myspill', r2, THOR))));
+);

thorlcr/slave/slavmain.cpp

@@ -849,7 +849,7 @@ class CKJService : public CSimpleInterfaceOf<IKJService>, implements IThreaded,
                 MemoryBufferBuilder aBuilder(diskFetchRowMb, 0);
                 LocalVirtualFieldCallback fieldCallback("<MORE>", requestHeader.fpos, 0);
                 translator->translate(aBuilder, fieldCallback, diskFetchRow);
-                diskFetchRow = reinterpret_cast<const byte *>(diskFetchRowMb.toByteArray());
+                diskFetchRow = aBuilder.getSelf();
             }
             size32_t fetchReplySz = sizeof(FetchReplyHeader);
             if (helper->fetchMatch(fetchKey, diskFetchRow))

tools/dumpkey/dumpkey.cpp

@@ -323,7 +323,7 @@ int main(int argc, const char **argv)
                         MemoryBufferBuilder aBuilder(buf, 0);
                         if (translator->translate(aBuilder, callback, buffer))
                         {
-                            outmeta->toXML((const byte *) buf.toByteArray(), *writer.get());
+                            outmeta->toXML(aBuilder.getSelf(), *writer.get());
                             printf("%s\n", writer->str());
                             writer->clear();
                         }