Merge branch 'candidate-7.8.x' into candidate-7.10.x

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

ecl/hthor/hthor.cpp

@@ -9013,6 +9013,12 @@ const void *CHThorDiskNormalizeActivity::nextRow()
             {
                 try
                 {
+                    if (unlikely(translator))
+                    {
+                        MemoryBufferBuilder aBuilder(translatedRow.clear(), 0);
+                        translator->translate(aBuilder, *this, next);
+                        next = aBuilder.getSelf();
+                    }
                     expanding = helper.first(next);
                 }
                 catch(IException * e)
@@ -9103,7 +9109,16 @@ const void *CHThorDiskAggregateActivity::nextRow()
                 const byte * next = prefetchBuffer.queryRow();
                 size32_t sizeRead = prefetchBuffer.queryRowSize();
                 if (segMonitorsMatch(next))
-                    helper.processRow(outBuilder, next);
+                {
+                    if (unlikely(translator))
+                    {
+                        MemoryBufferBuilder aBuilder(translatedRow.clear(), 0);
+                        translator->translate(aBuilder, *this, next);
+                        helper.processRow(outBuilder, aBuilder.getSelf());
+                    }
+                    else
+                        helper.processRow(outBuilder, next);
+                }
                 prefetchBuffer.finishedRow();
                 localOffset += sizeRead;
             }
@@ -9275,7 +9290,16 @@ const void *CHThorDiskGroupAggregateActivity::nextRow()
                     size32_t sizeRead = prefetchBuffer.queryRowSize();
 
                     if (segMonitorsMatch(next))
-                        helper.processRow(next, this);
+                   {
+                        if (unlikely(translator))
+                        {
+                            MemoryBufferBuilder aBuilder(translatedRow.clear(), 0);
+                            translator->translate(aBuilder, *this, next);
+                            helper.processRow(aBuilder.getSelf(), this);
+                        }
+                        else
+                            helper.processRow(next, this);
+                    }
 
                     prefetchBuffer.finishedRow();
                     localOffset += sizeRead;

ecl/hthor/hthor.ipp

@@ -2327,6 +2327,7 @@ protected:
     Owned<IOutputRowDeserializer> deserializer;
     CThorContiguousRowBuffer prefetchBuffer;
     CThorStreamDeserializerSource deserializeSource;
+    MemoryBuffer translatedRow;
 public:
     CHThorBinaryDiskReadBase(IAgentContext &agent, unsigned _activityId, unsigned _subgraphId, IHThorDiskReadBaseArg &_arg, IHThorCompoundBaseArg & _segHelper, ThorActivityKind _kind, IPropertyTree *_node);
 
@@ -2482,6 +2483,7 @@ class CHThorDiskAggregateActivity : public CHThorBinaryDiskReadBase
 protected:
     IHThorDiskAggregateArg &helper;
     RtlDynamicRowBuilder outBuilder;
+    MemoryBuffer translatedRow;
     bool finished;
 
     virtual void gatherInfo(IFileDescriptor * fileDesc);
@@ -2526,6 +2528,7 @@ class CHThorDiskGroupAggregateActivity : public CHThorBinaryDiskReadBase, implem
 protected:
     IHThorDiskGroupAggregateArg &helper;
     RowAggregator aggregated;
+    MemoryBuffer translatedRow;
     bool eof;
     bool gathered;
 

testing/regress/ecl/key/sqtrans.xml

@@ -0,0 +1,53 @@
+<Dataset name='Result 1'>
+ <Row><surname>Halliday            </surname><forename>Gavin     </forename></Row>
+ <Row><surname>Halliday            </surname><forename>Abigail   </forename></Row>
+ <Row><surname>Halliday            </surname><forename>Liz       </forename></Row>
+ <Row><surname>Windsor             </surname><forename>Elizabeth </forename></Row>
+ <Row><surname>Windsor             </surname><forename>Philip    </forename></Row>
+ <Row><surname>Flintstone          </surname><forename>Fred      </forename></Row>
+ <Row><surname>Grabdale            </surname><forename>John      </forename></Row>
+ <Row><surname>Jones               </surname><forename>Brian     </forename></Row>
+ <Row><surname>Jones               </surname><forename>Julia     </forename></Row>
+</Dataset>
+<Dataset name='Result 2'>
+ <Row><name>To kill a mocking bird</name><author>Harper Lee</author></Row>
+ <Row><name>Clarion Language Manual</name><author>Richard Taylor</author></Row>
+ <Row><name>The bible</name><author>Various</author></Row>
+ <Row><name>Compilers</name><author>Aho, Sethi, Ullman</author></Row>
+ <Row><name>Lord of the Rings</name><author>JRR Tolkien</author></Row>
+ <Row><name>The thinks you can think</name><author>Dr. Seuss</author></Row>
+ <Row><name>Where is flop?</name><author></author></Row>
+ <Row><name>David and Goliath</name><author></author></Row>
+ <Row><name>The story of Jonah</name><author></author></Row>
+ <Row><name>The Life of Pi</name><author>Yan Martel</author></Row>
+ <Row><name>BNF</name><author>Various</author></Row>
+ <Row><name>The Third Policeman</name><author>Flann O&apos;Brien</author></Row>
+ <Row><name>The bible</name><author>Various</author></Row>
+ <Row><name>The Prince</name><author>Machiavelli</author></Row>
+ <Row><name>Atlas of the world</name><author>Times</author></Row>
+ <Row><name>Girl guide annual</name><author>Various</author></Row>
+ <Row><name>Rwandan war dances</name><author>Unknown</author></Row>
+ <Row><name>Greek tragedies</name><author>Various</author></Row>
+ <Row><name>Social etiquette</name><author>RU Correct</author></Row>
+ <Row><name>The Rituals of Dinner: The Origins, Evolution, Eccentricities and the Meaning of Table Manners</name><author>Margaret Visser</author></Row>
+ <Row><name>The cat in the hat</name><author>Dr. Seuss</author></Row>
+ <Row><name>Dinosaur stews</name><author>Trog</author></Row>
+ <Row><name>Pride and prejudice, 1st edition</name><author>Jane Austen</author></Row>
+ <Row><name>Mein Kampf, 1st edition</name><author>Adolph Hitler</author></Row>
+ <Row><name>All quiet on the western front</name><author>Erich Maria Remarque</author></Row>
+ <Row><name>Fox in Socks</name><author>Dr. Seuss</author></Row>
+ <Row><name>Zen and the art of motorcyle maintenance</name><author>Robert Pirsig</author></Row>
+ <Row><name>Where the wild things are</name><author>Maurice Sendak</author></Row>
+ <Row><name>The bible</name><author>Various</author></Row>
+ <Row><name>The cat in the hat</name><author>Dr. Seuss</author></Row>
+</Dataset>
+<Dataset name='Result 3'>
+ <Row><_unnamed_max_surname1>Windsor             </_unnamed_max_surname1></Row>
+</Dataset>
+<Dataset name='Result 4'>
+ <Row><cnt>3</cnt><surname>Halliday            </surname></Row>
+ <Row><cnt>2</cnt><surname>Jones               </surname></Row>
+ <Row><cnt>2</cnt><surname>Windsor             </surname></Row>
+ <Row><cnt>1</cnt><surname>Flintstone          </surname></Row>
+ <Row><cnt>1</cnt><surname>Grabdale            </surname></Row>
+</Dataset>

testing/regress/ecl/sqtrans.ecl

@@ -0,0 +1,47 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//version multiPart=false
+
+import ^ as root;
+multiPart := #IFDEFINED(root.multiPart, false);
+
+//--- end of version configuration ---
+
+//check that compound disk read activities workcorrectly when the input file is translated
+//include keyed filters to ensure they are also translated correctly
+
+import $.setup;
+sq := setup.sq(multiPart);
+
+r1 := recordof(sq.simplePersonBookDs);
+r2 := r1 - [surname];
+r3 := RECORD
+    r2;
+    string surname;
+END;
+
+translatedFile := DATASET(__NAMEOF__(sq.simplePersonBookDs), r3, thor);
+//translatedFile := sq.simplePersonBookDs;
+filtered := translatedFile(KEYED(forename != 'Wilma'));
+
+sequential(
+    output(TABLE(filtered, { surname, forename })),
+    output(TABLE(translatedFile.books, { name, author })),
+    output(TABLE(filtered, { max(group, surname) })),
+    output(SORT(NOFOLD(TABLE(filtered, { unsigned cnt := count(group), surname }, surname, FEW )), -cnt, surname))
+);