Browse Source

HPCC-17789 Avoid hthor spilling datasets used inside LOOP to workunit

Signed-off-by: Gavin Halliday <gavin.halliday@lexisnexis.com>
Gavin Halliday 8 years ago
parent
commit
f86de98de4
3 changed files with 83 additions and 1 deletions
  1. 1 1
      ecl/hqlcpp/hqlresource.cpp
  2. 42 0
      testing/regress/ecl/key/loop10.xml
  3. 40 0
      testing/regress/ecl/loop10.ecl

+ 1 - 1
ecl/hqlcpp/hqlresource.cpp

@@ -1054,7 +1054,7 @@ void ActivityInvariantHoister::gatherChildSplitPoints(IHqlExpression * expr, Act
         alwaysOnce = true;
         break;
     case no_loop:
-        if ((options.targetClusterType == ThorLCRCluster) && !options.isChildQuery)
+        if ((options.targetClusterType != RoxieCluster) && !options.isChildQuery)
         {
             //This is ugly!  The body is executed in parallel, so don't force that as a work unit result
             //It means some child query expressions within loops don't get forced into work unit writes

+ 42 - 0
testing/regress/ecl/key/loop10.xml

@@ -0,0 +1,42 @@
+<Dataset name='Result 1'>
+ <Row><i>4</i></Row>
+ <Row><i>5</i></Row>
+ <Row><i>9</i></Row>
+ <Row><i>10</i></Row>
+ <Row><i>14</i></Row>
+ <Row><i>15</i></Row>
+ <Row><i>19</i></Row>
+ <Row><i>20</i></Row>
+ <Row><i>24</i></Row>
+ <Row><i>25</i></Row>
+ <Row><i>29</i></Row>
+ <Row><i>30</i></Row>
+ <Row><i>34</i></Row>
+ <Row><i>35</i></Row>
+ <Row><i>39</i></Row>
+ <Row><i>40</i></Row>
+ <Row><i>44</i></Row>
+ <Row><i>45</i></Row>
+ <Row><i>49</i></Row>
+ <Row><i>50</i></Row>
+ <Row><i>54</i></Row>
+ <Row><i>55</i></Row>
+ <Row><i>59</i></Row>
+ <Row><i>60</i></Row>
+ <Row><i>64</i></Row>
+ <Row><i>65</i></Row>
+ <Row><i>69</i></Row>
+ <Row><i>70</i></Row>
+ <Row><i>74</i></Row>
+ <Row><i>75</i></Row>
+ <Row><i>79</i></Row>
+ <Row><i>80</i></Row>
+ <Row><i>84</i></Row>
+ <Row><i>85</i></Row>
+ <Row><i>89</i></Row>
+ <Row><i>90</i></Row>
+ <Row><i>94</i></Row>
+ <Row><i>95</i></Row>
+ <Row><i>99</i></Row>
+ <Row><i>100</i></Row>
+</Dataset>

+ 40 - 0
testing/regress/ecl/loop10.ecl

@@ -0,0 +1,40 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2017 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//noroxie   - roxie doesn't support spilling to disk with the results being read from a child graph, the spill file is too large for memory
+
+//generate an error if a workunit spill is used
+#option ('outputLimitMb', 1);
+
+r := record
+  integer i;
+end;
+
+r t(integer c) := transform
+  SELF.i := c;
+end;
+
+ds1 := NOCOMBINE(dataset(1000000, t(COUNTER)));
+ds2 := sort(ds1, i)(i != 5);
+
+ds3 := DATASET(100, t(COUNTER));
+
+f(dataset(r) inDs) := join(inDs, ds2, LEFT.i % 10 = RIGHT.i, t(LEFT.i+1));
+
+l := loop(ds3, 3, f(ROWS(LEFT)));
+
+sort(l, i);