Merge pull request #760 from ghalliday/pipeflat

Fix gh-704 Allow FLAT (and THOR) on PIPE

Reviewed-By: Richard Chapman <rchapman@hpccsystems.com>

ecl/hql/hqlatoms.cpp

@@ -124,6 +124,7 @@ _ATOM firstRightAtom;
 _ATOM fixedAtom;
 _ATOM flagAtom;
 _ATOM flagsAtom;
+_ATOM flatAtom;
 _ATOM _folded_Atom;
 _ATOM formatAtom;
 _ATOM forwardAtom;
@@ -332,6 +333,7 @@ _ATOM tempAtom;
 _ATOM templateAtom;
 _ATOM terminateAtom;
 _ATOM terminatorAtom;
+_ATOM thorAtom;
 _ATOM thresholdAtom;
 _ATOM timeoutAtom;
 _ATOM timeLimitAtom;
@@ -496,6 +498,7 @@ MODULE_INIT(INIT_PRIORITY_HQLATOM)
     MAKEATOM(fixed);
     MAKEATOM(flag);
     MAKEATOM(flags);
+    MAKEATOM(flat);
     MAKESYSATOM(folded);
     MAKEATOM(format);
     MAKEATOM(forward);
@@ -704,6 +707,7 @@ MODULE_INIT(INIT_PRIORITY_HQLATOM)
     MAKEATOM(template);
     MAKEATOM(terminate);
     MAKEATOM(terminator);
+    MAKEATOM(thor);
     MAKEATOM(threshold);
     MAKEATOM(timeout);
     MAKEATOM(timeLimit);

ecl/hql/hqlatoms.hpp

@@ -127,6 +127,7 @@ extern HQL_API _ATOM firstRightAtom;
 extern HQL_API _ATOM fixedAtom;
 extern HQL_API _ATOM flagAtom;
 extern HQL_API _ATOM flagsAtom;
+extern HQL_API _ATOM flatAtom;
 extern HQL_API _ATOM _folded_Atom;
 extern HQL_API _ATOM formatAtom;
 extern HQL_API _ATOM forwardAtom;
@@ -336,6 +337,7 @@ extern HQL_API _ATOM tempAtom;
 extern HQL_API _ATOM templateAtom;
 extern HQL_API _ATOM terminateAtom;
 extern HQL_API _ATOM terminatorAtom;
+extern HQL_API _ATOM thorAtom;
 extern HQL_API _ATOM thresholdAtom;
 extern HQL_API _ATOM timeoutAtom;
 extern HQL_API _ATOM timeLimitAtom;

ecl/hql/hqlgram.y

@@ -7885,9 +7885,18 @@ simpleDataSet
                             parser->transferOptions($3, $8);
                             parser->normalizeExpression($3, type_string, false);
 
-                            IHqlExpression *mode = $7.getExpr();
+                            OwnedHqlExpr mode = $7.getExpr();
                             OwnedHqlExpr options = $8.getExpr();
                             OwnedHqlExpr filename = $3.getExpr();
+                            if (mode->getOperator() == no_comma)
+                            {
+                                //Handle a messy common-grammar production which isn't in the format we want
+                                assertex(mode->queryChild(0)->getOperator() == no_pipe);
+                                HqlExprArray args;
+                                unwindChildren(args, mode->queryChild(0));
+                                mode->queryChild(1)->unwindList(args, no_comma);
+                                mode.setown(createValue(no_pipe, makeNullType(), args));
+                            }
 
                             //LOCAL(expression) is now an overloaded the syntax, so disambiguate here...
                             if (filename->getOperator() == no_forcelocal)
@@ -7895,7 +7904,7 @@ simpleDataSet
                                 filename.setown(createValue(no_assertconstant, filename->getType(), LINK(filename->queryChild(0))));
                                 options.setown(createComma(options.getClear(), createAttribute(localUploadAtom)));
                             }
-                            IHqlExpression * dataset = createNewDataset(filename.getClear(), $5.getExpr(), mode, NULL, NULL, options.getClear());
+                            IHqlExpression * dataset = createNewDataset(filename.getClear(), $5.getExpr(), mode.getClear(), NULL, NULL, options.getClear());
                             parser->checkValidRecordMode(dataset, $4, $7);
                             $$.setExpr(dataset);
                             $$.setPosition($1);
@@ -9246,6 +9255,8 @@ pipeFormatOption
                             $3.unwindCommaList(args);
                             $$.setExpr(createExprAttribute(xmlAtom, args), $1);
                         }
+    | FLAT              {   $$.setExpr(createAttribute(flatAtom), $1);   }
+    | THOR              {   $$.setExpr(createAttribute(thorAtom), $1);   }
     ;
 
 setCountList

ecl/hqlcpp/hqlttcpp.cpp

@@ -10251,8 +10251,10 @@ IHqlExpression * HqlTreeNormalizer::transformTable(IHqlExpression * untransforme
 
     OwnedHqlExpr modeThor = createValue(no_thor);
     IHqlExpression * diskRead = replaceChild(transformed, 2, modeThor);
-    IHqlExpression * pipe = mode->queryChild(0);
-    return createDataset(no_pipe, diskRead, LINK(pipe));
+    HqlExprArray args;
+    args.append(*diskRead);
+    unwindChildren(args, mode);
+    return createDataset(no_pipe, args);
 }
 
 IHqlExpression * HqlTreeNormalizer::optimizeAssignSkip(HqlExprArray & children, IHqlExpression * expr, IHqlExpression * cond, unsigned depth)

ecl/regress/pipe.ecl

@@ -18,6 +18,6 @@
 
 import std.system.thorlib;
 
-d := dataset('CFB::rawfile', {integer4 aa}, pipe('hello ' + thorlib.node()));
+d := dataset('CFB::rawfile', {integer4 aa}, pipe('hello ' + thorlib.node(), CSV));
 output(d);
 

ecl/regress/pipe2.ecl

@@ -29,8 +29,6 @@ p3 := CHOOSEN(PIPE(dt(l!='p3'), 'sort', { string lout }, csv, output(csv)), 1);
 p4 := CHOOSEN(PIPE(dt(l!='p4'), 'sort', { string lout }, REPEAT, csv, output(csv)), 1);
 
 output(p1, { lout, '\n' } );
-/*output(p2, { string20 l := Str.FindReplace(lout, '\r', ' ') } );
+output(p2, { string20 l := Str.FindReplace(lout, '\r', ' ') } );
 output(p3, { string20 l := Str.FindReplace(lout, '\r', ' ') } );
 output(p4, { string20 l := Str.FindReplace(lout, '\r', ' ') } );
-*/
-

ecl/regress/pipe2a.ecl

@@ -0,0 +1,34 @@
+/*##############################################################################
+
+    Copyright (C) 2011 HPCC Systems.
+
+    All rights reserved. This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as
+    published by the Free Software Foundation, either version 3 of the
+    License, or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+############################################################################## */
+
+import lib_stringlib;
+import std.str;
+
+d := dataset([{ 'Hello there'}, {'what a nice day'}, {'1234'}], { varstring20 line}) : stored('nofold');
+
+dt := TABLE(d, { string l := line });
+
+p1 := PIPE(dt(l!='p1'), 'sort', { string lout }, flat, output(csv));
+p2 := PIPE(dt(l!='p2'), 'sort', { string lout }, csv, output(flat), REPEAT);
+p3 := CHOOSEN(PIPE(dt(l!='p3'), 'sort', { string lout }, thor, output(csv)), 1);
+p4 := CHOOSEN(PIPE(dt(l!='p4'), 'sort', { string lout }, REPEAT, csv, output(thor)), 1);
+
+output(p1, { lout, '\n' } );
+output(p2, { string20 l := Str.FindReplace(lout, '\r', ' ') } );
+output(p3, { string20 l := Str.FindReplace(lout, '\r', ' ') } );
+output(p4, { string20 l := Str.FindReplace(lout, '\r', ' ') } );