HPCC-13088 Fix FORMAT as an attribute on HEADING

Note: test cases already existed -ee csvheading4.ecl.

Signed-off-by: Gavin Halliday <gavin.halliday@lexisnexis.com>

ecl/hql/hqlgram.y

@@ -102,6 +102,11 @@ static void eclsyntaxerror(HqlGram * parser, const char * s, short yystate, int
 /* remember to add any new tokens to the error reporter and lexer too! */
 /* If they clash with other #defines etc then use TOK_ as a prefix */
 
+// NB: Very occassionally the same keyword in the source (e.g., MERGE, PARTITION may return a different token
+// (MERGE_ATTR, PARTITION_ATTR) depending on the context it is used in.  This is because there would be a s/r
+// error, so the _ATTR form is only allowed in the situations where the attibute is valid - enabled by a
+// call to enableAttributes() from a production in the grammar.
+
   ABS
   ACOS
   AFTER
@@ -212,10 +217,9 @@ static void eclsyntaxerror(HqlGram * parser, const char * s, short yystate, int
   FIRST
   TOK_FIXED
   FLAT
-  FROM
-  FORMAT
   FORMAT_ATTR
   FORWARD
+  FROM
   FROMJSON
   FROMUNICODE
   FROMXML
@@ -1612,15 +1616,15 @@ failure
                             parser->normalizeExpression($5, type_string, true);
                             $$.setExpr(createValueF(no_persist, makeVoidType(), $3.getExpr(), $5.getExpr(), $6.getExpr(), NULL), $1);
                         }
-    | STORED '(' expression ',' fewMany optStoredFieldFormat ')'
+    | STORED '(' startStoredAttrs expression ',' fewMany optStoredFieldFormat ')'
                         {
-                            parser->normalizeStoredNameExpression($3);
-                            $$.setExpr(createValue(no_stored, makeVoidType(), $3.getExpr(), $5.getExpr(), $6.getExpr()), $1);
+                            parser->normalizeStoredNameExpression($4);
+                            $$.setExpr(createValue(no_stored, makeVoidType(), $4.getExpr(), $6.getExpr(), $7.getExpr()), $1);
                         }
-    | STORED '(' expression optStoredFieldFormat ')'
+    | STORED '(' startStoredAttrs expression optStoredFieldFormat ')'
                         {
-                            parser->normalizeStoredNameExpression($3);
-                            $$.setExpr(createValue(no_stored, makeVoidType(), $3.getExpr(), $4.getExpr()), $1);
+                            parser->normalizeStoredNameExpression($4);
+                            $$.setExpr(createValue(no_stored, makeVoidType(), $4.getExpr(), $5.getExpr()), $1);
                         }
     | CHECKPOINT '(' constExpression ')'
                         {
@@ -1732,7 +1736,7 @@ optStoredFieldFormat
     :                   {
                             $$.setNullExpr();
                         }
-    | ',' FORMAT '(' hintList ')'
+    | ',' FORMAT_ATTR '(' hintList ')'
                         {
                             HqlExprArray args;
                             $4.unwindCommaList(args);
@@ -12386,6 +12390,7 @@ featureModifiers
 
 startDistributeAttrs :  { parser->enableAttributes(DISTRIBUTE); $$.clear(); } ;
 startHeadingAttrs:      { parser->enableAttributes(HEADING); $$.clear(); } ;
+startStoredAttrs:       { parser->enableAttributes(STORED); $$.clear(); } ;
 
 
 //================================== end of syntax section ==========================

ecl/hql/hqlgram2.cpp

@@ -152,6 +152,8 @@ MODULE_INIT(INIT_PRIORITY_STANDARD)
 
     setAttributes(DISTRIBUTE, MERGE_ATTR, PARTITION_ATTR, 0);
     setAttributes(HEADING, FORMAT_ATTR, 0);
+    setAttributes(STORED, FORMAT_ATTR, 0);
+
     alreadyAssignedNestedTag = createAttribute(_alreadyAssignedNestedTag_Atom);
     return true;
 }
@@ -10507,7 +10509,6 @@ static void getTokenText(StringBuffer & msg, int token)
     case FIRST: msg.append("FIRST"); break;
     case TOK_FIXED: msg.append("FIXED"); break;
     case FLAT: msg.append("FLAT"); break;
-    case FORMAT: msg.append("FORMAT"); break;
     case FORMAT_ATTR: msg.append("FORMAT"); break;
     case FORWARD: msg.append("FORWARD"); break;
     case FROM: msg.append("FROM"); break;

ecl/hql/hqllex.l

@@ -705,10 +705,9 @@ FILTERED            { RETURNSYM(FILTERED); }
 FIRST               { RETURNSYM(FIRST); }
 FIXED               { RETURNSYM(TOK_FIXED); }
 FLAT                { RETURNSYM(FLAT); }
-FORMAT              { RETURNSYM(FORMAT); }
-FROM                { RETURNSYM(FROM); }
-FORMAT              { RETURNSYM(FORMAT_ATTR); }
+FORMAT              { RETURNSYM(FORMAT_ATTR); }             // Dynamically enabled based on the context
 FORWARD             { RETURNSYM(FORWARD); }
+FROM                { RETURNSYM(FROM); }
 FROMUNICODE         { RETURNSYM(FROMUNICODE); }
 FROMJSON            { RETURNSYM(FROMJSON); }
 FROMXML             { RETURNSYM(FROMXML); }
@@ -785,7 +784,7 @@ MAX                 { RETURNSYM(MAX); }
 MAXCOUNT            { RETURNSYM(MAXCOUNT); }
 MAXLENGTH           { RETURNSYM(MAXLENGTH); }
 MAXSIZE             { RETURNSYM(MAXSIZE); }
-MERGE               { RETURNSYM(MERGE); }
+MERGE               { RETURNSYM(MERGE); }                   // May be dynamically converted to MERGE_ATTR
 MERGEJOIN           { RETURNSYM(MERGEJOIN); }
 MIN                 { RETURNSYM(MIN); }
 MODULE              { RETURNHARD(MODULE); }
@@ -831,7 +830,7 @@ __OWNED__           { RETURNSYM(__OWNED__); }
 PACKED              { RETURNSYM(PACKED); }
 PARALLEL            { RETURNSYM(PARALLEL); }
 PARSE               { RETURNSYM(PARSE); }
-PARTITION           { RETURNSYM(PARTITION); }
+PARTITION           { RETURNSYM(PARTITION); }               // May be dynamically converted to PARTITION_ATTR
 PATTERN             { RETURNSYM(TOK_PATTERN); }
 PENALTY             { RETURNSYM(PENALTY); }
 PERSIST             { RETURNSYM(PERSIST); }

ecl/hqlcpp/hqlcse.cpp

@@ -106,6 +106,7 @@ bool canCreateTemporary(IHqlExpression * expr)
     case no_thisnode:
     case no_libraryscopeinstance:
     case no_loopbody:
+    case no_external:
         return false;
     }
     ITypeInfo * type = expr->queryType();
@@ -1223,6 +1224,8 @@ static bool canHoistInvariant(IHqlExpression * expr)
     }
     if (!expr->isPure())
         return false;
+    if (expr->isFunction())
+        return false;
     switch (expr->getOperator())
     {
     case no_list:

ecl/regress/format.ecl

@@ -0,0 +1,5 @@
+import lib_stringlib;
+
+format := 3;
+
+output(format);