Merge pull request #8278 from jamienoss/issue15018-update-reservedwrods.cpp

HPCC-15018 Update reservedwords.cpp with missing reserved words

Reviewed-by: Gavin Halliday <ghalliday@hpccsystems.com>

ecl/eclcc/CMakeLists.txt

@@ -26,8 +26,6 @@
 project( eclcc ) 
 
 set (    SRCS
-         reservedwords.hpp
-         reservedwords.cpp
          eclcc.hpp
          eclcc.cpp
     )

ecl/eclcc/eclcc.cpp

@@ -276,7 +276,7 @@ public:
     }
 
     bool printKeywordsToXml();
-    bool parseCommandLineOptions(int argc, const char* argv[]);
+    int parseCommandLineOptions(int argc, const char* argv[]);
     void loadOptions();
     void loadManifestOptions();
     bool processFiles();
@@ -435,8 +435,9 @@ static int doMain(int argc, const char *argv[])
         return doSelfTest(argc, argv);
 
     EclCC processor(argc, argv);
-    if (!processor.parseCommandLineOptions(argc, argv))
-        return 1;
+    int ret = processor.parseCommandLineOptions(argc, argv);
+    if (ret != 0)
+        return ret;
 
     try
     {
@@ -1893,12 +1894,12 @@ bool EclCC::allowAccess(const char * category, bool isSigned)
 }
 
 //=========================================================================================
-bool EclCC::parseCommandLineOptions(int argc, const char* argv[])
+int EclCC::parseCommandLineOptions(int argc, const char* argv[])
 {
     if (argc < 2)
     {
         usage();
-        return false;
+        return 1;
     }
 
     ArgvIterator iter(argc, argv);
@@ -1976,7 +1977,15 @@ bool EclCC::parseCommandLineOptions(int argc, const char* argv[])
         else if (strcmp(arg, "-internal")==0)
         {
             outputSizeStmts();
-            testHqlInternals();
+            int error = testHqlInternals() + testReservedWords();  //NOTE: testReservedWords() depends on testHqlInternals() so must be be called after.
+            // report test result
+            if (error)
+            {
+                printf("%d error%s found!\n", error, error<=1?"":"s");
+                return 300;
+            }
+            else
+                printf("No errors\n");
         }
         else if (iter.matchFlag(tempBool, "-save-cpps"))
         {
@@ -2076,7 +2085,7 @@ bool EclCC::parseCommandLineOptions(int argc, const char* argv[])
             if (!checkFileExists(optIniFilename))
             {
                 ERRLOG("Error: INI file '%s' does not exist",optIniFilename.get());
-                return false;
+                return 1;
             }
         }
         else if (iter.matchFlag(optShowPaths, "-showpaths"))
@@ -2085,7 +2094,7 @@ bool EclCC::parseCommandLineOptions(int argc, const char* argv[])
         else if (iter.matchOption(optManifestFilename, "-manifest"))
         {
             if (!isManifestFileValid(optManifestFilename))
-                return false;
+                return 1;
         }
         else if (iter.matchOption(tempArg, "-split"))
         {
@@ -2094,7 +2103,7 @@ bool EclCC::parseCommandLineOptions(int argc, const char* argv[])
             if (!split)
             {
                 ERRLOG("Error: syntax is -split=part:splits\n");
-                return false;
+                return 1;
             }
             batchSplit = atoi(split+1);
             if (batchSplit == 0)
@@ -2108,7 +2117,7 @@ bool EclCC::parseCommandLineOptions(int argc, const char* argv[])
         else if (iter.matchOption(tempArg, "-platform") || /*deprecated*/ iter.matchOption(tempArg, "-target"))
         {
             if (!setTargetPlatformOption(tempArg.get(), optTargetClusterType))
-                return false;
+                return 1;
         }
         else if (iter.matchFlag(logVerbose, "-v") || iter.matchFlag(logVerbose, "--verbose"))
         {
@@ -2118,7 +2127,7 @@ bool EclCC::parseCommandLineOptions(int argc, const char* argv[])
         else if (strcmp(arg, "--version")==0)
         {
             fprintf(stdout,"%s %s\n", LANGUAGE_VERSION, BUILD_TAG);
-            return false;
+            return 1;
         }
         else if (startsWith(arg, "-Wc,"))
         {
@@ -2150,7 +2159,7 @@ bool EclCC::parseCommandLineOptions(int argc, const char* argv[])
         {
             ERRLOG("Error: unrecognised option %s",arg);
             usage();
-            return false;
+            return 1;
         }
         else
             inputFileNames.append(arg);
@@ -2158,7 +2167,7 @@ bool EclCC::parseCommandLineOptions(int argc, const char* argv[])
     if (showHelp)
     {
         usage();
-        return false;
+        return 1;
     }
 
     if (optComponentName.length())
@@ -2175,9 +2184,9 @@ bool EclCC::parseCommandLineOptions(int argc, const char* argv[])
     if (inputFileNames.ordinality() == 0 && !optKeywords)
     {
         if (optGenerateHeader || optShowPaths || (!optBatchMode && optQueryRepositoryReference))
-            return true;
+            return 0;
         ERRLOG("No input filenames supplied");
-        return false;
+        return 1;
     }
 
     if (optDebugMemLeak)
@@ -2187,7 +2196,7 @@ bool EclCC::parseCommandLineOptions(int argc, const char* argv[])
         initLeakCheck(title);
     }
 
-    return true;
+    return 0;
 }
 
 //=========================================================================================

ecl/hql/CMakeLists.txt

@@ -27,34 +27,35 @@
 project( hql ) 
 
 set (   SRCS 
-        hqlatoms.cpp 
-        hqlattr.cpp 
+        hqlatoms.cpp
+        hqlattr.cpp
         hqlcollect.cpp
         hqldesc.cpp
-        hqldsparam.cpp 
-        hqlerror.cpp 
-        hqlesp.cpp 
-        hqlexpr.cpp 
-        hqlfold.cpp 
-        hqlgram2.cpp 
-        hqlmanifest.cpp 
-        hqlmeta.cpp 
-        hqlopt.cpp 
-        hqlparse.cpp 
-        hqlpmap.cpp 
+        hqldsparam.cpp
+        hqlerror.cpp
+        hqlesp.cpp
+        hqlexpr.cpp
+        hqlfold.cpp
+        hqlgram2.cpp
+        hqlmanifest.cpp
+        hqlmeta.cpp
+        hqlopt.cpp
+        hqlparse.cpp
+        hqlpmap.cpp
         hqlplugininfo.cpp
-        hqlpregex.cpp 
+        hqlpregex.cpp
         hqlrepository.cpp
-        hqlscope.cpp 
-        hqlstack.cpp 
-        hqlthql.cpp 
-        hqltrans.cpp 
+        hqlscope.cpp
+        hqlstack.cpp
+        hqlthql.cpp
+        hqltrans.cpp
         hqlusage.cpp
-        hqlutil.cpp 
+        hqlutil.cpp
         hqlir.cpp
-        hqlvalid.cpp 
-        hqlwuerr.cpp 
-        hqlxmldb.cpp 
+        hqlvalid.cpp
+        hqlwuerr.cpp
+        hqlxmldb.cpp
+        reservedwords.cpp
 
         hqlgram.y
         hqllex.l
@@ -87,6 +88,7 @@ set (   SRCS
         hqlvalid.hpp
         hqlwuerr.hpp
         hqlxmldb.hpp
+        reservedwords.hpp
     )
 
 include_directories ( 
@@ -134,11 +136,11 @@ ADD_DEFINITIONS( -D_USRDLL -DHQL_EXPORTS -DHQLFOLD_EXPORTS -DHQLTRANS_EXPORTS )
 
 HPCC_ADD_LIBRARY( hql SHARED ${SRCS} ${CMAKE_CURRENT_BINARY_DIR}/hqlgram.cpp ${CMAKE_CURRENT_BINARY_DIR}/hqllex.cpp  )
 install ( TARGETS hql RUNTIME DESTINATION ${EXEC_DIR} LIBRARY DESTINATION ${LIB_DIR} )
-target_link_libraries ( hql  
+target_link_libraries ( hql
          jlib
-         nbcd 
-         eclrtl 
-         deftype 
+         nbcd
+         eclrtl
+         deftype
          ${CPPUNIT_LIBRARIES}
     )
 

ecl/hql/hqlgram.hpp

@@ -1234,6 +1234,7 @@ private:
 
 IHqlExpression *reparseTemplateFunction(IHqlExpression * funcdef, IHqlScope *scope, HqlLookupContext & ctx, bool hasFieldMap);
 extern HQL_API void resetLexerUniqueNames();        // to make regression suite consistent
-extern HQL_API void testHqlInternals();
+extern HQL_API int testHqlInternals();
+extern HQL_API int testReservedWords();
 
 #endif

ecl/hql/hqlgram2.cpp

@@ -48,6 +48,7 @@
 #include "hqlvalid.hpp"
 #include "hqlrepository.hpp"
 #include "hqlir.hpp"
+#include "reservedwords.hpp"
 
 #define ADD_IMPLICIT_FILEPOS_FIELD_TO_INDEX         TRUE
 #define FAST_FIND_FIELD
@@ -10551,7 +10552,7 @@ static void getTokenText(StringBuffer & msg, int token)
     case ENCRYPT: msg.append("ENCRYPT"); break;
     case ENCRYPTED: msg.append("ENCRYPTED"); break;
     case END: msg.append("END"); break;
-    case ENDCPP: msg.append("ENDCPP"); break;
+    case ENDCPP: msg.append("ENDC++"); break;
     case ENDEMBED: msg.append("ENDEMBED"); break;
     case ENTH: msg.append("ENTH"); break;
     case ENUM: msg.append("ENUM"); break;
@@ -10698,7 +10699,7 @@ static void getTokenText(StringBuffer & msg, int token)
     case ONLY: msg.append("ONLY"); break;
     case ONWARNING: msg.append("ONWARNING"); break;
     case OPT: msg.append("OPT"); break;
-    case OR : msg.append("OR "); break;
+    case OR : msg.append("OR"); break;
     case ORDER: msg.append("ORDER"); break;
     case ORDERED: msg.append("ORDERED"); break;
     case OUTER: msg.append("OUTER"); break;
@@ -11808,7 +11809,7 @@ void parseAttribute(IHqlScope * scope, IFileContents * contents, HqlLookupContex
     attrCtx.noteEndAttribute();
 }
 
-void testHqlInternals()
+int testHqlInternals()
 {
     printf("Sizes: const(%u) expr(%u) select(%u) dataset(%u) annotation(%u) prop(%u)\n",
             (unsigned)sizeof(CHqlConstant),
@@ -11865,12 +11866,33 @@ void testHqlInternals()
         }
     }
 
-    // 
-    // report test result
-    if (error)
-        printf("%d error%s found!\n", error, error<=1?"":"s");
-    else
-        printf("No errors\n");
+    return error;
+}
+
+int testReservedWords()
+{
+    printf("Testing --keywords is complete...\n");
+    int error = 0;
+
+    for (int token = 258; token < YY_LAST_TOKEN; token++)
+    {
+        try
+        {
+            StringBuffer tokenText;
+            getTokenText(tokenText, token);
+            tokenText.toLowerCase();
+            if (!searchReservedWords(tokenText.str()))
+            {
+                error++;
+                printf("   Error: '%s' is missing from reservedWords.cpp\n", tokenText.str());
+            }
+        }
+        catch (...)
+        {
+            printf("   Error: complete test not possible - getTokenText() does not handle expected: %d\n", token);
+        }
+    }
+    return error;
 }
 
 IHqlExpression *HqlGram::yyParse(bool _parsingTemplateAttribute, bool catchAbort)

ecl/eclcc/reservedwords.cpp

@@ -105,6 +105,7 @@ static const char * eclReserved3[] = { //Template language
     "#trace",
     "#uniquename",
     "#warning",
+    "#webservice",
     "#workunit",
     "loadxml",
     NULL
@@ -119,7 +120,9 @@ static const char * eclReserved4[] = { //String functions
     "length",
     "realformat",
     "regexfind",
+    "regexfindset",
     "regexreplace",
+    "tojson",
     "tounicode",
     "toxml",
     "trim",
@@ -161,13 +164,15 @@ static const char * eclReserved5[] = { //Math
     NULL
 };
 
-static const char * eclReserved6[] = {
+static const char * eclReserved6[] = { //bit ops
     "&",
     "|",
     //"^", ambiguous group location
     "bnot",
     "<<",
     ">>",
+    "(>",
+    "<)",
     NULL
 };
 
@@ -176,7 +181,8 @@ static const char * eclReserved7[] = { //Comparison ops
     "<",
     "<=",
     ">",
-    //">=",     //x.<y>= n  really wants to be processed as x.<y> = n, not x.<y >=
+    //">=", //x.<y>= n  really wants to be processed as x.<y> = n, not x.<y >= (this token has been added to the internalTokens list instead)
+
     "!=",
     "<>",
     "between",
@@ -196,6 +202,15 @@ static const char * eclReserved8[] = { //Binary ops
     NULL
 };
 
+static const char * eclReserved19[] = { //Misc ops
+    ":=",
+    "<?>",
+    "<\?\?>",
+    "..",
+    "=>",
+    NULL
+};
+
 static const char * eclReserved9[] = {//ECL fundamental types and associates
     "ascii",
     "big_endian",
@@ -261,6 +276,7 @@ static const char * eclReserved11[] = {//Activities
     "enth",
     "fetch",
     "fromxml",
+    "fromjson",
     "graph",
     "group",
     "having",
@@ -272,9 +288,11 @@ static const char * eclReserved11[] = {//Activities
     "map",
     "merge",
     "mergejoin",
+    "nocombine",
     "nonempty",
     "normalize",
     "parse",
+    "pattern",
     "process",
     "project",
     "quantile",
@@ -286,11 +304,13 @@ static const char * eclReserved11[] = {//Activities
     "rollup",
     "row",
     "sample",
+    "score",
     "sort",
     "stepped",
     "subsort",
     "table",
     "topn",
+    "trace",
     "ungroup",
     "which",
     "within",
@@ -318,6 +338,7 @@ static const char * eclReserved12[] = {//Attributes
     "best",
     "bitmap",
     "blob",
+    "c++",
     "choosen:all",
     "const",
     "counter",
@@ -398,12 +419,14 @@ static const char * eclReserved14[] = { //Attribute functions (some might actual
     "compressed",
     "default",
     "escape",
+    "format",
     "global",
     "groupby",
     "guard",
     "httpheader",
     "internal",
     "joined",
+    "json",
     "keep",
     "keyed",
     "limit",
@@ -425,6 +448,7 @@ static const char * eclReserved14[] = { //Attribute functions (some might actual
     "penalty",
     "prefetch",
     "proxyaddress",
+    "refresh",
     "repeat",
     "response",
     "retry",
@@ -446,6 +470,7 @@ static const char * eclReserved14[] = { //Attribute functions (some might actual
     "width",
     "wild",
     "xml",
+    "xmlns",
     "xmldefault",
     "xpath",
     "xmlproject",
@@ -455,6 +480,7 @@ static const char * eclReserved14[] = { //Attribute functions (some might actual
 };
 
 static const char * eclReserved15[] = { //Actions and statements
+    "action",
     "apply",
     "as",
     "build",
@@ -554,7 +580,8 @@ static const Keywords keywordList[] = {
    {15, eclReserved15},
    {16, eclReserved16},
    {17, eclReserved17},
-   {18, eclReserved18}
+   {18, eclReserved18},
+   {19, eclReserved19}
 };
 
 void printKeywordsToXml()
@@ -577,3 +604,64 @@ void printKeywordsToXml()
      buffer.append("</xml>");
      fprintf(stdout, "%s\n", buffer.str());
 }
+
+static const char * internalTokens[] = { //tokens present in the grammar that aren't in the language or are waiting to be deprecated
+    "",
+    ">=", //x.<y>= n  really wants to be processed as x.<y> = n, not x.<y >= (this token would otherwise belong to group7 - comparison ops)
+    "#elif",
+    "&&",
+    "constant",
+    "complex-macro",
+    "datarow",
+    "expression",
+    "feature-name",
+    "field list",
+    "field reference",
+    "function-name",
+    "hole",
+    "identifier",
+    "implements",
+    "inline",
+    "macro-name",
+    "module-name",
+    "number",
+    "of",
+    "or",
+    "order",
+    "pattern-name",
+    "physicalfilename",
+    "record-name",
+    "relationship",
+    "right2",
+    "skipped",
+    "swapped",
+    "transform-name",
+    "type name",
+    "type-name",
+    "unicode-string",
+    NULL
+};
+
+
+bool searchReservedWords(const char * tokenText)
+{
+    unsigned int nGroups = sizeof(keywordList)/sizeof(keywordList[0]);
+    for (unsigned i = 0; i < nGroups; ++i)
+    {
+        int j = 0;
+        while(keywordList[i].keywords[j])
+        {
+            if (strcmp(tokenText, keywordList[i].keywords[j]) == 0)
+                return true;
+            j++;
+        }
+    }
+    int i = 0;
+    while(internalTokens[i])
+    {
+        if (strcmp(tokenText, internalTokens[i]) == 0)
+            return true;
+        i++;
+    }
+    return false;
+}

ecl/eclcc/reservedwords.hpp

@@ -18,5 +18,6 @@
 #define RESERVEDWORDS_HPP
 
 void printKeywordsToXml();
+bool searchReservedWords(const char * tokenText);
 
 #endif