Merge pull request #8839 from jamienoss/issue15209-limit-compile-errors

HPCC-15209 Limit eclcc compile errors

Reviewed-by: Gavin Halliday <ghalliday@hpccsystems.com>

ecl/eclcc/eclcc.cpp

@@ -191,12 +191,8 @@ struct EclCompileInstance
 {
 public:
     EclCompileInstance(IFile * _inputFile, IErrorReceiver & _errorProcessor, FILE * _errout, const char * _outputFilename, bool _legacyImport, bool _legacyWhen) :
-      inputFile(_inputFile), errorProcessor(&_errorProcessor), errout(_errout), outputFilename(_outputFilename)
-    {
-        legacyImport = _legacyImport;
-        legacyWhen = _legacyWhen;
-        ignoreUnknownImport = false;
-        fromArchive = false;
+      inputFile(_inputFile), errorProcessor(&_errorProcessor), errout(_errout), outputFilename(_outputFilename), legacyImport(_legacyImport), legacyWhen(_legacyWhen)
+{
         stats.parseTime = 0;
         stats.generateTime = 0;
         stats.xmlSize = 0;
@@ -223,8 +219,8 @@ public:
     Owned<IPropertyTree> globalDependTree;
     bool legacyImport;
     bool legacyWhen;
-    bool fromArchive;
-    bool ignoreUnknownImport;
+    bool fromArchive = false;
+    bool ignoreUnknownImport = false;
     struct {
         unsigned parseTime;
         unsigned generateTime;
@@ -244,37 +240,6 @@ public:
     {
         argc = _argc;
         argv = _argv;
-        logVerbose = false;
-        logTimings = false;
-        optArchive = false;
-        optCheckEclVersion = true;
-        optEvaluateResult = false;
-        optGenerateMeta = false;
-        optGenerateDepend = false;
-        optIncludeMeta = false;
-        optKeywords = false;
-        optLegacyImport = false;
-        optLegacyWhen = false;
-        optShared = false;
-        optWorkUnit = false;
-        optNoCompile = false;
-        optNoLogFile = false;
-        optNoStdInc = false;
-        optNoBundles = false;
-        optOnlyCompile = false;
-        optBatchMode = false;
-        optSaveQueryText = false;
-        optSaveQueryArchive = false;
-        optGenerateHeader = false;
-        optShowPaths = false;
-        optNoSourcePath = false;
-        optTargetClusterType = RoxieCluster;
-        optTargetCompiler = DEFAULT_COMPILER;
-        optThreads = 0;
-        optLogDetail = 0;
-        batchPart = 0;
-        batchSplit = 1;
-        batchLog = NULL;
         cclogFilename.append("cc.").append((unsigned)GetCurrentProcessId()).append(".log");
         defaultAllowed[false] = true;  // May want to change that?
         defaultAllowed[true] = true;
@@ -344,7 +309,7 @@ protected:
     StringAttr optOutputFilename;
     StringAttr optQueryRepositoryReference;
     StringAttr optComponentName;
-    FILE * batchLog;
+    FILE * batchLog = nullptr;
 
     StringAttr optManifestFilename;
     StringArray resourceManifestFiles;
@@ -364,38 +329,40 @@ protected:
     StringArray deniedPermissions;
     bool defaultAllowed[2];
 
-    ClusterType optTargetClusterType;
-    CompilerType optTargetCompiler;
-    unsigned optThreads;
-    unsigned batchPart;
-    unsigned batchSplit;
-    unsigned optLogDetail;
-    bool logVerbose;
-    bool logTimings;
-    bool optArchive;
-    bool optCheckEclVersion;
+    ClusterType optTargetClusterType = RoxieCluster;
+    CompilerType optTargetCompiler = DEFAULT_COMPILER;
+    unsigned optThreads = 0;
+    unsigned batchPart = 0;
+    unsigned batchSplit = 1;
+    unsigned optLogDetail = 0;
+    unsigned optMaxErrors = 0;
+    bool optUnsuppressImmediateSyntaxErrors = false;
+    bool logVerbose = false;
+    bool logTimings = false;
+    bool optArchive = false;
+    bool optCheckEclVersion = true;
     bool optDebugMemLeak = false;
-    bool optEvaluateResult;
-    bool optGenerateMeta;
-    bool optGenerateDepend;
-    bool optIncludeMeta;
-    bool optKeywords;
+    bool optEvaluateResult = false;
+    bool optGenerateMeta = false;
+    bool optGenerateDepend = false;
+    bool optIncludeMeta = false;
+    bool optKeywords = false;
     bool optLeakCheck = false;
-    bool optWorkUnit;
-    bool optNoCompile;
-    bool optNoLogFile;
-    bool optNoStdInc;
-    bool optNoBundles;
-    bool optBatchMode;
-    bool optShared;
-    bool optOnlyCompile;
-    bool optSaveQueryText;
-    bool optSaveQueryArchive;
-    bool optLegacyImport;
-    bool optLegacyWhen;
-    bool optGenerateHeader;
-    bool optShowPaths;
-    bool optNoSourcePath;
+    bool optWorkUnit = false;
+    bool optNoCompile = false;
+    bool optNoLogFile = false;
+    bool optNoStdInc = false;
+    bool optNoBundles = false;
+    bool optBatchMode = false;
+    bool optShared = false;
+    bool optOnlyCompile = false;
+    bool optSaveQueryText = false;
+    bool optSaveQueryArchive = false;
+    bool optLegacyImport = false;
+    bool optLegacyWhen = false;
+    bool optGenerateHeader = false;
+    bool optShowPaths = false;
+    bool optNoSourcePath = false;
     int argc;
     const char **argv;
 };
@@ -488,7 +455,7 @@ int main(int argc, const char *argv[])
     if (!optReleaseAllMemory)
     {
         //In release mode exit without calling all the clean up code.
-        //It is faster, and it helps avoids potential crashes if there are active objects which depend on objects in file hook dlls.
+        //It is faster, and it helps avoid potential crashes if there are active objects which depend on objects in file hook dlls.
         fflush(NULL);
         _exit(exitCode);
     }
@@ -1132,6 +1099,9 @@ void EclCC::processSingleQuery(EclCompileInstance & instance,
     {
         //Minimize the scope of the parse context to reduce lifetime of cached items.
         HqlParseContext parseCtx(instance.dataServer, this, instance.archive);
+        if (optMaxErrors > 0)
+            parseCtx.maxErrors = optMaxErrors;
+        parseCtx.unsuppressImmediateSyntaxErrors = optUnsuppressImmediateSyntaxErrors;
         if (!instance.archive)
             parseCtx.globalDependTree.setown(createPTree(ipt_none)); //to locate associated manifests, keep separate from user specified MetaOptions
         if (optGenerateMeta || optIncludeMeta)
@@ -2112,6 +2082,12 @@ int EclCC::parseCommandLineOptions(int argc, const char* argv[])
         {
             setDebugOption("syntaxCheck", tempBool);
         }
+        else if (iter.matchOption(optMaxErrors, "--maxErrors"))
+        {
+        }
+        else if (iter.matchFlag(optUnsuppressImmediateSyntaxErrors, "--unsuppressImmediateSyntaxErrors"))
+        {
+        }
         else if (iter.matchOption(optIniFilename, "-specs"))
         {
             if (!checkFileExists(optIniFilename))

ecl/eclcc/eclcc.hpp

@@ -93,6 +93,7 @@ const char * const helpText[] = {
     "!   -m            Enable leak checking",
 #endif
     "    --nosourcepath Compile as if the source came from stdin",
+    "!   --maxErrors=<n> Limit the number of errors, aborting on the nth (default = 5)",
 #ifndef _WIN32
     "!   -pch          Generate precompiled header for eclinclude4.hpp",
 #endif

ecl/hql/hqlerror.hpp

@@ -21,6 +21,7 @@
 #include "jexcept.hpp"
 #include "hql.hpp"
 
+#define DEFAULT_MAX_ERRORS 5
 #define HQLERR_ErrorAlreadyReported             4799            // special case...
 
 interface IWorkUnit;

ecl/hql/hqlexpr.hpp

@@ -899,6 +899,8 @@ public:
     IErrorArray orphanedWarnings;
     HqlExprArray defaultFunctionCache;
     CIArrayOf<ForwardScopeItem> forwardLinks;
+    unsigned maxErrors = DEFAULT_MAX_ERRORS;
+    bool unsuppressImmediateSyntaxErrors = false;
     bool expandCallsWhenBound;
     bool ignoreUnknownImport;
     bool aborting;

ecl/hql/hqlgram.hpp

@@ -31,7 +31,6 @@
 
 #include "hqlxmldb.hpp"
 
-#define DEFAULT_MAX_ERRORS 100
 #define EXPORT_FLAG 1
 #define VIRTUAL_FLAG 2
 #define SHARED_FLAG 4
@@ -411,7 +410,6 @@ public:
     IHqlScope * queryGlobalScope();
 
     bool canFollowCurrentState(int tok, const short * yyps);
-    void syntaxError(const char *s, int token, int *expected);
     int mapToken(int lexToken) const;
     IHqlExpression *lookupSymbol(IIdAtom * name, const attribute& errpos);
     IHqlExpression *lookupSymbol(IHqlScope * scope, IIdAtom * name);
@@ -578,6 +576,18 @@ public:
     IHqlExpression * nextEnumValue();
 
 // Error handling
+    void syntaxError(const char *s, int token, int *expected);
+    bool checkErrorCountAndAbort();
+    bool exceedsMaxCompileErrors();
+    unsigned getMaxCompileErrors()
+    {
+        return lookupCtx.queryParseContext().maxErrors;
+    }
+    bool unsuppressImmediateSyntaxErrors()
+    {
+        return lookupCtx.queryParseContext().unsuppressImmediateSyntaxErrors;
+    }
+    void reportTooManyErrors();
     void doReportWarning(WarnErrorCategory category, int warnNo, const char *msg, const char *filename, int lineno, int column, int pos);
     void reportError(int errNo, const attribute& a, const char* format, ...) __attribute__((format(printf, 4, 5)));
     void reportError(int errNo, const ECLlocation & pos, const char* format, ...) __attribute__((format(printf, 4, 5)));
@@ -691,8 +701,6 @@ public:
     IHqlExpression * processIfProduction(attribute & condAttr, attribute & trueAttr, attribute * falseAttr);
 
     IHqlExpression * createSymbolFromValue(IHqlExpression * primaryExpr, IHqlExpression * value);
-    unsigned getMaxErrorsAllowed() { return m_maxErrorsAllowed; }
-    void setMaxErrorsAllowed(unsigned n) { m_maxErrorsAllowed = n; } 
     void setAssociateWarnings(bool value) { associateWarnings = value; }
     IHqlExpression* clearFieldMap(IHqlExpression* expr);
     void setExpectedAttribute(IIdAtom * _expectedAttribute)             { expectedAttribute = _expectedAttribute; current_id = _expectedAttribute; }
@@ -753,7 +761,6 @@ protected:
 
     void canNotAssignTypeError(ITypeInfo* expected, ITypeInfo* given, const attribute& errpos);
     void canNotAssignTypeWarn(ITypeInfo* expected, ITypeInfo* given, const attribute& errpos);
-    void abortParsing();
     bool isExceptionalCase(attribute& defineid, attribute& object, attribute& failure);
     void checkSvcAttrNoValue(IHqlExpression* attr, const attribute& errpos);
     void checkFormals(IIdAtom * name, HqlExprArray & parms, HqlExprArray & defaults, attribute& object);
@@ -776,7 +783,17 @@ protected:
 
     void disableError() { errorDisabled = true; }
     void enableError() { errorDisabled = false; }
-    bool isAborting() { return errorDisabled; }
+    void abortParsing();
+    bool checkAborting()
+    {
+        if (lookupCtx.isAborting())
+        {
+            abortParsing();//Ensure a consistent abort by propagating the aborting state.
+            return true;
+        }
+        return false;
+    }
+
     IIdAtom * fieldMapTo(IHqlExpression* expr, IIdAtom * name);
     IIdAtom * fieldMapFrom(IHqlExpression* expr, IIdAtom * name);
     bool requireLateBind(IHqlExpression* funcdef, const HqlExprArray & actuals);
@@ -854,7 +871,6 @@ protected:
     bool isQuery;
     bool parseConstantText;
     bool expandingMacroPosition;
-    unsigned m_maxErrorsAllowed;
     bool inSignedModule;
 
     IErrorArray pendingWarnings;
@@ -911,7 +927,6 @@ protected:
     ConstPointerArray validAttributesStack;
     unsigned minimumScopeIndex;
     const TokenMap * pendingAttributes;
-    bool aborting;
 
     void setIdUnknown(bool expected) { expectedUnknownId = expected; }
     bool getIdUnknown() { return expectedUnknownId; }
@@ -1138,7 +1153,7 @@ class HqlLex
         inline ISourcePath * querySourcePath() const { return sourcePath; }
 
         bool isMacroActive(IHqlExpression *expr);
-        bool isAborting();
+        bool checkAborting();
         void pushMacro(IHqlExpression *expr);
         void pushText(IFileContents * text, int startLineNo, int startColumn);
         void pushText(const char *s, int startLineNo, int startColumn);

ecl/hql/hqlgram.y

@@ -92,7 +92,7 @@ inline int eclyylex(attribute * yylval, HqlGram* parser, const short int * yyssp
 
 
 static void eclsyntaxerror(HqlGram * parser, const char * s, short yystate, int token);
-#define eclyyerror(parser, s)       eclsyntaxerror(parser, s, yystate, yychar)
+#define eclyyerror(parser, s) eclsyntaxerror(parser, s, yystate, yychar); if (parser->unsuppressImmediateSyntaxErrors()) yyerrok; if (parser->checkErrorCountAndAbort()) YYABORT;
 #define ignoreBisonWarning(x)
 #define ignoreBisonWarnings2(x,y)
 #define ignoreBisonWarnings3(x,y,z)
@@ -105,9 +105,9 @@ static void eclsyntaxerror(HqlGram * parser, const char * s, short yystate, int
 /* remember to add any new tokens to the error reporter and lexer too! */
 /* If they clash with other #defines etc then use TOK_ as a prefix */
 
-// NB: Very occassionally the same keyword in the source (e.g., MERGE, PARTITION may return a different token
+// NB: Very occasionally the same keyword in the source (e.g., MERGE, PARTITION may return a different token
 // (MERGE_ATTR, PARTITION_ATTR) depending on the context it is used in.  This is because there would be a s/r
-// error, so the _ATTR form is only allowed in the situations where the attibute is valid - enabled by a
+// error, so the _ATTR form is only allowed in the situations where the attribute is valid - enabled by a
 // call to enableAttributes() from a production in the grammar.
 
   ABS

ecl/hql/hqlgram2.cpp

@@ -430,7 +430,6 @@ void HqlGram::init(IHqlScope * _globalScope, IHqlScope * _containerScope)
 
     outerScopeAccessDepth = 0;
     inType = false;
-    aborting = lookupCtx.isAborting();
     errorHandler = NULL;
     moduleName = NULL;
     resolveSymbols = true;
@@ -455,9 +454,8 @@ void HqlGram::init(IHqlScope * _globalScope, IHqlScope * _containerScope)
     fieldMapUsed = false;
     associateWarnings = true;
 
-    errorDisabled = false;
+    errorDisabled = lookupCtx.isAborting();//NOTE: If in an aborting state, disable future error reports.
     setIdUnknown(false);
-    m_maxErrorsAllowed = DEFAULT_MAX_ERRORS;
     sortDepth = 0;
     serviceScope.clear();
 
@@ -484,7 +482,7 @@ HqlGram::~HqlGram()
 
 int HqlGram::yyLex(attribute * yylval, const short * activeState)
 {
-    if (aborting) return 0;
+    if (checkAborting()) return 0;
     return lexObject->yyLex(*yylval, resolveSymbols, activeState);
 }
 
@@ -3065,7 +3063,7 @@ void HqlGram::processForwardModuleDefinition(const attribute & errpos)
             reportError(ERR_EXPECTED, errpos, "Missing END in FORWARD module definition");
             abortParsing();
             return;
-        case COMPLEX_MACRO: 
+        case COMPLEX_MACRO:
         case MACRO:
         case SIMPLE_TYPE:
         case CPPBODY:
@@ -3391,8 +3389,7 @@ IHqlExpression *HqlGram::lookupSymbol(IIdAtom * searchName, const attribute& err
         return NULL;
 
     //Check periodically if parsing a referenced identifier has caused the compile to abort.
-    if (lookupCtx.isAborting())
-        aborting = true;
+    checkAborting();//NOTE: checkAborting() checks whether the parseContext is aborting and implicitly propagates this state, thus consistently triggering the parser to abort.
 
     try
     {
@@ -3556,7 +3553,7 @@ IHqlExpression *HqlGram::lookupSymbol(IIdAtom * searchName, const attribute& err
     }
     catch(IError* error)
     {
-        if(errorHandler && !errorDisabled)
+        if(errorHandler && !errorDisabled && !checkAborting())
             errorHandler->report(error);
         error->Release();
         // recover: to avoid reload the definition again and again
@@ -6029,7 +6026,7 @@ void HqlGram::exportMappings(IWorkUnit * wu) const
 
 void HqlGram::report(IError* error)
 {
-    if (errorHandler && !errorDisabled)
+    if (errorHandler && !errorDisabled && !checkAborting())
     {
         //Severity of warnings are not mapped here.  Fatal errors are reported directly.  Others are delayed
         //(and may possibly be disabled by local onWarnings etc.)
@@ -6047,12 +6044,7 @@ void HqlGram::report(IError* error)
         else
         {
             errorHandler->report(error);
-
-            if (getMaxErrorsAllowed()>0 && errorHandler->errCount() >= getMaxErrorsAllowed())
-            {
-                errorHandler->reportError(ERR_ERROR_TOOMANY,"Too many errors; parsing aborted",error->getFilename(),error->getLine(),error->getColumn(),error->getPosition());
-                abortParsing();
-            }
+            checkErrorCountAndAbort();
         }
 
         reportMacroExpansionPosition(error, lexObject);
@@ -11050,9 +11042,39 @@ void HqlGram::simplifyExpected(int *expected)
     simplify(expected, END, '}', 0);
 }
 
+bool HqlGram::exceedsMaxCompileErrors()
+{
+    unsigned errorCount = errorHandler ? errorHandler->errCount() : 0;
+    return errorCount >= getMaxCompileErrors();
+}
+
+bool HqlGram::checkErrorCountAndAbort()
+{
+    if (checkAborting())
+        return true;
+
+    if (exceedsMaxCompileErrors())
+    {
+        reportTooManyErrors();
+        abortParsing();
+        return true;
+    }
+    return false;
+}
+
+void HqlGram::reportTooManyErrors()
+{
+    if (errorHandler && !errorDisabled && !checkAborting())
+    {
+        StringBuffer msg("Too many errors");
+        msg.append(" (max = ").append(getMaxCompileErrors()).append("); Aborting...");
+        errorHandler->reportError(ERR_ERROR_TOOMANY, msg.str(), str(lexObject->queryActualSourcePath()), lexObject->getActualLineNo(), lexObject->getActualColumn(), lexObject->get_yyPosition());
+    }
+}
+
 void HqlGram::syntaxError(const char *s, int token, int *expected)
-{ 
-    if (errorDisabled || !s || !errorHandler)
+{
+    if (errorDisabled || !s || !errorHandler || checkAborting())
         return;
 
     int lineno = lexObject->getActualLineNo();
@@ -11165,10 +11187,8 @@ void HqlGram::syntaxError(const char *s, int token, int *expected)
 
 void HqlGram::abortParsing()
 {
-    // disable more error report
-    disableError();
     lookupCtx.setAborting();
-    aborting = true;
+    disableError();//NOTE: aborting the parser may not be immediate, disable future errors to aid the illusion of aborting.
 }
 
 IHqlExpression * HqlGram::createCheckMatchAttr(attribute & attr, type_t tc)

ecl/hql/hqllex.l

@@ -1715,7 +1715,7 @@ FUNCTIONMACRO|MACRO {
                                     StringBuffer msg("Can not terminate a string with escape char '\\': ");
                                     msg.append(CUR_TOKEN_TEXT);
                                     lexer->reportError(returnToken, RRR_ESCAPE_ENDWITHSLASH, "%s", msg.str());
-                                    if (lexer->isAborting())
+                                    if (lexer->checkAborting())
                                         return EOF;
                                 }
                                 else if (next == '\'' || next == '\\' || next == '?' || next == '"') 
@@ -1775,7 +1775,7 @@ FUNCTIONMACRO|MACRO {
                                         StringBuffer msg;
                                         msg.append("3-digit numeric escape sequence contained non-octal digit: ").append(next);
                                         lexer->reportError(returnToken, ERR_ESCAPE_UNKNOWN, "%s", msg.str());
-                                        if (lexer->isAborting())
+                                        if (lexer->checkAborting())
                                             return EOF;
                                     }
                                     *bf++ = value;
@@ -1783,7 +1783,7 @@ FUNCTIONMACRO|MACRO {
                                     {
                                         returnToken.setPosition(lexer->yyLineNo, oldColumn+delta, oldPosition+delta, lexer->querySourcePath());
                                         lexer->reportWarning(CategoryCast, returnToken, ERR_STRING_NON_ASCII, "Character in string literal is not defined in encoding " ASCII_LIKE_CODEPAGE);
-                                        if (lexer->isAborting())
+                                        if (lexer->checkAborting())
                                             return EOF;
                                     }
                                     finger += count;
@@ -1796,7 +1796,7 @@ FUNCTIONMACRO|MACRO {
                                     msg.append("\\").append(finger[1]);
                                     returnToken.setPosition(lexer->yyLineNo, oldColumn+delta, oldPosition+delta, lexer->querySourcePath());
                                     lexer->reportError(returnToken, ERR_ESCAPE_UNKNOWN, "%s", msg.str());
-                                    if (lexer->isAborting())
+                                    if (lexer->checkAborting())
                                         return EOF;
                                 }
                                 *bf++ = next;
@@ -1805,7 +1805,7 @@ FUNCTIONMACRO|MACRO {
                             {
                                 returnToken.setPosition(lexer->yyLineNo, oldColumn+delta, oldPosition+delta, lexer->querySourcePath());
                                 lexer->reportError(returnToken, ERR_STRING_NEEDESCAPE,"' needs to be escaped by \\ inside string");
-                                if (lexer->isAborting())
+                                if (lexer->checkAborting())
                                     return EOF;
                             }
                             else if (next >= 128)
@@ -1819,7 +1819,7 @@ FUNCTIONMACRO|MACRO {
                                     //because two characters > 128 are next to each other.
                                     returnToken.setPosition(lexer->yyLineNo, oldColumn+delta, oldPosition+delta, lexer->querySourcePath());
                                     lexer->reportError(returnToken, ERR_STRING_NON_ASCII, "Character in string literal is not legal UTF-8");
-                                    if (lexer->isAborting())
+                                    if (lexer->checkAborting())
                                         return EOF;
                                     *bf = next;
                                 }
@@ -1841,7 +1841,7 @@ FUNCTIONMACRO|MACRO {
                                 {
                                     returnToken.setPosition(lexer->yyLineNo, oldColumn+delta, oldPosition+delta, lexer->querySourcePath());
                                     lexer->reportError(returnToken, ERR_STRING_NON_ASCII, "Character in string literal is not defined in encoding " ASCII_LIKE_CODEPAGE);
-                                    if (lexer->isAborting())
+                                    if (lexer->checkAborting())
                                         return EOF;
                                 }
                             }

ecl/hql/hqlparse.cpp

@@ -1535,7 +1535,7 @@ void HqlLex::doIsValid(YYSTYPE & returnToken)
 
 void HqlLex::checkNextLoop(const YYSTYPE & errpos, bool first, int startLine, int startCol)
 {
-    if (yyParser->aborting)
+    if (yyParser->checkAborting())
         return;
     if (loopTimes++ > MAX_LOOP_TIMES)
     {
@@ -2003,9 +2003,9 @@ bool HqlLex::checkUnicodeLiteral(char const * str, unsigned length, unsigned & e
 
 //====================================== Error Reporting  ======================================
 
-bool HqlLex::isAborting()
+bool HqlLex::checkAborting()
 {
-    return yyParser->isAborting();
+    return yyParser->checkAborting();
 }
 
 void HqlLex::reportError(const YYSTYPE & returnToken, int errNo, const char *format, ...)