Merge pull request #6608 from AttilaVamos/HPCC-11905-fix

HPCC-11905 Regression test engine sensitive to parameter order

Reviewed-By: Richard Chapman <rchapman@hpccsystems.com>

testing/regress/README.rst

@@ -15,13 +15,18 @@ Command:
 Result:
 
 |
-|       usage: ecl-test [-h] [--version] [--config [CONFIG]]
-|                       [--loglevel [{info,debug}]] [--suiteDir [SUITEDIR]]
-|                       [--timeout [TIMEOUT]] [--keyDir [KEYDIR]]
+|       usage: ecl-test [-h] [--config [CONFIG]]
+|                       [--loglevel [{info,debug}]]
+|                       [--suiteDir [SUITEDIR]]
+|                       [--timeout [TIMEOUT]]
+|                       [--keyDir [KEYDIR]]
 |                       [--ignoreResult]
 |                       [-X name1=value1[,name2=value2...]]
 |                       [-f optionA=valueA[,optionB=valueB...]]
+|                       [--pq threadNumber]
 |                       [--dynamic source=cluster_list|all]
+|                       [--runclass class[,class,...]]
+|                       [--excludeclass class[,class,...]]
 |                       {list,setup,run,query} ...
 | 
 |       HPCC Platform Regression suite
@@ -35,7 +40,6 @@ Result:
 |
 |       optional arguments:
 |        -h, --help               show this help message and exit
-|        --version, -v            show program's version number and exit
 |        --config [CONFIG]        config file to use. Default: ecl-test.json.
 |        --loglevel [{info,debug}]
 |                                 set the log level. Use debug for more detailed logfile.
@@ -44,13 +48,19 @@ Result:
 |        --timeout [TIMEOUT]      timeout for query execution in sec. Use -1 to disable timeout. Default value defined in ecl-test.json config file (see: 9.)
 |        --keyDir [KEYDIR], -k [KEYDIR]
 |                                 key file directory to compare test output. Default value defined in regress.json config file.
-|        --ignoreResult, -i    completely ignore the result.
+|        --ignoreResult, -i       completely ignore the result.
 |        -X name1=value1[,name2=value2...]
 |                                 sets the stored input value (stored('name')).
 |        -f optionA=valueA[,optionB=valueB...]
 |                                 set an ECL option (equivalent to #option).
+|        --pq threadNumber        parallel query execution with threadNumber threads. (If threadNumber is '-1' on a single node system then threadNumber = numberOfLocalCore * 2)
 |        --dynamic source=cluster_list|all
 |                                 execute ECL query through a generated stub with source cluster(s).
+|        --runclass class[,class,...], -r class[,class,...]
+|                                 run subclass(es) of the suite. Default value is 'all'
+|        --excludeclass class[,class,...], -e class[,class,...]
+|                                 exclude subclass(es) of the suite. Default value is 'none'
+|
 
 Important!
     There is a bug in Python argparse library whichis impacts the quoted parameters. So either in -X or -f or both contains a value with space(s) inside then the whole argument should be put in double quote!
@@ -76,13 +86,17 @@ Command:
 Result:
 
 |
-|       usage: ecl-test list [-h]
+|       usage: ecl-test list [-h] [--config [CONFIG]]
+|                            [--loglevel [{info,debug}]]
 |
 |       positional arguments:
 |        targets                  print target clusters from config (ecl-test.json by default).
 |
 |       optional arguments:
 |        -h, --help               show this help message and exit
+|        --config [CONFIG]        config file to use. Default: ecl-test.json
+|        --loglevel [{info,debug}]
+|                                 set the log level. Use debug for more detailed logfile.
 |
 
 Parameters of Regression Suite setup sub-command:
@@ -95,14 +109,44 @@ Command:
 Result:
 
 |
-|       usage: ecl-test setup [-h] [--target [target_cluster_list | all]]
+|       usage: ecl-test setup [-h] [--config [CONFIG]]
+|                             [--loglevel [{info,debug}]]
+|                             [--suiteDir [SUITEDIR]]
+|                             [--timeout [TIMEOUT]]
+|                             [--keyDir [KEYDIR]]
+|                             [--ignoreResult]
+|                             [-X name1=value1[,name2=value2...]]
+|                             [-f optionA=valueA[,optionB=valueB...]]
 |                             [--pq threadNumber]
+|                             [--dynamic source=cluster_list|all]
+|                             [--runclass class[,class,...]]
+|                             [--excludeclass class[,class,...]]
+|                             [--target [target_cluster_list | all]]
 |
 |       optional arguments:
 |        -h, --help               show this help message and exit
+|        --config [CONFIG]        config file to use. Default: ecl-test.json.
+|        --loglevel [{info,debug}]
+|                                 set the log level. Use debug for more detailed logfile.
+|        --suiteDir [SUITEDIR], -s [SUITEDIR]
+|                                 suiteDir to use. Default value is the current directory and it can handle relative path.
+|        --timeout [TIMEOUT]      timeout for query execution in sec. Use -1 to disable timeout. Default value defined in ecl-test.json config file (see: 9.)
+|        --keyDir [KEYDIR], -k [KEYDIR]
+|                                 key file directory to compare test output. Default value defined in regress.json config file.
+|        --ignoreResult, -i       completely ignore the result.
+|        -X name1=value1[,name2=value2...]
+|                                 sets the stored input value (stored('name')).
+|        -f optionA=valueA[,optionB=valueB...]
+|                                 set an ECL option (equivalent to #option).
+|        --pq threadNumber        parallel query execution with threadNumber threads. (If threadNumber is '-1' on a single node system then threadNumber = numberOfLocalCore * 2)
+|        --dynamic source=cluster_list|all
+|                                 execute ECL query through a generated stub with source cluster(s).
+|        --runclass class[,class,...], -r class[,class,...]
+|                                 run subclass(es) of the suite. Default value is 'all'
+|        --excludeclass class[,class,...], -e class[,class,...]
+|                                 exclude subclass(es) of the suite. Default value is 'none'
 |        --target [target_cluster_list | all], -t [target_cluster_list | all]
 |                                 run the setup on target cluster(s). If target = 'all' then run setup on all clusters. If undefined the config 'defaultSetupClusters' value will be used.
-|        --pq threadNumber        parallel query execution with threadNumber threads. (If threadNumber is '-1' on a single node system then threadNumber = numberOfLocalCore * 2)
 |
 
 Parameters of Regression Suite run sub-command:
@@ -115,20 +159,47 @@ Command:
 Result:
 
 |
-|       usage: ecl-test run [-h] [--target [target_cluster_list | all]] [--publish]
+|       usage: ecl-test run [-h][--config [CONFIG]]
+|                           [--loglevel [{info,debug}]]
+|                           [--suiteDir [SUITEDIR]]
+|                           [--timeout [TIMEOUT]]
+|                           [--keyDir [KEYDIR]]
+|                           [--ignoreResult]
+|                           [-X name1=value1[,name2=value2...]]
+|                           [-f optionA=valueA[,optionB=valueB...]]
 |                           [--pq threadNumber]
-|                           [--runclass [class]] [--excludeclass [class]]
+|                           [--dynamic source=cluster_list|all]
+|                           [--runclass class[,class,...]]
+|                           [--excludeclass class[,class,...]]
+|                           [--target [target_cluster_list | all]]
+|                           [--publish]
 |
 |       optional arguments:
 |        -h, --help               show this help message and exit
-|        --target [target_cluster_list | all], -t [target_cluster_list | all]
-|                                 run the suite on target cluster(s). If target = 'all' then run suite on all clusters. If undefined the config 'defaultTargetClusters' value will be used.
-|        --publish, -p            publish compiled query instead of run.
+|        --config [CONFIG]        config file to use. Default: ecl-test.json.
+|        --loglevel [{info,debug}]
+|                                 set the log level. Use debug for more detailed logfile.
+|        --suiteDir [SUITEDIR], -s [SUITEDIR]
+|                                 suiteDir to use. Default value is the current directory and it can handle relative path.
+|        --timeout [TIMEOUT]      timeout for query execution in sec. Use -1 to disable timeout. Default value defined in ecl-test.json config file (see: 9.)
+|        --keyDir [KEYDIR], -k [KEYDIR]
+|                                 key file directory to compare test output. Default value defined in regress.json config file.
+|        --ignoreResult, -i       completely ignore the result.
+|        -X name1=value1[,name2=value2...]
+|                                 sets the stored input value (stored('name')).
+|        -f optionA=valueA[,optionB=valueB...]
+|                                 set an ECL option (equivalent to #option).
 |        --pq threadNumber        parallel query execution with threadNumber threads. (If threadNumber is '-1' on a single node system then threadNumber = numberOfLocalCore * 2)
+|        --dynamic source=cluster_list|all
+|                                 execute ECL query through a generated stub with source cluster(s).
 |        --runclass class[,class,...], -r class[,class,...]
 |                                 run subclass(es) of the suite. Default value is 'all'
 |        --excludeclass class[,class,...], -e class[,class,...]
 |                                 exclude subclass(es) of the suite. Default value is 'none'
+|        --target [target_cluster_list | all], -t [target_cluster_list | all]
+|                                 run the setup on target cluster(s). If target = 'all' then run setup on all clusters. If undefined the config 'defaultSetupClusters' value will be used.
+|        --publish, -p            publish compiled query instead of run.
+|
 
 
 Parameters of Regression Suite query sub-command:
@@ -141,8 +212,20 @@ Command:
 Result:
 
 |
-|       usage: ecl-test query [-h] [--target [target_cluster_list | all]] [--publish]
+|       usage: ecl-test query [-h] [--config [CONFIG]]
+|                             [--loglevel [{info,debug}]]
+|                             [--suiteDir [SUITEDIR]]
+|                             [--timeout [TIMEOUT]]
+|                             [--keyDir [KEYDIR]]
+|                             [--ignoreResult]
+|                             [-X name1=value1[,name2=value2...]]
+|                             [-f optionA=valueA[,optionB=valueB...]]
 |                             [--pq threadNumber]
+|                             [--dynamic source=cluster_list|all]
+|                             [--runclass class[,class,...]]
+|                             [--excludeclass class[,class,...]]
+|                             [--target [target_cluster_list | all]]
+|                             [--publish]
 |                             ECL_query [ECL_query ...]
 |
 |       positional arguments:
@@ -150,10 +233,29 @@ Result:
 |
 |       optional arguments:
 |        -h, --help               show this help message and exit
+|        --config [CONFIG]        config file to use. Default: ecl-test.json.
+|        --loglevel [{info,debug}]
+|                                 set the log level. Use debug for more detailed logfile.
+|        --suiteDir [SUITEDIR], -s [SUITEDIR]
+|                                 suiteDir to use. Default value is the current directory and it can handle relative path.
+|        --timeout [TIMEOUT]      timeout for query execution in sec. Use -1 to disable timeout. Default value defined in ecl-test.json config file (see: 9.)
+|        --keyDir [KEYDIR], -k [KEYDIR]
+|                                 key file directory to compare test output. Default value defined in regress.json config file.
+|        --ignoreResult, -i       completely ignore the result.
+|        -X name1=value1[,name2=value2...]
+|                                 sets the stored input value (stored('name')).
+|        -f optionA=valueA[,optionB=valueB...]
+|                                 set an ECL option (equivalent to #option).
+|        --pq threadNumber        parallel query execution with threadNumber threads. (If threadNumber is '-1' on a single node system then threadNumber = numberOfLocalCore * 2)
+|        --dynamic source=cluster_list|all
+|                                 execute ECL query through a generated stub with source cluster(s).
+|        --runclass class[,class,...], -r class[,class,...]
+|                                 run subclass(es) of the suite. Default value is 'all'
+|        --excludeclass class[,class,...], -e class[,class,...]
+|                                 exclude subclass(es) of the suite. Default value is 'none'
 |        --target [target_cluster_list | all], -t [target_cluster_list | all]
-|                                 run the query on target cluster(s). If target = 'all' then run query on all clusters. If undefined the config 'defaultTargetClusters' value will be used.
+|                                 run the setup on target cluster(s). If target = 'all' then run setup on all clusters. If undefined the config 'defaultSetupClusters' value will be used.
 |        --publish, -p            publish compiled query instead of run.
-|        --pq threadNumber        parallel query execution for multiple test cases specified in CLI with threadNumber threads. (If threadNumber is '-1' on a single node system then threadNumber = numberOfLocalCore * 2 )
 |
 
 Steps to run Regression Suite

testing/regress/ecl-test

@@ -147,55 +147,59 @@ class RegressMain:
         if (pythonVer['main'] >= 2) and (pythonVer['minor'] >= 7):
             atexit.register(logging.shutdown)
 
-        parser = argparse.ArgumentParser(prog=prog, description=description)
-        parser.add_argument('--config', help="config file to use. Default: ecl-test.json",
+        helperParser=argparse.ArgumentParser(add_help=False)
+        helperParser.add_argument('--config', help="config file to use. Default: ecl-test.json",
                             nargs='?', default="ecl-test.json")
-        parser.add_argument('--loglevel', help="set the log level. Use debug for more detailed logfile.",
+        helperParser.add_argument('--loglevel', help="set the log level. Use debug for more detailed logfile.",
                             nargs='?', default="info",
                             choices=['info', 'debug'])
-        parser.add_argument('--suiteDir', '-s', help="suiteDir to use. Default value is the current directory and it can handle relative path.",
+
+        commonParser=argparse.ArgumentParser(add_help=False)
+        commonParser.add_argument('--suiteDir', '-s', help="suiteDir to use. Default value is the current directory and it can handle relative path.",
                             nargs='?', default=".")
-        parser.add_argument('--timeout', help="timeout for query execution in sec. Use -1 to disable timeout. Default value defined in ecl-test.json config file.",
+        commonParser.add_argument('--timeout', help="timeout for query execution in sec. Use -1 to disable timeout. Default value defined in ecl-test.json config file.",
                             nargs='?', default="0")
-        parser.add_argument('--keyDir', '-k', help="key file directory to compare test output. Default value defined in ecl-test.json config file.",
+        commonParser.add_argument('--keyDir', '-k', help="key file directory to compare test output. Default value defined in ecl-test.json config file.",
                             nargs='?', default="ecl/key")
-        parser.add_argument('--ignoreResult', '-i', help="completely ignore the result.",
+        commonParser.add_argument('--ignoreResult', '-i', help="completely ignore the result.",
                             action='store_true')
-        parser.add_argument('-X', help="sets the stored input value (stored('name')).",
+        commonParser.add_argument('-X', help="sets the stored input value (stored('name')).",
                             nargs=1, type=checkXParam,  default='None',  metavar="name1=value1[,name2=value2...]")
-        parser.add_argument('-f', help="set an ECL option (equivalent to #option).",
+        commonParser.add_argument('-f', help="set an ECL option (equivalent to #option).",
                             nargs=1, type=checkXParam,  default='None',  metavar="optionA=valueA[,optionB=valueB...]")
-        parser.add_argument('--dynamic', help="execute ECL query through a generated stub with source cluster(s).",
+        commonParser.add_argument('--pq', help="Parallel query execution with threadNumber threads. (If threadNumber is '-1' on a single node system then threadNumber = numberOfLocalCore * 2 )",
+                                type=checkPqParam,  default = 0,   metavar="threadNumber")
+        commonParser.add_argument('--dynamic', help="execute ECL query through a generated stub with source cluster(s).",
                             nargs=1, type=checkXParam,  default='None',  metavar="source=cluster_list|all")
 
+        executionParser=argparse.ArgumentParser(add_help=False)
+        executionParser.add_argument('--runclass', '-r', help="run subclass(es) of the suite. Default value is 'all'",
+                                nargs=1,  default = ['all'],   metavar="class[,class,...]")
+        executionParser.add_argument('--excludeclass', '-e', help="exclude subclass(es) of the suite. Default value is 'none'",
+                                nargs=1,  default = ['none'],   metavar="class[,class,...]")
+
+        parser = argparse.ArgumentParser(prog=prog, description=description,  parents=[helperParser, commonParser,  executionParser])
+
         subparsers = parser.add_subparsers(help='sub-command help')
 
-        parser_list = subparsers.add_parser('list', help='list help')
+        parser_list = subparsers.add_parser('list', help='list help', parents=[helperParser])
         parser_list.set_defaults(func='list')
         parser_list.add_argument('targets', help="Print target clusters from config (ecl-test.json by default).",
                                  action='store_true')
 
-        parser_setup = subparsers.add_parser('setup', help='setup help')
+        parser_setup = subparsers.add_parser('setup', help='setup help',  parents=[helperParser, commonParser, executionParser])
         parser_setup.set_defaults(func='setup')
         parser_setup.add_argument('--target', '-t', help="Run the setup on target cluster(s). If target = 'all' then run setup on all clusters. If not defined then default value(s) come from config (ecl-test.json by default).",
                                 nargs='?', type=str,  default='', metavar="target_cluster_list | all")
-        parser_setup.add_argument('--pq', help="Parallel query execution with threadNumber threads. (If threadNumber is '-1' on a single node system then threadNumber = numberOfLocalCore * 2 )",
-                                type=checkPqParam,  default = 0,   metavar="threadNumber")
 
-        parser_run = subparsers.add_parser('run', help='run help')
+        parser_run = subparsers.add_parser('run', help='run help',  parents=[helperParser, commonParser, executionParser])
         parser_run.set_defaults(func='run')
         parser_run.add_argument('--target', '-t', help="Run the cluster(s) suite. If target = 'all' then run suite on all clusters. If not defined then default value(s) come from config (ecl-test.json by default).",
                                 nargs='?', type=str,  default='', metavar="target_cluster_list | all")
         parser_run.add_argument('--publish', '-p', help="Publish compiled query instead of run.",
                                 action='store_true')
-        parser_run.add_argument('--pq', help="Parallel query execution with threadNumber threads. (If threadNumber is '-1' on a single node system then threadNumber = numberOfLocalCore * 2 )",
-                                type=checkPqParam,  default = 0,   metavar="threadNumber")
-        parser_run.add_argument('--runclass', '-r', help="run subclass(es) of the suite. Default value is 'all'",
-                                nargs=1,  default = ['all'],   metavar="class[,class,...]")
-        parser_run.add_argument('--excludeclass', '-e', help="exclude subclass(es) of the suite. Default value is 'none'",
-                                nargs=1,  default = ['none'],   metavar="class[,class,...]")
 
-        parser_query = subparsers.add_parser('query', help='query help')
+        parser_query = subparsers.add_parser('query', help='query help',  parents=[helperParser, commonParser, executionParser])
         parser_query.set_defaults(func='query')
         parser_query.add_argument('query', help="One or more ECL file(s). It can contain wildcards. (mandatory).",
                                   nargs='+', metavar="ECL_query")
@@ -203,8 +207,6 @@ class RegressMain:
                                 nargs='?', default='', metavar="target_cluster_list | all")
         parser_query.add_argument('--publish', '-p', help="Publish compiled query instead of run.",
                                 action='store_true')
-        parser_query.add_argument('--pq', help="Parallel query execution with threadNumber threads. (If threadNumber is '-1' on a single node system then threadNumber = numberOfLocalCore * 2 )",
-                                type=checkPqParam,  default = 0,   metavar="threadNumber")
 
         self.args = parser.parse_args()
         try: