HPCC-10366 Add parallel query support to regression suite

Update after review.

Signed-off-by: Attila Vamos <attila.vamos@gmail.com>

testing/regress/README.rst

@@ -98,8 +98,15 @@ The result:
 -------------------------------------------------------------
 Command:
 
-        ./regress run thor
+        ./regress run cluster [-h] [--pq threadNumber]
 
+Positional arguments:
+  cluster            Run the cluster suite (default: setup).
+
+Optional arguments:
+  -h, --help         show help message and exit
+  --pq threadNumber  Parallel query execution with threadNumber threads.
+                    ('-1' can be use to calculate usable thread count on a single node system)
 
 The result is a list of test cases and their result. 
 
@@ -107,34 +114,125 @@ The first and last couple of lines look like this:
 
 |
 |        [Action] Suite: thor
-|        [Action] Queries: 253
-|        [Action] 
-|        [Action] 1. Test: groupglobal3b.ecl
-|        [Pass] Pass W20131029-165658
-|        [Pass] URL http://127.0.0.1:8010/WsWorkunits/WUInfo?Wuid=W20131029-165658
-|        [Action] 2. Test: realround.ecl
-|        [Pass] Pass W20131029-165701
-|        [Pass] URL http://127.0.0.1:8010/WsWorkunits/WUInfo?Wuid=W20131029-165701
-|        [Action] 3. Test: patmin.ecl
+|        [Action] Queries: 257
+|        [Action]
+|        [Action]   1. Test: agglist.ecl
+|        [Pass]   1. Pass W20131119-173524 (2 sec)
+|        [Pass]   1. URL http://127.0.0.1:8010/WsWorkunits/WUInfo?Wuid=W20131119-173524
+|        [Action]   2. Test: aggregate.ecl
+|        [Pass]   2. Pass W20131119-173527 (1 sec)
+|        [Pass]   2. URL http://127.0.0.1:8010/WsWorkunits/WUInfo?Wuid=W20131119-173527
+|        [Action]   3. Test: aggsq1.ecl
+
 |        .
 |        .
 |        .
-|        [Action] 252. Test: ds_map.ecl
-|        [Pass] Pass W20131029-171831
-|        [Pass] URL http://127.0.0.1:8010/WsWorkunits/WUInfo?Wuid=W20131029-171831
-|        [Action] 253. Test: lookupjoin.ecl
-|        [Pass] Pass W20131029-171833
-|        [Pass] URL http://127.0.0.1:8010/WsWorkunits/WUInfo?Wuid=W20131029-171833
+|        [Action] 256. Test: xmlout2.ecl
+|        [Pass] Pass W20131119-182536 (1 sec)
+|        [Pass] URL http://127.0.0.1:8010/WsWorkunits/WUInfo?Wuid=W20131119-182536
+|        [Action] 257. Test: xmlparse.ecl
+|        [Pass] Pass W20131119-182537 (1 sec)
+|        [Pass] URL http://127.0.0.1:8010/WsWorkunits/WUInfo?Wuid=W20131119-182537
 |        
 |         Results
 |         `-------------------------------------------------`
-|         Passing: 253
+|         Passing: 257
 |         Failure: 0
 |         `-------------------------------------------------`
-|         Log: /var/log/HPCCSystems/regression/thor.13-10-29-16-56.log
+|         Log: /home/ati/HPCCSystems-regression/log/thor.13-11-19-17-52-27.log
+|         `-------------------------------------------------`
+|         Elapsed time: 1992 sec  (00:33:12)
 |         `-------------------------------------------------`
 |
 
+If --pq option used (in this case with 16 threads) then then the content of the console log will be different like this:
+
+|
+|        [Action] Suite: thor
+|        [Action] Queries: 257
+|        [Action]
+|        [Action]   1. Test: agglist.ecl
+|        [Action]   2. Test: aggregate.ecl
+|        [Action]   3. Test: aggsq1.ecl
+|        [Action]   4. Test: aggsq1seq.ecl
+|        [Action]   5. Test: aggsq2.ecl
+|        [Action]   6. Test: aggsq2seq.ecl
+|        [Action]   7. Test: aggsq4.ecl
+|        [Action]   8. Test: aggsq4seq.ecl
+|        [Action]   9. Test: alljoin.ecl
+|        [Action]  10. Test: apply3.ecl
+|        [Action]  11. Test: atmost2.ecl
+|        [Action]  12. Test: bcd1.ecl
+|        [Action]  13. Test: bcd2.ecl
+|        [Action]  14. Test: bcd4.ecl
+|        [Action]  15. Test: betweenjoin.ecl
+|        [Action]  16. Test: bigrecs.ecl
+|        [Pass]   2. Pass W20131119-150514 (4 sec)
+|        [Pass]   2. URL http://127.0.0.1:8010/WsWorkunits/WUInfo?Wuid=W20131119-150514
+|        [Pass]   1. Pass W20131119-150513 (4 sec)
+|        [Pass]   1. URL http://127.0.0.1:8010/WsWorkunits/WUInfo?Wuid=W20131119-150513
+|        [Action]  17. Test: bloom2.ecl
+|        [Action]  18. Test: bug8688.ecl
+|        [Pass]   3. Pass W20131119-150514-5 (5 sec)
+|        [Pass]   3. URL http://127.0.0.1:8010/WsWorkunits/WUInfo?Wuid=W20131119-150514-5
+|        [Action]  19. Test: builtin.ecl
+|        [Pass]  12. Pass W20131119-150517 (5 sec)
+|        [Pass]  12. URL http://127.0.0.1:8010/WsWorkunits/WUInfo?Wuid=W20131119-150517
+|        [Action]  20. Test: casts.ecl
+|        [Pass]  14. Pass W20131119-150517-2 (6 sec)
+|        [Pass]  14. URL http://127.0.0.1:8010/WsWorkunits/WUInfo?Wuid=W20131119-150517-2
+|        [Action]  21. Test: catchexpr.ecl
+|        .
+|        .
+|        .
+|        [Action] 257. Test: xmlparse.ecl
+|        [Pass] 240. Pass W20131119-160614 (9 sec)
+|        [Pass] 240. URL http://127.0.0.1:8010/WsWorkunits/WUInfo?Wuid=W20131119-160614
+|        [Pass] 241. Pass W20131119-160614-3 (10 sec)
+|        [Pass] 241. URL http://127.0.0.1:8010/WsWorkunits/WUInfo?Wuid=W20131119-160614-3
+|        [Pass] 254. Pass W20131119-160622-1 (2 sec)
+|        [Pass] 254. URL http://127.0.0.1:8010/WsWorkunits/WUInfo?Wuid=W20131119-160622-1
+|        [Pass] 191. Pass W20131119-160058-2 (327 sec)
+|        [Pass] 191. URL http://127.0.0.1:8010/WsWorkunits/WUInfo?Wuid=W20131119-160058-2
+|        [Pass] 245. Pass W20131119-160617-3 (9 sec)
+|        [Pass] 245. URL http://127.0.0.1:8010/WsWorkunits/WUInfo?Wuid=W20131119-160617-3
+|        [Pass] 248. Pass W20131119-160619-4 (7 sec)
+|        [Pass] 248. URL http://127.0.0.1:8010/WsWorkunits/WUInfo?Wuid=W20131119-160619-4
+|        [Pass] 249. Pass W20131119-160619-3 (9 sec)
+|        [Pass] 249. URL http://127.0.0.1:8010/WsWorkunits/WUInfo?Wuid=W20131119-160619-3
+|        [Pass] 250. Pass W20131119-160620 (10 sec)
+|        [Pass] 250. URL http://127.0.0.1:8010/WsWorkunits/WUInfo?Wuid=W20131119-160620
+|        [Pass] 252. Pass W20131119-160620-3 (10 sec)
+|        [Pass] 252. URL http://127.0.0.1:8010/WsWorkunits/WUInfo?Wuid=W20131119-160620-3
+|        [Pass] 253. Pass W20131119-160622 (8 sec)
+|        [Pass] 253. URL http://127.0.0.1:8010/WsWorkunits/WUInfo?Wuid=W20131119-160622
+|        [Pass] 255. Pass W20131119-160623 (8 sec)
+|        [Pass] 255. URL http://127.0.0.1:8010/WsWorkunits/WUInfo?Wuid=W20131119-160623
+|        [Pass] 256. Pass W20131119-160623-1 (9 sec)
+|        [Pass] 256. URL http://127.0.0.1:8010/WsWorkunits/WUInfo?Wuid=W20131119-160623-1
+|        [Pass] 257. Pass W20131119-160624 (9 sec)
+|        [Pass] 257. URL http://127.0.0.1:8010/WsWorkunits/WUInfo?Wuid=W20131119-160624
+|        [Pass] 213. Pass W20131119-160138-4 (305 sec)
+|        [Pass] 213. URL http://127.0.0.1:8010/WsWorkunits/WUInfo?Wuid=W20131119-160138-4
+|        [Pass] 127. Pass W20131119-155918 (462 sec)
+|        [Pass] 127. URL http://127.0.0.1:8010/WsWorkunits/WUInfo?Wuid=W20131119-155918
+|        [Pass] 100. Pass W20131119-155713 (600 sec)
+|        [Pass] 100. URL http://127.0.0.1:8010/WsWorkunits/WUInfo?Wuid=W20131119-155713
+|        [Action]
+|        [Action]
+|         Results
+|         `-------------------------------------------------`
+|         Passing: 257
+|         Failure: 0
+|         `-------------------------------------------------`
+|         Log: /home/ati/HPCCSystems-regression/log/thor.13-11-19-15-55-32.log
+|         `-------------------------------------------------`
+|         Elapsed time: 701 sec  (00:11:41)
+|         `-------------------------------------------------`
+|
+
+The logfile generated into the HPCCSystems-regression/log subfolder of the user personal folder and sorted by the test case number.
+
 
 5. To run Regression Suite with selected test case on a selected cluster (e.g. Thor): 
 -------------------------------------------------------------------------------------
@@ -143,7 +241,15 @@ The first and last couple of lines look like this:
 
 Command:
 
-        ./regress query test_name [cluster]
+        ./regress query [-h] [--publish] test_name [target cluster | all]
+
+Positional arguments:
+        test_name               Name of a single ECL query (mandatory).
+        target cluster | all    Cluster for single query run (default: thor).
+                                If cluster = 'all' then run ECL query on all clusters.
+Optional arguments:
+        -h, --help            Show help message and exit
+        --publish             Publish compiled query instead of run.
 
 
 The format of result is same as above:

testing/regress/hpcc/regression/regress.py

@@ -34,7 +34,7 @@ from ..util.expandcheck import ExpandCheck
 
 
 class Regression:
-    def __init__(self, config="regress.json", level='info', suiteDir=None):
+    def __init__(self, config="regress.json", level='info', suiteDir=None,  numOfThreads=1):
         self.config = Config(config).configObj
         self.suites = {}
         self.log = Logger(level)
@@ -63,14 +63,15 @@ class Regression:
         logging.debug("Archive Dir    : %s", self.dir_a)
 
         self.loggermutex = thread.allocate_lock()
-        self.numOfCpus = 1
-
-        if 'linux' in sys.platform :
-            command = 'grep cores /proc/cpuinfo | sort -u'
-            cpuInfo = os.popen(command).read()
-            self.numOfCpus = int(cpuInfo.split()[3])
-
-        self.maxthreads = self.numOfCpus * 2
+        if numOfThreads == -1:
+            self.numOfCpus = 1
+            if 'linux' in sys.platform :
+                command = 'grep cores /proc/cpuinfo | sort -u'
+                cpuInfo = os.popen(command).read()
+                self.numOfCpus = int(cpuInfo.split()[3])
+            numOfThreads = self.numOfCpus  * 2
+
+        self.maxthreads = numOfThreads
         self.maxtasks = 0
         self.exitmutexes = [thread.allocate_lock() for i in range(self.maxthreads)]
 

testing/regress/hpcc/util/expandcheck.py

@@ -1,7 +1,25 @@
+'''
+/*#############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################ */
+'''
+
 import os
 import logging
 
-
 class ExpandCheck:
 
     @staticmethod

testing/regress/hpcc/util/util.py

@@ -0,0 +1,36 @@
+'''
+/*#############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################ */
+'''
+
+import argparse
+
+def isPositiveIntNum(string):
+    for i in range(0,  len(string)):
+        if (string[i] < '0') or (string[i] > '9'):
+            return False
+    return True
+
+def checkPqParam(string):
+    param = str(string)
+    if isPositiveIntNum(string) or (string == '-1'):
+        value = int(string)
+    else:
+        msg = "Wrong value of threadNumber parameter: '"+string+"' !"
+        raise argparse.ArgumentTypeError(msg)
+
+    return value

testing/regress/regress

@@ -26,6 +26,7 @@ import atexit
 
 from hpcc.regression.regress import Regression
 from hpcc.util.ecl.file import ECLFile
+from hpcc.util.util import checkPqParam
 
 if __name__ == "__main__":
     atexit.register(logging.shutdown)
@@ -41,15 +42,18 @@ if __name__ == "__main__":
                         choices=['info', 'debug'])
     parser.add_argument('--suiteDir', '-s', help="suiteDir to use.",
                         nargs='?', default=".")
+
     subparsers = parser.add_subparsers(help='sub-command help')
     parser_list = subparsers.add_parser('list', help='list help')
     parser_list.add_argument('clusters', help="Print clusters from config.",
                              action='store_true')
+
     parser_run = subparsers.add_parser('run', help='run help')
     parser_run.add_argument('cluster', help="Run the cluster suite.",
-                            nargs='?', default='setup')
-    parser_run.add_argument('--pq', help="Parallel query.",
-                            action='store_true')
+                            nargs='?', type=str,  default='setup')
+    parser_run.add_argument('--pq', help="Parallel query execution with threadNumber threads. ('-1' can be use to calculate usable thread count on a single node system)",
+                            type=checkPqParam,  default = 1,   metavar="threadNumber")
+
     parser_query = subparsers.add_parser('query', help='query help')
     parser_query.add_argument('query', help="Name of a single ECL query (mandatory).",
                               nargs='?', metavar="ECL query")
@@ -57,13 +61,13 @@ if __name__ == "__main__":
                             nargs='?', default='thor', metavar="target cluster | all")
     parser_query.add_argument('--publish', help="Publish compiled query instead of run.",
                             action='store_true')
+
     args = parser.parse_args()
 
     suiteDir = ""
     if 'suiteDir' in args:
         suiteDir = args.suiteDir
     try:
-        regress = Regression(args.config, args.loglevel, suiteDir)
         if 'clusters' in args:
             Clusters = ['setup']
             for cluster in regress.config.Clusters:
@@ -76,6 +80,7 @@ if __name__ == "__main__":
                 print "\nMissing ECL query file!\n"
                 parser_query.print_help()
                 exit()
+            regress = Regression(args.config, args.loglevel, suiteDir)
             ecl = os.path.join(regress.dir_ec, args.query)
             eclfile = ECLFile(ecl, regress.dir_a, regress.dir_ex,
                               regress.dir_r)
@@ -108,6 +113,7 @@ if __name__ == "__main__":
                     logging.error("%s. Query %s does not exist!" % (1,  args.query))
                     exit()
         elif 'cluster' in args:
+            regress = Regression(args.config, args.loglevel, suiteDir,  args.pq)
             regress.bootstrap(args.cluster)
             if 'setup' in args.cluster:
                 regress.runSuite('setup', regress.setup)