HPCC-22504 Ensure NumStarts matches NumStops in Thor

Signed-off-by: Shamser Ahmed <shamser.ahmed@lexisnexis.co.uk>

testing/esp/wudetails/wucheckstartstops.py

@@ -0,0 +1,179 @@
+#! /usr/bin/python3
+################################################################################
+#    HPCC SYSTEMS software Copyright (C) 2019 HPCC Systems®.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+################################################################################
+
+import wucommon
+import logging
+from wucommon import TestCase
+
+def execTestCase(jobname, wuid, tcase):
+    logging.debug('Executing %s',jobname)
+    wuresp = wutest.makeWuDetailsRequest(jobname, wuid, tcase)
+
+    # Check numstop and numstarts
+    if (wuresp['Scopes'] != None):
+        for scope in wuresp['Scopes']['Scope']:
+            properties = scope['Properties']
+            if (properties == None):
+                continue
+            numStarts = ''
+            numStops = ''
+            for property in properties['Property']:
+                if (property['Name'] == 'NumStarts'):
+                    numStarts = property['Formatted']
+                elif (property['Name'] == 'NumStops'):
+                    numStops = property['Formatted']
+            if (numStarts != numStops):
+                logging.error('Job %s WUID %s ScopeName %s NumStarts %s NumStops %s', jobname, wuid, scope['ScopeName'], numStarts, numStops)
+                return False
+    return True
+
+###############################################################################
+print('wucheckstartstops.py - Check NumStarts matches NumStops')
+print('-------------------------------------------------------')
+print('')
+
+requiredJobs = ( ('childds1',      ['thor']),
+                 ('sort',          ['thor']),
+                 ('key',           ['thor']),
+                 ('dict1',         ['thor']),
+                 ('indexread2-multiPart(true)',['thor']),
+                 ('sets',          ['thor']),
+                 ('HeadingExample',['thor']),
+                 ('aaawriteresult',['thor']),
+                 ('action1',       ['thor']),
+                 ('action1a',      ['thor']),
+                 ('action2',       ['thor']),
+                 ('action4',       ['thor']),
+                 ('action5',       ['thor']),
+                 ('aggds1-multiPart(true)',                      ['thor']),
+                 ('aggds1-multiPart(false)-useSequential(true)', ['thor']),
+                 ('aggds2-multiPart(true)',                      ['thor']),
+                 ('aggds2-multiPart(false)-useSequential(true)', ['thor']),
+                 ('aggds3-multiPart(true)',                      ['thor']),
+                 ('aggds3-multiPart(false)-useSequential(true)', ['thor']),
+                 ('aggds3-keyedFilters(true)',                   ['thor']),
+                 ('diskread-multiPart(false)',                   ['thor']),
+                 ('diskGroupAggregate-multiPart(false)',         ['thor']),
+                 ('diskAggregate-multiPart(false)',              ['thor']),
+                 ('dict_once',     ['thor']),
+                 ('dict_null',     ['thor']),
+                 ('dict_matrix',   ['thor']),
+                 ('dict_map',      ['thor']),
+                 ('dict_keyed-multiPart(true)',  ['thor']),
+                 ('dict_keyed-multiPart(false)', ['thor']),
+                 ('dict_int',      ['thor']),
+                 ('dict_indep',    ['thor']),
+                 ('dict_if',       ['thor']),
+                 ('dict_choose',   ['thor']),
+                 ('dict_case',     ['thor']),
+                 ('dict_dsout',    ['thor']),
+                 ('dict_dups',     ['thor']),
+                 ('dict_field',    ['thor']),
+                 ('dict_field2',   ['thor']),
+                 ('dict_func',     ['thor']),
+                 ('dict5c',        ['thor']),
+                 ('dict5b',        ['thor']),
+                 ('dict5a',        ['thor']),
+                 ('dict5',         ['thor']),
+                 ('dict3a',        ['thor']),
+                 ('dict3',         ['thor']),
+                 ('dict2',         ['thor']),
+                 ('dict17',        ['thor']),
+                 ('dict16',        ['thor']),
+                 ('dict15c-multiPart(true)',   ['thor']),
+                 ('dict15c-multiPart(false)',  ['thor']),
+                 ('dict15b-multiPart(true)',   ['thor']),
+                 ('dict15b-multiPart(false)',  ['thor']),
+                 ('dict15a-multiPart(true)',   ['thor']),
+                 ('dict15a-multiPart(false)',  ['thor']),
+                 ('dict15-multiPart(true)',    ['thor']),
+                 ('dict15-multiPart(false)',   ['thor']),
+                 ('dict12',         ['thor']),
+                 ('dict11',         ['thor']),
+                 ('dict10',         ['thor']),
+                 ('dict1',          ['thor']),
+                 ('dfsrecordof',    ['thor']),
+                 ('dfsj',           ['thor']),
+                 ('dfsirecordof',   ['thor']),
+                 ('groupread-multiPart(true)', ['thor']),
+                 ('groupread-multiPart(false)',['thor']),
+                 ('groupjoin1',     ['thor']),
+                 ('grouphashdedup2',['thor']),
+                 ('grouphashdedup', ['thor']),
+                 ('grouphashagg',   ['thor']),
+                 ('groupglobal3c',  ['thor']),
+                 ('groupglobal3b',  ['thor']),
+                 ('groupglobal3a',  ['thor']),
+                 ('groupglobal2c',  ['thor']),
+                 ('groupglobal2b',  ['thor']),
+                 ('groupglobal2a',  ['thor']),
+                 ('groupglobal1c',  ['thor']),
+                 ('groupglobal1b',  ['thor']),
+                 ('groupglobal1a',  ['thor']),
+                 ('groupchild',     ['thor']),
+                 ('group',          ['thor']),
+                 ('globals',        ['thor']),
+                 ('globalmerge',    ['thor']),
+                 ('globalid',       ['thor']),
+                 ('globalfile',     ['thor']),
+                 ('global',         ['thor']),
+                 ('genjoin3',       ['thor']),
+                 ('fullkeyed-multiPart(true)', ['thor']),
+                 ('fullkeyed-multiPart(false)',['thor']),
+                 ('full_test',      ['thor']),
+                 ('fromxml5',       ['thor']),
+                 ('fromjson4',      ['thor']),
+                 ('formatstored',   ['thor']),
+                 ('filterproject2', ['thor']),
+                 ('filtergroup',    ['thor']),
+                 ('fileservice',    ['thor']),
+                 ('diskGroupAggregate-multiPart(false)', ['thor']),
+                 ('denormalize1',   ['thor']),
+                 ('dataset_transform_inline', ['thor']),
+                 ('choosesets',     ['thor']),
+                 ('bloom2',         ['thor']),
+                 ('badindex-newIndexReadMapping(false)', ['thor']),
+                 ('all_denormalize-multiPart(true)',         ['thor']))
+
+wutest = wucommon.WuTest()
+
+logging.info('Gathering workunits')
+wu = wutest.getTestWorkunits(requiredJobs)
+
+if (wutest.getMatchedJobCount()==0):
+    logging.error('There are no matching jobs.  Has the performance regression suite been executed?')
+    logging.error('Aborting')
+    exit(1)
+
+wuDetailsReq = TestCase(wutest.scopeFilter(MaxDepth='999', ScopeTypes={'ScopeType':['edge']}),
+                        wutest.nestedFilter(),
+                        wutest.propertiesToReturn(Properties={'Property':['NumStarts','NumStops']}),
+                        wutest.scopeOptions(IncludeMatchedScopesInResults='1', IncludeScope='1', IncludeId='0', IncludeScopeType='0'),
+                        wutest.propertyOptions(IncludeName='1', IncludeRawValue='0', IncludeFormatted='1', IncludeMeasure='0', IncludeCreator='0', IncludeCreatorType='0'))
+
+logging.info('Checking NumStart and NumStop matches')
+stats = wucommon.Statistics()
+for jobname, wuid in wu.items():
+    success = execTestCase(jobname, wuid, wuDetailsReq)
+    logging.debug('Job %-33s WUID %-20s Success: %s', jobname, wuid, success)
+    stats.addCount(success)
+
+logging.info('Missing count: %d', wutest.getMissingJobCount(requiredJobs))
+logging.info('Matched jobs:  %d', len(wu))
+logging.info('Success count: %d', stats.successCount)
+logging.info('Failure count: %d', stats.failureCount)
+

testing/esp/wudetails/wucommon.py

@@ -0,0 +1,267 @@
+#! /usr/bin/python3
+################################################################################
+#    HPCC SYSTEMS software Copyright (C) 2019 HPCC Systems®.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+################################################################################
+
+import argparse
+import logging
+import datetime
+import traceback
+import requests.packages.urllib3
+import sys
+import inspect
+from requests import Session
+from zeep import Client
+from zeep.transports import Transport
+from pathlib import Path
+from collections import namedtuple
+from zeep.cache import SqliteCache
+
+TestCase = namedtuple('testCase', ['ScopeFilter',
+                                   'NestedFilter',
+                                   'PropertiesToReturn',
+                                   'ScopeOptions',
+                                   'PropertyOptions'])
+
+def safeMkdir(path):
+    try:
+        path.mkdir(parents=True)
+    except FileExistsError as e:
+        pass
+    except (FileNotFoundError, PermissionError) as e:
+        logging.error("'%s' \nExit." % (str(e)))
+        exit(-1)
+    except:
+        print("Unexpected error:" + str(sys.exc_info()[0]) + " (line: " + str(inspect.stack()[0][2]) + ")" )
+        traceback.print_stack()
+        exit(-1)
+
+class DebugTransport(Transport):
+    def post(self, address, message, headers):
+        self.xml_request = message.decode('utf-8')
+        response = super().post(address, message, headers)
+        self.response = response
+        return response
+
+class Statistics:
+    def __init__(self):
+        self.successCount = 0
+        self.failureCount = 0
+
+    def addCount(self, success):
+        if (success):
+            self.successCount += 1
+        else:
+            self.failureCount += 1
+
+class WuTest:
+    def connect(self):
+        # Consume WSDL and generate soap structures
+        try:
+            session = Session()
+            if (self.args.nosslverify):
+                session.verify = False
+                session.sslverify = False
+            self.transport = DebugTransport(cache=SqliteCache(), session=session)
+            self.wudetails = Client(self.wudetailsservice_wsdl_url, transport=self.transport)
+        except:
+            logging.critical ('Unable to connect/obtain WSDL from ' + self.wudetailsservice_wsdl_url)
+            raise
+
+    def __init__(self, maskFields=(), maskMeasureTypes=(), maskVersion=False, maskWUID=False):
+        argumentparser = argparse.ArgumentParser()
+        argumentparser.add_argument('-o', '--outdir', help='Results directory', default='results', metavar='dir')
+        argumentparser.add_argument('-d', '--debug', help='Enable debug', action='store_true', default=False)
+        argumentparser.add_argument('--logresp', help='Log wudetails responses (in results directory)', action='store_true', default=False)
+        argumentparser.add_argument('--logreq', help='Log wudetails requests (in results directory)', action='store_true', default=False)
+        argumentparser.add_argument('-n', '--lastndays', help="Use workunits from last 'n' days", type=int, default=1, metavar='days')
+        argumentparser.add_argument('--nosslverify', help="Disable SSL certificate verification", action='store_true', default=False)
+        argumentparser.add_argument('--baseurl', help="Base url for both WUQuery and WUDetails", default='http://localhost:8010', metavar='url')
+        argumentparser.add_argument('--user', help='Username for authentication', metavar='username')
+        argumentparser.add_argument('--pw', help='Password for authentication', metavar='password')
+        argumentparser.add_argument('--httpdigestauth', help='User HTTP digest authentication(Basic auth default)', action='store_true')
+        self.args = argumentparser.parse_args()
+
+        if (self.args.debug):
+            loglevel = logging.DEBUG
+        else:
+            loglevel=logging.INFO
+        logging.basicConfig(level=loglevel, format='[%(levelname)s] %(message)s')
+        requests.packages.urllib3.disable_warnings()
+        self.keydir = 'key'
+        self.wuqueryservice_wsdl_url = self.args.baseurl + '/WsWorkunits/WUQuery.json?ver_=1.71&wsdl'
+        self.wudetailsservice_wsdl_url = self.args.baseurl + '/WsWorkunits/WUDetails.json?ver_=1.71&wsdl'
+        self.outdir = self.args.outdir
+
+        if (len(maskFields)==0 and len(maskMeasureTypes)==0 and maskWUID==False and maskVersion==False):
+            self.enableMasking = False
+        else:
+            self.enableMasking = True
+
+        self.maskFields = maskFields
+        self.maskMeasureTypes = maskMeasureTypes
+        self.maskVersion = maskVersion
+        self.maskWUID = maskWUID
+
+        self.resultdir = Path(self.args.outdir)
+        safeMkdir(self.resultdir)
+
+        self.tcasekeydir = Path(self.keydir)
+        safeMkdir(self.tcasekeydir)
+        try:
+            self.connect()
+            try:
+                self.scopeFilter = self.wudetails.get_type('ns0:WUScopeFilter')
+                self.nestedFilter = self.wudetails.get_type('ns0:WUNestedFilter')
+                self.propertiesToReturn = self.wudetails.get_type('ns0:WUPropertiesToReturn')
+                self.scopeOptions = self.wudetails.get_type('ns0:WUScopeOptions')
+                self.propertyOptions = self.wudetails.get_type('ns0:WUPropertyOptions')
+                self.extraProperties = self.wudetails.get_type('ns0:WUExtraProperties')
+            except:
+                logging.critical ('WSDL different from expected')
+                raise
+        except:
+            sys.exit('Aborting!')
+
+        self.wuquery = Client(self.wuqueryservice_wsdl_url, transport=self.transport)
+
+    # Mask out fields in the response structure
+    #
+    def maskoutFields(self, wudetails_resp, wuid):
+        try:
+            if (self.maskWUID and wudetails_resp['WUID']==wuid):
+                wudetails_resp['WUID'] = '{masked WUID - matches request}'
+
+            if (self.maskVersion and wudetails_resp['MaxVersion'].isnumeric()):
+                wudetails_resp['MaxVersion'] = '{masked number}'
+
+            if (wudetails_resp['Scopes'] != None):
+                for scope in wudetails_resp['Scopes']['Scope']:
+                    properties = scope['Properties']
+                    if (properties == None):
+                        continue
+                    for property in properties['Property']:
+                        if ((property['Name'] in self.maskFields) or (property['Measure'] in self.maskMeasureTypes)):
+                            if (property['RawValue'] != None):
+                                property['RawValue'] = '{masked}'
+                            if (property['Formatted'] != None):
+                                property['Formatted'] = '{masked}'
+                        property['Creator'] = '{masked}'
+        except:
+            logging.critical('Unable to process WUDetails response: %s', wuid)
+            raise
+
+    def makeWuDetailsRequest(self,testfilename,wuid,tcase):
+        outfile = (self.resultdir / testfilename).with_suffix('.json')
+        errfile = outfile.with_suffix('.err')
+
+        if (outfile.exists()): outfile.unlink()
+        if (errfile.exists()): errfile.unlink()
+
+        try:
+            wuresp = self.wudetails.service.WUDetails(WUID=wuid,
+                                            ScopeFilter=tcase.ScopeFilter,
+                                            NestedFilter=tcase.NestedFilter,
+                                            PropertiesToReturn=tcase.PropertiesToReturn,
+                                            ScopeOptions=tcase.ScopeOptions,
+                                            PropertyOptions=tcase.PropertyOptions)
+        except:
+            logging.critical('Unable to submit WUDetails request: %s', testfilename)
+            raise
+        finally:
+            if (self.args.logreq):
+                reqfile = outfile.with_suffix('.req')
+                try:
+                    if (reqfile.exists()): reqfile.unlink()
+                    with reqfile.open(mode='w') as f:
+                        print (self.transport.xml_request, file=f)
+                except:
+                    logging.critical('Unable write logrequest to file: %s', reqfile)
+                    pass
+
+        if (self.args.logresp):
+            respfile = outfile.with_suffix('.resp')
+            if (respfile.exists()): respfile.unlink()
+            with respfile.open(mode='w') as f:
+                print (wuresp, file=f)
+
+        if (self.enableMasking):
+            self.maskoutFields(wuresp, wuid)
+        return wuresp
+
+    # Get a list of workunits that will be used for testing wudetails
+    def getTestWorkunits(self, requiredJobs):
+        # Calculate date range (LastNDays not processed correctly by wuquery)
+        enddate = datetime.datetime.now()
+        startdate = enddate - datetime.timedelta(days=self.args.lastndays)
+        self.matchedWU = {}
+
+        logging.debug ('Gathering Workunits')
+        for reqjob in requiredJobs:
+            reqJobname = reqjob[0]
+            reqClusters = reqjob[1]
+            nextPage = 0
+            while (nextPage >=0):
+                wuqueryresp = self.wuquery.service.WUQuery(Owner='regress',
+                                                    State='completed',
+                                                    PageSize=500,
+                                                    PageStartFrom=nextPage,
+                                                    LastNDays=self.args.lastndays,
+                                                    StartDate=startdate.strftime('%Y-%m-%dT00:00:00'),
+                                                    EndDate=enddate.strftime('%Y-%m-%dT23:59:59'),
+                                                    Jobname=reqJobname + '*',
+                                                    Descending='1')
+                try:
+                    nextPage = wuqueryresp['NextPage']
+                    workunits = wuqueryresp['Workunits']['ECLWorkunit']
+                except:
+                    break
+
+                try:
+                    logging.debug('jobname %s count: %d', reqJobname,len(workunits))
+                    workunits.sort(key=lambda k: k['Jobname'], reverse=True)
+
+                    # Extract jobname from jobname with date postfix
+                    for wu in workunits:
+                        s = wu['Jobname'].split('-')
+                        cluster = wu['Cluster']
+                        if (len(s) >2):
+                            sep = '-'
+                            job = sep.join(s[0:len(s)-2])
+                        else:
+                            job = wu['Jobname']
+                        key = job + '_' + cluster
+                        if ( (job == reqJobname) and (cluster in reqClusters) and (key not in self.matchedWU)):
+                            self.matchedWU[key] = wu['Wuid']
+                except:
+                    logging.error('Unexpected response from WUQuery: %s', wuqueryresp)
+                    raise
+
+        return self.matchedWU
+
+    def getMissingJobCount(self,requiredJobs):
+        missingjobcount = 0
+        for reqjob in requiredJobs:
+            jobname = reqjob[0]
+            for cluster in reqjob[1]:
+                key = jobname + '_' + cluster
+                if (key not in self.matchedWU):
+                    logging.error('Missing job: %s (%s)', jobname, cluster)
+                    missingjobcount += 1
+        return missingjobcount
+
+    def getMatchedJobCount(self):
+        return len(self.matchedWU)
+

testing/esp/wudetails/wutest.py

@@ -1,6 +1,6 @@
 #! /usr/bin/python3
 ################################################################################
-#    HPCC SYSTEMS software Copyright (C) 2018 HPCC Systems®.
+#    HPCC SYSTEMS software Copyright (C) 2019 HPCC Systems®.
 #
 #    Licensed under the Apache License, Version 2.0 (the "License");
 #    you may not use this file except in compliance with the License.
@@ -15,51 +15,40 @@
 #    limitations under the License.
 ################################################################################
 
-import argparse
-import filecmp
+import wucommon
 import logging
-import logging.config
-import datetime
-import requests.packages.urllib3
-import inspect
-import traceback
-import sys
-from requests import Session
-from requests.auth import HTTPBasicAuth
-from requests.auth import HTTPDigestAuth
-from zeep import Client 
-from zeep.transports import Transport
-from zeep.cache import SqliteCache
-from collections import namedtuple
-from pathlib import Path
+import filecmp
+from wucommon import TestCase
 
-print('WUDetails Regression (wutest.py)')
-print('--------------------------------')
-print('')
+def execTestCase(jobname, wuid, tcase, tcasename):
+    testfilename = jobname + '_' + tcasename
+    logging.debug('Executing %s',testfilename)
 
-argumentparser = argparse.ArgumentParser()
-argumentparser.add_argument('-o', '--outdir', help='Results directory', default='results', metavar='dir')
-argumentparser.add_argument('-d', '--debug', help='Enable debug', action='store_true', default=False)
-argumentparser.add_argument('--logresp', help='Log wudetails responses (in results directory)', action='store_true', default=False)
-argumentparser.add_argument('--logreq', help='Log wudetails requests (in results directory)', action='store_true', default=False)
-argumentparser.add_argument('-n', '--lastndays', help="Use workunits from last 'n' days", type=int, default=1, metavar='days')
-argumentparser.add_argument('--nosslverify', help="Disable SSL certificate verification", action='store_true', default=False)
-argumentparser.add_argument('-u', '--baseurl', help="Base url for both WUQuery and WUDetails", default='http://localhost:8010', metavar='url')
-argumentparser.add_argument('--user', help='Username for authentication', metavar='username')
-argumentparser.add_argument('--pw', help='Password for authentication', metavar='password')
-argumentparser.add_argument('--httpdigestauth', help='User HTTP digest authentication(Basic auth default)', action='store_true')
-args = argumentparser.parse_args()
+    wuresp = wutest.makeWuDetailsRequest(testfilename, wuid, tcase)
 
-if (args.debug):
-    loglevel = logging.DEBUG
-else:
-    loglevel=logging.INFO
-logging.basicConfig(level=loglevel, format='[%(levelname)s] %(message)s')
+    outfile = (wutest.resultdir / testfilename).with_suffix('.json')
+    if (outfile.exists()): outfile.unlink()
+    with outfile.open(mode='w') as f:
+        print (tcase, file=f)
+        print (wuresp, file=f)
+
+    keyfile = (wutest.tcasekeydir / testfilename).with_suffix('.json')
+    if (not keyfile.exists()):
+        logging.error('Missing key file %s', str(keyfile))
+        return False
+
+    # Compare actual and expectetd
+    if (not filecmp.cmp(str(outfile),str(keyfile))):
+        logging.error('Regression check Failed: %s', testfilename)
+        return False
+    else:
+        logging.debug('PASSED %s', testfilename)
+        return True
 
-keydir = 'key'
-wuqueryservice_url = args.baseurl + '/WsWorkunits/WUQuery.json?ver_=1.71'
-wuqueryservice_wsdl_url = args.baseurl + '/WsWorkunits/WUQuery.json?ver_=1.71&wsdl'
-wudetailsservice_wsdl_url = args.baseurl + '/WsWorkunits/WUDetails.json?ver_=1.71&wsdl'
+###############################################################################
+print('WUDetails Regression (wutest.py)')
+print('--------------------------------')
+print('')
 
 requiredJobs = ( ('childds1',      ('roxie','thor','hthor')),
                  ('dedup_all',     ('roxie','hthor')),
@@ -69,381 +58,249 @@ requiredJobs = ( ('childds1',      ('roxie','thor','hthor')),
                  ('indexread2-multiPart(true)',('roxie', 'thor','hthor')),
                  ('sets',           ('roxie','thor','hthor')) )
 
-maskValueFields = ('Definition','DefinitionList','SizePeakMemory', 'WhenFirstRow', 'TimeElapsed', 'TimeTotalExecute', 'TimeFirstExecute', 'TimeLocalExecute',
+maskFields = ('Definition','DefinitionList','SizePeakMemory', 'WhenFirstRow', 'TimeElapsed', 'TimeTotalExecute', 'TimeFirstExecute', 'TimeLocalExecute',
                    'WhenStarted', 'TimeMinLocalExecute', 'TimeMaxLocalExecute', 'TimeAvgLocalExecute', 'SkewMinLocalExecute', 'SkewMaxLocalExecute',
                    'NodeMaxLocalExecute', 'NodeMaxDiskWrites', 'NodeMaxLocalExecute', 'NodeMaxLocalExecute', 'NodeMaxSortElapsed', 'NodeMinDiskWrites',
                    'NodeMinLocalExecute', 'NodeMinLocalExecute', 'NodeMinLocalExecute', 'NodeMinSortElapsed', 'SkewMaxDiskWrites', 'SkewMaxLocalExecute',
                    'SkewMaxLocalExecute', 'SkewMaxSortElapsed', 'SkewMinDiskWrites', 'SkewMinLocalExecute', 'SkewMinLocalExecute', 'SkewMinSortElapsed',
                    'TimeAvgSortElapsed', 'TimeMaxSortElapsed', 'TimeMinSortElapsed')
-maskMeasureTypes = ('ts','ns', 'skw', 'node')
-
-requests.packages.urllib3.disable_warnings()
-
-class DebugTransport(Transport):
-    def post(self, address, message, headers):
-        self.xml_request = message.decode('utf-8')
-        response = super().post(address, message, headers)
-        self.response = response
-        
-        return response
-
-# Get a list of workunits that will be used for testing wudetails
-#
-def GetTestWorkunits():
-    try:
-        session = Session()
-        if (args.nosslverify):
-            session.verify = False
-            session.sslverify = False
-        if (args.pw and args.user):
-            if (args.httpdigestauth):
-                session.auth = HTTPDigestAuth(args.user, args.pw)
-            else:
-                session.auth = HTTPBasicAuth(args.user, args.pw)
-
-        transport = DebugTransport(cache=SqliteCache(), session=session)
-        wuquery = Client(wuqueryservice_wsdl_url, transport=transport)
-    except:
-        logging.critical ('Unable to obtain WSDL from %s', wuqueryservice_wsdl_url)
-        raise
-   
-    # Calculate date range (LastNDays not processed correctly by wuquery)
-    enddate = datetime.datetime.now()
-    startdate = enddate - datetime.timedelta(days=args.lastndays)
-    matchedWU = {}
-
-    logging.debug ('Gathering Workunits')
-    for reqjob in requiredJobs:
-        reqJobname = reqjob[0]
-        reqClusters = reqjob[1]
-        nextPage = 0 
-        while (nextPage >=0):
-            wuqueryresp = wuquery.service.WUQuery(Owner='regress',
-                                                  State='completed',
-                                                  PageSize=500,
-                                                  PageStartFrom=nextPage,
-                                                  LastNDays=args.lastndays, 
-                                                  StartDate=startdate.strftime('%Y-%m-%dT00:00:00'),
-                                                  EndDate=enddate.strftime('%Y-%m-%dT23:59:59'),
-                                                  Jobname=reqJobname + '*',
-                                                  Descending='1')
-                
-            try:
-                nextPage = wuqueryresp['NextPage']
-                workunits = wuqueryresp['Workunits']['ECLWorkunit']
-            except:
-                return matchedWU
-
-            try:
-                logging.debug('Workunit count: %d', len(workunits))
-                workunits.sort(key=lambda k: k['Jobname'], reverse=True)
-
-                # Extract jobname from jobname with date postfix
-                for wu in workunits:
-                    s = wu['Jobname'].split('-')
-                    cluster = wu['Cluster']
-                    if (len(s) >2):
-                        sep = '-'
-                        job = sep.join(s[0:len(s)-2])
-                    else:
-                        job = wu['Jobname']
-                    key = job + '_' + cluster
-                    if ( (job == reqJobname) and (cluster in reqClusters) and (key not in matchedWU)):
-                        matchedWU[key] = wu['Wuid']
-            except:
-                logging.error('Unexpected response from WUQuery: %s', wuqueryresp) 
-                raise
-
-    return matchedWU
 
-def GetMissingJobCount(wu):
-    missingjobs = 0
-    for reqjob in requiredJobs:
-        jobname = reqjob[0]
-        for cluster in reqjob[1]:
-            key = jobname + '_' + cluster
-            if (key not in wu):
-                logging.error('Missing job: %s (%s)', jobname, cluster)
-                missingjobs += 1
-    return missingjobs
-        
-# Mask out fields in the response structure
-#
-def maskoutFields(wudetails_resp) :
-    if (wudetails_resp['MaxVersion'].isnumeric()):
-        wudetails_resp['MaxVersion'] = '{masked number}'
-
-    if (wudetails_resp['Scopes'] != None):
-        for scope in wudetails_resp['Scopes']['Scope']:
-            properties = scope['Properties']
-            if (properties == None):
-                continue
-            for property in properties['Property']:
-                if ((property['Name'] in maskValueFields) or (property['Measure'] in maskMeasureTypes)):
-                    if (property['RawValue'] != None):
-                        property['RawValue'] = '{masked}'
-                    if (property['Formatted'] != None):
-                        property['Formatted'] = '{masked}'
-            
-                property['Creator'] = '{masked}'
-
-# Main
-#
-# Consume WSDL and generate soap structures
-try:
-    session = Session()
-    if (args.nosslverify):
-        session.verify = False
-        session.sslverify = False
-    transport = DebugTransport(cache=SqliteCache(), session=session)
-    wudetails = Client(wudetailsservice_wsdl_url, transport=transport)
-except:
-    logging.critical ('Unable to obtain WSDL from ' +wudetailsservice_wsdl_url)
-    raise
-
-try:
-    scopeFilter = wudetails.get_type('ns0:WUScopeFilter')
-    nestedFilter = wudetails.get_type('ns0:WUNestedFilter')
-    propertiesToReturn = wudetails.get_type('ns0:WUPropertiesToReturn')
-    scopeOptions = wudetails.get_type('ns0:WUScopeOptions')
-    propertyOptions = wudetails.get_type('ns0:WUPropertyOptions')
-    extraProperties = wudetails.get_type('ns0:WUExtraProperties')
-except:
-    logging.critical ('WSDL different from expected')
-    raise
+maskMeasureTypes = ('ts','ns', 'skw', 'node')
 
+wutest = wucommon.WuTest(maskFields, maskMeasureTypes, True, True)
 
-# Generate Test cases
-testCase = namedtuple('testCase', ['ScopeFilter',
-                                   'NestedFilter',
-                                   'PropertiesToReturn',
-                                   'ScopeOptions',
-                                   'PropertyOptions'])
+scopeFilter = wutest.scopeFilter
+nestedFilter = wutest.nestedFilter
+propertiesToReturn = wutest.propertiesToReturn
+scopeOptions = wutest.scopeOptions
+propertyOptions = wutest.propertyOptions
+extraProperties = wutest.extraProperties
 
+# Test cases
 #scopeFilter(MaxDepth='999', Scopes=set(), Ids=set(), ScopeTypes=set()),
 #nestedFilter(Depth='999', ScopeTypes=set()),
 #propertiesToReturn(AllProperties='1', MinVersion='0', Measure='', Properties=set(), ExtraProperties=set()),
 #scopeOptions(IncludeMatchedScopesInResults='1', IncludeScope='1', IncludeId='1', IncludeScopeType='1'),
 #propertyOptions(IncludeName='1', IncludeRawValue='1', IncludeFormatted='1', IncludeMeasure='1', IncludeCreator='1', IncludeCreatorType='1')
-testCases = [ 
-             testCase(
+TestCases = [
+             TestCase(
                  scopeFilter(MaxDepth='999'),
                  nestedFilter(),
                  propertiesToReturn(AllProperties='1'),
                  scopeOptions(IncludeMatchedScopesInResults='1', IncludeScope='1', IncludeId='1', IncludeScopeType='1'),
                  propertyOptions(IncludeName='1', IncludeRawValue='1', IncludeFormatted='1', IncludeMeasure='1', IncludeCreator='1', IncludeCreatorType='1')
              ),
-             testCase(
+             TestCase(
                  scopeFilter(MaxDepth='999'),
                  nestedFilter(),
                  propertiesToReturn(AllStatistics='1'),
                  scopeOptions(IncludeMatchedScopesInResults='1', IncludeScope='1', IncludeId='1', IncludeScopeType='1'),
                  propertyOptions(IncludeName='1', IncludeRawValue='1', IncludeMeasure='1', IncludeCreator='1', IncludeCreatorType='1')
              ),
-             testCase(
+             TestCase(
                  scopeFilter(MaxDepth='999'),
                  nestedFilter(),
                  propertiesToReturn(AllAttributes='1'),
                  scopeOptions(IncludeMatchedScopesInResults='1', IncludeScope='1', IncludeId='1', IncludeScopeType='1'),
                  propertyOptions(IncludeName='1', IncludeRawValue='1', IncludeMeasure='1', IncludeCreator='1', IncludeCreatorType='1')
              ),
-             testCase(
+             TestCase(
                  scopeFilter(MaxDepth='999'),
                  nestedFilter(),
                  propertiesToReturn(AllHints='1'),
                  scopeOptions(IncludeMatchedScopesInResults='1', IncludeScope='1', IncludeId='1', IncludeScopeType='1'),
                  propertyOptions(IncludeName='1', IncludeRawValue='1', IncludeMeasure='1', IncludeCreator='1', IncludeCreatorType='1')
              ),
-             testCase(
+             TestCase(
                  scopeFilter(MaxDepth='999', Scopes={'Scope':'w1:graph1'}),
                  nestedFilter(),
                  propertiesToReturn(AllProperties='1'),
                  scopeOptions(IncludeMatchedScopesInResults='1', IncludeScope='1', IncludeId='1', IncludeScopeType='1'),
                  propertyOptions(IncludeName='1', IncludeRawValue='1', IncludeMeasure='1', IncludeCreator='1', IncludeCreatorType='1')
              ),
-             testCase(
+             TestCase(
                  scopeFilter(MaxDepth='1', ScopeTypes={'ScopeType':'graph'}),
                  nestedFilter(Depth='1'),
                  propertiesToReturn(AllProperties='1'),
                  scopeOptions(IncludeMatchedScopesInResults='1', IncludeScope='1', IncludeId='1', IncludeScopeType='1'),
                  propertyOptions(IncludeName='1', IncludeRawValue='1', IncludeMeasure='1', IncludeCreator='1', IncludeCreatorType='1')
              ),
-             testCase(
+             TestCase(
                  scopeFilter(MaxDepth='999', ScopeTypes={'ScopeType':'subgraph'}),
                  nestedFilter(),
                  propertiesToReturn(AllStatistics='1'),
                  scopeOptions(IncludeMatchedScopesInResults='1', IncludeScope='1', IncludeId='1', IncludeScopeType='1'),
                  propertyOptions(IncludeName='1', IncludeRawValue='1', IncludeMeasure='1', IncludeCreator='1', IncludeCreatorType='1')
              ),
-             testCase(
+             TestCase(
                  scopeFilter(MaxDepth='999', ScopeTypes={'ScopeType':'global'}),
                  nestedFilter(),
                  propertiesToReturn(AllStatistics='1'),
                  scopeOptions(IncludeMatchedScopesInResults='1', IncludeScope='1', IncludeId='1', IncludeScopeType='1'),
                  propertyOptions(IncludeName='1', IncludeRawValue='1', IncludeMeasure='1', IncludeCreator='1', IncludeCreatorType='1')
              ),
-             testCase(
+             TestCase(
                  scopeFilter(MaxDepth='999', ScopeTypes={'ScopeType':'activity'}),
                  nestedFilter(),
                  propertiesToReturn(AllStatistics='1'),
                  scopeOptions(IncludeMatchedScopesInResults='1', IncludeScope='1', IncludeId='1', IncludeScopeType='1'),
                  propertyOptions(IncludeName='1', IncludeRawValue='1', IncludeMeasure='1', IncludeCreator='1', IncludeCreatorType='1')
              ),
-             testCase(
+             TestCase(
                  scopeFilter(MaxDepth='999', ScopeTypes={'ScopeType':'allocator'}),
                  nestedFilter(),
                  propertiesToReturn(AllStatistics='1'),
                  scopeOptions(IncludeMatchedScopesInResults='1', IncludeScope='1', IncludeId='1', IncludeScopeType='1'),
                  propertyOptions(IncludeName='1', IncludeRawValue='1', IncludeMeasure='1', IncludeCreator='1', IncludeCreatorType='1')
              ),
-             testCase(
+             TestCase(
                  scopeFilter(MaxDepth='2'),
                  nestedFilter(),
                  propertiesToReturn(AllStatistics='1', AllAttributes='1'),
                  scopeOptions(IncludeMatchedScopesInResults='1', IncludeScope='1', IncludeId='1', IncludeScopeType='1'),
                  propertyOptions()
              ),
-             testCase(
+             TestCase(
                  scopeFilter(MaxDepth='1'),
                  nestedFilter(Depth='1'),
                  propertiesToReturn(AllStatistics='1', AllAttributes='1'),
                  scopeOptions(),
                  propertyOptions()
              ),
-             testCase(
+             TestCase(
                  scopeFilter(MaxDepth='2'),
                  nestedFilter(),
                  propertiesToReturn(Properties={'Property':['WhenStarted','WhenCreated']}),
                  scopeOptions(IncludeMatchedScopesInResults='1', IncludeScope='1', IncludeId='1', IncludeScopeType='1'),
                  propertyOptions()
              ),
-             testCase(
+             TestCase(
                  scopeFilter(MaxDepth='2'),
                  nestedFilter(),
                  propertiesToReturn(Measure='ts'),
                  scopeOptions(IncludeMatchedScopesInResults='1', IncludeScope='1', IncludeId='1', IncludeScopeType='1'),
                  propertyOptions(IncludeName='1', IncludeRawValue='1', IncludeMeasure='1', IncludeCreator='1', IncludeCreatorType='1')
              ),
-             testCase(
+             TestCase(
                  scopeFilter(MaxDepth='2'),
                  nestedFilter(),
                  propertiesToReturn(Measure='cnt'),
                  scopeOptions(IncludeMatchedScopesInResults='1', IncludeScope='1', IncludeId='1', IncludeScopeType='1'),
                  propertyOptions()
              ),
-             testCase(
+             TestCase(
                  scopeFilter(MaxDepth='999', ScopeTypes={'ScopeType':'subgraph'}),
                  nestedFilter(),
                  propertiesToReturn(AllStatistics='1', AllAttributes='1'),
                  scopeOptions(IncludeMatchedScopesInResults='1'),
                  propertyOptions(IncludeName='1', IncludeRawValue='1', IncludeMeasure='1', IncludeCreator='1', IncludeCreatorType='1')
              ),
-             testCase(
+             TestCase(
                  scopeFilter(MaxDepth='999', ScopeTypes={'ScopeType':'subgraph'}),
                  nestedFilter(),
                  propertiesToReturn(AllStatistics='1', AllAttributes='1'),
                  scopeOptions(IncludeScope='1'),
                  propertyOptions(IncludeName='1', IncludeRawValue='1', IncludeMeasure='1', IncludeCreator='1', IncludeCreatorType='1')
              ),
-             testCase(
+             TestCase(
                  scopeFilter(MaxDepth='999', ScopeTypes={'ScopeType':'subgraph'}),
                  nestedFilter(),
                  propertiesToReturn(AllStatistics='1', AllAttributes='1'),
                  scopeOptions(IncludeId='1'),
                  propertyOptions(IncludeName='1', IncludeRawValue='1', IncludeMeasure='1', IncludeCreator='1', IncludeCreatorType='1')
              ),
-             testCase(
+             TestCase(
                  scopeFilter(MaxDepth='999', ScopeTypes={'ScopeType':'subgraph'}),
                  nestedFilter(),
                  propertiesToReturn(AllStatistics='1', AllAttributes='1'),
                  scopeOptions(),
                  propertyOptions(IncludeName='1', IncludeRawValue='1', IncludeMeasure='1', IncludeCreator='1', IncludeCreatorType='1')
              ),
-             testCase(
+             TestCase(
                  scopeFilter(MaxDepth='999', ScopeTypes={'ScopeType':'subgraph'}),
                  nestedFilter(),
                  propertiesToReturn(AllStatistics='1', AllAttributes='1'),
                  scopeOptions(IncludeMatchedScopesInResults='1', IncludeScope='1', IncludeId='1', IncludeScopeType='1'),
                  propertyOptions(IncludeName='0')
              ),
-             testCase(
+             TestCase(
                  scopeFilter(MaxDepth='999', ScopeTypes={'ScopeType':'subgraph'}),
                  nestedFilter(),
                  propertiesToReturn(AllStatistics='1', AllAttributes='1'),
                  scopeOptions(IncludeMatchedScopesInResults='1', IncludeScope='1', IncludeId='1', IncludeScopeType='1'),
                  propertyOptions(IncludeRawValue='1')
              ),
-             testCase(
+             TestCase(
                  scopeFilter(MaxDepth='999', ScopeTypes={'ScopeType':'subgraph'}),
                  nestedFilter(),
                  propertiesToReturn(AllStatistics='1', AllAttributes='1'),
                  scopeOptions(IncludeMatchedScopesInResults='1', IncludeScope='1', IncludeId='1', IncludeScopeType='1'),
                  propertyOptions(IncludeMeasure='0')
              ),
-             testCase(
+             TestCase(
                  scopeFilter(MaxDepth='999', ScopeTypes={'ScopeType':'subgraph'}),
                  nestedFilter(),
                  propertiesToReturn(AllStatistics='1', AllAttributes='1'),
                  scopeOptions(IncludeMatchedScopesInResults='1', IncludeScope='1', IncludeId='1', IncludeScopeType='1'),
                  propertyOptions(IncludeCreator='0')
              ),
-             testCase(
+             TestCase(
                  scopeFilter(MaxDepth='999', ScopeTypes={'ScopeType':'subgraph'}),
                  nestedFilter(),
                  propertiesToReturn(AllStatistics='1', AllAttributes='1'),
                  scopeOptions(IncludeMatchedScopesInResults='1', IncludeScope='1', IncludeId='1', IncludeScopeType='1'),
                  propertyOptions(IncludeCreatorType='0')
              ),
-             testCase(
+             TestCase(
                  scopeFilter(MaxDepth='2', Scopes={'Scope':'w1:graph1:sg1'}),
                  nestedFilter(Depth=0),
                  propertiesToReturn(AllStatistics='1', AllAttributes='1'),
                  scopeOptions(IncludeMatchedScopesInResults='1', IncludeScope='1', IncludeId='1', IncludeScopeType='1'),
                  propertyOptions(IncludeName='1', IncludeRawValue='1', IncludeMeasure='1', IncludeCreator='1', IncludeCreatorType='1')
              ),
-             testCase(
+             TestCase(
                  scopeFilter(MaxDepth='2', Scopes={'Scope':'w1:graph1:sg1'}),
                  nestedFilter(Depth=1),
                  propertiesToReturn(Properties={'Property':['WhenStarted','WhenCreated']}, ExtraProperties={'Extra':{'scopeType':'edge','Properties':{'Property':['NumStarts','NumStops']}}}),
                  scopeOptions(IncludeMatchedScopesInResults='1', IncludeScope='1', IncludeId='1', IncludeScopeType='1'),
                  propertyOptions(IncludeName='1', IncludeRawValue='1', IncludeMeasure='1', IncludeCreator='1', IncludeCreatorType='1')
              ),
-             testCase(
+             TestCase(
                  scopeFilter(MaxDepth='999', PropertyFilters={'PropertyFilter':{'Name':'NumRowsProcessed','MinValue':'10000','MaxValue':'20000'}}),
                  nestedFilter(),
                  propertiesToReturn(AllStatistics='1', AllAttributes='1'),
                  scopeOptions(IncludeMatchedScopesInResults='1', IncludeScope='1', IncludeId='1', IncludeScopeType='1'),
                  propertyOptions(IncludeName='1', IncludeRawValue='1', IncludeMeasure='1', IncludeCreator='1', IncludeCreatorType='1')
              ),
-             testCase(
+             TestCase(
                  scopeFilter(MaxDepth='999', PropertyFilters={'PropertyFilter':{'Name':'NumIndexSeeks','MaxValue':'3'}}),
                  nestedFilter(),
                  propertiesToReturn(AllStatistics='1', AllAttributes='1'),
                  scopeOptions(IncludeMatchedScopesInResults='1', IncludeScope='1', IncludeId='1', IncludeScopeType='1'),
                  propertyOptions(IncludeName='1', IncludeRawValue='1', IncludeMeasure='1', IncludeCreator='1', IncludeCreatorType='1')
              ),
-             testCase(
+             TestCase(
                  scopeFilter(MaxDepth='999', PropertyFilters={'PropertyFilter':{'Name':'NumIndexSeeks','ExactValue':'4'}}),
                  nestedFilter(),
                  propertiesToReturn(AllStatistics='1', AllAttributes='1'),
                  scopeOptions(IncludeMatchedScopesInResults='1', IncludeScope='1', IncludeId='1', IncludeScopeType='1'),
                  propertyOptions(IncludeName='1', IncludeRawValue='1', IncludeMeasure='1', IncludeCreator='1', IncludeCreatorType='1')
              ),
-             testCase(
+             TestCase(
                  scopeFilter(MaxDepth='999', PropertyFilters={'PropertyFilter':[{'Name':'NumIndexSeeks','ExactValue':'4'},{'Name':'NumAllocations','MinValue':'5','MaxValue':'10'}]}),
                  nestedFilter(),
                  propertiesToReturn(AllStatistics='1', AllAttributes='1'),
                  scopeOptions(IncludeMatchedScopesInResults='1', IncludeScope='1', IncludeId='1', IncludeScopeType='1'),
                  propertyOptions(IncludeName='1', IncludeRawValue='1', IncludeMeasure='1', IncludeCreator='1', IncludeCreatorType='1')
              ),
-             testCase(
+             TestCase(
                  scopeFilter(ScopeTypes={'ScopeType':'workflow'}, MaxDepth='999',),
                  nestedFilter(Depth='0'),
                  propertiesToReturn(AllAttributes='1', Properties=[{'Property':'IdDependencyList'}]),
                  scopeOptions(IncludeMatchedScopesInResults='1', IncludeScope='1', IncludeId='1', IncludeScopeType='1'),
                  propertyOptions(IncludeName='1', IncludeRawValue='1', IncludeMeasure='1', IncludeCreator='1', IncludeCreatorType='1')
              ),
-             testCase(
+             TestCase(
                  scopeFilter(ScopeTypes={'ScopeType':'workflow'}, MaxDepth='999',),
                  nestedFilter(Depth='0'),
                  propertiesToReturn(Properties=[{'Property':'IdDependency'}]),
@@ -452,136 +309,36 @@ testCases = [
              ),
             ]
 
-def ExecTestCase(jobname, wuid, tcase, tcasename):
-    testfilename = jobname + '_' + tcasename
-    logging.debug('Executing %s',testfilename)
- 
-    keyfile = (tcasekeydir / testfilename).with_suffix('.json')
-    outfile = (resultdir / testfilename).with_suffix('.json')
-    errfile = outfile.with_suffix('.err')
-    reqfile = outfile.with_suffix('.req')
-    respfile = outfile.with_suffix('.resp')
-
-    if (outfile.exists()): outfile.unlink()
-    if (errfile.exists()): errfile.unlink()
-    if (reqfile.exists()): reqfile.unlink()
-    if (respfile.exists()): respfile.unlink()
-
-    try:
-        wuresp = wudetails.service.WUDetails(WUID=wuid,
-                                             ScopeFilter=tcase.ScopeFilter,
-                                             NestedFilter=tcase.NestedFilter,
-                                             PropertiesToReturn=tcase.PropertiesToReturn,
-                                             ScopeOptions=tcase.ScopeOptions,
-                                             PropertyOptions=tcase.PropertyOptions)
-    except:
-        logging.critical('Unable to submit WUDetails request')
-        raise
-
-    if (args.logreq):
-        with reqfile.open(mode='w') as f:
-            print (transport.xml_request, file=f)
-    if (args.logresp):
-        with respfile.open(mode='w') as f:
-            print (wuresp, file=f)
-    try:
-        if (wuresp['WUID']==wuid):
-            wuresp['WUID'] = '{masked WUID - matches request}'
-
-        maskoutFields(wuresp)
-    except:
-        logging.critical('Unable to process WUDetails response')
-        logging.critical('Request & response content in %s', str(errfile))
-        with errfile.open(mode='w') as f:
-            print ('====== Request ===========', file=f)
-            print (transport.xml_request, file=f)
-            print ('====== Response ==========', file=f)
-            print (wuresp, file=f)
-        return False
-
-    with outfile.open(mode='w') as f:
-        print (tcase, file=f)
-        print (wuresp, file=f)
-
-    if (not keyfile.exists()):       
-        logging.error('FAILED %s', testfilename)
-        logging.error('Missing key file %s', str(keyfile))
-        return False
-
-    # Compare actual and expectetd
-    if (not filecmp.cmp(str(outfile),str(keyfile))):
-        logging.error('FAILED %s', testfilename)
-        logging.error('WUDetails response %s', str(outfile))
-        return False
-    else:
-        logging.debug('PASSED %s', testfilename)
-        return True
-
-class Statistics:
-    def __init__(self):
-        self.successCount = 0
-        self.failureCount = 0 
-       
-    def addCount(self, success):
-        if (success):
-            self.successCount += 1
-        else:
-            self.failureCount += 1
-
-# To make this Python3.4 compatible
-def safeMkdir(path):
-    try:
-        path.mkdir(parents=True)
-    except FileExistsError as e:
-        # It is ok if alrady exists
-        pass
-    except (FileNotFoundError, PermissionError) as e:
-        logging.error("'%s' \nExit." % (str(e)))
-        exit(-1)
-    except:
-        print("Unexpected error:" + str(sys.exc_info()[0]) + " (line: " + str(inspect.stack()[0][2]) + ")" )
-        traceback.print_stack()
-        exit(-1)
-
-resultdir = Path(args.outdir)
-safeMkdir(resultdir)
-
-tcasekeydir = Path(keydir)
-safeMkdir(tcasekeydir)
-
 logging.info('Gathering workunits')
-try:
-    wu = GetTestWorkunits()
-except:
-    raise
+wu = wutest.getTestWorkunits(requiredJobs)
 
-logging.info('Matched job count: %d', len(wu))
-if (len(wu)==0):
+logging.info('Matched job count: %d', wutest.getMatchedJobCount())
+if (wutest.getMatchedJobCount()==0):
     logging.error('There are no matching jobs.  Has the regression suite been executed?')
     logging.error('Aborting')
     exit(1)
 
-missingjobs = GetMissingJobCount(wu)
+missingjobs = wutest.getMissingJobCount(requiredJobs)
 if (missingjobs > 0):
     logging.warning('There are %d missing jobs.  Full regression will not be executed', missingjobs)
 
 logging.info('Executing regression test cases')
-stats = Statistics()
+stats = wucommon.Statistics()
 for jobname, wuid in wu.items():
     logging.debug('Job %s (WUID %s)', jobname, wuid)
 
     if (jobname == 'sort_thor'):
-        for index, t in enumerate(testCases):
+        for index, t in enumerate(TestCases):
             tcasename = 'testcase' + str(index+1)
-            success = ExecTestCase(jobname, wuid, t, tcasename)
+            success = execTestCase(jobname, wuid, t, tcasename)
             stats.addCount(success)
     elif (jobname in ['sets_thor','sets_roxie', 'sets_hthor']):
-        success = ExecTestCase(jobname, wuid, testCases[30], 'testcase31')
+        success = execTestCase(jobname, wuid, TestCases[30], 'testcase31')
         stats.addCount(success)
-        success = ExecTestCase(jobname, wuid, testCases[31], 'testcase32')
+        success = execTestCase(jobname, wuid, TestCases[31], 'testcase32')
         stats.addCount(success)
     else:
-        success = ExecTestCase(jobname, wuid, testCases[0], 'testcase1')
+        success = execTestCase(jobname, wuid, TestCases[0], 'testcase1')
         stats.addCount(success)
 logging.info('Success count: %d', stats.successCount)
 logging.info('Failure count: %d', stats.failureCount)

thorlcr/graph/thgraphmaster.cpp

@@ -2958,19 +2958,22 @@ CTimingInfo::CTimingInfo(CJobBase &ctx) : CThorStats(ctx, StTimeLocalExecute)
 
 ProgressInfo::ProgressInfo(CJobBase &ctx) : CThorStats(ctx, StNumRowsProcessed)
 {
-    startcount = stopcount = 0;
+    startCount = stopCount = 0;
 }
 void ProgressInfo::processInfo() // reimplement as counts have special flags (i.e. stop/start)
 {
     reset();
-    startcount = stopcount = 0;
+    startCount = stopCount = 0;
     ForEachItemIn(n, counts)
     {
         unsigned __int64 thiscount = counts.item(n);
-        if (thiscount & THORDATALINK_STARTED)
-            startcount++;
         if (thiscount & THORDATALINK_STOPPED)
-            stopcount++;
+        {
+            startCount++;
+            stopCount++;
+        }
+        else if (thiscount & THORDATALINK_STARTED)
+            startCount++;
         thiscount = thiscount & THORDATALINK_COUNT_MASK;
         tallyValue(thiscount, n+1);
     }
@@ -2982,8 +2985,8 @@ void ProgressInfo::getStats(IStatisticGatherer & stats)
     CThorStats::getStats(stats, true);
     stats.addStatistic(kind, tot);
     stats.addStatistic(StNumSlaves, counts.ordinality());
-    stats.addStatistic(StNumStarts, startcount);
-    stats.addStatistic(StNumStops, stopcount);
+    stats.addStatistic(StNumStarts, startCount);
+    stats.addStatistic(StNumStops, stopCount);
 }
 
 

thorlcr/graph/thgraphmaster.ipp

@@ -119,7 +119,7 @@ public:
 
 class graphmaster_decl ProgressInfo : public CThorStats
 {
-    unsigned startcount, stopcount;
+    unsigned startCount, stopCount;
 public:
     ProgressInfo(CJobBase &ctx);
 

thorlcr/graph/thgraphslave.hpp

@@ -78,7 +78,8 @@ public:
 
     inline void dataLinkStop()
     {
-        count = (count & THORDATALINK_COUNT_MASK) | THORDATALINK_STOPPED;
+        if (hasStarted())
+            count = (count & THORDATALINK_COUNT_MASK) | THORDATALINK_STOPPED;
 #ifdef _TESTING
         owner.ActPrintLog("ITDL output %d stopped, count was %" RCPF "d", outputId, getDataLinkCount());
 #endif