HPCC-13003 Regression Test Engine system check won't work on remote cluster.

Add code to check the ESP server can reach on every target.

Reorganise ecl-test main to provide targets to checking.

Add new, remote related error code.

Signed-off-by: Attila Vamos <attila.vamos@gmail.com>

testing/regress/ecl-test

@@ -28,8 +28,9 @@ import glob
 from hpcc.util import argparse
 from hpcc.regression.regress import Regression
 from hpcc.util.ecl.file import ECLFile
-from hpcc.util.util import checkPqParam, getVersionNumbers, checkXParam, convertPath, getRealIPAddress, parentPath, checkClusters, checkHpccStatus
+from hpcc.util.util import setConfig, checkPqParam, getVersionNumbers, checkXParam, convertPath, getRealIPAddress, parentPath, checkClusters, checkHpccStatus
 from hpcc.common.error import Error
+from hpcc.common.config import Config
 
 # For coverage
 if ('coverage' in os.environ) and (os.environ['coverage'] == '1'):
@@ -113,11 +114,6 @@ class RegressMain:
                 self.regress.runSuite(cluster, self.regress.suites[cluster])
 
     def main(self):
-        try:
-            checkHpccStatus()
-        except Error as e:
-            exit(e.getErrorCode());
-
         prog = "ecl-test"
         description = 'HPCC Platform Regression suite'
         pythonVer = getVersionNumbers()
@@ -199,6 +195,49 @@ class RegressMain:
                                 action='store_true')
 
         self.args = parser.parse_args()
+
+        # Process config parameter
+        self.config = Config(self.args.config).configObj
+        setConfig(self.config)
+
+        # Process target parameter
+        self.targetClusters = []
+        if 'target' in self.args:
+            if '' == self.args.target:
+                # Target not specified, use default from config
+                try:
+                    if self.args.func == 'setup':
+                        defaultTargets = self.config.defaultSetupClusters
+                        targetSet='defaultSetupClusters'
+                    else:
+                        defaultTargets = self.config.defaultTargetClusters
+                        targetSet='defaultTargetClusters'
+
+                    self.targetClusters = checkClusters(defaultTargets, targetSet)
+
+                except AttributeError:
+                    # It seems there is no defaultSetupClusters|defaultTargetClusters array in the config file
+                    # use the first one of cluster list in config file
+                    self.targetClusters.append(self.config.Clusters[0])
+            elif 'all' == self.args.target:
+                for cluster in self.config.Clusters:
+                    self.targetClusters.append(str(cluster))
+            else:
+                if ',' in self.args.target:
+                    # target is a list, process it
+                    targets = self.args.target.split(',')
+                    self.targetClusters = checkClusters(targets, 'target')
+                elif self.args.target in self.config.Clusters:
+                    self.targetClusters.append(self.args.target)
+                else:
+                    logging.error("%s. Unknown target cluster:'%s'!" % (1,  self.args.target))
+                    raise Error("4000")
+
+        try:
+            checkHpccStatus(self.targetClusters)
+        except Error as e:
+            exit(e.getErrorCode());
+
         try:
             self.regress = None
             if self.args.X[0]== "5000":
@@ -222,38 +261,6 @@ class RegressMain:
             self.regress = Regression(self.args)
             logging.debug("Suite full path:%s",  regressionSuiteFullPath)
 
-            self.targetClusters = []
-            if 'target' in self.args:
-                if '' == self.args.target:
-                    # Target not specified, use default from config
-                    try:
-                        if self.args.func == 'setup':
-                            defaultTargets = self.regress.config.defaultSetupClusters
-                            targetSet='defaultSetupClusters'
-                        else:
-                            defaultTargets = self.regress.config.defaultTargetClusters
-                            targetSet='defaultTargetClusters'
-
-                        self.targetClusters = checkClusters(defaultTargets, targetSet)
-
-                    except AttributeError:
-                        # It seems there is no defaultSetupClusters|defaultTargetClusters array in the config file
-                        # use the first one of cluster list in config file
-                        self.targetClusters.append(self.regress.config.Clusters[0])
-                elif 'all' == self.args.target:
-                    for cluster in self.regress.config.Clusters:
-                        self.targetClusters.append(str(cluster))
-                else:
-                    if ',' in self.args.target:
-                        # target is a list, process it
-                        targets = self.args.target.split(',')
-                        self.targetClusters = checkClusters(targets, 'target')
-                    elif self.args.target in self.regress.config.Clusters:
-                        self.targetClusters.append(self.args.target)
-                    else:
-                        logging.error("%s. Unknown target cluster:'%s'!" % (1,  self.args.target))
-                        raise Error("4000")
-
             if self.args.func == 'list':
                 self.listClusters()
             elif self.args.func == 'query':

testing/regress/hpcc/common/error.py

@@ -33,6 +33,7 @@ ERROR = {
     "6001": "HPCC System doesn't run!",
     "6002": "OS error when try to call ecl command!",
     "6003": "Parameter error when try to call ecl command!",
+    "6004": "Can't connect to remote HPCC System!",
 }
 
 

testing/regress/hpcc/regression/regress.py

@@ -32,7 +32,7 @@ from ..regression.suite import Suite
 from ..util.ecl.cc import ECLCC
 from ..util.ecl.command import ECLcmd
 from ..util.expandcheck import ExpandCheck
-from ..util.util import setConfig,  queryWuid,  abortWorkunit, getVersionNumbers
+from ..util.util import getConfig, queryWuid,  abortWorkunit, getVersionNumbers
 
 
 class Regression:
@@ -48,8 +48,7 @@ class Regression:
 
     def __init__(self, args):
         self.args = args
-        self.config = Config(args.config).configObj
-        setConfig(self.config)
+        self.config = getConfig()
         self.suites = {}
         self.log = Logger(args.loglevel)
         if args.timeout == '0':

testing/regress/hpcc/util/util.py

@@ -25,6 +25,7 @@ import subprocess
 
 from ..common.error import Error
 from ..common.shell import Shell
+from ..common.config import Config
 
 def isPositiveIntNum(string):
     for i in range(0,  len(string)):
@@ -180,44 +181,79 @@ def checkClusters(clusters,  targetSet):
 
     return  targetClusters
 
-def checkHpccStatus():
-    status='OK'
-    try:
-        myProc = subprocess.Popen(["ecl --version"],  shell=True,  bufsize=8192,  stdout=subprocess.PIPE,  stderr=subprocess.PIPE)
-        result = myProc.stdout.read() + myProc.stderr.read()
-        results = result.split('\n')
-        for line in results:
-            if 'not found' in line:
-                err = Error("6000")
-                logging.error("%s. %s:'%s'" % (1,  err,  line))
-                raise  err
-                break
-
-        myProc = subprocess.Popen("ecl getname --wuid 'W*'",  shell=True,  bufsize=8192,  stdout=subprocess.PIPE,  stderr=subprocess.PIPE)
-        result  = myProc.stdout.read() + myProc.stderr.read()
-        results = result.split('\n')
-        for line in results:
-            if "Error connecting" in line:
-                err = Error("6001")
-                logging.error("%s. %s:'%s'" % (1,  err,  line))
-                raise (err)
-                break
-
-            if "command not found" in line:
-                err = Error("6002")
-                logging.error("%s. %s:'%s'" % (1,  err,  line))
-                raise (err)
-                break
-
-    except  OSError:
-        err = Error("6002")
-        logging.error("%s. checkHpccStatus error:%s!" % (1,  err))
-        raise Error(err)
-
-    except ValueError:
-        err = Error("6003")
-        logging.error("%s. checkHpccStatus error:%s!" % (1,  err))
-        raise Error(err)
-
-    finally:
-        pass
+def isLocalIP(ip):
+    retVal=False
+    if '127.0.0.1' == ip:
+        retVal = True
+    elif ip == getRealIPAddress():
+        retVal = True
+
+    return retVal
+
+def checkHpccStatus(targets):
+    # Check HPCC Systems status on all local/remote target
+    isLocal = False
+    isLocalChecked = False
+    isIpChecked={}
+    config = getConfig()
+    for target in targets:
+        ip = config.IpAddress[target]
+
+        if not ip in isIpChecked:
+            isIpChecked[ip] = False
+
+        isLocal = isLocalIP(ip)
+        if isIpChecked[ip] or (isLocal and isLocalChecked):
+            continue
+
+        try:
+            if isLocal:
+                # There is no remote version (yet)
+                myProc = subprocess.Popen(["ecl --version"],  shell=True,  bufsize=8192,  stdout=subprocess.PIPE,  stderr=subprocess.PIPE)
+                result = myProc.stdout.read() + myProc.stderr.read()
+                results = result.split('\n')
+                for line in results:
+                    if 'not found' in line:
+                        err = Error("6000")
+                        logging.error("%s. %s:'%s'" % (1,  err,  line))
+                        raise  err
+                        break
+
+            myProc = subprocess.Popen("ecl getname --wuid 'W*' --limit=5 --server="+ip,  shell=True,  bufsize=8192,  stdout=subprocess.PIPE,  stderr=subprocess.PIPE)
+            result  = myProc.stdout.read() + myProc.stderr.read()
+            results = result.split('\n')
+            for line in results:
+                if "Error connecting" in line:
+                    if isLocal:
+                        err = Error("6001")
+                        logging.error("%s. %s:'%s local %s target!'" % (1,  err,  line,  target))
+                        raise (err)
+                    else:
+                        err = Error("6004")
+                        logging.error("%s. %s:'%s remote %s target!'" % (1,  err,  line,  target))
+                        raise (err)
+                    break
+
+                if "command not found" in line:
+                    err = Error("6002")
+                    logging.error("%s. %s:'%s'" % (1,  err,  line))
+                    raise (err)
+                    break
+
+            if isLocal:
+                isLocalChecked = True
+
+            isIpChecked[ip] = True
+
+        except  OSError:
+            err = Error("6002")
+            logging.error("%s. checkHpccStatus error:%s!" % (1,  err))
+            raise Error(err)
+
+        except ValueError:
+            err = Error("6003")
+            logging.error("%s. checkHpccStatus error:%s!" % (1,  err))
+            raise Error(err)
+
+        finally:
+            pass