HPCC-11606 ecl-test no longer starting

Resolve merge problems

Signed-off-by: Attila Vamos <attila.vamos@gmail.com>

testing/regress/README.rst

@@ -5,8 +5,8 @@ To use Regression Suite change directory to HPCC-Platform/testing/regress subdir
 
 Regression Suite requires Python environment version >=2.6.6 and < 3.x
 
-Parameters of Regression Suite:
--------------------------------
+Global parameters of Regression Suite:
+--------------------------------------
 
 Command:
  
@@ -582,6 +582,93 @@ So if you have a new test case and it works well on all clusters (or some of the
         "timeout":"600",                                - Default test case timeout in sec. Can be override by command line parameter or //timeout tag in ECL file
         "maxAttemptCount":"3"                           - Max retry count to reset timeout if a testcase in any early stage (compiled, blocked) of execution pipeline.
 
+Optionally the config file can contain a section of default values for stored parameters like this:
+
+    "Params":[
+                "querya.ecl:param1=value1,param2=value2",
+                "queryb.ecl:param1=value3",
+                "some*.ecl:paramforsome=value4",
+                "*.ecl:globalparam=blah"
+            ]
+
+The Regression Suite processes the Params definition(s) sequentially. The -Xname=value command line parameter overrides any values defined in this section.
+Examples:
+
+We have an ECL source called PassTest.ecl with these lines:
+
+|    //nokey        # To avoid result comparison error
+|    string bla := 'EN' : STORED('bla');
+|    output(bla);
+
+1. For the purposes of this example, we assume there is no Params section in the testing/regress/ecl_test.json file or it is empty and there are no PassTest.ecl related global entries.
+
+If we execute it with query mode:
+
+|     ./ecl_test query PassTest.ecl -t hthor
+
+The result is:
+
+|     [Action] Target: hthor
+|     [Action] Queries: 1
+|     [Action]   1. Test: PassTest.ecl
+|     [Pass]   1. Pass W20140508-180241 (1 sec)
+|     [Pass]   1. URL http://127.0.0.1:8010/WsWorkunits/WUInfo?Wuid=W20140508-180241
+|     [Action]
+|         Results
+|         -------------------------------------------------
+|         Passing: 1
+|         Failure: 0
+|         -------------------------------------------------
+|         u"Output of PassTest.ecl test is:\n\t<Dataset name='Result 1'>\n <Row><Result_1>EN</Result_1></Row>\n</Dataset>\n"
+|         -------------------------------------------------
+|         Log: /home/ati/HPCCSystems-regression/log/hthor.14-05-08-18-02-41.log
+|         -------------------------------------------------
+|         Elapsed time: 4 sec  (00:00:04)
+|         -------------------------------------------------
+
+2. Same as 1. but execute it in query mode with -X parameter:
+
+|     ./ecl_test -Xbla=blabla query PassTest.ecl -t hthor
+
+then the output of PassTest.ecl changes in the result:
+|         -------------------------------------------------
+|         u"Output of PassTest.ecl test is:\n\t<Dataset name='Result 1'>\n <Row><Result_1>blabla</Result_1></Row>\n</Dataset>\n"
+|         -------------------------------------------------
+
+3. If we want to apply same stored value every execution then we can put it into the ecl_test.json configuration file:
+
+|    "Params":[
+|                "PassTest.ecl:bla='A value'"
+|          ]
+
+We can execute it with a simple query mode:
+
+|     ./ecl_test query PassTest.ecl -t hthor
+
+then the output of PassTest.ecl changes in the result accordingly with the value from the Params option:
+|         -------------------------------------------------
+|         u"Output of PassTest.ecl test is:\n\t<Dataset name='Result 1'>\n <Row><Result_1>A value</Result_1></Row>\n</Dataset>\n"
+|         -------------------------------------------------
+
+4. Finally we have value(s) in the config file, but we want to run PassTest.ecl with another input value.
+
+In this case we can use same command as in 2. with a new value:
+
+|     ./ecl_test -Xbla='Another value' query PassTest.ecl -t hthor
+
+then the output of PassTest.ecl changes in the result:
+|         -------------------------------------------------
+|         u"Output of PassTest.ecl test is:\n\t<Dataset name='Result 1'>\n <Row><Result_1>Another value</Result_1></Row>\n</Dataset>\n"
+|         -------------------------------------------------
+
+We can use as many values as we need in this form:
+|       -Xname1=value1,name2=value2...
+
+Important!
+    There should not be any spaces before or after the commas.
+    If there is more than one -X in the command line, the last will be the active and all other discarded.
+
+
 
 10. Authentication:
 -------------------

testing/regress/ecl-test

@@ -28,7 +28,7 @@ import glob
 from hpcc.util import argparse
 from hpcc.regression.regress import Regression
 from hpcc.util.ecl.file import ECLFile
-from hpcc.util.util import checkPqParam,  getVersionNumbers
+from hpcc.util.util import checkPqParam,  getVersionNumbers,  checkXParam
 from hpcc.common.error import Error
 
 prog_version = "0.0.21"
@@ -88,15 +88,15 @@ class RegressMain:
             try:
                 if len(eclfiles) > 1:
                     #Execute multiple ECL files like RUN to generates summary results and diff report.
-                    self.regress.bootstrap(cluster, eclfiles)
-                    if  'pq' in self.args:
+                    self.regress.bootstrap(cluster, self.args,  eclfiles)
+                    if  self.args.pq:
                         self.regress.runSuiteP(cluster, self.regress.suites[cluster])
                     else:
                         self.regress.runSuite(cluster, self.regress.suites[cluster])
                 elif len(eclfiles) == 1:
                     # Execute one ECL file on the cluster
                     for ecl in eclfiles:
-                        eclfile = ECLFile(ecl, self.regress.dir_a, self.regress.dir_ex, self.regress.dir_r, cluster)
+                        eclfile = ECLFile(ecl, self.regress.dir_a, self.regress.dir_ex, self.regress.dir_r, cluster,  self.args)
                         # Check if this query is not skip on this cluster and not part of setup
                         if (not eclfile.testSkip(cluster)['skip']) and (not eclfile.testSkip('setup')['skip'] ):
                             if not eclfile.testExclusion(cluster):
@@ -115,16 +115,16 @@ class RegressMain:
     def setup(self):
         if self.args.target in self.regress.config.Clusters:
             if  self.args.pq :
-                self.regress.runSuiteP(self.args.target, self.regress.Setup(self.args.target))
+                self.regress.runSuiteP(self.args.target, self.regress.Setup(self.args))
             else:
-                self.regress.runSuite(self.args.target, self.regress.Setup(self.args.target))
+                self.regress.runSuite(self.args.target, self.regress.Setup(self.args))
         else:
             logging.error("%s. Unknown target cluster:'%s'!" % (1,  self.args.target))
             raise Error("4000")
 
     def run(self):
         if self.args.target in self.regress.config.Clusters:
-            self.regress.bootstrap(self.args.target)
+            self.regress.bootstrap(self.args.target, self.args)
             if  self.args.pq :
                 self.regress.runSuiteP(self.args.target, self.regress.suites[self.args.target])
             else:
@@ -208,18 +208,21 @@ class RegressMain:
                                 type=checkPqParam,  default = 0,   metavar="threadNumber")
 
         self.args = parser.parse_args()
+        try:
+            if self.args.X[0]== "5000":
+                self.regress = None
+                raise Error(self.args.X[0])
 
-        # Resolve Regression Suite starting path for ecl-test.json config file
-        # It is necessary when Regression Suite doesn't started from its home directory
-        regressionSuiteMainDir = os.path.dirname(__file__)
-        regressionSuiteFullPath = os.path.realpath(regressionSuiteMainDir)
-        self.args.config = str(os.path.join(regressionSuiteFullPath, self.args.config))
+            # Resolve Regression Suite starting path for ecl-test.json config file
+            # It is necessary when Regression Suite doesn't started from its home directory
+            regressionSuiteMainDir = os.path.dirname(__file__)
+            regressionSuiteFullPath = os.path.realpath(regressionSuiteMainDir)
+            self.args.config = str(os.path.join(regressionSuiteFullPath, self.args.config))
 
-        self.regress = Regression(self.args)
-        logging.debug("Suite version:%s",  versionStr)
-        logging.debug("Suite full path:%s",  regressionSuiteFullPath)
+            self.regress = Regression(self.args)
+            logging.debug("Suite version:%s",  versionStr)
+            logging.debug("Suite full path:%s",  regressionSuiteFullPath)
 
-        try:
             if self.args.func == 'list':
                 self.listClusters()
             elif self.args.func == 'query':
@@ -237,7 +240,8 @@ class RegressMain:
         except KeyboardInterrupt:
             logging.critical("Keyboard Interrupt Caught.")
         finally:
-            self.regress.StopTimeoutThread()
+            if self.regress:
+                self.regress.StopTimeoutThread()
         exit()
 
 if __name__ == "__main__":

testing/regress/ecl-test.json

@@ -19,6 +19,9 @@
             "roxie"
         ],
         "timeout":"600",
-        "maxAttemptCount":"3"
+        "maxAttemptCount":"3",
+        "Params":[
+            "PassTest.ecl:bla='A value'"
+        ]
     }
 }

testing/regress/hpcc/common/error.py

@@ -27,7 +27,8 @@ ERROR = {
     "3000": "Return is null",
     "3001": "Return diff does not match.",
     "4000": "Unknown cluster!",
-    "4001": "No ECl file!"
+    "4001": "No ECl file!",
+    "5000": "Missing argument of -X parameter!\nIt should be 'name=val[,name2=val2..]'"
 }
 
 

testing/regress/hpcc/regression/regress.py

@@ -121,27 +121,27 @@ class Regression:
     def setLogLevel(self, level):
         self.log.setLevel(level)
 
-    def bootstrap(self, cluster,  fileList=None):
+    def bootstrap(self, cluster, args,   fileList=None):
         self.createDirectory(self.regressionDir)
         self.createDirectory(self.dir_a)
         self.createDirectory(self.dir_r)
         self.createDirectory(self.logDir)
 
-        self.suites[cluster] = Suite(cluster, self.dir_ec, self.dir_a, self.dir_ex, self.dir_r, self.logDir,  False,  fileList)
+        self.suites[cluster] = Suite(cluster, self.dir_ec, self.dir_a, self.dir_ex, self.dir_r, self.logDir, args, False, fileList)
         self.maxtasks = len(self.suites[cluster].getSuite())
 
     def createDirectory(self, dir_n):
         if not os.path.isdir(dir_n):
             os.makedirs(dir_n)
 
-    def Setup(self,  cluster):
+    def Setup(self,  args):
         self.createDirectory(self.regressionDir)
         self.createDirectory(self.dir_a)
         self.createDirectory(self.dir_r)
         self.createDirectory(self.logDir)
         self.setupDir = ExpandCheck.dir_exists(os.path.join(self.suiteDir, self.config.setupDir), True)
         logging.debug("Setup Dir      : %s", self.setupDir)
-        self.setupSuite = Suite(cluster, self.setupDir, self.dir_a, self.dir_ex, self.dir_r, self.logDir,  True)
+        self.setupSuite = Suite(args.target, self.setupDir, self.dir_a, self.dir_ex, self.dir_r, self.logDir, args, True)
         self.maxtasks = len(self.setupSuite.getSuite())
         return self.setupSuite
 

testing/regress/hpcc/regression/suite.py

@@ -25,7 +25,7 @@ from ..util.ecl.file import ECLFile
 from ..common.error import Error
 
 class Suite:
-    def __init__(self, name, dir_ec, dir_a, dir_ex, dir_r, logDir,  isSetup=False,  fileList = None):
+    def __init__(self, name, dir_ec, dir_a, dir_ex, dir_r, logDir, args, isSetup=False,  fileList = None):
         self.name = name
         self.suite = []
         self.dir_ec = dir_ec
@@ -36,7 +36,7 @@ class Suite:
         self.exclude = []
         self.publish = []
 
-        self.buildSuite(isSetup,  fileList)
+        self.buildSuite(args, isSetup, fileList)
 
         if len(self.exclude):
             curTime = time.strftime("%y-%m-%d-%H-%M")
@@ -47,7 +47,7 @@ class Suite:
                 self.log.write(item+"\n")
             self.log.close();
 
-    def buildSuite(self,  isSetup,  fileList):
+    def buildSuite(self, args, isSetup,  fileList):
         if fileList == None:
             if not os.path.isdir(self.dir_ec):
                 raise Error("2001", err="Not Found: %s" % self.dir_ec)
@@ -60,7 +60,7 @@ class Suite:
             if file.endswith(".ecl"):
                 ecl = os.path.join(self.dir_ec, file)
                 eclfile = ECLFile(ecl, self.dir_a, self.dir_ex,
-                                  self.dir_r,  self.name)
+                                  self.dir_r,  self.name, args)
                 if isSetup:
                     skipResult = eclfile.testSkip('setup')
                 else:

testing/regress/hpcc/util/ecl/command.py

@@ -70,6 +70,9 @@ class ECLcmd(Shell):
                 name = eclfile.getJobname()
 
             args.append("--name=" + name)
+
+            args = args + eclfile.getStoredInputParameters()
+
             args.append(eclfile.getArchive())
         data = ""
         wuid = "N/A"

testing/regress/hpcc/util/ecl/file.py

@@ -21,8 +21,9 @@ import difflib
 import logging
 import os
 import traceback
+import re
 
-from ...util.util import isPositiveIntNum
+from ...util.util import isPositiveIntNum, getConfig
 
 class ECLFile:
     ecl = None
@@ -42,7 +43,7 @@ class ECLFile:
     taskId = -1
     ignoreResult=False
 
-    def __init__(self, ecl, dir_a, dir_ex, dir_r,  cluster):
+    def __init__(self, ecl, dir_a, dir_ex, dir_r,  cluster, args):
         self.dir_ec = os.path.dirname(ecl)
         self.dir_ex = dir_ex
         self.dir_r = dir_r

testing/regress/hpcc/util/util.py

@@ -21,6 +21,8 @@ import argparse
 import platform
 import logging
 
+from ..common.error import Error
+
 def isPositiveIntNum(string):
     for i in range(0,  len(string)):
         if (string[i] < '0') or (string[i] > '9'):
@@ -37,6 +39,20 @@ def checkPqParam(string):
 
     return value
 
+def checkXParam(string):
+    param=str(string)
+    if len(param):
+        if ('=' in param) or ('None' == param):
+            value = param
+        else:
+            #logging.error("%s. Missing or wrong argument '%s' after -X parameter!\nIt should be 'name=val[,name2=val2..]'\n5000\n" % (1,  param))
+            value="5000"
+    else:
+        msg = "Missing argument of -X parameter!"
+        raise argparse.ArgumentTypeError(msg)
+
+    return value
+
 def getVersionNumbers():
     version = platform.python_version_tuple()
     verNum = {'main':0,  'minor':0,  'patch':0}
@@ -57,6 +73,9 @@ def setConfig(config):
     global gConfig
     gConfig = config
 
+def getConfig():
+    return gConfig
+
 def queryWuid(jobname,  taskId):
     server = gConfig.server
     host = "http://"+server+"/WsWorkunits/WUQuery.json?Jobname="+jobname