Merge pull request #15314 from AttilaVamos/HPCC-26462-impr-8.2.x

HPCC-26462 Make ecl command '--poll' parameter configurable via config file.

Reviewed-By: Richard Chapman <rchapman@hpccsystems.com>

testing/regress/ecl-test

@@ -368,10 +368,13 @@ class RegressMain:
         self.config.set('preAbort', self.args.preAbort)
 
         if not self.config.has('wuStatusTimeout'):
-            self.config.set('wuStatusTimeout', 30)
+            self.config.set('wuStatusTimeout', "30")
 
         if not self.config.has('wuAbortTimeout'):
-            self.config.set('wuAbortTimeout', 30)
+            self.config.set('wuAbortTimeout', "30")
+
+        if not self.config.has('usePoll'):
+            self.config.set('usePoll', "False")
 
         self.config.set('log',  self.log)
         setConfig(self.config)

testing/regress/ecl-test-azure.json

@@ -0,0 +1,72 @@
+{
+    "Regress":{
+        "roxieTestSocket": ":9876",
+        "espIp" : "127.0.0.1",
+        "espSocket": "8010",
+        "dropzoneIp" : "127.0.0.1",
+        "dropzonePath" : "/var/lib/HPCCSystems/mydropzone/",
+        "espFileSprayService" : "FileSpray",
+        "useSsl" : "False",
+        "username": "regress",
+        "password": "regress",
+        "suiteDir": "",
+        "eclDir": "ecl",
+        "setupDir": "ecl/setup",
+        "keyDir": "ecl/key",
+        "archiveDir": "archives",
+        "resultDir": "results",
+        "zapDir":"zap",
+        "regressionDir": "~/HPCCSystems-regression",
+        "logDir": "~/HPCCSystems-regression/log",
+        "Engines": [
+            "hthor",
+            "thor",
+            "roxie-workunit"
+        ],
+        "Clusters": {
+            "hthor" : "hthor",
+            "thor"  : "thor",
+            "roxie-workunit" : "roxie-workunit"
+        },
+        "ClusterNames": {
+            "hthor" : "myhthor",
+            "thor"  : "mythor",
+            "roxie-workunit" : "myroxie"
+        },
+        "timeout":"3600",
+        "maxAttemptCount":"1",
+        "wuStatusTimeout":"3600",
+        "wuAbortTimeout":"720",
+        "usePoll":"True",
+        "defaultSetupTargets": [
+            "hthor",
+            "thor",
+            "roxie-workunit"
+        ],
+        "defaultTargets": [
+            "hthor",
+            "thor",
+            "roxie-workunit"
+        ],
+        "Params":[
+            "PassTest.ecl:bla='A value'",
+            "httpcall_multiheader.ecl:TargetIP=.",
+            "soapcall_multihttpheader.ecl:TargetIP=."
+        ],
+        "engineParams":[
+            "failOnLeaks",
+            "disableLocalOptimizations",
+            "thorConnectTimeout=36000"
+        ],
+        "setupExtraParams":{
+            "OriginalTextFilesOsPath" : "/opt/HPCCSystems/testing/regress",
+            "OriginalTextFilesIp" : "127.0.0.1"
+        },
+        "setupExtraDParams":{
+            "HPCCBaseDir" : "/opt/HPCCSystems/"
+        },
+        "FileExclusion":[
+            "*_blahblah.ecl"
+        ]
+    }
+}

testing/regress/ecl-test.json

@@ -37,6 +37,7 @@
         "maxAttemptCount":"3",
         "wuStatusTimeout":"30",
         "wuAbortTimeout":"30",
+        "usePoll":"False",
         "defaultSetupTargets": [
             "thor",
             "roxie"

testing/regress/hpcc/regression/regress.py

@@ -251,7 +251,7 @@ class Regression:
                     query.setIgnoreResult(self.args.ignoreResult)
                     query.setJobname(time.strftime("%y%m%d-%H%M%S"))
                     timeout = query.getTimeout()
-                    logger.debug("Query timeout:%d", -1, timeout)
+                    logger.debug("%3d. Query timeout:%d", -1, timeout)
                     oldCnt = cnt
 
                 started = False

testing/regress/hpcc/util/ecl/command.py

@@ -83,6 +83,9 @@ class ECLcmd(Shell):
             args.append('--exception-level=warning')
             args.append('--noroot')
 
+            if self.config.usePoll.lower() == 'true':
+                args.append('--poll')
+                
             name = kwargs.pop('name', False)
             if not name:
                 name = eclfile.getJobname()
@@ -145,8 +148,10 @@ class ECLcmd(Shell):
                             i = ET.tostring(xml).decode("utf-8")
                         logger.debug("%3d. ret:'%s'", eclfile.getTaskId(),  i )
                         pass
-                    try:    
-                        result += i + "\n"
+                    try:
+                        # "ecl run .... --poll" inserts 2 or 3 more lines and <Result> and </Result> tags into the result, filter them out
+                        if not ( i.startswith("Polling for") or i.startswith("Getting Workunit") or i.startswith("Getting Results") or i.startswith("Retrieving Results") or i.startswith("<Result>") or i.startswith("</Result>")):
+                            result += i + "\n"
                     except:
                         logger.error("%3d. type of i: '%s', i: '%s'", eclfile.getTaskId(), type(i), i )
                 cnt += 1
@@ -164,6 +169,15 @@ class ECLcmd(Shell):
             raise err
         finally:
             res = queryWuid(eclfile.getJobname(), eclfile.getTaskId())
+            if not res['wuid'].strip().startswith('W'):
+                tryCount = 5
+                while  tryCount > 0:
+                    tryCount -= 1
+                    res = queryWuid(eclfile.getJobname(), eclfile.getTaskId())
+                    if res['wuid'].strip().startswith('W'):
+                        break
+                    logger.debug("%3d. in finally -> 'wuid':'%s', 'state':'%s', attempt: %d, ", eclfile.getTaskId(), res['wuid'], res['state'],  tryCount)
+
             logger.debug("%3d. in finally -> 'wuid':'%s', 'state':'%s', data':'%s', ", eclfile.getTaskId(), res['wuid'], res['state'], data)
             if wuid ==  'N/A':
                 logger.debug("%3d. in finally queryWuid() -> 'result':'%s', 'wuid':'%s', 'state':'%s'", eclfile.getTaskId(),  res['result'],  res['wuid'],  res['state'])
@@ -223,13 +237,16 @@ class ECLcmd(Shell):
                     logger.debug("%3d. in state == failed 'wuid':'%s', 'state':'%s', data':'%s', ", eclfile.getTaskId(), res['wuid'], res['state'], data)
                     resultLines = data.strip().split('\n')
                     resultLineIndex = 0;
+                    #                                                   It has some output what should compare    
                     while resultLineIndex < len(resultLines) and not resultLines[resultLineIndex].startswith('<'):
                         resultLineIndex += 1
                     logger.debug("%3d. State is fail (resultLineIndex:%d, resultLines:'%s' )", eclfile.getTaskId(), resultLineIndex,  resultLines)
                     data = '\n'.join(resultLines[resultLineIndex:])+ "\n"
                     eclfile.addResults(data, wuid)
                     logger.debug("%3d. State is fail (resultLineIndex:%d, data:'%s' )", eclfile.getTaskId(), resultLineIndex,  data)
-                    test = eclfile.testResults()
+                    test = False
+                    if not resultLines[resultLineIndex].startswith('Error ('):
+                        test = eclfile.testResults()
                 else:
                     test = eclfile.testResults()
             report.addResult(eclfile)

testing/regress/hpcc/util/ecl/file.py

@@ -60,6 +60,7 @@ class ECLFile:
 
     def __init__(self, ecl, dir_a, dir_ex, dir_r, dir_inc, cluster, args):
         logger.debug("%3d. ECLFile(ecl: '%s', cluster: '%s').", self.taskId, ecl,  cluster)
+        self.config = getConfig()
         self.dir_ec = os.path.dirname(ecl)
         self.dir_ex = dir_ex
         self.dir_r = dir_r
@@ -87,6 +88,8 @@ class ECLFile:
         self.version=''
         self.versionId=0
         self.timeout = self.checkFileTimeout(int(args.timeout))
+        if self.timeout == -1:
+            self.timeout = int(self.config.timeout) # use default from config
         self.args = args
         self.eclccWarning = ''
         self.eclccWarningChanges = ''
@@ -110,7 +113,6 @@ class ECLFile:
         # -X in the CLI is the highest.
         self.optXHash=self.checkQueryxmlFile()
 
-        self.config = getConfig()
         try:
             # Process definitions of stored input value(s) from config
             for param in self.config.Params:

testing/regress/hpcc/util/util.py

@@ -128,6 +128,7 @@ def queryWuid(jobname,  taskId):
     args.append('-v')
     args.append('-n=' + jobname)
     args.append('--wait-read='+ gConfig.wuStatusTimeout )  # Timeout while reading from socket (in seconds)
+    args.append('--wait-connect=%d' % (int(gConfig.wuStatusTimeout) * 1000) )  # Timeout while reading from socket (in milliseconds)
     addCommonEclArgs(args)
 
     res, stderr = shell.command(cmd, *defaults)(*args)