Merge pull request #14469 from AttilaVamos/HPCC-25175-fix-7.12.x

HPCC-25175 Down-merge latest RTE to 7.12.x

Reviewed-by: Gavin Halliday <ghalliday@hpccsystems.com>
Merged-by: Gavin Halliday <ghalliday@hpccsystems.com>

testing/regress/ecl-test

@@ -155,8 +155,8 @@ class RegressMain:
                 if os.path.isfile(eclPath):
                     eclFiles.append(eclPath)
                 else:
-                    logging.error("%s. ECL file '%s' doesn't exist!" % (1,  eclPath))
-                    raise Error("4001")
+                    # Do not raise an error if file not exists
+                    logger.debug("To be excluded ECL file '%s' not exists!" % ( eclPath))
 
         if len(eclFiles) > 1:
             # Remove duplicates
@@ -287,6 +287,11 @@ class RegressMain:
             logging.critical(e)
             exit(e.getErrorCode());
 
+        if self.args.loglevel == 'info':
+            logger.setLevel(logging.INFO)
+        elif self.args.loglevel == 'debug':
+            logger.setLevel(logging.DEBUG)
+            
         regressionSuiteMainDir = os.path.dirname(__file__)
         regressionSuiteFullPath = os.path.realpath(regressionSuiteMainDir)
 
@@ -408,7 +413,7 @@ class RegressMain:
             self.args.setupExtraX.append('espIp=' + self.config.espIp )
 
             self.regress = Regression(self.args)
-            logging.debug("Suite full path:%s",  regressionSuiteFullPath)
+            logger.debug("Suite full path:%s",  regressionSuiteFullPath)
 
             # process file exclusion
             excludeItems = []

testing/regress/hpcc/regression/regress.py

@@ -527,14 +527,6 @@ class Regression:
                     query.setWuid(wuid)
                     query.diff = query.getEclccWarningChanges()
                     report[0].addResult(query)
-                elif query.testFail():
-                    logger.debug("Intentionally fails",  extra={'taskId':cnt})
-                    res = True
-                    wuid="No WUID"
-                    url = "N/A (Intentionally fails)"
-                    query.setWuid(wuid)
-                    query.diff = ''
-                    report[0].addResult(query)
                 else:
                     eclCmd = ECLcmd()
                     try:
@@ -569,13 +561,23 @@ class Regression:
                             PrintException(repr(e) + " runQuery() ")
 
                     wuid = query.getWuid()
-                    logger.debug("CMD result: '%s', wuid:'%s'"  % ( res,  wuid),  extra={'taskId':cnt})
                     if wuid == 'Not found':
                         res = False
+                        wuid="No WUID"
+                    if query.testFail():
+                        logger.debug("Intentionally fails",  extra={'taskId':cnt})
+                        if res == False:
+                            res = True
+                    logger.debug("CMD result: '%s', wuid:'%s'"  % ( res,  wuid),  extra={'taskId':cnt})
+                   
             else:
-                res = False
-                report[0].addResult(query)
                 wuid="N/A"
+                if query.testFail():
+                    res = True
+                    report[0].addResult(query)
+                else:
+                    res = False
+                    report[0].addResult(query)
 
             if wuid and wuid.startswith("W"):
                 if self.config.useSsl.lower() == 'true':
@@ -586,6 +588,7 @@ class Regression:
                 url += "/?Widget=WUDetailsWidget&Wuid="
                 url += wuid
             elif query.testFail():
+                url = "N/A"
                 res = True
             else:
                 url = "N/A"
@@ -595,6 +598,8 @@ class Regression:
             elapsTime = time.time()-startTime
             if res:
                 logger.info("%3d. Pass %s - %s (%d sec)" % (cnt, query.getBaseEclRealName(), wuid,  elapsTime),  extra={'taskId':cnt})
+                if query.testFail():
+                    logger.info("%3d. Intentionally fails" % (cnt),  extra={'taskId':cnt})
                 logger.info("%3d. URL %s" % (cnt,url))
             else:
                 if not wuid or not wuid.startswith("W"):
@@ -608,6 +613,7 @@ class Regression:
             query.setElapsTime(elapsTime)
             self.exitmutexes[th].release()
         except Exception as e:
+            PrintException(repr(e) + " runQuery()")
             logger.error("Unexpected error:'%s' (line: %s ) :%s " %( sys.exc_info()[0], str(inspect.stack()[0][2]),  repr(e) ) ,  extra={'taskId':cnt})
             elapsTime = time.time()-startTime
             query.setElapsTime(elapsTime)

testing/regress/hpcc/regression/suite.py

@@ -99,7 +99,7 @@ class Suite:
                         exclude=False
                         exclusionReason=''
                         if isSetup:
-                            exclude = eclfile.testExclusion('setup')
+                            exclude = eclfile.testExclusion('setup') or eclfile.testExclusion(self.clusterName)
                             exclusionReason=' setup'
                         elif ( 'all' not in  classIncluded ) or ('none' not in classExcluded):
                             included = True

testing/regress/hpcc/util/ecl/cc.py

@@ -59,10 +59,10 @@ class ECLCC(Shell):
                 os.unlink(filename)
             result, stderr = self.getArchive(ecl)
 
-            if result.startswith( 'Error()'):
+            if result.startswith( 'Error('):
                 if ecl.testFail():
                     logger.debug("%3d. Fail is the expected result (ecl:'%s')", ecl.getTaskId(),  ecl.getBaseEclRealName())
-                    retVal = True
+                    retVal = False # True
                 else:
                     retVal = False
                     ecl.diff += ("%3d. Test: %s\n") % (ecl.getTaskId(), ecl.getBaseEclRealName())

testing/regress/hpcc/util/ecl/command.py

@@ -74,8 +74,10 @@ class ECLcmd(Shell):
             name = kwargs.pop('name', False)
             if not name:
                 name = eclfile.getBaseEclName()
+                jname = eclfile.getJobname()
 
             args.append("--name=" + name)
+            args.append("-Dname=" + jname)
 
         else:
             args.append('--exception-level=warning')
@@ -166,7 +168,10 @@ class ECLcmd(Shell):
             if wuid ==  'N/A':
                 logger.debug("%3d. in finally queryWuid() -> 'result':'%s', 'wuid':'%s', 'state':'%s'", eclfile.getTaskId(),  res['result'],  res['wuid'],  res['state'])
                 wuid = res['wuid']
-                if res['result'] != "OK":
+                if eclfile.testFail():
+                    #eclfile.diff=eclfile.getBaseEcl()+'\n\t'+data+'\n'
+                    pass
+                elif res['result'] != "OK":
                     eclfile.diff=eclfile.getBaseEcl()+'\n\t'+res['state']+'\n'
                     logger.error("%3d. %s in queryWuid(%s)",  eclfile.getTaskId(),  res['state'],  eclfile.getJobname())
 

testing/regress/hpcc/util/ecl/file.py

@@ -75,7 +75,7 @@ class ECLFile:
         self.ecl = self.baseEcl
         self.xml_e = self.baseXml
         self.xml_r = self.baseXml
-        self.xml_a = 'archive_' + self.baseXml
+        self.xml_a = 'archive_' + self.cluster + '_' + self.baseXml
         self.jobname = self.basename
         self.diff = ''
         self.aborted = False
@@ -257,7 +257,7 @@ class ECLFile:
         #print("ECLFile.getArchiveName()")
         logger.debug("%3d. getArchiveName(isVersions:'%s')", self.taskId, self.isVersions )
         if self.isVersions:
-            dynamicFilename='archive_' + self.basename
+            dynamicFilename='archive_' + self.cluster + '_' + self.basename
             dynamicFilename+= '_v'+ str(self.versionId)
             dynamicFilename += '.xml'
             return os.path.join(self.dir_a, dynamicFilename)
@@ -649,6 +649,7 @@ class ECLFile:
                 open(expectedKeyPath, 'w').write("\n".join(self.eclccWarning))
                 pass
         try:
+            logger.debug("%3d.  self.eclccWarning: " +  self.eclccWarning,  self.taskId )
             diffLines = ''
             d = list(difflib.unified_diff(eclccKeyContent, self.eclccWarning, fromfile=expectedKeyPath, tofile="eclcc warning",  lineterm = ""))
             diffLines = "\n".join(d)
@@ -659,8 +660,8 @@ class ECLFile:
                 self.eclccWarningChanges += "\tEclcc generated warning changed\n"
                 logger.debug( "type(diffLines) is %s: ",  repr(type(diffLines)), extra={'taskId':self.taskId})
                 if type(diffLines) == type(' '):
-                    diffLines = unicodedata.normalize('NFKD', diffLines).encode('ascii','ignore').replace('\'','').replace('\\u', '\\\\u')
-                    diffLines = str(diffLines)
+                    diffLines = unicodedata.normalize('NFKD', diffLines).encode('ascii','ignore') #.replace('\'','').replace('\\u', '\\\\u')
+                    diffLines = str(diffLines).replace('\'','').replace('\\u', '\\\\u')
                 else:
                     diffLines = str(diffLines)
                 self.eclccWarningChanges += str(diffLines)

testing/regress/hpcc/util/util.py

@@ -160,7 +160,7 @@ def queryEngineProcess(engine,  taskId):
             if len(items) >= 12:
                 if engine in items[10]:
                     myProc2 = subprocess.Popen(["sudo readlink -f /proc/" + items[1] + "/exe"],  shell=True,  bufsize=8192,  stdout=subprocess.PIPE,  stderr=subprocess.PIPE)
-                    result2 = myProc2.stdout.read().replace ('\n', '')
+                    result2 = myProc2.stdout.read().decode("utf-8").replace ('\n', '')
                     binPath = os.path.dirname(result2)
                     logger.debug("%3d. queryEngineProcess(engine: %s): binary: '%s', binPath:'%s'",  taskId,  engine, result2,  binPath)
                     if 'slavenum' in line: