Merge pull request #9664 from ghalliday/issue17170

HPCC-17170 For regression tests split timing information into a separate file

Reviewed-By: Richard Chapman <rchapman@hpccsystems.com>

common/workunit/workunit.cpp

@@ -9012,7 +9012,7 @@ extern WORKUNIT_API StringBuffer &exportWorkUnitToXML(const IConstWorkUnit *wu,
     return str;
 }
 
-extern WORKUNIT_API void exportWorkUnitToXMLFile(const IConstWorkUnit *wu, const char * filename, unsigned extraXmlFlags, bool unpack, bool includeProgress, bool hidePasswords)
+extern WORKUNIT_API void exportWorkUnitToXMLFile(const IConstWorkUnit *wu, const char * filename, unsigned extraXmlFlags, bool unpack, bool includeProgress, bool hidePasswords, bool splitStats)
 {
     const IExtendedWUInterface *ewu = queryExtendedWU(wu);
     if (ewu)
@@ -9024,6 +9024,17 @@ extern WORKUNIT_API void exportWorkUnitToXMLFile(const IConstWorkUnit *wu, const
             p.set(ewu->queryPTree());
         if (hidePasswords && p->hasProp("Variables/Variable[Format/@password]"))
             return exportWorkUnitToXMLFileWithHiddenPasswords(p, filename, extraXmlFlags);
+        if (splitStats)
+        {
+            StringBuffer statsFilename;
+            statsFilename.append(filename).append(".stats");
+            IPropertyTree * stats = p->queryPropTree("Statistics");
+            if (stats)
+            {
+                saveXML(statsFilename, stats, 0, XML_Format|XML_SortTags|extraXmlFlags);
+                p->removeProp("Statistics");
+            }
+        }
         saveXML(filename, p, 0, XML_Format|XML_SortTags|extraXmlFlags);
     }
     else

common/workunit/workunit.hpp

@@ -1412,7 +1412,7 @@ extern WORKUNIT_API IWorkUnitFactory * getWorkUnitFactory(ISecManager *secmgr, I
 extern WORKUNIT_API ILocalWorkUnit* createLocalWorkUnit(const char *XML);
 extern WORKUNIT_API IConstWorkUnitInfo *createConstWorkUnitInfo(IPropertyTree &p);
 extern WORKUNIT_API StringBuffer &exportWorkUnitToXML(const IConstWorkUnit *wu, StringBuffer &str, bool unpack, bool includeProgress, bool hidePasswords);
-extern WORKUNIT_API void exportWorkUnitToXMLFile(const IConstWorkUnit *wu, const char * filename, unsigned extraXmlFlags, bool unpack, bool includeProgress, bool hidePasswords);
+extern WORKUNIT_API void exportWorkUnitToXMLFile(const IConstWorkUnit *wu, const char * filename, unsigned extraXmlFlags, bool unpack, bool includeProgress, bool hidePasswords, bool splitStats);
 extern WORKUNIT_API void submitWorkUnit(const char *wuid, const char *username, const char *password);
 extern WORKUNIT_API void abortWorkUnit(const char *wuid);
 extern WORKUNIT_API void submitWorkUnit(const char *wuid, ISecManager *secmgr, ISecUser *secuser);

common/workunit/workunit.ipp

@@ -213,7 +213,7 @@ template <>  struct CachedTags<CLocalWUAppValue, IConstWUAppValue>
 class WORKUNIT_API CLocalWorkUnit : implements IWorkUnit , implements IExtendedWUInterface, public CInterface
 {
     friend StringBuffer &exportWorkUnitToXML(const IConstWorkUnit *wu, StringBuffer &str, bool decodeGraphs, bool includeProgress, bool hidePasswords);
-    friend void exportWorkUnitToXMLFile(const IConstWorkUnit *wu, const char * filename, unsigned extraXmlFlags, bool decodeGraphs, bool includeProgress, bool hidePasswords);
+    friend void exportWorkUnitToXMLFile(const IConstWorkUnit *wu, const char * filename, unsigned extraXmlFlags, bool decodeGraphs, bool includeProgress, bool hidePasswords, bool splitStats);
 
 protected:
     Owned<IPropertyTree> p;

dali/daliadmin/daliadmin.cpp

@@ -2506,7 +2506,7 @@ static void dumpWorkunit(const char *wuid, bool includeProgress)
 {
     Owned<IWorkUnitFactory> factory = getWorkUnitFactory();
     Owned<IConstWorkUnit> workunit = factory->openWorkUnit(wuid);
-    exportWorkUnitToXMLFile(workunit, "stdout:", 0, true, includeProgress, true);
+    exportWorkUnitToXMLFile(workunit, "stdout:", 0, true, includeProgress, true, false);
 }
 
 static void dumpProgress(const char *wuid, const char * graph)

ecl/eclcc/eclcc.cpp

@@ -1187,8 +1187,7 @@ void EclCC::processSingleQuery(EclCompileInstance & instance,
             unsigned __int64 parseTimeNs = cycle_to_nanosec(get_cycles_now() - startCycles);
             instance.stats.parseTime = (unsigned)nanoToMilli(parseTimeNs);
 
-            if (instance.wu->getDebugValueBool("addTimingToWorkunit", true))
-                updateWorkunitTimeStat(instance.wu, SSTcompilestage, "compile:parseTime", StTimeElapsed, NULL, parseTimeNs);
+            updateWorkunitTimeStat(instance.wu, SSTcompilestage, "compile:parseTime", StTimeElapsed, NULL, parseTimeNs);
 
             if (exportDependencies)
             {
@@ -1290,8 +1289,7 @@ void EclCC::processSingleQuery(EclCompileInstance & instance,
     unsigned __int64 totalTimeNs = cycle_to_nanosec(get_cycles_now() - startCycles);
     instance.stats.generateTime = (unsigned)nanoToMilli(totalTimeNs) - instance.stats.parseTime;
     //MORE: This is done too late..
-    if (instance.wu->getDebugValueBool("addTimingToWorkunit", true))
-        updateWorkunitTimeStat(instance.wu, SSTcompilestage, "compile", StTimeElapsed, NULL, totalTimeNs);
+    updateWorkunitTimeStat(instance.wu, SSTcompilestage, "compile", StTimeElapsed, NULL, totalTimeNs);
 }
 
 void EclCC::processDefinitions(EclRepositoryArray & repositories)
@@ -1668,7 +1666,7 @@ void EclCC::generateOutput(EclCompileInstance & instance)
         else
             xmlFilename.append(DEFAULT_OUTPUTNAME);
         xmlFilename.append(".xml");
-        exportWorkUnitToXMLFile(instance.wu, xmlFilename, 0, true, false, false);
+        exportWorkUnitToXMLFile(instance.wu, xmlFilename, 0, true, false, false, false);
     }
 }
 
@@ -2285,7 +2283,7 @@ void EclCC::processBatchedFile(IFile & file, bool multiThreaded)
             if (info.wu &&
                 (info.wu->getDebugValueBool("generatePartialOutputOnError", false) || info.queryErrorProcessor().errCount() == 0))
             {
-                exportWorkUnitToXMLFile(info.wu, xmlFilename, XML_NoBinaryEncode64, true, false, false);
+                exportWorkUnitToXMLFile(info.wu, xmlFilename, XML_NoBinaryEncode64, true, false, false, true);
                 Owned<IFile> xml = createIFile(xmlFilename);
                 info.stats.xmlSize = xml->size();
             }

ecl/eclccserver/eclccserver.cpp

@@ -202,16 +202,14 @@ class EclccCompileThread : implements IPooledThread, implements IErrorReporter,
                 timings.findstr(max, 3);
                 timings.findstr(count, 4);
                 timings.findstr(ave, 5);
-                if (workunit->getDebugValueBool("addTimingToWorkunit", true))
-                {
-                    unsigned __int64 nval = atoi64(total) * 1000000; // in milliseconds
-                    unsigned __int64 nmax = atoi64(max) * 1000; // in microseconds
-                    unsigned __int64 cnt = atoi64(count);
-                    const char * scope = section.str();
-                    StatisticScopeType scopeType = SSTcompilestage;
-                    StatisticKind kind = StTimeElapsed;
-                    workunit->setStatistic(queryStatisticsComponentType(), queryStatisticsComponentName(), scopeType, scope, kind, NULL, nval, cnt, nmax, StatsMergeReplace);
-                }
+
+                unsigned __int64 nval = atoi64(total) * 1000000; // in milliseconds
+                unsigned __int64 nmax = atoi64(max) * 1000; // in microseconds
+                unsigned __int64 cnt = atoi64(count);
+                const char * scope = section.str();
+                StatisticScopeType scopeType = SSTcompilestage;
+                StatisticKind kind = StTimeElapsed;
+                workunit->setStatistic(queryStatisticsComponentType(), queryStatisticsComponentName(), scopeType, scope, kind, NULL, nval, cnt, nmax, StatsMergeReplace);
             }
             else
             {
@@ -330,8 +328,7 @@ class EclccCompileThread : implements IPooledThread, implements IErrorReporter,
             eclccCmd.append(" -");
         if (mainDefinition.length())
             eclccCmd.append(" -main ").append(mainDefinition);
-        if (workunit->getDebugValueBool("addTimingToWorkunit", true))
-            eclccCmd.append(" --timings");
+        eclccCmd.append(" --timings");
 
         Owned<IPipeProcess> pipe = createPipeProcess();
         pipe->setenv("ECLCCSERVER_THREAD_INDEX", idxStr.str());
@@ -415,8 +412,7 @@ class EclccCompileThread : implements IPooledThread, implements IErrorReporter,
                 queryDllServer().registerDll(realdllname.str(), "Workunit DLL", dllurl.str());
 
                 cycle_t elapsedCycles = get_cycles_now() - startCycles;
-                if (workunit->getDebugValueBool("addTimingToWorkunit", true))
-                    updateWorkunitTimeStat(workunit, SSTcompilestage, "compile", StTimeElapsed, NULL, cycle_to_nanosec(elapsedCycles));
+                updateWorkunitTimeStat(workunit, SSTcompilestage, "compile", StTimeElapsed, NULL, cycle_to_nanosec(elapsedCycles));
 
                 workunit->commit();
                 return true;

ecl/hqlcpp/hqlcpp.cpp

@@ -1602,7 +1602,6 @@ void HqlCppTranslator::cacheOptions()
 #else
         DebugOption(options.regressionTest,"regressionTest", false),
 #endif
-        DebugOption(options.addTimingToWorkunit, "addTimingToWorkunit", true),
         //recreating case can cause duplicate branches in weird situations.
         DebugOption(options.recreateMapFromIf,"recreateMapFromIf", !targetThor()),
 

ecl/hqlcpp/hqlcpp.ipp

@@ -607,7 +607,6 @@ struct HqlCppOptions
     bool                allowAmbiguousSelector;
     bool                regressionTest;
     bool                recreateMapFromIf;
-    bool                addTimingToWorkunit;
     bool                reduceNetworkTraffic;
     bool                optimizeProjectsPreservePersists;
     bool                showMetaText;
@@ -1086,8 +1085,7 @@ public:
     ITimeReporter * queryTimeReporter() const { return timeReporter; }
     void noteFinishedTiming(const char * name, cycle_t startCycles)
     {
-        if (options.addTimingToWorkunit)
-            timeReporter->addTiming(name, get_cycles_now()-startCycles);
+        timeReporter->addTiming(name, get_cycles_now()-startCycles);
     }
 
     void updateClusterType();

ecl/hqlcpp/hqlecl.cpp

@@ -467,11 +467,9 @@ bool HqlDllGenerator::generateCode(HqlQueryContext & query)
             setSaveGeneratedFiles(true);
 
         doExpand(translator);
-        if (wu->getDebugValueBool("addTimingToWorkunit", true))
-        {
-            unsigned __int64 elapsed = cycle_to_nanosec(get_cycles_now() - startCycles);
-            updateWorkunitTimeStat(wu, SSTcompilestage, "compile:generate c++", StTimeElapsed, NULL, elapsed);
-        }
+        unsigned __int64 elapsed = cycle_to_nanosec(get_cycles_now() - startCycles);
+        updateWorkunitTimeStat(wu, SSTcompilestage, "compile:generate c++", StTimeElapsed, NULL, elapsed);
+
         if (wu->getDebugValueBool("addMemoryToWorkunit", true))
         {
             memsize_t peakVm, peakResident;
@@ -525,8 +523,7 @@ void HqlDllGenerator::doExpand(HqlCppTranslator & translator)
     }
 
     cycle_t elapsedCycles = get_cycles_now() - startCycles;
-    if (wu->getDebugValueBool("addTimingToWorkunit", true))
-        updateWorkunitTimeStat(wu, SSTcompilestage, "compile:write c++", StTimeElapsed, NULL, cycle_to_nanosec(elapsedCycles));
+    updateWorkunitTimeStat(wu, SSTcompilestage, "compile:write c++", StTimeElapsed, NULL, cycle_to_nanosec(elapsedCycles));
 }
 
 bool HqlDllGenerator::abortRequested()

ecl/hqlcpp/hqlhtcpp.cpp

@@ -5970,7 +5970,7 @@ protected:
 
 void HqlCppTranslator::ensureWorkUnitUpdated()
 {
-    if (timeReporter && options.addTimingToWorkunit)
+    if (timeReporter)
     {
         WuTimingUpdater updater(wu());
         timeReporter->report(updater);

ecl/hqlcpp/hqlnlp.cpp

@@ -684,7 +684,7 @@ ABoundActivity * HqlCppTranslator::doBuildActivityParse(BuildCtx & ctx, IHqlExpr
 
     //This will become conditional on the flags....
     cycle_t startPrepareCycles = get_cycles_now();
-    ITimeReporter * reporter = options.addTimingToWorkunit ? timeReporter : NULL;
+    ITimeReporter * reporter = timeReporter;
     if (expr->hasAttribute(tomitaAtom))
         nlpParse = createTomitaContext(expr, code->workunit, options, reporter);
     else

ecl/regress/regress.sh

@@ -113,7 +113,7 @@ fi
 
 if [[ $eclcc != '' ]]; then
     ## Set flags
-    default_flags="-P$target_dir -legacy -platform=thorlcr -fforceGenerate -fregressionTest -faddTimingToWorkunit=0 -b -S -shared"
+    default_flags="-P$target_dir -legacy -platform=thorlcr -fforceGenerate -fregressionTest -b -S -shared"
     flags="$default_flags $include_dir -fshowMetaInGraph -fspanMultipleCpp- $userflags"
 
     ## Prepare target directory