HPCC-17839 Avoid duplicating stats from eclccserver

Also adds timing for c++ compile time

Signed-off-by: Gavin Halliday <gavin.halliday@lexisnexis.com>

common/workunit/workunit.cpp

@@ -1386,8 +1386,8 @@ public:
             { c->requestAbort(); }
     virtual unsigned calculateHash(unsigned prevHash)
             { return queryExtendedWU(c)->calculateHash(prevHash); }
-    virtual void copyWorkUnit(IConstWorkUnit *cached, bool all)
-            { queryExtendedWU(c)->copyWorkUnit(cached, all); }
+    virtual void copyWorkUnit(IConstWorkUnit *cached, bool copyStats, bool all)
+            { queryExtendedWU(c)->copyWorkUnit(cached, copyStats, all); }
     virtual IPropertyTree *queryPTree() const
             { return queryExtendedWU(c)->queryPTree(); }
     virtual IPropertyTree *getUnpackedTree(bool includeProgress) const
@@ -5341,7 +5341,7 @@ IPropertyTree *CLocalWorkUnit::queryPTree() const
     return p;
 }
 
-void CLocalWorkUnit::copyWorkUnit(IConstWorkUnit *cached, bool all)
+void CLocalWorkUnit::copyWorkUnit(IConstWorkUnit *cached, bool copyStats, bool all)
 {
     CLocalWorkUnit *from = QUERYINTERFACE(cached, CLocalWorkUnit);
     if (!from)
@@ -5395,10 +5395,8 @@ void CLocalWorkUnit::copyWorkUnit(IConstWorkUnit *cached, bool all)
     copyTree(p, fromP, "Graphs");
     copyTree(p, fromP, "Workflow");
     copyTree(p, fromP, "WebServicesInfo");
-    if (all)
+    if (copyStats)
     {
-        // 'all' mode is used when setting up a dali WU from the embedded wu in a workunit dll
-
         // Merge timing info from both branches
         pt = fromP->getBranch("Statistics");
         if (pt)

common/workunit/workunit.hpp

@@ -1343,7 +1343,7 @@ interface IWorkflowScheduleConnection : extends IInterface
 interface IExtendedWUInterface
 {
     virtual unsigned calculateHash(unsigned prevHash) = 0;
-    virtual void copyWorkUnit(IConstWorkUnit *cached, bool all) = 0;
+    virtual void copyWorkUnit(IConstWorkUnit *cached, bool copyStats, bool all) = 0;
     virtual bool archiveWorkUnit(const char *base,bool del,bool ignoredllerrors,bool deleteOwned,bool exportAssociatedFiles) = 0;
     virtual IPropertyTree *getUnpackedTree(bool includeProgress) const = 0;
     virtual IPropertyTree *queryPTree() const = 0;

common/workunit/workunit.ipp

@@ -324,7 +324,7 @@ public:
     virtual IWorkUnit& lock();
     virtual void requestAbort();
     virtual unsigned calculateHash(unsigned prevHash);
-    virtual void copyWorkUnit(IConstWorkUnit *cached, bool all);
+    virtual void copyWorkUnit(IConstWorkUnit *cached, bool copyStats, bool all);
     virtual IPropertyTree *queryPTree() const;
     virtual unsigned queryFileUsage(const char *filename) const;
     virtual IConstWUFileUsageIterator * getFieldUsage() const;

ecl/eclagent/eclagent.cpp

@@ -3426,7 +3426,7 @@ extern int HTHOR_API eclagent_main(int argc, const char *argv[], StringBuffer *
                 Owned<IWorkUnitFactory> factory = getWorkUnitFactory();
                 Owned<IWorkUnit> daliWu = factory->createWorkUnit("eclagent", "eclagent");
                 IExtendedWUInterface * extendedWu = queryExtendedWU(daliWu);
-                extendedWu->copyWorkUnit(standAloneWorkUnit, true);
+                extendedWu->copyWorkUnit(standAloneWorkUnit, true, true);
                 wuid.set(daliWu->queryWuid());
                 globals->setProp("WUID", wuid.str());
 

ecl/eclcc/eclcc.cpp

@@ -2573,7 +2573,7 @@ void EclCC::processBatchedFile(IFile & file, bool multiThreaded)
             if (info.generatedMeta)
                 saveXML(metaFilename, info.generatedMeta, 0, XML_Embed|XML_LineBreak);
 
-            info.logStats();
+            info.logStats(logTimings);
         }
     }
     catch (IException * e)

ecl/eclccserver/eclccserver.cpp

@@ -15,17 +15,17 @@
     limitations under the License.
 ############################################################################## */
 
-#include <jlib.hpp>
-#include <jmisc.hpp>
-#include <jisem.hpp>
-#include <jfile.hpp>
-#include <jencrypt.hpp>
-#include <jregexp.hpp>
-#include <mpbase.hpp>
-#include <daclient.hpp>
-#include <dasess.hpp>
-#include <danqs.hpp>
-#include <dalienv.hpp>
+#include "jlib.hpp"
+#include "jmisc.hpp"
+#include "jisem.hpp"
+#include "jfile.hpp"
+#include "jencrypt.hpp"
+#include "jregexp.hpp"
+#include "mpbase.hpp"
+#include "daclient.hpp"
+#include "dasess.hpp"
+#include "danqs.hpp"
+#include "dalienv.hpp"
 #include "workunit.hpp"
 #include "wujobq.hpp"
 #include "dllserver.hpp"
@@ -381,7 +381,7 @@ class EclccCompileThread : implements IPooledThread, implements IErrorReporter,
                     throw makeStringException(999, "Failed to extract workunit from query dll");
 
                 Owned<ILocalWorkUnit> embeddedWU = createLocalWorkUnit(wuXML);
-                queryExtendedWU(workunit)->copyWorkUnit(embeddedWU, true);
+                queryExtendedWU(workunit)->copyWorkUnit(embeddedWU, false, true);
                 workunit->setIsClone(false);
                 const char *jobname = embeddedWU->queryJobName();
                 if (jobname && *jobname) //let ECL win naming job during initial compile

ecl/hql/hqlerror.cpp

@@ -235,7 +235,6 @@ public:
         OwnedPTree tree = error->toTree();
 
         StringBuffer msg;
-        //toJSON(tree, msg, 0, JSON_IncludeRoot);
         toXML(tree, msg, 0, XML_Embed);
         fprintf(f, "%s\n", msg.str());
     }

ecl/hqlcpp/hqlecl.cpp

@@ -537,6 +537,7 @@ bool HqlDllGenerator::abortRequested()
 bool HqlDllGenerator::doCompile(ICppCompiler * compiler)
 {
     cycle_t startCycles = get_cycles_now();
+    addTimeStamp(wu, SSTcompilestage, "compile:compile c++", StWhenStarted);
     ForEachItemIn(i, sourceFiles)
         compiler->addSourceFile(sourceFiles.item(i));
 
@@ -596,9 +597,8 @@ bool HqlDllGenerator::doCompile(ICppCompiler * compiler)
             errs->report(&cur);
     }
 
-    cycle_t elapsedCycles = get_cycles_now() - startCycles;
-    //For eclcc the workunit has been written to the resource - so any further timings will not be preserved -> need to report differently
-    queryActiveTimer()->addTiming("compile:compile c++", elapsedCycles);
+    unsigned __int64 elapsed = cycle_to_nanosec(get_cycles_now() - startCycles);
+    updateWorkunitTimeStat(wu, SSTcompilestage, "compile:compile c++", StTimeElapsed, NULL, elapsed);
 
     //Keep the files if there was a compile error.
     if (ok && deleteGenerated)

esp/services/ws_ecl/ws_ecl_service.cpp

@@ -1840,7 +1840,7 @@ int CWsEclBinding::submitWsEclWorkunit(IEspContext & context, WsEclWuInfo &wsinf
     Owned <IWorkUnit> workunit = factory->createWorkUnit("wsecl", context.queryUserId(), context.querySecManager(), context.queryUser());
 
     IExtendedWUInterface *ext = queryExtendedWU(workunit);
-    ext->copyWorkUnit(sourceWorkUnit, false);
+    ext->copyWorkUnit(sourceWorkUnit, false, false);
 
     workunit->clearExceptions();
     workunit->resetWorkflow();

esp/services/ws_workunits/ws_workunitsHelpers.cpp

@@ -3209,7 +3209,7 @@ void WsWuHelpers::copyWsWorkunit(IEspContext &context, IWorkUnit &wu, const char
     Owned<IWorkUnitFactory> factory = getWorkUnitFactory(context.querySecManager(), context.queryUser());
     Owned<IConstWorkUnit> src(factory->openWorkUnit(srcWuid));
 
-    queryExtendedWU(&wu)->copyWorkUnit(src, false);
+    queryExtendedWU(&wu)->copyWorkUnit(src, false, false);
 
     SCMStringBuffer token;
     wu.setSecurityToken(createToken(wu.queryWuid(), context.queryUserId(), context.queryPassword(), token).str());

esp/services/ws_workunits/ws_workunitsService.cpp

@@ -769,7 +769,7 @@ bool CWsWorkunitsEx::onWUResubmit(IEspContext &context, IEspWUResubmitRequest &r
                     Owned<IConstWorkUnit> src(factory->openWorkUnit(wuid.str()));
                     NewWsWorkunit wu(factory, context);
                     wuid.set(wu->queryWuid());
-                    queryExtendedWU(wu)->copyWorkUnit(src, false);
+                    queryExtendedWU(wu)->copyWorkUnit(src, false, false);
 
                     SCMStringBuffer token;
                     wu->setSecurityToken(createToken(wuid.str(), context.queryUserId(), context.queryPassword(), token).str());
@@ -4311,7 +4311,7 @@ void deploySharedObject(IEspContext &context, StringBuffer &wuid, const char *fi
     if (getWorkunitXMLFromFile(dllpath.str(), dllXML))
     {
         Owned<ILocalWorkUnit> embeddedWU = createLocalWorkUnit(dllXML.str());
-        queryExtendedWU(wu)->copyWorkUnit(embeddedWU, true);
+        queryExtendedWU(wu)->copyWorkUnit(embeddedWU, true, true);
     }
 
     wu.associateDll(dllpath.str(), dllname.str());

plugins/cassandra/cassandrawu.cpp

@@ -2498,7 +2498,7 @@ public:
         noteDirty(xpath, wuExceptionsMappings);
         return result;
     }
-    virtual void copyWorkUnit(IConstWorkUnit *cached, bool all)
+    virtual void copyWorkUnit(IConstWorkUnit *cached, bool copyStats, bool all)
     {
         // Make sure that any required updates to the secondary files happen
         IPropertyTree *fromP = queryExtendedWU(cached)->queryPTree();
@@ -2506,7 +2506,7 @@ public:
             trackSecondaryChange(fromP->queryProp(*search), *search);
         for (const ChildTableInfo * const * table = childTables; *table != NULL; table++)
             checkChildLoaded(**table);
-        CPersistedWorkUnit::copyWorkUnit(cached, all);
+        CPersistedWorkUnit::copyWorkUnit(cached, copyStats, all);
         memset(childLoaded, 1, sizeof(childLoaded));
         allDirty = true;
         actionChanged = true;

roxie/ccd/ccddali.cpp

@@ -606,7 +606,7 @@ public:
             if (getEmbeddedWorkUnitXML(source, wuXML))
             {
                 Owned<ILocalWorkUnit> localWU = createLocalWorkUnit(wuXML);
-                queryExtendedWU(w)->copyWorkUnit(localWU, true);
+                queryExtendedWU(w)->copyWorkUnit(localWU, true, true);
             }
             else
                 throw MakeStringException(ROXIE_DALI_ERROR, "Failed to locate dll workunit info");

system/jlib/jptree.cpp

@@ -5722,7 +5722,7 @@ jlib_decl StringBuffer &toJSON(const IPropertyTree *tree, StringBuffer &ret, uns
         virtual size32_t write(size32_t len, const void * data) override { out.append(len, (const char *)data); return len; }
     } adapter(ret);
     bool delimit = false;
-    _toJSON(tree->queryBranch(NULL), adapter, indent, flags, delimit, (flags & JSON_IncludeRoot) == 0);
+    _toJSON(tree->queryBranch(NULL), adapter, indent, flags, delimit, true);
     return ret;
 }
 

system/jlib/jptree.hpp

@@ -253,7 +253,6 @@ jlib_decl void saveXML(IIOStream &stream, const IPropertyTree *tree, unsigned in
 #define JSON_Format   0x02
 #define JSON_Sanitize 0x08
 #define JSON_SanitizeAttributeValues 0x10
-#define JSON_IncludeRoot 0x20
 
 jlib_decl StringBuffer &toJSON(const IPropertyTree *tree, StringBuffer &ret, unsigned indent = 0, byte flags=JSON_Format);
 jlib_decl void toJSON(const IPropertyTree *tree, IIOStream &out, unsigned indent = 0, byte flags=JSON_Format);

testing/regress/ecl/apply.ecl

@@ -15,8 +15,6 @@
     limitations under the License.
 ############################################################################## */
 
-//noroxie
-
 import lib_logging;
 import lib_workunitservices;
 

tools/wutool/wutool.cpp

@@ -786,7 +786,7 @@ protected:
                 );
         StringBuffer xml1, xml2, xml3;
         exportWorkUnitToXML(embeddedWU, xml1, false, false, false);
-        queryExtendedWU(createWu)->copyWorkUnit(embeddedWU, true);
+        queryExtendedWU(createWu)->copyWorkUnit(embeddedWU, true, true);
         createWu->setState(WUStateCompleted);
         exportWorkUnitToXML(createWu, xml2, false, false, false);
         createWu->commit();