HPCC-20187 eclcc does not periodically trace memory usage

Also add eclcc log file to workunit helpers.

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

common/workunit/workunit.cpp

@@ -9613,6 +9613,7 @@ EnumMapping queryFileTypes[] = {
    { FileTypeResText, "res" },
    { FileTypeHintXml, "hint" },
    { FileTypeXml, "xml" },
+   { FileTypeLog, "log" },
    { FileTypeSize,  NULL },
 };
 

common/workunit/workunit.hpp

@@ -354,7 +354,8 @@ enum WUFileType
     FileTypeResText = 2,
     FileTypeHintXml = 3,
     FileTypeXml = 4,
-    FileTypeSize = 5
+    FileTypeLog = 5,
+    FileTypeSize = 6
 };
 
 

ecl/eclcc/eclcc.cpp

@@ -372,6 +372,7 @@ protected:
     unsigned batchPart = 0;
     unsigned batchSplit = 1;
     unsigned optLogDetail = 0;
+    unsigned optMonitorInterval = 60;
     unsigned optMaxErrors = 0;
     unsigned optDaliTimeout = 30000;
     bool optUnsuppressImmediateSyntaxErrors = false;
@@ -502,9 +503,8 @@ int main(int argc, const char *argv[])
     // Turn logging down (we turn it back up if -v option seen)
     Owned<ILogMsgFilter> filter = getCategoryLogMsgFilter(MSGAUD_user, MSGCLS_error);
     queryLogMsgManager()->changeMonitorFilter(queryStderrLogMsgHandler(), filter);
-
     unsigned exitCode = doMain(argc, argv);
-
+    stopPerformanceMonitor();
     if (!optReleaseAllMemory)
     {
         //In release mode exit without calling all the clean up code.
@@ -641,7 +641,10 @@ void EclCC::loadOptions()
             if (logVerbose)
                 fprintf(stdout, "Logging to '%s'\n",lf.str());
         }
+        if (optMonitorInterval)
+            startPerformanceMonitor(optMonitorInterval*1000, PerfMonStandard, nullptr);
     }
+
     if (hooksPath.length())
         installFileHooks(hooksPath.str());
 
@@ -2553,6 +2556,9 @@ int EclCC::parseCommandLineOptions(int argc, const char* argv[])
         else if (iter.matchOption(optLogDetail, "--logdetail"))
         {
         }
+        else if (iter.matchOption(optMonitorInterval, "--monitorinterval"))
+        {
+        }
         else if (iter.matchOption(optQueryRepositoryReference, "-main"))
         {
         }

ecl/eclccserver/eclccserver.cpp

@@ -314,6 +314,8 @@ class EclccCompileThread : implements IPooledThread, implements IErrorReporter,
         eclccCmd.append(" --timings --xml");
         eclccCmd.append(" --nostdinc");
         eclccCmd.append(" --metacache=");
+        VStringBuffer logfile("%s.eclcc.log", workunit->queryWuid());
+        eclccCmd.appendf(" --logfile=%s", logfile.str());
         if (globals->getPropBool("@enableEclccDali", true))
         {
             const char *daliServers = globals->queryProp("@daliServers");
@@ -405,19 +407,27 @@ class EclccCompileThread : implements IPooledThread, implements IErrorReporter,
 
                 createUNCFilename(realdllfilename.str(), dllurl);
                 unsigned crc = crc_file(realdllfilename.str());
-
                 Owned<IWUQuery> query = workunit->updateQuery();
+                associateLocalFile(query, FileTypeLog, logfile, "Compiler log", 0);
                 associateLocalFile(query, FileTypeDll, realdllfilename, "Workunit DLL", crc);
                 queryDllServer().registerDll(realdllname.str(), "Workunit DLL", dllurl.str());
                 workunit->commit();
                 return true;
             }
+            else
+            {
+                Owned<IWUQuery> query = workunit->updateQuery();
+                associateLocalFile(query, FileTypeLog, logfile, "Compiler log", 0);
+                workunit->commit();
+                return false;
+            }
         }
         catch (IException * e)
         {
             reportError(e);
             e->Release();
         }
+        workunit->commit();
         return false;
     }
 
@@ -790,6 +800,9 @@ int main(int argc, const char *argv[])
     {
         initClientProcess(serverGroup, DCR_EclCCServer);
         openLogFile();
+        unsigned optMonitorInterval = globals->getPropInt("@monitorInterval", 60);
+        if (optMonitorInterval)
+            startPerformanceMonitor(optMonitorInterval*1000, PerfMonStandard, nullptr);
         SCMStringBuffer queueNames;
         getEclCCServerQueueNames(queueNames, processName);
         if (!queueNames.length())
@@ -812,6 +825,7 @@ int main(int argc, const char *argv[])
     {
         ERRLOG("Terminating unexpectedly");
     }
+    stopPerformanceMonitor();
     globals.clear();
     UseSysLogForOperatorMessages(false);
     ::closedownClientProcess(); // dali client closedown

esp/scm/ws_workunits.ecm

@@ -391,6 +391,7 @@ ESPenum WUFileType : string
     CPP("cpp"),
     DLL("dll"),
     Res("res"),
+    LOG("log"),
     WUECL("WUECL"),
     ThorLog("ThorLog"),
     ThorSlaveLog("ThorSlaveLog"),

esp/services/ws_workunits/ws_workunitsHelpers.cpp

@@ -510,6 +510,7 @@ EnumMapping queryFileTypes[] = {
    { FileTypeResText, "res" },
    { FileTypeHintXml, "hint" },
    { FileTypeXml, "xml" },
+   { FileTypeLog, "log" },
    { FileTypeSize,  NULL },
 };
 
@@ -3904,6 +3905,7 @@ void CWsWuFileHelper::readWUFile(const char* wuid, const char* workingFolder, Ws
         writeToFileIOStream(workingFolder, "ArchiveQuery.xml", mb);
         break;
     case CWUFileType_CPP:
+    case CWUFileType_LOG:
     {
         const char *tail=pathTail(item.getName());
         fileName.set(tail ? tail : item.getName());

esp/services/ws_workunits/ws_workunitsHelpers.hpp

@@ -36,6 +36,7 @@ namespace ws_workunits {
 #define     OTHERS_WU_ACCESS   "OthersWorkunitsAccess"
 
 #define    File_Cpp "cpp"
+#define    File_Log "log"
 #define    File_ThorLog "ThorLog"
 #define    File_ThorSlaveLog "ThorSlaveLog"
 #define    File_EclAgentLog "EclAgentLog"

esp/services/ws_workunits/ws_workunitsService.cpp

@@ -3007,7 +3007,7 @@ bool CWsWorkunitsEx::onWUFile(IEspContext &context,IEspWULogFileRequest &req, IE
                 winfo.getWorkunitArchiveQuery(mb);
                 openSaveFile(context, opt, req.getSizeLimit(), "ArchiveQuery.xml", HTTP_TYPE_APPLICATION_XML, mb, resp);
             }
-            else if (strieq(File_Cpp,req.getType()) && notEmpty(req.getName()))
+            else if ((strieq(File_Cpp,req.getType()) || strieq(File_Log,req.getType())) && notEmpty(req.getName()))
             {
                 winfo.getWorkunitCpp(req.getName(), req.getDescription(), req.getIPAddress(),mb, opt > 0, nullptr);
                 openSaveFile(context, opt, req.getSizeLimit(), req.getName(), HTTP_TYPE_TEXT_PLAIN, mb, resp);

esp/src/eclwatch/HelpersWidget.js

@@ -64,6 +64,7 @@ define([
                     case "Workunit XML":
                         params = "/WUFile?Wuid=" + this.wu.Wuid + "&Type=XML";
                         break;
+                    case "log":
                     case "cpp":
                     case "hpp":
                         params = "/WUFile?Wuid=" + this.wu.Wuid + "&Name=" + item.Orig.Name + "&IPAddress=" + item.Orig.IPAddress + "&Description=" + item.Orig.Description + "&Type=" + item.Orig.Type;

initfiles/componentfiles/configxml/eclccserver.xsd

@@ -225,6 +225,13 @@
                     </xs:appinfo>
                 </xs:annotation>
             </xs:attribute>
+            <xs:attribute name="monitorInterval" type="xs:nonNegativeInteger" use="optional" default="60">
+                <xs:annotation>
+                    <xs:appinfo>
+                        <tooltip>Interval (in seconds) for reporting of memory usage stats. Set to 0 to disable.</tooltip>
+                    </xs:appinfo>
+                </xs:annotation>
+            </xs:attribute>
         </xs:complexType>
     </xs:element>
 </xs:schema>