Merge pull request #5353 from jakesmith/hpcc-10796

HPCC-10796 - Add option to dump workunit with/without progress

Reviewed-By: Richard Chapman <rchapman@hpccsystems.com>

common/workunit/workunit.cpp

@@ -533,8 +533,8 @@ template <>  struct CachedTags<CLocalWUAppValue, IConstWUAppValue>
 
 class CLocalWorkUnit : public CInterface, implements IConstWorkUnit , implements ISDSSubscription, implements IExtendedWUInterface
 {
-    friend StringBuffer &exportWorkUnitToXML(const IConstWorkUnit *wu, StringBuffer &str, bool decodeGraphs);
-    friend void exportWorkUnitToXMLFile(const IConstWorkUnit *wu, const char * filename, unsigned extraXmlFlags, bool decodeGraphs);
+    friend StringBuffer &exportWorkUnitToXML(const IConstWorkUnit *wu, StringBuffer &str, bool decodeGraphs, bool includeProgress);
+    friend void exportWorkUnitToXMLFile(const IConstWorkUnit *wu, const char * filename, unsigned extraXmlFlags, bool decodeGraphs, bool includeProgress);
 
     // NOTE - order is important - we need to construct connection before p and (especially) destruct after p
     Owned<IRemoteConnection> connection;
@@ -583,7 +583,7 @@ public:
     CLocalWorkUnit(IRemoteConnection *_conn, IPropertyTree* root, ISecManager *secmgr, ISecUser *secuser);
     ~CLocalWorkUnit();
     CLocalWorkUnit(const char *dummyWuid, const char *parentWuid, ISecManager *secmgr, ISecUser *secuser);
-    IPropertyTree *getUnpackedTree() const;
+    IPropertyTree *getUnpackedTree(bool includeProgress) const;
 
     ISecManager *querySecMgr(){return secMgr.get();}
     ISecUser *querySecUser(){return secUser.get();}
@@ -3151,7 +3151,7 @@ bool CLocalWorkUnit::archiveWorkUnit(const char *base,bool del,bool ignoredllerr
         return false;
 
     StringBuffer buf;
-    exportWorkUnitToXML(this, buf, false);
+    exportWorkUnitToXML(this, buf, false, false);
 
     StringBuffer extraWorkUnitXML;
     StringBuffer xpath("/GraphProgress/");
@@ -3454,7 +3454,7 @@ void CLocalWorkUnit::serialize(MemoryBuffer &tgt)
 {
     CriticalBlock block(crit);
     StringBuffer x;
-    tgt.append(exportWorkUnitToXML(this, x, false).str());
+    tgt.append(exportWorkUnitToXML(this, x, false, false).str());
 }
 
 void CLocalWorkUnit::deserialize(MemoryBuffer &src)
@@ -6554,14 +6554,14 @@ bool CLocalWorkUnit::switchThorQueue(const char *cluster, IQueueSwitcher *qs)
 
 //=================================================================================================
 
-IPropertyTree *CLocalWorkUnit::getUnpackedTree() const
+IPropertyTree *CLocalWorkUnit::getUnpackedTree(bool includeProgress) const
 {
     Owned<IPropertyTree> ret = createPTreeFromIPT(p);
     Owned<IConstWUGraphIterator> graphIter = &getGraphs(GraphTypeAny);
     ForEach(*graphIter)
     {
         IConstWUGraph &graph  = graphIter->query();
-        Owned<IPropertyTree> graphTree = graph.getXGMMLTree(false);
+        Owned<IPropertyTree> graphTree = graph.getXGMMLTree(includeProgress);
         SCMStringBuffer gName;
         graph.getName(gName);
         StringBuffer xpath("Graphs/Graph[@name=\"");
@@ -6575,7 +6575,6 @@ IPropertyTree *CLocalWorkUnit::getUnpackedTree() const
     }
     return ret.getClear();
 }
-
 void CLocalWorkUnit::loadGraphs() const
 {
     CriticalBlock block(crit);
@@ -8651,7 +8650,7 @@ extern WORKUNIT_API ILocalWorkUnit * createLocalWorkUnit()
     return ret;
 }
 
-extern WORKUNIT_API StringBuffer &exportWorkUnitToXML(const IConstWorkUnit *wu, StringBuffer &str, bool unpack)
+extern WORKUNIT_API StringBuffer &exportWorkUnitToXML(const IConstWorkUnit *wu, StringBuffer &str, bool unpack, bool includeProgress)
 {
     const CLocalWorkUnit *w = QUERYINTERFACE(wu, const CLocalWorkUnit);
     if (!w)
@@ -8663,8 +8662,8 @@ extern WORKUNIT_API StringBuffer &exportWorkUnitToXML(const IConstWorkUnit *wu,
     if (w)
     {
         Linked<IPropertyTree> p;
-        if (unpack)
-            p.setown(w->getUnpackedTree());
+        if (unpack||includeProgress)
+            p.setown(w->getUnpackedTree(includeProgress));
         else
             p.set(w->p);
         toXML(p, str, 0, XML_Format|XML_SortTags);
@@ -8674,14 +8673,14 @@ extern WORKUNIT_API StringBuffer &exportWorkUnitToXML(const IConstWorkUnit *wu,
     return str;
 }
 
-extern WORKUNIT_API IStringVal& exportWorkUnitToXML(const IConstWorkUnit *wu, IStringVal &str, bool unpack)
+extern WORKUNIT_API IStringVal& exportWorkUnitToXML(const IConstWorkUnit *wu, IStringVal &str, bool unpack, bool includeProgress)
 {
     StringBuffer x;
-    str.set(exportWorkUnitToXML(wu,x,unpack).str());
+    str.set(exportWorkUnitToXML(wu,x,unpack, includeProgress).str());
     return str;
 }
 
-extern WORKUNIT_API void exportWorkUnitToXMLFile(const IConstWorkUnit *wu, const char * filename, unsigned extraXmlFlags, bool unpack)
+extern WORKUNIT_API void exportWorkUnitToXMLFile(const IConstWorkUnit *wu, const char * filename, unsigned extraXmlFlags, bool unpack, bool includeProgress)
 {
     const CLocalWorkUnit *w = QUERYINTERFACE(wu, const CLocalWorkUnit);
     if (!w)
@@ -8693,8 +8692,8 @@ extern WORKUNIT_API void exportWorkUnitToXMLFile(const IConstWorkUnit *wu, const
     if (w)
     {
         Linked<IPropertyTree> p;
-        if (unpack)
-            p.setown(w->getUnpackedTree());
+        if (unpack||includeProgress)
+            p.setown(w->getUnpackedTree(includeProgress));
         else
             p.set(w->p);
         saveXML(filename, p, 0, XML_Format|XML_SortTags|extraXmlFlags);

common/workunit/workunit.hpp

@@ -1235,9 +1235,9 @@ extern WORKUNIT_API IWorkUnitFactory * getWorkUnitFactory();
 extern WORKUNIT_API IWorkUnitFactory * getSecWorkUnitFactory(ISecManager &secmgr, ISecUser &secuser);
 extern WORKUNIT_API IWorkUnitFactory * getWorkUnitFactory(ISecManager *secmgr, ISecUser *secuser);
 extern WORKUNIT_API ILocalWorkUnit* createLocalWorkUnit();
-extern WORKUNIT_API IStringVal& exportWorkUnitToXML(const IConstWorkUnit *wu, IStringVal &str, bool unpack);
-extern WORKUNIT_API StringBuffer &exportWorkUnitToXML(const IConstWorkUnit *wu, StringBuffer &str, bool unpack);
-extern WORKUNIT_API void exportWorkUnitToXMLFile(const IConstWorkUnit *wu, const char * filename, unsigned extraXmlFlags, bool unpack);
+extern WORKUNIT_API IStringVal& exportWorkUnitToXML(const IConstWorkUnit *wu, IStringVal &str, bool unpack, bool includeProgress);
+extern WORKUNIT_API StringBuffer &exportWorkUnitToXML(const IConstWorkUnit *wu, StringBuffer &str, bool unpack, bool includeProgress);
+extern WORKUNIT_API void exportWorkUnitToXMLFile(const IConstWorkUnit *wu, const char * filename, unsigned extraXmlFlags, bool unpack, bool includeProgress);
 extern WORKUNIT_API void submitWorkUnit(const char *wuid, const char *username, const char *password);
 extern WORKUNIT_API void abortWorkUnit(const char *wuid);
 extern WORKUNIT_API void submitWorkUnit(const char *wuid, ISecManager *secmgr, ISecUser *secuser);

dali/daliadmin/daliadmin.cpp

@@ -118,6 +118,7 @@ void usage(const char *exe)
   printf("  validatestore [fix=<true|false>]\n"
          "                [verbose=<true|false>]\n"
          "                [deletefiles=<true|false>]-- perform some checks on dali meta data an optionally fix or remove redundant info \n");
+  printf("  workunit <workunit> [true]      -- dump workunit xml, if 2nd parameter equals true, will also include progress data\n");
   printf("  wuidcompress <wildcard> <type>  --  scan workunits that match <wildcard> and compress resources of <type>\n");
   printf("  wuiddecompress <wildcard> <type> --  scan workunits that match <wildcard> and decompress resources of <type>\n");
   printf("  xmlsize <filename> [<percentage>] --  analyse size usage in xml file, display individual items above 'percentage' \n");
@@ -2372,6 +2373,13 @@ static void unlock(const char *pattern)
     }
 }
 
+static void dumpWorkunit(const char *wuid, bool includeProgress)
+{
+    Owned<IWorkUnitFactory> factory = getWorkUnitFactory();
+    Owned<IConstWorkUnit> workunit = factory->openWorkUnit(wuid, false);
+    exportWorkUnitToXMLFile(workunit, "stdout:", 0, true, includeProgress);
+}
+
 static void wuidCompress(const char *match, const char *type, bool compress)
 {
     if (0 != stricmp("graph", type))
@@ -2873,6 +2881,13 @@ int main(int argc, char* argv[])
                         bool deleteFiles = props->getPropBool("deletefiles");
                         validateStore(fix, deleteFiles, verbose);
                     }
+                    else if (stricmp(cmd, "workunit") == 0) {
+                        CHECKPARAMS(1,2);
+                        bool includeProgress=false;
+                        if (np>1)
+                            includeProgress = strToBool(params.item(2));
+                        dumpWorkunit(params.item(1), includeProgress);
+                    }
                     else if (stricmp(cmd,"wuidCompress")==0) {
                         CHECKPARAMS(2,2);
                         wuidCompress(params.item(1), params.item(2), true);

ecl/eclcc/eclcc.cpp

@@ -1479,7 +1479,7 @@ void EclCC::generateOutput(EclCompileInstance & instance)
         else
             xmlFilename.append(DEFAULT_OUTPUTNAME);
         xmlFilename.append(".xml");
-        exportWorkUnitToXMLFile(instance.wu, xmlFilename, 0, true);
+        exportWorkUnitToXMLFile(instance.wu, xmlFilename, 0, true, false);
     }
 }
 
@@ -2112,7 +2112,7 @@ void EclCC::processBatchedFile(IFile & file, bool multiThreaded)
             if (info.wu &&
                 (info.wu->getDebugValueBool("generatePartialOutputOnError", false) || info.errs->errCount() == 0))
             {
-                exportWorkUnitToXMLFile(info.wu, xmlFilename, XML_NoBinaryEncode64, true);
+                exportWorkUnitToXMLFile(info.wu, xmlFilename, XML_NoBinaryEncode64, true, false);
                 Owned<IFile> xml = createIFile(xmlFilename);
                 info.stats.xmlSize = xml->size();
             }

ecl/hqlcpp/hqlecl.cpp

@@ -451,7 +451,7 @@ bool HqlDllGenerator::generateCode(HqlQueryContext & query)
 void HqlDllGenerator::addWorkUnitAsResource()
 {
     SCMStringBuffer wuXML;
-    exportWorkUnitToXML(wu, wuXML, false);
+    exportWorkUnitToXML(wu, wuXML, false, false);
     code->addCompressResource("WORKUNIT", wuXML.length(), wuXML.str(), NULL, 1000);
 }
 

ecl/wutest/wutest.cpp

@@ -70,7 +70,7 @@ bool dump(IConstWorkUnit &w, IProperties *globals)
     else if (stricmp(action, "dump")==0)
     {
         SCMStringBuffer xml;
-        exportWorkUnitToXML(&w, xml, true);
+        exportWorkUnitToXML(&w, xml, true, false);
         printf("%s\n", xml.str());
     }
     else if (stricmp(action, "temporaries")==0)

esp/services/ws_workunits/ws_workunitsHelpers.cpp

@@ -1961,7 +1961,7 @@ void WsWuInfo::getWorkunitXml(const char* plainText, MemoryBuffer& buf)
         header = "<?xml version=\"1.0\" encoding=\"UTF-8\"?><?xml-stylesheet href=\"../esp/xslt/xmlformatter.xsl\" type=\"text/xsl\"?>";
 
     SCMStringBuffer xml;
-    exportWorkUnitToXML(cw, xml, true);
+    exportWorkUnitToXML(cw, xml, true, false);
 
     buf.append(strlen(header), header);
     buf.append(xml.length(), xml.str());

esp/services/ws_workunits/ws_workunitsService.cpp

@@ -3186,7 +3186,7 @@ bool CWsWorkunitsEx::onWUExport(IEspContext &context, IEspWUExportRequest &req,
         {
             Owned<IConstWorkUnit> cw = factory->openWorkUnit(it->c_str(), false);
             if (cw)
-                exportWorkUnitToXML(cw, xml, true);
+                exportWorkUnitToXML(cw, xml, true, false);
         }
         xml.append("</Workunits>");