Refactor WsWorkunits

General cleanup of code without dramatic changes in functionality,
minimize roxie specific code, and include preview of mechanism for
copying logical files to the clusters a query is being published to.

common/workunit/workunit.cpp

@@ -650,7 +650,7 @@ public:
     virtual IPropertyTreeIterator & getFileIterator() const;
     virtual bool archiveWorkUnit(const char *base,bool del,bool ignoredllerrors,bool deleteOwned);
     virtual void packWorkUnit(bool pack=true);
-    virtual IDateTime & getTimeScheduled(IDateTime &val) const;
+    virtual IJlibDateTime & getTimeScheduled(IJlibDateTime &val) const;
     virtual IPropertyTreeIterator & getFilesReadIterator() const;
     virtual void protect(bool protectMode);
     virtual IConstWULibrary * getLibraryByName(const char * name) const;
@@ -726,7 +726,7 @@ public:
     void resetBeforeGeneration();
     void deleteTempFiles(const char *graph, bool deleteOwned, bool deleteJobOwned);
     void addDiskUsageStats(__int64 avgNodeUsage, unsigned minNode, __int64 minNodeUsage, unsigned maxNode, __int64 maxNodeUsage, __int64 graphId);
-    void setTimeScheduled(const IDateTime &val);
+    void setTimeScheduled(const IJlibDateTime &val);
 
 // ILocalWorkUnit - used for debugging etc
     void loadXML(const char *xml);
@@ -1063,7 +1063,7 @@ public:
             { c->packWorkUnit(pack); }
     virtual unsigned queryFileUsage(const char *filename) const
             { return c->queryFileUsage(filename); }
-    virtual IDateTime & getTimeScheduled(IDateTime &val) const
+    virtual IJlibDateTime & getTimeScheduled(IJlibDateTime &val) const
             { return c->getTimeScheduled(val); }
     virtual unsigned getDebugAgentListenerPort() const
             { return c->getDebugAgentListenerPort(); }
@@ -1219,7 +1219,7 @@ public:
 
     virtual void setSnapshot(const char * value)
             { c->setSnapshot(value); }
-    virtual void setTimeScheduled(const IDateTime &val)
+    virtual void setTimeScheduled(const IJlibDateTime &val)
             { c->setTimeScheduled(val); }
     virtual void setDebugAgentListenerPort(unsigned port)
             { c->setDebugAgentListenerPort(port); }
@@ -2450,6 +2450,13 @@ extern WORKUNIT_API IWorkUnitFactory * getSecWorkUnitFactory(ISecManager &secmgr
     return new CSecureWorkUnitFactory(secmgr, secuser);
 }
 
+extern WORKUNIT_API IWorkUnitFactory * getWorkUnitFactory(ISecManager *secmgr, ISecUser *secuser)
+{
+    if (secmgr && secuser)
+        return getSecWorkUnitFactory(*secmgr, *secuser);
+    else
+        return getWorkUnitFactory();
+}
 
 //==========================================================================================
 
@@ -2668,14 +2675,14 @@ void CLocalWorkUnit::cleanupAndDelete(bool deldll,bool deleteOwned)
     connection.clear();
 }
 
-void CLocalWorkUnit::setTimeScheduled(const IDateTime &val)
+void CLocalWorkUnit::setTimeScheduled(const IJlibDateTime &val)
 {
     SCMStringBuffer strval;
     val.getGmtString(strval);
     p->setProp("@timescheduled",strval.str());
 }
 
-IDateTime & CLocalWorkUnit::getTimeScheduled(IDateTime &val) const
+IJlibDateTime & CLocalWorkUnit::getTimeScheduled(IJlibDateTime &val) const
 {
     StringBuffer str;
     p->getProp("@timescheduled",str);
@@ -3798,7 +3805,9 @@ class CEnvironmentClusterInfo: public CInterface, implements IConstWUClusterInfo
     StringAttr serverQueue;
     StringAttr agentQueue;
     StringAttr roxieQueue;
+    StringAttr roxieProcess;
     StringAttr thorQueue;
+    StringArray thorProcesses;
     StringAttr prefix;
     StringAttr platform;
     StringAttr querySetName;
@@ -3817,6 +3826,7 @@ public:
             ForEachItemIn(i,thors) 
             {
                 IPropertyTree &thor = thors.item(i);
+                thorProcesses.append(thor.queryProp("@name"));
                 unsigned ts = thor.getPropInt("@slaves");
                 if (clusterWidth && (ts!=clusterWidth)) 
                     throw MakeStringException(WUERR_MismatchClusterSize,"CEnvironmentClusterInfo: mismatched thor sizes in cluster");
@@ -3831,6 +3841,7 @@ public:
         }
         else if (roxie)
         {
+            roxieProcess.set(roxie->queryProp("@name"));
             clusterWidth = roxie->getPropInt("@numChannels", 1);
             platform.set("roxie");
         }
@@ -3891,6 +3902,15 @@ public:
         str.set(querySetName.get());
         return str;
     }
+    IStringVal & getRoxieProcess(IStringVal & str) const
+    {
+        str.set(roxieProcess.get());
+        return str;
+    }
+    const StringArray & getThorProcesses() const
+    {
+        return thorProcesses;
+    }
 
 };
 
@@ -4357,6 +4377,7 @@ void CLocalWorkUnit::copyWorkUnit(IConstWorkUnit *cached)
     updateProp(p, fromP, "allowedclusters");
     updateProp(p, fromP, "@submitID");
     updateProp(p, fromP, "CustomerID");
+    updateProp(p, fromP, "SNAPSHOT");
 
     //Variables may have been set up as parameters to the query - so need to preserve any values that were supplied.
     pt = fromP->getBranch("Variables");
@@ -7474,7 +7495,7 @@ extern WORKUNIT_API ILocalWorkUnit * createLocalWorkUnit()
     return ret;
 }
 
-StringBuffer &exportWorkUnitToXML(const IConstWorkUnit *wu, StringBuffer &str)
+extern WORKUNIT_API StringBuffer &exportWorkUnitToXML(const IConstWorkUnit *wu, StringBuffer &str)
 {
     const CLocalWorkUnit *w = QUERYINTERFACE(wu, const CLocalWorkUnit);
     if (!w)

common/workunit/workunit.hpp

@@ -35,6 +35,7 @@
 
 #include "jiface.hpp"
 #include "errorlist.h"
+#include "jtime.hpp"
 
 #define CHEAP_UCHAR_DEF
 #ifdef _WIN32
@@ -73,28 +74,6 @@ interface IScmIterator : extends IInterface
     virtual bool isValid() = 0;
 };
 
-interface IConstDateTime : extends IInterface
-{
-    virtual IStringVal & getGmtString(IStringVal & str) const = 0;
-    virtual IStringVal & getLocalString(IStringVal & str) const = 0;
-    virtual IStringVal & getGmtDateString(IStringVal & str) const = 0;
-    virtual IStringVal & getLocalDateString(IStringVal & str) const = 0;
-    virtual IStringVal & getGmtTimeString(IStringVal & str) const = 0;
-    virtual IStringVal & getLocalTimeString(IStringVal & str) const = 0;
-};
-
-
-interface IDateTime : extends IConstDateTime
-{
-    virtual void setGmtString(const char * pstr) = 0;
-    virtual void setLocalString(const char * pstr) = 0;
-    virtual void setGmtDateString(const char * pstr) = 0;
-    virtual void setLocalDateString(const char * pstr) = 0;
-    virtual void setGmtTimeString(const char * pstr) = 0;
-    virtual void setLocalTimeString(const char * pstr) = 0;
-};
-
-
 interface IQueueSwitcher : extends IInterface
 {
     virtual void * getQ(const char * qname, const char * wuid) = 0;
@@ -561,6 +540,8 @@ interface IConstWUClusterInfo : extends IInterface
     virtual IStringVal & getRoxieQueue(IStringVal & str) const = 0;
     virtual IStringVal & getServerQueue(IStringVal & str) const = 0;
     virtual IStringVal & getQuerySetName(IStringVal & str) const = 0;
+    virtual IStringVal & getRoxieProcess(IStringVal & str) const = 0;
+    virtual const StringArray & getThorProcesses() const = 0;
 };
 
 //! IWorkflowItem
@@ -909,7 +890,7 @@ interface IConstWorkUnit : extends IInterface
     virtual bool getCloneable() const = 0;
     virtual IUserDescriptor * getUserDescriptor() const = 0;
     virtual IStringVal & getSnapshot(IStringVal & str) const = 0;
-    virtual IDateTime & getTimeScheduled(IDateTime & val) const = 0;
+    virtual IJlibDateTime & getTimeScheduled(IJlibDateTime & val) const = 0;
     virtual IPropertyTreeIterator & getFilesReadIterator() const = 0;
     virtual void protect(bool protectMode) = 0;
     virtual IStringVal & getAllowedClusters(IStringVal & str) const = 0;
@@ -994,7 +975,7 @@ interface IWorkUnit : extends IConstWorkUnit
     virtual void addDiskUsageStats(__int64 avgNodeUsage, unsigned minNode, __int64 minNodeUsage, unsigned maxNode, __int64 maxNodeUsage, __int64 graphId) = 0;
     virtual void setCloneable(bool value) = 0;
     virtual void setIsClone(bool value) = 0;
-    virtual void setTimeScheduled(const IDateTime & val) = 0;
+    virtual void setTimeScheduled(const IJlibDateTime & val) = 0;
     virtual void noteFileRead(IDistributedFile * file) = 0;
     virtual void clearGraphProgress() = 0;
     virtual void resetBeforeGeneration() = 0;
@@ -1035,12 +1016,12 @@ interface IConstWorkUnitIterator : extends IScmIterator
 
 interface IWUTimers : extends IInterface
 {
-    virtual void setTrigger(const IDateTime & dt) = 0;
-    virtual IDateTime & getTrigger(IDateTime & dt) const = 0;
-    virtual void setExpiration(const IDateTime & dt) = 0;
-    virtual IDateTime & getExpiration(IDateTime & dt) const = 0;
-    virtual void setSubmission(const IDateTime & dt) = 0;
-    virtual IDateTime & getSubmission(IDateTime & dt) const = 0;
+    virtual void setTrigger(const IJlibDateTime & dt) = 0;
+    virtual IJlibDateTime & getTrigger(IJlibDateTime & dt) const = 0;
+    virtual void setExpiration(const IJlibDateTime & dt) = 0;
+    virtual IJlibDateTime & getExpiration(IJlibDateTime & dt) const = 0;
+    virtual void setSubmission(const IJlibDateTime & dt) = 0;
+    virtual IJlibDateTime & getSubmission(IJlibDateTime & dt) const = 0;
 };
 
 
@@ -1132,6 +1113,13 @@ interface IExtendedWUInterface
     
 };
 
+struct WorkunitUpdate : public Owned<IWorkUnit>
+{
+public:
+    WorkunitUpdate(IWorkUnit *wu) : Owned<IWorkUnit>(wu) { }
+    ~WorkunitUpdate() { if (get()) get()->commit(); }
+};
+
 extern WORKUNIT_API IStringVal &getEclCCServerQueueNames(IStringVal &ret, const char *process);
 extern WORKUNIT_API IStringVal &getEclServerQueueNames(IStringVal &ret, const char *process);
 extern WORKUNIT_API IStringVal &getEclSchedulerQueueNames(IStringVal &ret, const char *process);
@@ -1151,8 +1139,10 @@ extern WORKUNIT_API bool parseGraphTimerLabel(const char *label, StringBuffer &g
 extern WORKUNIT_API void addExceptionToWorkunit(IWorkUnit * wu, WUExceptionSeverity severity, const char * source, unsigned code, const char * text, const char * filename, unsigned lineno, unsigned column);
 extern WORKUNIT_API IWorkUnitFactory * getWorkUnitFactory();
 extern WORKUNIT_API IWorkUnitFactory * getSecWorkUnitFactory(ISecManager &secmgr, ISecUser &secuser);
+extern WORKUNIT_API IWorkUnitFactory * getWorkUnitFactory(ISecManager *secmgr, ISecUser *secuser);
 extern WORKUNIT_API ILocalWorkUnit* createLocalWorkUnit();
 extern WORKUNIT_API IStringVal& exportWorkUnitToXML(const IConstWorkUnit *wu, IStringVal &str);
+extern WORKUNIT_API StringBuffer &exportWorkUnitToXML(const IConstWorkUnit *wu, StringBuffer &str);
 extern WORKUNIT_API void exportWorkUnitToXMLFile(const IConstWorkUnit *wu, const char * filename, unsigned extraXmlFlags);
 extern WORKUNIT_API void submitWorkUnit(const char *wuid, const char *username, const char *password);
 extern WORKUNIT_API void abortWorkUnit(const char *wuid);

common/workunit/wujobq.cpp

@@ -1897,5 +1897,31 @@ extern bool WORKUNIT_API runWorkUnit(const char *wuid)
         return false;
 }
 
+extern bool WORKUNIT_API switchWorkUnitQueue(IWorkUnit* wu, const char *cluster)
+{
+    if (!wu)
+        return false;
+
+    class cQswitcher: public CInterface, implements IQueueSwitcher
+    {
+    public:
+        IMPLEMENT_IINTERFACE;
+        void * getQ(const char * qname, const char * wuid)
+        {
+            Owned<IJobQueue> q = createJobQueue(qname);
+            return q->take(wuid);
+        }
+        void putQ(const char * qname, const char * wuid, void * qitem)
+        {
+            Owned<IJobQueue> q = createJobQueue(qname);
+            q->enqueue((IJobQueueItem *)qitem);
+        }
+        bool isAuto()
+        {
+            return false;
+        }
+    } switcher;
 
+    return wu->switchThorQueue(cluster, &switcher);
+}
 

common/workunit/wujobq.hpp

@@ -140,5 +140,6 @@ extern WORKUNIT_API IJobQueue *createJobQueue(const char *name);
 extern bool WORKUNIT_API runWorkUnit(const char *wuid, const char *cluster);
 extern bool WORKUNIT_API runWorkUnit(const char *wuid);
 
+extern bool WORKUNIT_API switchWorkUnitQueue(IWorkUnit* wu, const char *cluster);
 
 #endif

common/wuwebview/wuwebview.cpp

@@ -449,11 +449,28 @@ void WuWebView::load(const char *wuid)
 
 extern WUWEBVIEW_API IWuWebView *createWuWebView(IConstWorkUnit &wu, const char *queryname, const char *dir, bool mapEspDirectories)
 {
-    return new WuWebView(wu, queryname, dir, mapEspDirectories);
+    try
+    {
+        return new WuWebView(wu, queryname, dir, mapEspDirectories);
+    }
+    catch (...)
+    {
+        SCMStringBuffer wuid;
+        DBGLOG("ERROR loading workunit %s shared object.", wu.getWuid(wuid).str());
+    }
+    return NULL;
 }
 
 extern WUWEBVIEW_API IWuWebView *createWuWebView(const char *wuid, const char *queryname, const char *dir, bool mapEspDirectories)
 {
-    return new WuWebView(wuid, queryname, dir, mapEspDirectories);
+    try
+    {
+        return new WuWebView(wuid, queryname, dir, mapEspDirectories);
+    }
+    catch (...)
+    {
+        DBGLOG("ERROR loading workunit %s shared object.", wuid);
+    }
+    return NULL;
 }
 

ecl/eclagent/agentctx.hpp

@@ -108,11 +108,4 @@ struct IAgentContext : extends IGlobalCodeContext
     virtual const char *queryWuid() = 0;
 };
 
-struct WorkunitUpdate : public Owned<IWorkUnit>
-{
-public:
-    WorkunitUpdate(IWorkUnit *wu) : Owned<IWorkUnit>(wu) { }
-    ~WorkunitUpdate() { if (get()) get()->commit(); }
-};
-
 #endif // AGENTCTX_HPP_INCL

esp/bindings/bindutil.cpp

@@ -625,10 +625,10 @@ static unsigned char isValidUrlChar[96] =
     0x1,0x1,0x1,0x1,0x1,0x1,0x1,0x1,0x1,0x1,0x1,0x0,0x0,0x0,0x0,0x0  /* pqrstuvwxyz{\}~DEL */
 };
 
-void Utils::url_encode(const char* url, StringBuffer& encoded_url)
+StringBuffer &Utils::url_encode(const char* url, StringBuffer& encoded_url)
 {
     if(!url)
-        return;
+        return encoded_url;
     unsigned char c;
     int i = 0;
     while((c = url[i]) != 0)
@@ -641,6 +641,7 @@ void Utils::url_encode(const char* url, StringBuffer& encoded_url)
             encoded_url.appendf("%%%02X", c);
         i++;
     }
+    return encoded_url;
 }
 
 static char translateHex(char hex) {

esp/bindings/bindutil.hpp

@@ -41,7 +41,7 @@ public:
     static StringBuffer& base64encode(const void *data, long length, StringBuffer& result);
     static int base64decode(int inlen, const char *in, StringBuffer& data);
     static unsigned int hash (register const char *str, register unsigned int len);
-    static void url_encode(const char* url, StringBuffer& encoded_url);
+    static StringBuffer &url_encode(const char* url, StringBuffer& encoded_url);
     static int url_decode(const char* param, StringBuffer& result);
     static  void SplitURL(const char* url, StringBuffer& Protocol,StringBuffer& Name,StringBuffer& Password,StringBuffer& FQDN, StringBuffer& Port, StringBuffer& Path);
 };

esp/eclwatch/ws_XSLT/WUCopyLogicalFiles.xslt

@@ -0,0 +1,40 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<xsl:stylesheet version="1.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform" >
+    <xsl:template match="/">
+        <xsl:apply-templates select="WUCopyLogicalFilesResponse"/>
+    </xsl:template>
+   <xsl:template match="WUCopyLogicalFilesResponse">
+       <xsl:variable name="wuid" select="Wuid"/>
+      <html>
+         <head>
+           <link rel="stylesheet" type="text/css" href="/esp/files/yui/build/fonts/fonts-min.css" />
+           <link rel="stylesheet" type="text/css" href="/esp/files/css/espdefault.css" />
+           <link rel="stylesheet" type="text/css" href="/esp/files/css/eclwatch.css" />
+           <link rel="stylesheet" type="text/css" href="/esp/files/yui/build/menu/assets/skins/sam/menu.css" />
+           <link rel="stylesheet" type="text/css" href="/esp/files/yui/build/button/assets/skins/sam/button.css" />
+           <script type="text/javascript" src="/esp/files/scripts/espdefault.js">&#160;</script>
+         </head>
+         <body class="yui-skin-sam">
+            <h3>Copying Workunit Files</h3>
+            <xsl:if test="ClusterFiles/Cluster/NotOnCluster">
+                The following files are being copyied to the specified clusters:
+                <xsl:for-each select="ClusterFiles/Cluster">
+                    <br/><br/><xsl:text disable-output-escaping="yes">&amp;nbsp;</xsl:text><b>Cluster: </b><xsl:value-of select="ClusterName"/><br/><br/>
+                    <xsl:for-each select="NotOnCluster/WULogicalFileCopyInfo">
+                        <xsl:text disable-output-escaping="yes">&amp;nbsp;&amp;nbsp;&amp;nbsp;&amp;nbsp;</xsl:text><xsl:value-of select="LogicalName"/>
+                        <xsl:text disable-output-escaping="yes">&amp;nbsp;&amp;nbsp;&amp;nbsp;&amp;nbsp;</xsl:text>
+                            <xsl:if test="string(DfuCopyWuid)">
+                                <a><xsl:attribute name="href">/FileSpray/GetDFUWorkunit?wuid=<xsl:value-of select="DfuCopyWuid"/></xsl:attribute><xsl:value-of select="DfuCopyWuid"/></a>
+                            </xsl:if>
+                        <xsl:value-of select="DfuCopyError"/>
+                    </xsl:for-each>
+                </xsl:for-each>
+            </xsl:if>
+            <br/>
+            <form action="none">
+                <input type="button" value="Back" onclick="history.back()"/>
+            </form>
+         </body>
+      </html>
+   </xsl:template>
+</xsl:stylesheet>

esp/eclwatch/ws_XSLT/WUDeployWorkunit.xslt

@@ -0,0 +1,38 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<xsl:stylesheet version="1.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform" >
+    <xsl:template match="/">
+        <xsl:apply-templates select="WUDeployWorkunitResponse"/>
+    </xsl:template>
+   <xsl:template match="WUDeployWorkunitResponse">
+       <xsl:variable name="wuid" select="Wuid"/>
+      <html>
+         <head>
+           <link rel="stylesheet" type="text/css" href="/esp/files/yui/build/fonts/fonts-min.css" />
+           <link rel="stylesheet" type="text/css" href="/esp/files/css/espdefault.css" />
+           <link rel="stylesheet" type="text/css" href="/esp/files/css/eclwatch.css" />
+           <link rel="stylesheet" type="text/css" href="/esp/files/yui/build/menu/assets/skins/sam/menu.css" />
+           <link rel="stylesheet" type="text/css" href="/esp/files/yui/build/button/assets/skins/sam/button.css" />
+           <script type="text/javascript" src="/esp/files/scripts/espdefault.js">&#160;</script>
+         </head>
+         <body class="yui-skin-sam">
+            <h3>Workunit Published!</h3>
+            <xsl:if test="ClusterFiles/Cluster/NotOnCluster">
+                The following files are located on different clusters in this environment, to copy them to the specified cluster press the "Copy Files" button:
+                <xsl:for-each select="ClusterFiles/Cluster">
+                    <br/><br/><xsl:text disable-output-escaping="yes">&amp;nbsp;</xsl:text><b>Cluster: </b><xsl:value-of select="ClusterName"/><br/><br/>
+                    <xsl:for-each select="NotOnCluster/WULogicalFileCopyInfo">
+                        <xsl:text disable-output-escaping="yes">&amp;nbsp;&amp;nbsp;&amp;nbsp;&amp;nbsp;</xsl:text><xsl:value-of select="LogicalName"/><br/>
+                    </xsl:for-each>
+                </xsl:for-each>
+            </xsl:if>
+            <br/>
+            <form action="WUCopyLogicalFiles" method="get">
+                <input type="hidden" name="Wuid" value="{$wuid}"/>
+                <input type="hidden" name="CopyLocal" value="1"/>
+                <input type="submit" value="Copy Files"/><xsl:text disable-output-escaping="yes">&amp;nbsp;&amp;nbsp;&amp;nbsp;&amp;nbsp;</xsl:text>
+                <input type="button" value="Back" onclick="history.back()"/>
+            </form>
+         </body>
+      </html>
+   </xsl:template>
+</xsl:stylesheet>

esp/scm/ws_workunits.ecm

@@ -961,17 +961,53 @@ ESPresponse [exceptions_inline] WUDebugResponse
     string Result;
 };
 
+ESPrequest WUCopyLogicalFilesRequest
+{
+    string Wuid;
+    string Cluster;
+    bool CopyLocal;
+};
+
+ESPStruct WULogicalFileCopyInfo
+{
+    bool IsIndex;
+    string LogicalName;
+    string DfuCopyWuid;
+    string DfuCopyError;
+    ESParray<string> Clusters;
+};
+
+ESPStruct WUCopyLogicalClusterFileSections
+{
+    string ClusterName;
+    ESParray<ESPstruct WULogicalFileCopyInfo> OnCluster;
+    ESParray<ESPstruct WULogicalFileCopyInfo> NotOnCluster;
+    ESParray<ESPstruct WULogicalFileCopyInfo> Foreign;
+    ESParray<ESPstruct WULogicalFileCopyInfo> NotFound;
+};
+
+ESPresponse [exceptions_inline] WUCopyLogicalFilesResponse
+{
+    string Wuid;
+    ESParray<ESPStruct WUCopyLogicalClusterFileSections, Cluster> ClusterFiles;
+};
+
+
 ESPrequest WUDeployWorkunitRequest
 {
     string Wuid;
     string JobName;
     int Activate;
     bool NotifyCluster(false);
+    bool showFiles(0);
+    bool CopyLocal(0);
 };
 
 ESPresponse [exceptions_inline] WUDeployWorkunitResponse
 {
+    string Wuid;
     string Result;
+    ESParray<ESPStruct WUCopyLogicalClusterFileSections, Cluster> ClusterFiles;
 };
 
 
@@ -1062,28 +1098,10 @@ ESPresponse [exceptions_inline] WUQuerySetActionAliasesResponse
     ESParray<ESPstruct QuerySetAliasAction> QuerysetAliasActions;
 };
 
-
 ESPservice [
     version("1.34"), default_client_version("1.34"),
     noforms,exceptions_inline("./smc_xslt/exceptions.xslt"),use_method_name] WsWorkunits
 {
-    ESPuses ESPStruct ECLException;
-    ESPuses ESPStruct ECLResult;
-    ESPuses ESPStruct ECLGraph;
-    ESPuses ESPStruct ECLGraphEx;
-    ESPuses ESPStruct ECLTimer; 
-    ESPuses ESPStruct ECLHelpFile; 
-    ESPuses ESPStruct ECLQuery;
-    ESPuses ESPStruct ECLWorkunit;
-    ESPuses ESPStruct ECLSourceFile;
-    ESPuses ESPStruct WUECLAttribute;
-    ESPuses ESPStruct DebugValue;
-    ESPuses ESPStruct ApplicationValue;
-    ESPuses ESPStruct ECLJob;
-    ESPuses ESPStruct QuerySet;
-    ESPuses ESPStruct QuerySetQueryAction;
-    ESPuses ESPStruct QuerySetAliasAction;
-
     ESPmethod [resp_xsl_default("/esp/xslt/workunits.xslt")]     WUQuery(WUQueryRequest, WUQueryResponse);
     ESPmethod [resp_xsl_default("/esp/xslt/wuid.xslt")]     WUInfo(WUInfoRequest, WUInfoResponse);
     ESPmethod [resp_xsl_default("/esp/xslt/wuiddetails.xslt")]     WUInfoDetails(WUInfoRequest, WUInfoResponse);
@@ -1136,17 +1154,15 @@ ESPservice [
 
     ESPmethod WUCDebug(WUDebugRequest, WUDebugResponse);
     
-    ESPmethod WUDeployWorkunit(WUDeployWorkunitRequest, WUDeployWorkunitResponse);
-    
+    ESPmethod [resp_xsl_default("/esp/xslt/WUDeployWorkunit.xslt")] WUDeployWorkunit(WUDeployWorkunitRequest, WUDeployWorkunitResponse);
     ESPmethod [resp_xsl_default("/esp/xslt/WUQuerysets.xslt")] WUQuerysets(WUQuerysetsRequest, WUQuerysetsResponse);
     ESPmethod [resp_xsl_default("/esp/xslt/WUQuerysetQueries.xslt")] WUQuerysetDetails(WUQuerySetDetailsRequest, WUQuerySetDetailsResponse);
     ESPmethod WUQuerysetActionQueries(WUQuerySetActionQueriesRequest, WUQuerySetActionQueriesResponse);
     ESPmethod WUQuerysetActionAliases(WUQuerySetActionAliasesRequest, WUQuerySetActionAliasesResponse);
-    
+    ESPmethod [resp_xsl_default("/esp/xslt/WUCopyLogicalFiles.xslt")] WUCopyLogicalFiles(WUCopyLogicalFilesRequest, WUCopyLogicalFilesResponse);
 };
 
 
 SCMexportdef(WSWU);
 
 SCMapi(WSWU) IClientWsWorkunits *createWsWorkunitsClient();
-

esp/services/ws_workunits/CMakeLists.txt

@@ -16,54 +16,60 @@
 ################################################################################
 
 
-# Component: ws_workunits 
+# Component: ws_workunits
 #####################################################
 # Description:
 # ------------
 #    Cmake Input File for ws_workunits
 #####################################################
 
-project( ws_workunits ) 
+project( ws_workunits )
 
 include(${HPCC_SOURCE_DIR}/esp/scm/smcscm.cmake)
 
-set (    SRCS 
-         ../../../dali/sasha/sacmd.cpp 
-         ${ESPSCM_GENERATED_DIR}/ws_workunits_esp.cpp 
+set (    SRCS
+         ../../../dali/sasha/sacmd.cpp
+         ${ESPSCM_GENERATED_DIR}/ws_workunits_esp.cpp
+         ${ESPSCM_GENERATED_DIR}/ws_fs_esp.cpp
          ${HPCC_SOURCE_DIR}/esp/scm/ws_workunits.ecm
-         ws_workunitsPlugin.cpp 
-         ws_workunitsService.cpp 
-         ws_workunitsService.hpp 
+         ws_workunitsPlugin.cpp
+         ws_workunitsService.cpp
+         ws_workunitsService.hpp
+         ws_workunitsHelpers.cpp
+         ws_workunitsHelpers.hpp
+         ws_workunitsAuditLogs.cpp
+         ws_workunitsQuerySets.cpp
     )
 
-include_directories ( 
-         ./../../esplib 
-         ./../../../system/mp 
-         ./../../platform 
-         ./../../../dali/sasha 
-         ./../../../common/roxiemanager 
-         ./../../../system/jlib 
-         ./../../../common/environment 
-         ./../../services 
-         ./../common 
-         ./../../../system/xmllib 
-         ./../../../common/deftype 
-         ./../../../ecl/hql 
-         ./../../../system/security/securesocket 
-         ./../../../system/security/shared   
-         ./../../../system/include 
-         ./../../../common/workunit 
+include_directories (
+         ./../../esplib
+         ./../../../system/mp
+         ./../../platform
+         ./../../../dali/dfu
+         ./../../../dali/sasha
+         ./../../../common/roxiemanager
+         ./../../../system/jlib
+         ./../../../common/environment
+         ./../../services
+         ./../common
+         ./../../../system/xmllib
+         ./../../../common/deftype
+         ./../../../ecl/hql
+         ./../../../system/security/securesocket
+         ./../../../system/security/shared
+         ./../../../system/include
+         ./../../../common/workunit
          ./../../../common/wuwebview
-         ./../../../ecl/schedulectrl 
-         ./../../clients 
-         ./../../../common/fileview2 
-         ./../../../dali/base 
-         ./../../../system/security/zcrypt 
-         ./../../../rtl/include 
-         ./../../../common/dllserver 
-         ./../../bindings 
-         ./../../smc/SMCLib 
-         ./../../bindings/SOAP/xpp 
+         ./../../../ecl/schedulectrl
+         ./../../clients
+         ./../../../common/fileview2
+         ./../../../dali/base
+         ./../../../system/security/zcrypt
+         ./../../../rtl/include
+         ./../../../common/dllserver
+         ./../../bindings
+         ./../../smc/SMCLib
+         ./../../bindings/SOAP/xpp
     )
 
 ADD_DEFINITIONS( -D_USRDLL -DWS_WORKUNITS_EXPORTS )
@@ -71,31 +77,31 @@ ADD_DEFINITIONS( -D_USRDLL -DWS_WORKUNITS_EXPORTS )
 HPCC_ADD_LIBRARY( ws_workunits SHARED ${SRCS} )
 add_dependencies ( ws_workunits espscm )
 install ( TARGETS ws_workunits DESTINATION ${OSSDIR}/lib )
-target_link_libraries ( ws_workunits 
+target_link_libraries ( ws_workunits
          jlib
-         mp 
-         hrpc 
-         remote 
-         xmllib 
-         esphttp 
-         dalibase 
-         environment 
-         dalift 
-         dllserver 
-         nbcd 
-         eclrtl 
-         deftype 
-         workunit 
+         mp
+         hrpc
+         remote
+         xmllib
+         esphttp
+         dalibase
+         environment
+         dalift
+         dllserver
+         nbcd
+         eclrtl
+         deftype
+         workunit
          wuwebview
-         SMCLib 
-         schedulectrl 
-         roxiecommlib 
-         roxiemanager 
-         LdapSecurity 
-         hql 
-         jhtree 
-         fileview2 
-         securesocket 
+         SMCLib
+         schedulectrl
+         roxiecommlib
+         roxiemanager
+         LdapSecurity
+         hql
+         jhtree
+         fileview2
+         securesocket
     )
 
 IF (USE_ZLIB)
@@ -104,4 +110,3 @@ IF (USE_ZLIB)
       zcrypt
       )
 ENDIF()
-

esp/services/ws_workunits/ws_workunitsHelpers.hpp

@@ -0,0 +1,322 @@
+/*##############################################################################
+
+    Copyright (C) 2011 HPCC Systems.
+
+    All rights reserved. This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as
+    published by the Free Software Foundation, either version 3 of the
+    License, or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+############################################################################## */
+
+#ifndef _WSWU_HELPERS_HPP__
+#define _WSWU_HELPERS_HPP__
+
+#include "ws_workunits_esp.ipp"
+#include "exception_util.hpp"
+
+#include "jtime.ipp"
+#include "workunit.hpp"
+#include "hqlerror.hpp"
+
+#include <list>
+#include <vector>
+
+
+#define     OWN_WU_ACCESS      "OwnWorkunitsAccess"
+#define     OTHERS_WU_ACCESS   "OthersWorkunitsAccess"
+
+#define    File_Cpp "cpp"
+#define    File_ThorLog "ThorLog"
+#define    File_ThorSlaveLog "ThorSlaveLog"
+#define    File_EclAgentLog "EclAgentLog"
+#define    File_XML "XML"
+#define    File_Res "res"
+#define    File_DLL "dll"
+#define    File_ArchiveQuery "ArchiveQuery"
+
+#define    TEMPZIPDIR "tempzipfiles"
+
+static const long MAXXLSTRANSFER = 5000000;
+const unsigned DATA_SIZE = 16;
+const unsigned AWUS_CACHE_SIZE = 16;
+const unsigned AWUS_CACHE_MIN_DEFAULT = 15;
+
+inline bool notEmpty(const char *val){return (val && *val);}
+inline bool isEmpty(const char *val){return (!val || !*val);}
+
+const char *getWuAccessType(IConstWorkUnit& cw, const char *user);
+
+SecAccessFlags chooseWuAccessFlagsByOwnership(const char *user, const char *owner, SecAccessFlags accessOwn, SecAccessFlags accessOthers);
+SecAccessFlags chooseWuAccessFlagsByOwnership(const char *user, IConstWorkUnit& cw, SecAccessFlags accessOwn, SecAccessFlags accessOthers);
+SecAccessFlags getWsWorkunitAccess(IEspContext& cxt, IConstWorkUnit& cw);
+
+void getUserWuAccessFlags(IEspContext& context, SecAccessFlags& accessOwn, SecAccessFlags& accessOthers, bool except);
+void ensureWsWorkunitAccess(IEspContext& cxt, IConstWorkUnit& cw, SecAccessFlags minAccess);
+void ensureWsWorkunitAccess(IEspContext& context, const char* wuid, SecAccessFlags minAccess);
+void ensureWsWorkunitAccessByOwnerId(IEspContext& context, const char* owner, SecAccessFlags minAccess);
+void ensureWsCreateWorkunitAccess(IEspContext& cxt);
+
+const char *getGraphNum(const char *s,unsigned &num);
+
+class WsWuDateTime : public CScmDateTime
+{
+public:
+    IMPLEMENT_IINTERFACE;
+    WsWuDateTime()
+    {
+        setSimpleLocal(0);
+    }
+
+    bool isValid()
+    {
+        unsigned year, month, day;
+        cdt.getDate(year, month, day, true);
+        return year>1969;
+    }
+};
+
+void formatDuration(StringBuffer &s, unsigned ms);
+
+struct WsWUExceptions
+{
+    WsWUExceptions(IConstWorkUnit& wu);
+
+    operator IArrayOf<IEspECLException>&() { return errors; }
+    int ErrCount() { return numerr; }
+    int WrnCount() { return numwrn; }
+    int InfCount() { return numinf; }
+
+private:
+    IArrayOf<IEspECLException> errors;
+    int numerr;
+    int numwrn;
+    int numinf;
+};
+
+#define WUINFO_TruncateEclTo64k         0x001
+#define WUINFO_IncludeExceptions        0x002
+#define WUINFO_IncludeGraphs            0x004
+#define WUINFO_IncludeResults           0x008
+#define WUINFO_IncludeVariables         0x010
+#define WUINFO_IncludeTimers            0x020
+#define WUINFO_IncludeDebugValues       0x040
+#define WUINFO_IncludeApplicationValues 0x080
+#define WUINFO_IncludeWorkflows         0x100
+#define WUINFO_IncludeEclSchemas        0x200
+#define WUINFO_IncludeSourceFiles       0x400
+#define WUINFO_IncludeResultsViewNames  0x800
+#define WUINFO_All                      0xFFF
+
+class WsWuInfo
+{
+public:
+    WsWuInfo(IEspContext &ctx, IConstWorkUnit *cw_) :
+      context(ctx), cw(cw_)
+    {
+        version = context.getClientVersion();
+        cw->getWuid(wuid);
+    }
+
+    WsWuInfo(IEspContext &ctx, const char *wuid_) :
+      context(ctx)
+    {
+        wuid.set(wuid_);
+        version = context.getClientVersion();
+        Owned<IWorkUnitFactory> factory = getWorkUnitFactory(ctx.querySecManager(), ctx.queryUser());
+        cw.setown(factory->openWorkUnit(wuid_, false));
+        if(!cw)
+            throw MakeStringException(ECLWATCH_CANNOT_UPDATE_WORKUNIT,"Cannot open workunit %s.", wuid_);
+    }
+
+    bool getResultViews(StringArray &resultViews, unsigned flags);
+
+    void getCommon(IEspECLWorkunit &info, unsigned flags);
+    void getInfo(IEspECLWorkunit &info, unsigned flags);
+
+    bool getResults(IEspECLWorkunit &info, unsigned flags);
+    bool getVariables(IEspECLWorkunit &info, unsigned flags);
+    bool getDebugValues(IEspECLWorkunit &info, unsigned flags);
+    bool getClusterInfo(IEspECLWorkunit &info, unsigned flags);
+    bool getApplicationValues(IEspECLWorkunit &info, unsigned flags);
+    void getExceptions(IEspECLWorkunit &info, unsigned flags);
+    bool getSourceFiles(IEspECLWorkunit &info, unsigned flags);
+    bool getTimers(IEspECLWorkunit &info, unsigned flags);
+    bool getHelpers(IEspECLWorkunit &info, unsigned flags);
+    bool getGraphInfo(IEspECLWorkunit &info, unsigned flags);
+    void getGraphTimingData(IArrayOf<IConstECLTimingData> &timingData, unsigned flags);
+
+    void getRoxieCluster(IEspECLWorkunit &info, unsigned flags);
+    bool getWorkflow(IEspECLWorkunit &info, unsigned flags);
+
+    void getHelpFiles(IConstWUQuery* query, WUFileType type, IArrayOf<IEspECLHelpFile>& helpers);
+    void getSubFiles(IPropertyTreeIterator* f, IEspECLSourceFile* eclSuperFile, StringArray& fileNames);
+    void getEclSchemaChildFields(IArrayOf<IEspECLSchemaItem>& schemas, IHqlExpression * expr, bool isConditional);
+    void getEclSchemaFields(IArrayOf<IEspECLSchemaItem>& schemas, IHqlExpression * expr, bool isConditional);
+    bool getResultEclSchemas(IConstWUResult &r, IArrayOf<IEspECLSchemaItem>& schemas);
+    void getResult(IConstWUResult &r, IArrayOf<IEspECLResult>& results, unsigned flags);
+
+    void getWorkunitEclAgentLog(MemoryBuffer& buf);
+    void getWorkunitThorLog(MemoryBuffer& buf);
+    void getWorkunitThorSlaveLog(const char *slaveip, MemoryBuffer& buf);
+    void getWorkunitResTxt(MemoryBuffer& buf);
+    void getWorkunitArchiveQuery(MemoryBuffer& buf);
+    void getWorkunitDll(MemoryBuffer& buf);
+    void getWorkunitXml(const char* plainText, MemoryBuffer& buf);
+    void getWorkunitCpp(const char* cppname, const char* description, const char* ipAddress, MemoryBuffer& buf);
+
+public:
+    IEspContext &context;
+    Linked<IConstWorkUnit> cw;
+    double version;
+    SCMStringBuffer clusterName;
+    SCMStringBuffer wuid;
+};
+
+void getSashaNode(SocketEndpoint &ep);
+
+struct WsWuSearch
+{
+    WsWuSearch(IEspContext& context,const char* owner=NULL,const char* state=NULL,const char* cluster=NULL,const char* startDate=NULL,const char* endDate=NULL,const char* ecl=NULL,const char* jobname=NULL,const char* appname=NULL,const char* appkey=NULL,const char* appvalue=NULL);
+
+    typedef std::vector<std::string>::iterator iterator;
+
+    iterator begin() { return wuids.begin(); }
+    iterator end()   { return wuids.end(); }
+
+    iterator locate(const char* wuid)
+    {
+        if(wuids.size() && *wuids.begin()>wuid)
+            return std::lower_bound(wuids.begin(),wuids.end(),wuid,std::greater<std::string>());
+        return wuids.begin();
+    }
+
+     __int64 getSize() { return wuids.size(); }
+
+private:
+
+    StringBuffer& createWuidFromDate(const char* timestamp,StringBuffer& s);
+
+    std::vector<std::string> wuids;
+};
+
+struct DataCacheElement: public CInterface, implements IInterface
+{
+    IMPLEMENT_IINTERFACE;
+    DataCacheElement(const char* filter, const char* data, const char* name, const char* logicalName, const char* wuid,
+        const char* resultName, unsigned seq,   __int64 start, unsigned count, __int64 requested, __int64 total):m_filter(filter),
+        m_data(data), m_name(name), m_logicalName(logicalName), m_wuid(wuid), m_resultName(resultName),
+        m_seq(seq), m_start(start), m_rowcount(count), m_requested(requested), m_total(total)
+    {
+        m_timeCached.setNow();
+    }
+
+    CDateTime m_timeCached;
+    std::string m_filter;
+    std::string m_data;
+    std::string m_name;
+    std::string m_logicalName;
+    std::string m_wuid;
+    std::string m_resultName;
+    unsigned m_seq;
+    __int64 m_start;
+    unsigned m_rowcount;
+    __int64 m_requested;
+    __int64 m_total;
+};
+
+struct DataCache: public CInterface, implements IInterface
+{
+    IMPLEMENT_IINTERFACE;
+
+    DataCache(size32_t _cacheSize=0): cacheSize(_cacheSize){}
+
+    DataCacheElement* lookup(IEspContext &context, const char* filter, unsigned timeOutMin);
+
+    void add(const char* filter, const char* data, const char* name, const char* localName, const char* wuid,
+    const char* resultName, unsigned seq,   __int64 start, unsigned count, __int64 requested, __int64 total);
+
+    std::list<StlLinked<DataCacheElement> > cache;
+    CriticalSection crit;
+    size32_t cacheSize;
+};
+
+struct ArchivedWuCacheElement: public CInterface, implements IInterface
+{
+    IMPLEMENT_IINTERFACE;
+    ArchivedWuCacheElement(const char* filter, const char* sashaUpdatedWhen, bool hasNextPage, /*const char* data,*/ IArrayOf<IEspECLWorkunit>& wus):m_filter(filter),
+        m_sashaUpdatedWhen(sashaUpdatedWhen), m_hasNextPage(hasNextPage)/*, m_data(data)*/
+    {
+        m_timeCached.setNow();
+        if (wus.length() > 0)
+
+        for (unsigned i = 0; i < wus.length(); i++)
+        {
+            Owned<IEspECLWorkunit> info= createECLWorkunit("","");
+            IEspECLWorkunit& info0 = wus.item(i);
+            info->copy(info0);
+
+            m_results.append(*info.getClear());
+        }
+    }
+
+    std::string m_filter;
+    std::string m_sashaUpdatedWhen;
+    bool m_hasNextPage;
+
+    CDateTime m_timeCached;
+    IArrayOf<IEspECLWorkunit> m_results;
+};
+
+struct ArchivedWuCache: public CInterface, implements IInterface
+{
+    IMPLEMENT_IINTERFACE;
+
+    ArchivedWuCache(size32_t _cacheSize=0): cacheSize(_cacheSize){}
+    ArchivedWuCacheElement* lookup(IEspContext &context, const char* filter, const char* sashaUpdatedWhen, unsigned timeOutMin);
+
+    void add(const char* filter, const char* sashaUpdatedWhen, bool hasNextPage, IArrayOf<IEspECLWorkunit>& wus);
+
+    std::list<StlLinked<ArchivedWuCacheElement> > cache;
+    CriticalSection crit;
+    size32_t cacheSize;
+};
+
+class WsWuJobQueueAuditInfo
+{
+public:
+    WsWuJobQueueAuditInfo() {  };
+    WsWuJobQueueAuditInfo(IEspContext &context, const char *cluster, const char *from , const char *to, CHttpResponse* response, const char *xls);
+
+    void getAuditLineInfo(const char* line, unsigned& longestQueue, unsigned& maxConnected, unsigned maxDisplay, unsigned showAll, IArrayOf<IEspThorQueue>& items);
+    bool checkSameStrings(const char* s1, const char* s2);
+    bool checkNewThorQueueItem(IEspThorQueue* tq, unsigned showAll, IArrayOf<IEspThorQueue>& items);
+};
+
+StringBuffer &getWuidFromLogicalFileName(IEspContext &context, const char *logicalName, StringBuffer &wuid);
+const char *getThorQueueName(const char *cluster);
+
+bool addToQueryString(StringBuffer &queryString, const char *name, const char *value, const char delim = '&');
+
+void xsltTransform(const char* xml, const char* sheet, IProperties *params, StringBuffer& ret);
+
+class WUSchedule : public Thread
+{
+    IEspContainer* m_container;
+
+public:
+    virtual int run();
+    virtual void setContainer(IEspContainer * container)
+    {
+        m_container = container;
+    }
+};
+
+#endif

esp/services/ws_workunits/ws_workunitsPlugin.cpp

@@ -43,8 +43,8 @@ ESP_FACTORY IEspService * esp_service_factory(const char *name, const char* type
    }
    return NULL;
 }
- 
-   
+
+
 
 ESP_FACTORY IEspRpcBinding * esp_binding_factory(const char *name, const char* type, IPropertyTree *cfg, const char *process)
 {
@@ -57,7 +57,7 @@ ESP_FACTORY IEspRpcBinding * esp_binding_factory(const char *name, const char* t
 #endif
         return new CWsWorkunitsSoapBindingEx(cfg, name, process, llevel);
     }
-    
+
     return NULL;
 }
 

esp/services/ws_workunits/ws_workunitsQuerySets.cpp

@@ -0,0 +1,368 @@
+/*##############################################################################
+
+    Copyright (C) 2011 HPCC Systems.
+
+    All rights reserved. This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as
+    published by the Free Software Foundation, either version 3 of the
+    License, or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+############################################################################## */
+
+#include "ws_workunitsService.hpp"
+#include "ws_fs.hpp"
+#include "jlib.hpp"
+#include "daclient.hpp"
+#include "dalienv.hpp"
+#include "dadfs.hpp"
+#include "dfuwu.hpp"
+#include "eclhelper.hpp"
+
+const unsigned roxieQueryRoxieTimeOut = 60000;
+
+#define SDS_LOCK_TIMEOUT (5*60*1000) // 5mins, 30s a bit short
+
+bool isRoxieProcess(const char *process)
+{
+    if (!process)
+        return false;
+    Owned<IRemoteConnection> conn = querySDS().connect("Environment", myProcessSession(), RTM_LOCK_READ, SDS_LOCK_TIMEOUT);
+    if (!conn)
+        return false;
+    VStringBuffer xpath("Software/RoxieCluster[@name=\"%s\"]", process);
+    return conn->queryRoot()->hasProp(xpath.str());
+}
+
+void doWuFileCopy(IClientFileSpray &fs, IEspWULogicalFileCopyInfo &info, const char *logicalname, const char *cluster, bool isRoxie, bool supercopy)
+{
+    try
+    {
+        Owned<IClientCopy> req = fs.createCopyRequest();
+        req->setSourceLogicalName(logicalname);
+        req->setDestLogicalName(logicalname);
+        req->setDestGroup(cluster);
+        req->setSuperCopy(supercopy);
+        if (isRoxie)
+            req->setDestGroupRoxie("Yes");
+
+        Owned<IClientCopyResponse> resp = fs.Copy(req);
+        info.setDfuCopyWuid(resp->getResult());
+    }
+    catch (IException *e)
+    {
+        StringBuffer msg;
+        info.setDfuCopyError(e->errorMessage(msg).str());
+    }
+}
+
+bool copyWULogicalFiles(IEspContext &context, IConstWorkUnit &cw, const char *cluster, bool copyLocal, IEspWUCopyLogicalClusterFileSections &lfinfo)
+{
+    if (isEmpty(cluster))
+        throw MakeStringException(ECLWATCH_INVALID_CLUSTER_NAME, "copyWULogicalFiles Cluster parameter not set.");
+
+    Owned<IUserDescriptor> udesc = createUserDescriptor();
+    udesc->set(context.queryUserId(), context.queryPassword());
+
+    IArrayOf<IEspWULogicalFileCopyInfo> foreign;
+    IArrayOf<IEspWULogicalFileCopyInfo> onCluster;
+    IArrayOf<IEspWULogicalFileCopyInfo> notOnCluster;
+    IArrayOf<IEspWULogicalFileCopyInfo> notFound;
+
+    Owned<IClientFileSpray> fs;
+    if (copyLocal)
+    {
+        fs.setown(createFileSprayClient());
+        VStringBuffer url("http://.:%d/FileSpray", 8010);
+        fs->addServiceUrl(url.str());
+    }
+
+    bool isRoxie = isRoxieProcess(cluster);
+
+    Owned<IConstWUGraphIterator> graphs = &cw.getGraphs(GraphTypeActivities);
+    ForEach(*graphs)
+    {
+        Owned <IPropertyTree> xgmml = graphs->query().getXGMMLTree(false);
+        Owned<IPropertyTreeIterator> iter = xgmml->getElements(".//node");
+        ForEach(*iter)
+        {
+            try
+            {
+                IPropertyTree &node = iter->query();
+                ThorActivityKind kind = (ThorActivityKind) node.getPropInt("att[@name='_kind']/@value", TAKnone);
+
+                if(kind==TAKdiskwrite || kind==TAKindexwrite || kind==TAKcsvwrite || kind==TAKxmlwrite)
+                    continue;
+                if (node.getPropBool("att[@name='_isSpill']/@value") || node.getPropBool("att[@name='_isTransformSpill']/@value"))
+                    continue;
+
+                Owned<IEspWULogicalFileCopyInfo> info = createWULogicalFileCopyInfo();
+                const char *logicalname = node.queryProp("att[@name='_indexFileName']/@value");
+                if (logicalname)
+                    info->setIsIndex(true);
+                else
+                    logicalname = node.queryProp("att[@name='_fileName']/@value");
+                info->setLogicalName(logicalname);
+                if (logicalname)
+                {
+                    if (!strnicmp("~foreign::", logicalname, 10))
+                        foreign.append(*info.getClear());
+                    else
+                    {
+                        Owned<IDistributedFile> df = queryDistributedFileDirectory().lookup(logicalname, udesc);
+                        if(!df)
+                            notFound.append(*info.getClear());
+                        else if (df->findCluster(cluster)!=NotFound)
+                        {
+                            onCluster.append(*info.getClear());
+                        }
+                        else
+                        {
+                            StringArray clusters;
+                            df->getClusterNames(clusters);
+                            info->setClusters(clusters);
+                            if (copyLocal)
+                            {
+                                StringBuffer wuid;
+                                bool supercopy = queryDistributedFileDirectory().isSuperFile(logicalname, NULL, udesc);
+                                doWuFileCopy(*fs, *info, logicalname, cluster, isRoxie, supercopy);
+                            }
+                            notOnCluster.append(*info.getClear());
+                        }
+                    }
+                }
+            }
+            catch(IException *e)
+            {
+                e->Release();
+            }
+        }
+        lfinfo.setClusterName(cluster);
+        lfinfo.setNotOnCluster(notOnCluster);
+        lfinfo.setOnCluster(onCluster);
+        lfinfo.setForeign(foreign);
+        lfinfo.setNotFound(notFound);
+    }
+
+    return true;
+}
+
+void copyWULogicalFilesToTarget(IEspContext &context, IConstWUClusterInfo &clusterInfo, IConstWorkUnit &cw, IArrayOf<IConstWUCopyLogicalClusterFileSections> &clusterfiles, bool doLocalCopy)
+{
+    const StringArray &thors = clusterInfo.getThorProcesses();
+    ForEachItemIn(i, thors)
+    {
+        Owned<IEspWUCopyLogicalClusterFileSections> files = createWUCopyLogicalClusterFileSections();
+        copyWULogicalFiles(context, cw, thors.item(i), doLocalCopy, *files);
+        clusterfiles.append(*files.getClear());
+    }
+    SCMStringBuffer roxie;
+    clusterInfo.getRoxieProcess(roxie);
+    if (roxie.length())
+    {
+        Owned<IEspWUCopyLogicalClusterFileSections> files = createWUCopyLogicalClusterFileSections();
+        copyWULogicalFiles(context, cw, roxie.str(), doLocalCopy, *files);
+        clusterfiles.append(*files.getClear());
+    }
+}
+
+bool CWsWorkunitsEx::onWUCopyLogicalFiles(IEspContext &context, IEspWUCopyLogicalFilesRequest &req, IEspWUCopyLogicalFilesResponse &resp)
+{
+    if (isEmpty(req.getWuid()))
+        throw MakeStringException(ECLWATCH_CANNOT_OPEN_WORKUNIT, "WUCopyLogicalFiles WUID parameter not set.");
+
+    Owned<IWorkUnitFactory> factory = getWorkUnitFactory(context.querySecManager(), context.queryUser());
+    Owned<IConstWorkUnit> cw = factory->openWorkUnit(req.getWuid(), false);
+    if (!cw)
+        throw MakeStringException(ECLWATCH_CANNOT_OPEN_WORKUNIT,"Cannot open workunit %s", req.getWuid());
+
+    resp.setWuid(req.getWuid());
+
+    SCMStringBuffer cluster;
+    if (notEmpty(req.getCluster()))
+        cluster.set(req.getCluster());
+    else
+        cw->getClusterName(cluster);
+
+    Owned <IConstWUClusterInfo> clusterInfo = getTargetClusterInfo(cluster.str());
+
+    IArrayOf<IConstWUCopyLogicalClusterFileSections> clusterfiles;
+    copyWULogicalFilesToTarget(context, *clusterInfo, *cw, clusterfiles, req.getCopyLocal());
+    resp.setClusterFiles(clusterfiles);
+
+    return true;
+}
+
+bool CWsWorkunitsEx::onWUDeployWorkunit(IEspContext &context, IEspWUDeployWorkunitRequest & req, IEspWUDeployWorkunitResponse & resp)
+{
+    if (isEmpty(req.getWuid()))
+        throw MakeStringException(ECLWATCH_NO_WUID_SPECIFIED,"No Workunit ID has been specified.");
+
+    Owned<IWorkUnitFactory> factory = getWorkUnitFactory(context.querySecManager(), context.queryUser());
+    Owned<IConstWorkUnit> cw = factory->openWorkUnit(req.getWuid(), false);
+    if (!cw)
+        throw MakeStringException(ECLWATCH_CANNOT_OPEN_WORKUNIT,"Cannot find the workunit %s", req.getWuid());
+
+    resp.setWuid(req.getWuid());
+
+    SCMStringBuffer queryName;
+    if (notEmpty(req.getJobName()))
+        queryName.set(req.getJobName());
+    else
+        cw->getJobName(queryName).str();
+
+    SCMStringBuffer cluster;
+    cw->getClusterName(cluster);
+
+    Owned <IConstWUClusterInfo> clusterInfo = getTargetClusterInfo(cluster.str());
+
+    SCMStringBuffer queryset;
+    clusterInfo->getQuerySetName(queryset);
+
+    WorkunitUpdate wu(&cw->lock());
+    if (notEmpty(req.getJobName()))
+        wu->setJobName(req.getJobName());
+
+    StringBuffer queryId;
+    addQueryToQuerySet(wu, queryset.str(), queryName.str(), NULL, (WUQueryActivationOptions)req.getActivate(), queryId);
+    wu->commit();
+    wu.clear();
+
+    if (req.getCopyLocal() || req.getShowFiles())
+    {
+        IArrayOf<IConstWUCopyLogicalClusterFileSections> clusterfiles;
+        copyWULogicalFilesToTarget(context, *clusterInfo, *cw, clusterfiles, req.getCopyLocal());
+        resp.setClusterFiles(clusterfiles);
+    }
+
+    return true;
+}
+
+bool CWsWorkunitsEx::onWUQuerysets(IEspContext &context, IEspWUQuerysetsRequest & req, IEspWUQuerysetsResponse & resp)
+{
+    Owned<IPropertyTree> queryRegistry = getQueryRegistryRoot();
+    if (!queryRegistry)
+        return false;
+
+    IArrayOf<IEspQuerySet> querySets;
+    Owned<IPropertyTreeIterator> it = queryRegistry->getElements("QuerySet");
+    ForEach(*it)
+    {
+        Owned<IEspQuerySet> qs = createQuerySet("", "");
+        qs->setQuerySetName(it->query().queryProp("@id"));
+        querySets.append(*qs.getClear());
+
+    }
+    resp.setQuerysets(querySets);
+    return true;
+}
+
+bool CWsWorkunitsEx::onWUQuerysetDetails(IEspContext &context, IEspWUQuerySetDetailsRequest & req, IEspWUQuerySetDetailsResponse & resp)
+{
+    resp.setQuerySetName(req.getQuerySetName());
+
+    Owned<IPropertyTree> registry = getQueryRegistry(req.getQuerySetName(), true);
+    if (!registry)
+        return false;
+
+    IArrayOf<IEspQuerySetQuery> querySetQueries;
+    Owned<IPropertyTreeIterator> queries = registry->getElements("Query");
+    ForEach(*queries)
+    {
+        IPropertyTree &query = queries->query();
+        Owned<IEspQuerySetQuery> q = createQuerySetQuery("", "");
+        q->setId(query.queryProp("@id"));
+        q->setName(query.queryProp("@name"));
+        q->setDll(query.queryProp("@dll"));
+        q->setWuid(query.queryProp("@wuid"));
+        q->setSuspended(query.getPropBool("@suspended", false));
+        querySetQueries.append(*q.getLink());
+
+    }
+    resp.setQuerysetQueries(querySetQueries);
+
+    IArrayOf<IEspQuerySetAlias> querySetAliases;
+    Owned<IPropertyTreeIterator> aliases = registry->getElements("Alias");
+    ForEach(*aliases)
+    {
+        IPropertyTree &alias = aliases->query();
+        Owned<IEspQuerySetAlias> a = createQuerySetAlias("", "");
+        a->setName(alias.queryProp("@name"));
+        a->setId(alias.queryProp("@id"));
+        querySetAliases.append(*a.getClear());
+    }
+    resp.setQuerysetAliases(querySetAliases);
+    return true;
+}
+
+bool CWsWorkunitsEx::onWUQuerysetActionQueries(IEspContext &context, IEspWUQuerySetActionQueriesRequest & req, IEspWUQuerySetActionQueriesResponse & resp)
+{
+    resp.setQuerySetName(req.getQuerySetName());
+    resp.setRemove(req.getRemove());
+
+    Owned<IPropertyTree> queryRegistry = getQueryRegistry(req.getQuerySetName(), false);
+
+    IArrayOf<IEspQuerySetQueryAction> actions;
+    ForEachItemIn(qa, req.getQuerysetQueryActions())
+    {
+        IConstQuerySetQueryAction& item=req.getQuerysetQueryActions().item(qa);
+        if(notEmpty(item.getId()))
+        {
+            if (req.getRemove())
+            {
+                removeAliasesFromNamedQuery(queryRegistry, item.getId());
+                removeNamedQuery(queryRegistry, item.getId());
+
+                Owned<IEspQuerySetQueryAction> action = createQuerySetQueryAction("", "");
+                action->setId(item.getId());
+                action->setStatus("Completed");
+                actions.append(*action.getClear());
+            }
+            if (req.getToggleSuspend())
+            {
+                setQuerySuspendedState(queryRegistry, item.getId(), !item.getSuspended());
+
+                Owned<IEspQuerySetQueryAction> action = createQuerySetQueryAction("", "");
+                action->setId(item.getId());
+                action->setStatus("Completed");
+                actions.append(*action.getClear());
+            }
+        }
+    }
+    resp.setQuerysetQueryActions(actions);
+    return true;
+}
+
+bool CWsWorkunitsEx::onWUQuerysetActionAliases(IEspContext &context, IEspWUQuerySetActionAliasesRequest & req, IEspWUQuerySetActionAliasesResponse & resp)
+{
+    resp.setQuerySetName(req.getQuerySetName());
+    resp.setRemove(req.getRemove());
+
+    Owned<IPropertyTree> queryRegistry = getQueryRegistry(req.getQuerySetName(), false);
+
+    IArrayOf<IEspQuerySetAliasAction> actions;
+    ForEachItemIn(aa, req.getQuerysetAliasActions())
+    {
+        IConstQuerySetAliasAction& item=req.getQuerysetAliasActions().item(aa);
+        if (req.getRemove())
+        {
+            if(notEmpty(item.getId()))
+            {
+                removeAliasesFromNamedQuery(queryRegistry, item.getId());
+                Owned<IEspQuerySetAliasAction> action = createQuerySetAliasAction("", "");
+                action->setId(item.getId());
+                action->setStatus("Completed");
+                actions.append(*action.getClear());
+
+            }
+        }
+    }
+    resp.setQuerysetAliasActions(actions);
+    return true;
+}

esp/services/ws_workunits/ws_workunitsService.hpp

@@ -21,223 +21,15 @@
 
 #include "ws_workunits_esp.ipp"
 #include "workunit.hpp"
-#include "WUWrapper.hpp"
-#include "WUXMLInfo.hpp"
+#include "ws_workunitsHelpers.hpp"
 
-#include <list>
-
-#include "jwrapper.hpp"
-
-#include "fileview.hpp"
-#include "hqlerror.hpp"
-
-#include <vector>
-
-using namespace esp;
-
-typedef enum wsEclTypes_
-{
-    wsEclTypeUnknown,
-    xsdString,
-    xsdBoolean,
-    xsdDecimal,
-    xsdFloat,
-    xsdDouble,
-    xsdDuration,
-    xsdDateTime,
-    xsdTime,
-    xsdDate,
-    xsdYearMonth,
-    xsdYear,
-    xsdMonthDay,
-    xsdDay,
-    xsdMonth,
-    xsdHexBinary,
-    xsdBase64Binary,
-    xsdAnyURI,
-    xsdQName,
-    xsdNOTATION,
-    xsdNormalizedString,
-    xsdToken,
-    xsdLanguage,
-    xsdNMTOKEN,
-    xsdNMTOKENS, 
-    xsdName,
-    xsdNCName,
-    xsdID,
-    xsdIDREF,
-    xsdIDREFS, 
-    xsdENTITY,
-    xsdENTITIES,
-    xsdInteger,
-    xsdNonPositiveInteger,
-    xsdNegativeInteger,
-    xsdLong,
-    xsdInt,
-    xsdShort,
-    xsdByte,
-    xsdNonNegativeInteger,
-    xsdUnsignedLong,
-    xsdUnsignedInt,
-    xsdUnsignedShort,
-    xsdUnsignedByte,
-    xsdPositiveInteger,
-
-    tnsRawDataFile,
-    tnsCsvDataFile,
-    tnsEspStringArray,
-    tnsEspIntArray,
-    tnsXmlDataSet,
-
-    maxWsEclType
-
-} wsEclType;
-
-class WUSchedule : public Thread
+class CWsWorkunitsEx : public CWsWorkunits
 {
-    IEspContainer* m_container;
-    bool m_allowNewRoxieOnDemandQuery;
-
 public:
-    virtual int run();
-    virtual void setContainer(IEspContainer * container)
-    {
-        m_container = container;
-    }
-    
-    void setAllowNewRoxieOnDemandQuery(bool allowNewRoxieOnDemandQuery)
-    {
-        m_allowNewRoxieOnDemandQuery = allowNewRoxieOnDemandQuery;
-    }
-};
-
-struct DataCacheElement: public CInterface, implements IInterface
-{
-    IMPLEMENT_IINTERFACE;
-    DataCacheElement(const char* filter, const char* data, const char* name, const char* logicalName, const char* wuid, 
-        const char* resultName, unsigned seq,   __int64 start, unsigned count, __int64 requested, __int64 total):m_filter(filter), 
-        m_data(data), m_name(name), m_logicalName(logicalName), m_wuid(wuid), m_resultName(resultName), 
-        m_seq(seq), m_start(start), m_rowcount(count), m_requested(requested), m_total(total)
-    {
-        m_timeCached.setNow();
-    }
-
-    CDateTime m_timeCached;
-    std::string m_filter;
-
-    std::string m_data;
-    std::string m_name;
-    std::string m_logicalName;
-    std::string m_wuid;
-    std::string m_resultName;
-    unsigned m_seq;
-    __int64 m_start;
-    unsigned m_rowcount;
-    __int64 m_requested;
-    __int64 m_total;
-};
-
-struct DataCache: public CInterface, implements IInterface
-{
-    IMPLEMENT_IINTERFACE;
-    
-    DataCache(size32_t _cacheSize=0): cacheSize(_cacheSize)
-    {
-    }
-
-
-    DataCacheElement* lookup(IEspContext &context, const char* filter, unsigned timeOutMin);
-
-     void add(const char* filter, const char* data, const char* name, const char* localName, const char* wuid, 
-        const char* resultName, unsigned seq,   __int64 start, unsigned count, __int64 requested, __int64 total);
-
-    std::list<StlLinked<DataCacheElement> > cache;
-    CriticalSection crit;
-    size32_t cacheSize;
-};
-
-struct ArchivedWUsCacheElement: public CInterface, implements IInterface
-{
     IMPLEMENT_IINTERFACE;
-    ArchivedWUsCacheElement(const char* filter, const char* sashaUpdatedWhen, bool hasNextPage, /*const char* data,*/ IArrayOf<IEspECLWorkunit>& wus):m_filter(filter), 
-        m_sashaUpdatedWhen(sashaUpdatedWhen), m_hasNextPage(hasNextPage)/*, m_data(data)*/ 
-    {
-        m_timeCached.setNow();
-        if (wus.length() > 0)
-
-        for (unsigned i = 0; i < wus.length(); i++)
-        {
-            Owned<IEspECLWorkunit> info= createECLWorkunit("","");
-            IEspECLWorkunit& info0 = wus.item(i);
-            info->copy(info0);
-
-            m_results.append(*info.getClear());
-        }
-    }
-
-    std::string m_filter;
-    std::string m_sashaUpdatedWhen;
-    bool m_hasNextPage;
-
-    CDateTime m_timeCached;
-    IArrayOf<IEspECLWorkunit> m_results;
-    //std::string m_data;
-};
-
-struct ArchivedWUsCache: public CInterface, implements IInterface
-{
-    IMPLEMENT_IINTERFACE;
-    
-    ArchivedWUsCache(size32_t _cacheSize=0): cacheSize(_cacheSize)
-    {
-    }
-
-
-    ArchivedWUsCacheElement* lookup(IEspContext &context, const char* filter, const char* sashaUpdatedWhen, unsigned timeOutMin);
-     //void getQueryFileListFromTree(IPropertyTreeIterator* queries, const char* fileType, const char* cluster, 
-    //  IArrayOf<IEspRoxieDFULogicalFile>& queryFileList);
 
-     void add(const char* filter, const char* sashaUpdatedWhen, bool hasNextPage, IArrayOf<IEspECLWorkunit>& wus);
+    CWsWorkunitsEx(){port=8010;}
 
-    std::list<StlLinked<ArchivedWUsCacheElement> > cache;
-    CriticalSection crit;
-    size32_t cacheSize;
-};
-
-class CWsWorkunitsSoapBindingEx : public CWsWorkunitsSoapBinding
-{
-public:
-    CWsWorkunitsSoapBindingEx(IPropertyTree *cfg, const char *name, const char *process, http_soap_log_level llevel) : CWsWorkunitsSoapBinding(cfg, name, process, llevel)
-    {
-        StringBuffer xpath;
-        xpath.appendf("Software/EspProcess[@name=\"%s\"]/EspBinding[@name=\"%s\"]/BatchWatch", process, name);
-        m_bBatchWatch = cfg->getPropBool(xpath.str(), false);
-    }
-
-    virtual int onGetForm(IEspContext &context, CHttpRequest* request, CHttpResponse* response, const char *service, const char *method);
-    virtual int onGet(CHttpRequest* request, CHttpResponse* response);
-
-    virtual void getNavigationData(IEspContext &context, IPropertyTree & data)
-    {
-        if (!m_bBatchWatch)
-        {
-            IPropertyTree *folder = ensureNavFolder(data, "ECL Workunits", "ECL Workunits", NULL, false, 2);
-            ensureNavLink(*folder, "Search", "/WsWorkunits/WUQuery?form_", "Search Workunits", NULL, NULL, 1);
-            ensureNavLink(*folder, "Browse", "/WsWorkunits/WUQuery", "Browse Workunits", NULL, NULL, 2);
-
-            IPropertyTree *folderQueryset = ensureNavFolder(data, "Query Sets", "Queryset Management", NULL, false, 3);
-            ensureNavLink(*folderQueryset, "Browse", "/WsWorkunits/WUQuerySets", "Browse Querysets");
-        }
-    }
-private:
-    void addLogicalClusterByName(const char* clusterName, StringArray& clusters, StringBuffer& x);
-    bool m_bBatchWatch;
-};
-
-class CWsWorkunitsEx : public CWsWorkunits
-{
-public:
-   IMPLEMENT_IINTERFACE;
     virtual ~CWsWorkunitsEx(){};
     virtual void init(IPropertyTree *cfg, const char *process, const char *service);
     virtual void setContainer(IEspContainer * container)
@@ -252,6 +44,8 @@ public:
     bool onWUQuerysetDetails(IEspContext &context, IEspWUQuerySetDetailsRequest & req, IEspWUQuerySetDetailsResponse & resp);
     bool onWUQuerysetActionQueries(IEspContext &context, IEspWUQuerySetActionQueriesRequest & req, IEspWUQuerySetActionQueriesResponse & resp);
     bool onWUQuerysetActionAliases(IEspContext &context, IEspWUQuerySetActionAliasesRequest & req, IEspWUQuerySetActionAliasesResponse & resp);
+    bool onWUCopyLogicalFiles(IEspContext &context, IEspWUCopyLogicalFilesRequest &req, IEspWUCopyLogicalFilesResponse &resp);
+
     bool onWUInfo(IEspContext &context, IEspWUInfoRequest &req, IEspWUInfoResponse &resp);
     bool onWUInfoDetails(IEspContext &context, IEspWUInfoRequest &req, IEspWUInfoResponse &resp);
     bool onWUFile(IEspContext &context,IEspWULogFileRequest &req, IEspWULogFileResponse &resp);
@@ -259,10 +53,10 @@ public:
     bool onWUResultView(IEspContext &context, IEspWUResultViewRequest &req, IEspWUResultViewResponse &resp);
     bool onWUResultSummary(IEspContext &context, IEspWUResultSummaryRequest &req, IEspWUResultSummaryResponse &resp);
     bool onWUResultBin(IEspContext &context, IEspWUResultBinRequest &req, IEspWUResultBinResponse &resp);
-   bool onWUGraphInfo(IEspContext &context,IEspWUGraphInfoRequest &req, IEspWUGraphInfoResponse &resp);
-   bool onWUGVCGraphInfo(IEspContext &context,IEspWUGVCGraphInfoRequest &req, IEspWUGVCGraphInfoResponse &resp);
+    bool onWUGraphInfo(IEspContext &context,IEspWUGraphInfoRequest &req, IEspWUGraphInfoResponse &resp);
+    bool onWUGVCGraphInfo(IEspContext &context,IEspWUGVCGraphInfoRequest &req, IEspWUGVCGraphInfoResponse &resp);
     bool onWUProcessGraph(IEspContext &context,IEspWUProcessGraphRequest &req, IEspWUProcessGraphResponse &resp);
-    bool onGVCAjaxGraph(IEspContext &context, IEspGVCAjaxGraphRequest &req, IEspGVCAjaxGraphResponse &resp); 
+    bool onGVCAjaxGraph(IEspContext &context, IEspGVCAjaxGraphRequest &req, IEspGVCAjaxGraphResponse &resp);
 
     bool onWUAction(IEspContext &context, IEspWUActionRequest &req, IEspWUActionResponse &resp);
     bool onWUShowScheduled(IEspContext &context, IEspWUShowScheduledRequest &req, IEspWUShowScheduledResponse &resp);
@@ -300,84 +94,53 @@ public:
 
     bool onWUCDebug(IEspContext &context, IEspWUDebugRequest &req, IEspWUDebugResponse &resp);
 
-    static void xsltTransform(const char* xml,const char* xsl,IProperties *params,StringBuffer& ret);
+    void setPort(unsigned short _port){port=_port;}
 
 private:
-    void getHelpFiles(double version, IConstWUQuery* query, WUFileType type, IArrayOf<IEspECLHelpFile>& helpers);
-    void getWorkunitCluster(IEspContext &context, const char* wuid, StringBuffer& cluster, bool checkArchiveWUs = false);
-    void getWorkunitXml(IEspContext &context, const char* wuid, const char* plainText, MemoryBuffer& buf);
-    void getWorkunitCpp(IEspContext &context, const char* cppname, const char* description, const char* ipAddress,MemoryBuffer& buf);
-    void getWorkunitResTxt(IEspContext &context, const char* wuid,MemoryBuffer& buf);
-    void getWorkunitDll(IEspContext &context, const char* wuid,MemoryBuffer& buf);
-    void getWorkunitArchiveQuery(IEspContext &context, const char* wuid,MemoryBuffer& buf);
-    void getWorkunitResults(IEspContext &context, const char* wuid, unsigned index,__int64 start, unsigned& count,__int64& total,IStringVal& resname,bool raw,MemoryBuffer& buf);
-    void getWorkunitEclAgentLog(IEspContext &context, const char *wuid, MemoryBuffer& log);
-    void getWorkunitThorLog(IEspContext &context, const char* type, const char *wuid,MemoryBuffer& log);
-    void getWorkunitThorSlaveLog(IEspContext &context, const char *wuid,const char *slaveip,MemoryBuffer& buf);
-    void getResult(IEspContext &context,IConstWUResult &r,IArrayOf<IEspECLResult>& results, const char* wuid = NULL, bool SuppressSchemas=false);
-    void getInfo(IEspContext &context,const char* wuid,IEspECLWorkunit *info, bool bTruncateEclTo64k, bool IncludeExceptions=true, bool IncludeGraphs=true, bool IncludeSourceFiles=true, bool IncludeResults=true, bool IncludeVariables=true, bool IncludeTimers=true, bool IncludeDebugValues=true, bool IncludeApplicationValues=true, bool IncludeWorkflows=true, bool SuppressSchemas=false, StringArray *resultViews=NULL);
-    bool getInfoFromSasha(IEspContext &context,const char *sashaServer,const char* wuid,IEspECLWorkunit *info);
-    void getResultView(INewResultSet* result, __int64 start, unsigned& count,__int64& total,IStringVal& resname,bool raw,MemoryBuffer& buf);
-    void getFileResults(IEspContext &context, const char* logicalName, const char* cluster,__int64 start, unsigned& count,__int64& total,IStringVal& resname,bool raw,MemoryBuffer& buf);
-    void getScheduledWUs(IEspContext &context, const char *serverName, const char *eventName, IArrayOf<IEspScheduledWU> & results);
-    void getArchivedWUInfo(IEspContext &context, IEspWUInfoRequest &req, IEspWUInfoResponse &resp);
-
-    void submitWU(IEspContext& context, const char* wuid, const char* cluster, const char* snapshot, int maxruntime, bool compile, bool resetWorkflow);
-    void resubmitWU(IEspContext& context, const char* wuid, const char* cluster, IStringVal& newWuid, bool forceRecompile);
-    void scheduleWU(IEspContext& context, const char* wuid, const char* cluster, const char* when, const char* snapshot, int maxruntime);
-    void pauseWU(IEspContext& context, const char* wuid, bool now = true);
-    void resumeWU(IEspContext& context, const char* wuid);
+    unsigned awusCacheMinutes;
 
-    void processWorkunit(IConstWorkUnit *workunit, const char* wuid, SCMStringBuffer &queryName,  SCMStringBuffer &clusterName, SCMStringBuffer &querySetName, int activateOption);
+    Owned<DataCache> dataCache;
+    Owned<ArchivedWuCache> archivedWuCache;
+    WUSchedule m_sched;
+    unsigned short port;
+};
 
-    bool doAction(IEspContext&, StringArray& wuids , int action, IProperties* params=NULL, IArrayOf<IConstWUActionResult>* results=NULL);
+class CWsWorkunitsSoapBindingEx : public CWsWorkunitsSoapBinding
+{
+public:
+    CWsWorkunitsSoapBindingEx(IPropertyTree *cfg, const char *name, const char *process, http_soap_log_level llevel) : CWsWorkunitsSoapBinding(cfg, name, process, llevel)
+    {
+        VStringBuffer xpath("Software/EspProcess[@name=\"%s\"]/EspBinding[@name=\"%s\"]/BatchWatch", process, name);
+        batchWatchFeaturesOnly = cfg->getPropBool(xpath.str(), false);
+    }
 
-    void getPopups(IEspContext &context, const char *wuid, const char *graphname, const char* popupId, StringBuffer &script);
-    void getGJSGraph(IEspContext &context, const char *wuid, const char *graphname, IProperties* params, StringBuffer &script);
+    virtual void getNavigationData(IEspContext &context, IPropertyTree & data)
+    {
+        if (!batchWatchFeaturesOnly)
+        {
+            IPropertyTree *folder = ensureNavFolder(data, "ECL Workunits", "ECL Workunits", NULL, false, 2);
+            ensureNavLink(*folder, "Search", "/WsWorkunits/WUQuery?form_", "Search Workunits", NULL, NULL, 1);
+            ensureNavLink(*folder, "Browse", "/WsWorkunits/WUQuery", "Browse Workunits", NULL, NULL, 2);
 
-    void doWUQueryByXPath(IEspContext &context, IEspWUQueryRequest &req, IEspWUQueryResponse &resp);
-    void doWUQueryWithSort(IEspContext &context, IEspWUQueryRequest &req, IEspWUQueryResponse &resp);
-    void doWUQueryForArchivedWUs(IEspContext &context, IEspWUQueryRequest &req, IEspWUQueryResponse &resp, const char *sashaAddress);
-    void addToQueryString(StringBuffer &queryString, const char *name, const char *value);
+            IPropertyTree *folderQueryset = ensureNavFolder(data, "Query Sets", "Queryset Management", NULL, false, 3);
+            ensureNavLink(*folderQueryset, "Browse", "/WsWorkunits/WUQuerySets", "Browse Querysets");
+        }
+    }
 
-    void gatherFields(IArrayOf<IEspECLSchemaItem>& schemas, IHqlExpression * expr, bool isConditional);
-    void gatherChildFields(IArrayOf<IEspECLSchemaItem>& schemas, IHqlExpression * expr, bool isConditional);
-    bool getResultSchemas(IConstWUResult &r, IArrayOf<IEspECLSchemaItem>& schemas);
-    bool checkFileContent(IEspContext &context, const char * logicalName);
+    int onGetForm(IEspContext &context, CHttpRequest* request, CHttpResponse* response, const char *service, const char *method);
+    int onGet(CHttpRequest* request, CHttpResponse* response);
 
-    void ensureArchivedWorkunitAccess(IEspContext& context, const char *owner, SecAccessFlags minAccess);
-    void ensureWorkunitAccess(IEspContext& context, IConstWorkUnit& wu, SecAccessFlags minAccess);
-    void ensureWorkunitAccess(IEspContext& context, const char* wuid, SecAccessFlags minAccess)
+    virtual void addService(const char * name, const char * host, unsigned short port, IEspService & service)
     {
-        CWUWrapper wu(wuid, context);
-        ensureWorkunitAccess(context, *wu, minAccess);
+        CWsWorkunitsEx* srv = dynamic_cast<CWsWorkunitsEx*>(&service);
+        if (srv)
+            srv->setPort(port);
+        CWsWorkunitsSoapBinding::addService(name, host, port, service);
     }
-    void lookupAccess(IEspContext& context, SecAccessFlags& accessOwn, SecAccessFlags& accessOthers);
-    SecAccessFlags getAccess(IEspContext& context, IConstWorkUnit& wu, 
-                             SecAccessFlags accessOwn, SecAccessFlags accessOthers);
-     SecAccessFlags getWorkunitAccess(IEspContext& context, IConstWorkUnit& wu);
 
-    bool getClusterJobQueueXLS(double version, IStringVal &ret, const char* cluster, const char* startDate, const char* endDate, const char* showType);
-    int loadFile(const char* fname, int& len, unsigned char* &buf, bool binary=true);
-
-    void addSubFiles(IPropertyTreeIterator* f, IEspECLSourceFile* eclSuperFile, StringArray& fileNames);
-    void openSaveFile(IEspContext &context, int opt, const char* filename, const char* origMimeType, MemoryBuffer& buf, IEspWULogFileResponse &resp);
-#if 0 //not use for now
-    void getSubFiles(IUserDescriptor* userdesc, const char *fileName, IEspECLSourceFile* eclSourceFile0);
-    bool checkFileInECLSourceFile(const char* file, IConstECLSourceFile& eclfile);
-    bool checkFileInECLSourceFiles(const char* file, IArrayOf<IEspECLSourceFile>& eclfiles);
-#endif
 
 private:
-    StringBuffer m_GraphUpdateGvcXSLT;
-    bool m_allowNewRoxieOnDemandQuery;
-    unsigned m_AWUS_cache_minutes;
-
-    Owned<DataCache> m_dataCache;
-    Owned<ArchivedWUsCache> m_archivedWUsCache;
-    WUSchedule m_sched;
+    bool batchWatchFeaturesOnly;
 };
 
-
-#endif 
-
+#endif

install_directory/esp.install

@@ -1780,6 +1780,12 @@ Install ( FILES ${CMAKE_CURRENT_SOURCE_DIR}/esp/eclwatch/ws_XSLT/filerelationsea
 ### name=smc_xslt/filerelationlist.xslt
 Install ( FILES ${CMAKE_CURRENT_SOURCE_DIR}/esp/eclwatch/ws_XSLT/filerelationlist.xslt DESTINATION ${OSSDIR}/componentfiles/smc_xslt
     PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE )
+### name=smc_xslt/WUDeployWorkunit.xslt
+Install ( FILES ${CMAKE_CURRENT_SOURCE_DIR}/esp/eclwatch/ws_XSLT/WUDeployWorkunit.xslt DESTINATION ${OSSDIR}/componentfiles/smc_xslt
+    PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE )
+### name=smc_xslt/WUCopyLogicalFiles.xslt
+Install ( FILES ${CMAKE_CURRENT_SOURCE_DIR}/esp/eclwatch/ws_XSLT/WUCopyLogicalFiles.xslt DESTINATION ${OSSDIR}/componentfiles/smc_xslt
+    PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE )
 ### name=smc_xslt/WUQuerysets.xslt
 Install ( FILES ${CMAKE_CURRENT_SOURCE_DIR}/esp/eclwatch/ws_XSLT/WUQuerysets.xslt DESTINATION ${OSSDIR}/componentfiles/smc_xslt
     PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE )

roxie/ccd/ccddali.hpp

@@ -26,15 +26,6 @@
 #include "dadfs.hpp"
 #include "daclient.hpp"
 
-// MORE - this is duplicated here and eclagent - move into workunit ?
-
-struct WorkunitUpdate : public Owned<IWorkUnit>
-{
-public:
-    WorkunitUpdate(IWorkUnit *wu) : Owned<IWorkUnit>(wu) { }
-    ~WorkunitUpdate() { if (get()) get()->commit(); }
-};
-
 extern void addWuException(IConstWorkUnit *workUnit, IException *E);
 
 interface IDaliPackageWatcher : extends IInterface

system/jlib/jlog.cpp

@@ -2582,3 +2582,11 @@ extern jlib_decl void UseSysLogForOperatorMessages(bool use)
     }
 }
 
+extern jlib_decl void AuditSystemAccess(const char *userid, bool success, char const * msg,...)
+{
+    va_list args;
+    va_start(args, msg);
+    VStringBuffer s("User %s: ", userid);
+    SYSLOG((success) ? AUDIT_TYPE_ACCESS_SUCCESS : AUDIT_TYPE_ACCESS_FAILURE, s.valist_appendf(msg, args).str());
+    va_end(args);
+}

system/jlib/jlog.hpp

@@ -953,6 +953,7 @@ extern jlib_decl void UseSysLogForOperatorMessages(bool use=true);
 #define SYSLOG querySysLogEventLogger()->log
 #define AUDIT SYSLOG                               // bwd compatibility
 
+extern jlib_decl void AuditSystemAccess(const char *userid, bool success, char const * msg,...) __attribute__((format(printf, 3, 4)));
 
 /***************************************************************************/
 /* The simplest logging commands are:                                      */

system/jlib/jstring.hpp

@@ -365,7 +365,8 @@ inline const char *encodeUtf8XML(const char *x, StringBuffer &ret, unsigned flag
 inline StringBuffer &appendXMLTag(StringBuffer &xml, const char *tag, const char *value, unsigned flags=0, unsigned len=(unsigned)-1, bool utf8=true)
 {
     xml.append('<').append(tag).append('>');
-    encodeXML(value, xml, flags, len, utf8);
+    if (value && *value)
+        encodeXML(value, xml, flags, len, utf8);
     return xml.append("</").append(tag).append('>');
 }