HPCC-11754 Support copy entire queryset from one environment to another

Also enhance behavior whenever copying queries from a remote environment:

Will try and reuse an existing local copy of the workunit, matching WUID,
hash, and dllCrc.

Otherwise try and reuse the WUID from the remote environment.

Also try and identify existing copies of the same query dll.

Signed-off-by: Anthony Fishbeck <anthony.fishbeck@lexisnexis.com>

common/workunit/workunit.cpp

@@ -10364,12 +10364,14 @@ void setQueryCommentForNamedQuery(const char *querySetName, const char *id, cons
     setQueryCommentForNamedQuery(queryRegistry, id, queryComment);
 }
 
-const char *queryIdFromQuerySetWuid(IPropertyTree *queryRegistry, const char *wuid, IStringVal &id)
+const char *queryIdFromQuerySetWuid(IPropertyTree *queryRegistry, const char *wuid, const char *queryName, IStringVal &id)
 {
     if (!queryRegistry)
         return NULL;
     StringBuffer xpath;
     xpath.appendf("Query[@wuid='%s']", wuid);
+    if (queryName && *queryName)
+        xpath.appendf("[@name='%s']", queryName);
     IPropertyTree *q = queryRegistry->queryPropTree(xpath.str());
     if (q)
     {
@@ -10378,10 +10380,10 @@ const char *queryIdFromQuerySetWuid(IPropertyTree *queryRegistry, const char *wu
     return id.str();
 }
 
-const char *queryIdFromQuerySetWuid(const char *querySetName, const char *wuid, IStringVal &id)
+const char *queryIdFromQuerySetWuid(const char *querySetName, const char *wuid, const char *queryName, IStringVal &id)
 {
     Owned<IPropertyTree> queryRegistry = getQueryRegistry(querySetName, true);
-    return queryIdFromQuerySetWuid(queryRegistry, wuid, id);
+    return queryIdFromQuerySetWuid(queryRegistry, wuid, queryName, id);
 }
 
 extern WORKUNIT_API void gatherLibraryNames(StringArray &names, StringArray &unresolved, IWorkUnitFactory &workunitFactory, IConstWorkUnit &cw, IPropertyTree *queryset)

common/workunit/workunit.hpp

@@ -1347,8 +1347,8 @@ extern WORKUNIT_API bool removeQuerySetAlias(const char *querySetName, const cha
 extern WORKUNIT_API void addQuerySetAlias(const char *querySetName, const char *alias, const char *id);
 extern WORKUNIT_API void setSuspendQuerySetQuery(const char *querySetName, const char *id, bool suspend, const char *userid);
 extern WORKUNIT_API void deleteQuerySetQuery(const char *querySetName, const char *id);
-extern WORKUNIT_API const char *queryIdFromQuerySetWuid(IPropertyTree *queryRegistry, const char *wuid, IStringVal &id);
-extern WORKUNIT_API const char *queryIdFromQuerySetWuid(const char *querySetName, const char *wuid, IStringVal &id);
+extern WORKUNIT_API const char *queryIdFromQuerySetWuid(IPropertyTree *queryRegistry, const char *wuid, const char *queryName, IStringVal &id);
+extern WORKUNIT_API const char *queryIdFromQuerySetWuid(const char *querySetName, const char *wuid, const char *queryName, IStringVal &id);
 extern WORKUNIT_API void removeQuerySetAliasesFromNamedQuery(const char *querySetName, const char * id);
 extern WORKUNIT_API void setQueryCommentForNamedQuery(const char *querySetName, const char *id, const char *comment);
 extern WORKUNIT_API void gatherLibraryNames(StringArray &names, StringArray &unresolved, IWorkUnitFactory &workunitFactory, IConstWorkUnit &cw, IPropertyTree *queryset);

ecl/eclcmd/queries/ecl-queries.cpp

@@ -699,9 +699,13 @@ public:
             "By default only active queries will be copied.  Use --all to copy all queries.\n"
             "\n"
             "ecl queries copy-set <source_target> <destination_target> [--clone-active-state]\n"
+
+            "ecl queries copy-set target1 target2\n"
+            "ecl queries copy-set //ip:port/target1 target2 --clone-active-state\n"
             "\n"
             " Options:\n"
-            "   <source_target>        Target cluster to copy queries from\n"
+            "   <source_target>        Name of local (or path to remote) target cluster to"
+            "                          copy queries from\n"
             "   <destination_target>   Target cluster to copy queries to\n"
             "   --all                  Copy both active and inactive queries\n"
             "   --no-files             Do not copy files referenced by query\n"

esp/services/ws_workunits/CMakeLists.txt

@@ -73,6 +73,7 @@ include_directories (
          ./../../../rtl/include
          ./../../../common/dllserver
          ./../../bindings
+         ./../../bindings/http/client
          ./../../smc/SMCLib
          ./../../bindings/SOAP/xpp
          ${HPCC_SOURCE_DIR}/dali/dfu

esp/services/ws_workunits/ws_workunitsHelpers.hpp

@@ -379,20 +379,29 @@ class NewWsWorkunit : public Owned<IWorkUnit>
 public:
     NewWsWorkunit(IWorkUnitFactory *factory, IEspContext &context)
     {
-        create(factory, context);
+        create(factory, context, NULL);
     }
 
     NewWsWorkunit(IEspContext &context)
     {
         Owned<IWorkUnitFactory> factory = getWorkUnitFactory(context.querySecManager(), context.queryUser());
-        create(factory, context);
+        create(factory, context, NULL);
+    }
+
+    NewWsWorkunit(IEspContext &context, const char *wuid)
+    {
+        Owned<IWorkUnitFactory> factory = getWorkUnitFactory(context.querySecManager(), context.queryUser());
+        create(factory, context, wuid);
     }
 
     ~NewWsWorkunit() { if (get()) get()->commit(); }
 
-    void create(IWorkUnitFactory *factory, IEspContext &context)
+    void create(IWorkUnitFactory *factory, IEspContext &context, const char *wuid)
     {
-        setown(factory->createWorkUnit(NULL, "ws_workunits", context.queryUserId()));
+        if (wuid && *wuid)
+            setown(factory->createNamedWorkUnit(wuid, NULL, "ws_workunits", context.queryUserId()));
+        else
+            setown(factory->createWorkUnit(NULL, "ws_workunits", context.queryUserId()));
         if(!get())
           throw MakeStringException(ECLWATCH_CANNOT_CREATE_WORKUNIT,"Could not create workunit.");
         get()->setUser(context.queryUserId());

esp/services/ws_workunits/ws_workunitsQuerySets.cpp

@@ -27,6 +27,7 @@
 #include "dfuutil.hpp"
 #include "dautils.hpp"
 #include "referencedfilelist.hpp"
+#include "httpclient.hpp"
 
 #define DALI_FILE_LOOKUP_TIMEOUT (1000*15*1)  // 15 seconds
 
@@ -59,20 +60,33 @@ void checkUseEspOrDaliIP(SocketEndpoint &ep, const char *ip, const char *esp)
         ep.ipset(esp);
 }
 
-IClientWUQuerySetDetailsResponse *fetchQueryDetails(IEspContext &context, IClientWsWorkunits *_ws, const char *target, const char *queryid)
+static IClientWsWorkunits *ensureWsWorkunitsClient(IClientWsWorkunits *ws, IEspContext *ctx, const char *netAddress)
 {
-    Linked<IClientWsWorkunits> ws = _ws;
-    if (!ws)
+    if (ws)
+        return LINK(ws);
+    StringBuffer url;
+    if (netAddress && *netAddress)
+        url.appendf("http://%s%s/WsWorkunits", netAddress, (!strchr(netAddress, ':')) ? ":8010" : "");
+    else
     {
-        StringBuffer host;
-        short port=0;
-        context.getServAddress(host, port);
-        VStringBuffer url("http://%s:%d/WsWorkunits", host.str(), port);
-        ws.setown(createWsWorkunitsClient());
-        ws->addServiceUrl(url.str());
-        if (context.queryUserId() && *context.queryUserId())
-            ws->setUsernameToken(context.queryUserId(), context.queryPassword(), NULL);
+        if (!ctx)
+            throw MakeStringException(ECLWATCH_INVALID_IP, "Missing WsWorkunits service address");
+        StringBuffer ip;
+        short port = 0;
+        ctx->getServAddress(ip, port);
+        url.appendf("http://%s:%d/WsWorkunits", ip.str(), port);
     }
+    Owned<IClientWsWorkunits> cws = createWsWorkunitsClient();
+    cws->addServiceUrl(url);
+    if (ctx && ctx->queryUserId() && *ctx->queryUserId())
+        cws->setUsernameToken(ctx->queryUserId(), ctx->queryPassword(), NULL);
+    return cws.getClear();
+}
+
+IClientWUQuerySetDetailsResponse *fetchQueryDetails(IClientWsWorkunits *_ws, IEspContext *ctx, const char *netAddress, const char *target, const char *queryid)
+{
+    Owned<IClientWsWorkunits> ws = ensureWsWorkunitsClient(_ws, ctx, netAddress);
+
     //using existing WUQuerysetDetails rather than extending WUQueryDetails, to support copying query meta data from prior releases
     Owned<IClientWUQuerySetDetailsRequest> reqQueryInfo = ws->createWUQuerysetDetailsRequest();
     reqQueryInfo->setClusterName(target);
@@ -82,16 +96,9 @@ IClientWUQuerySetDetailsResponse *fetchQueryDetails(IEspContext &context, IClien
     return ws->WUQuerysetDetails(reqQueryInfo);
 }
 
-void fetchRemoteWorkunit(IEspContext &context, const char *netAddress, const char *queryset, const char *query, const char *wuid, StringBuffer &name, StringBuffer &xml, StringBuffer &dllname, MemoryBuffer &dll, StringBuffer &daliServer, Owned<IClientWUQuerySetDetailsResponse> &respQueryInfo)
+void fetchRemoteWorkunit(IClientWsWorkunits *_ws, IEspContext *ctx, const char *netAddress, const char *queryset, const char *query, const char *wuid, StringBuffer &name, StringBuffer &xml, StringBuffer &dllname, MemoryBuffer &dll, StringBuffer &daliServer)
 {
-    Owned<IClientWsWorkunits> ws;
-    ws.setown(createWsWorkunitsClient());
-    VStringBuffer url("http://%s%s/WsWorkunits", netAddress, (!strchr(netAddress, ':')) ? ":8010" : "");
-    ws->addServiceUrl(url.str());
-
-    if (context.queryUserId() && *context.queryUserId())
-        ws->setUsernameToken(context.queryUserId(), context.queryPassword(), NULL);
-
+    Owned<IClientWsWorkunits> ws = ensureWsWorkunitsClient(_ws, ctx, netAddress);
     Owned<IClientWULogFileRequest> req = ws->createWUFileRequest();
     if (queryset && *queryset)
         req->setQuerySet(queryset);
@@ -116,8 +123,13 @@ void fetchRemoteWorkunit(IEspContext &context, const char *netAddress, const cha
     checkUseEspOrDaliIP(ep, resp->getDaliServer(), netAddress);
     if (!ep.isNull())
         ep.getUrlStr(daliServer);
+}
 
-    respQueryInfo.setown(fetchQueryDetails(context, ws, queryset, query));
+void fetchRemoteWorkunitAndQueryDetails(IClientWsWorkunits *_ws, IEspContext *ctx, const char *netAddress, const char *queryset, const char *query, const char *wuid, StringBuffer &name, StringBuffer &xml, StringBuffer &dllname, MemoryBuffer &dll, StringBuffer &daliServer, Owned<IClientWUQuerySetDetailsResponse> &respQueryInfo)
+{
+    Owned<IClientWsWorkunits> ws = ensureWsWorkunitsClient(_ws, ctx, netAddress);
+    fetchRemoteWorkunit(ws, ctx, netAddress, queryset, query, wuid, name, xml, dllname, dll, daliServer);
+    respQueryInfo.setown(fetchQueryDetails(ws, ctx, netAddress, queryset, query));
 }
 
 void doWuFileCopy(IClientFileSpray &fs, IEspWULogicalFileCopyInfo &info, const char *logicalname, const char *cluster, bool isRoxie, bool supercopy)
@@ -1927,7 +1939,7 @@ bool nextQueryPathNode(const char *&path, StringBuffer &node)
     return (*path && *++path);
 }
 
-bool splitQueryPath(const char *path, StringBuffer &netAddress, StringBuffer &queryset, StringBuffer &query)
+bool splitQueryPath(const char *path, StringBuffer &netAddress, StringBuffer &queryset, StringBuffer *query)
 {
     if (!path || !*path)
         return false;
@@ -1938,21 +1950,53 @@ bool splitQueryPath(const char *path, StringBuffer &netAddress, StringBuffer &qu
             return false;
     }
     if (!nextQueryPathNode(path, queryset))
+        return (query==NULL);
+    if (!query)
         return false;
-    if (nextQueryPathNode(path, query))
+    if (nextQueryPathNode(path, *query))
         return false; //query path too deep
     return true;
 }
 
+IPropertyTree *fetchRemoteQuerySetInfo(IEspContext *context, const char *srcAddress, const char *srcTarget)
+{
+    if (!srcAddress || !*srcAddress || !srcTarget || !*srcTarget)
+        return NULL;
+
+    VStringBuffer url("http://%s%s/WsWorkunits/WUQuerysetDetails.xml?ver_=1.51&QuerySetName=%s&FilterType=All", srcAddress, (!strchr(srcAddress, ':')) ? ":8010" : "", srcTarget);
+
+    Owned<IHttpClientContext> httpCtx = getHttpClientContext();
+    Owned<IHttpClient> httpclient = httpCtx->createHttpClient(NULL, url);
+
+    const char *user = context->queryUserId();
+    if (user && *user)
+        httpclient->setUserID(user);
+
+    const char *pw = context->queryPassword();
+    if (pw && *pw)
+         httpclient->setPassword(pw);
+
+    StringBuffer request; //empty
+    StringBuffer response;
+    StringBuffer status;
+    if (0 > httpclient->sendRequest("GET", NULL, request, response, status) || !response.length() || strncmp("200", status, 3))
+         throw MakeStringException(-1, "Error fetching remote queryset information: %s %s %s", srcAddress, srcTarget, status.str());
+
+    return createPTreeFromXMLString(response);
+}
+
 class QueryCloner
 {
 public:
-    QueryCloner(IEspContext *_context, const char *source, const char *_target) :
-        context(_context), cloneFilesEnabled(false), target(_target), overwriteDfs(false)
+    QueryCloner(IEspContext *_context, const char *address, const char *source, const char *_target) :
+        context(_context), cloneFilesEnabled(false), target(_target), overwriteDfs(false), srcAddress(address)
     {
-        srcQuerySet.setown(getQueryRegistry(source, true));
+        if (srcAddress.length())
+            srcQuerySet.setown(fetchRemoteQuerySetInfo(context, srcAddress, source));
+        else
+            srcQuerySet.setown(getQueryRegistry(source, true));
         if (!srcQuerySet)
-            throw MakeStringException(ECLWATCH_QUERYSET_NOT_FOUND, "Source Queryset %s not found", source);
+            throw MakeStringException(ECLWATCH_QUERYSET_NOT_FOUND, "Source Queryset %s %s not found", srcAddress.str(), source);
 
         destQuerySet.setown(getQueryRegistry(target, false));
         if (!destQuerySet) // getQueryRegistry should have created if not found
@@ -1960,26 +2004,72 @@ public:
 
         factory.setown(getWorkUnitFactory(context->querySecManager(), context->queryUser()));
     }
-    void clone(const char *name, const char *id, bool makeActive)
+
+    void cloneQueryRemote(IPropertyTree *query, bool makeActive)
     {
-        Owned<IPropertyTree> query = getQueryById(srcQuerySet, id);
-        if (!query)
+        StringBuffer wuid = query->queryProp("Wuid");
+        if (!wuid.length())
+            return;
+        const char *queryName = query->queryProp("Name");
+        if (!queryName || !*queryName)
             return;
+
+        StringBuffer xml;
+        MemoryBuffer dll;
+        StringBuffer dllname;
+        StringBuffer fetchedName;
+        StringBuffer remoteDfs;
+        fetchRemoteWorkunit(NULL, context, srcAddress.str(), NULL, NULL, wuid, fetchedName, xml, dllname, dll, remoteDfs);
+        deploySharedObject(*context, wuid, dllname, target, queryName, dll, NULL, xml.str());
+
+        SCMStringBuffer existingQueryId;
+        queryIdFromQuerySetWuid(destQuerySet, wuid, queryName, existingQueryId);
+        if (existingQueryId.length())
+        {
+            existingQueryIds.append(existingQueryId.str());
+            if (makeActive)
+                activateQuery(destQuerySet, ACTIVATE_SUSPEND_PREVIOUS, queryName, existingQueryId.str(), context->queryUserId());
+            return;
+        }
+        StringBuffer newQueryId;
+        Owned<IWorkUnit> workunit = factory->updateWorkUnit(wuid);
+        addQueryToQuerySet(workunit, destQuerySet, queryName, makeActive ? ACTIVATE_SUSPEND_PREVIOUS : DO_NOT_ACTIVATE, newQueryId, context->queryUserId());
+        copiedQueryIds.append(newQueryId);
+        Owned<IPropertyTree> destQuery = getQueryById(destQuerySet, newQueryId);
+        if (destQuery)
+        {
+            Owned<IAttributeIterator> aiter = query->getAttributes();
+            ForEach(*aiter)
+            {
+                const char *atname = aiter->queryName();
+                if (!destQuery->hasProp(atname))
+                    destQuery->setProp(atname, aiter->queryValue());
+            }
+            if (cloneFilesEnabled && wufiles)
+                wufiles->addFilesFromQuery(workunit, pm, newQueryId);
+        }
+    }
+
+    void cloneQueryLocal(IPropertyTree *query, bool makeActive)
+    {
         const char *wuid = query->queryProp("@wuid");
         if (!wuid || !*wuid)
             return;
+        const char *queryName = query->queryProp("@name");
+        if (!queryName || !*queryName)
+            return;
         SCMStringBuffer existingQueryId;
-        queryIdFromQuerySetWuid(destQuerySet, wuid, existingQueryId);
+        queryIdFromQuerySetWuid(destQuerySet, wuid, queryName, existingQueryId);
         if (existingQueryId.length())
         {
             existingQueryIds.append(existingQueryId.str());
             if (makeActive)
-                activateQuery(destQuerySet, ACTIVATE_SUSPEND_PREVIOUS, name, existingQueryId.str(), context->queryUserId());
+                activateQuery(destQuerySet, ACTIVATE_SUSPEND_PREVIOUS, queryName, existingQueryId.str(), context->queryUserId());
             return;
         }
         StringBuffer newQueryId;
         Owned<IWorkUnit> workunit = factory->updateWorkUnit(wuid);
-        addQueryToQuerySet(workunit, destQuerySet, name, makeActive ? ACTIVATE_SUSPEND_PREVIOUS : DO_NOT_ACTIVATE, newQueryId, context->queryUserId());
+        addQueryToQuerySet(workunit, destQuerySet, queryName, makeActive ? ACTIVATE_SUSPEND_PREVIOUS : DO_NOT_ACTIVATE, newQueryId, context->queryUserId());
         copiedQueryIds.append(newQueryId);
         Owned<IPropertyTree> destQuery = getQueryById(destQuerySet, newQueryId);
         if (destQuery)
@@ -2002,35 +2092,78 @@ public:
         }
     }
 
-    void cloneActive(bool makeActive)
+    void cloneActiveRemote(bool makeActive)
+    {
+        Owned<IPropertyTreeIterator> activeQueries = srcQuerySet->getElements("QuerysetAliases/QuerySetAlias");
+        ForEach(*activeQueries)
+        {
+            IPropertyTree &alias = activeQueries->query();
+            VStringBuffer xpath("QuerysetQueries/QuerySetQuery[Id='%s'][1]", alias.queryProp("Id"));
+            IPropertyTree *query = srcQuerySet->queryPropTree(xpath);
+            if (!query)
+                continue;
+            cloneQueryRemote(query, makeActive);
+        }
+    }
+
+    void cloneActiveLocal(bool makeActive)
     {
         Owned<IPropertyTreeIterator> activeQueries = srcQuerySet->getElements("Alias");
         ForEach(*activeQueries)
         {
             IPropertyTree &alias = activeQueries->query();
-            const char *name = alias.queryProp("@name");
-            const char *id = alias.queryProp("@id");
-            clone(name, id, makeActive);
+            Owned<IPropertyTree> query = getQueryById(srcQuerySet, alias.queryProp("@id"));
+            if (!query)
+                return;
+            cloneQueryLocal(query, makeActive);
         }
     }
 
-    void cloneAll(bool cloneActiveState)
+    void cloneActive(bool makeActive)
+    {
+        if (srcAddress.length())
+            cloneActiveRemote(makeActive);
+        else
+            cloneActiveLocal(makeActive);
+    }
+
+    void cloneAllRemote(bool cloneActiveState)
+    {
+        Owned<IPropertyTreeIterator> allQueries = srcQuerySet->getElements("QuerysetQueries/QuerySetQuery");
+        ForEach(*allQueries)
+        {
+            IPropertyTree &query = allQueries->query();
+            bool makeActive = false;
+            if (cloneActiveState)
+            {
+                VStringBuffer xpath("QuerysetAliases/QuerySetAlias[Id='%s']", query.queryProp("Id"));
+                makeActive = srcQuerySet->hasProp(xpath);
+            }
+            cloneQueryRemote(&query, makeActive);
+        }
+    }
+    void cloneAllLocal(bool cloneActiveState)
     {
         Owned<IPropertyTreeIterator> allQueries = srcQuerySet->getElements("Query");
         ForEach(*allQueries)
         {
             IPropertyTree &query = allQueries->query();
-            const char *name = query.queryProp("@name");
-            const char *id = query.queryProp("@id");
             bool makeActive = false;
             if (cloneActiveState)
             {
-                VStringBuffer xpath("Alias[@id='%s']", id);
+                VStringBuffer xpath("Alias[@id='%s']", query.queryProp("@id"));
                 makeActive = srcQuerySet->hasProp(xpath);
             }
-            clone(name, id, makeActive);
+            cloneQueryLocal(&query, makeActive);
         }
     }
+    void cloneAll(bool cloneActiveState)
+    {
+        if (srcAddress.length())
+            cloneAllRemote(cloneActiveState);
+        else
+            cloneAllLocal(cloneActiveState);
+    }
     void enableFileCloning(const char *dfsServer, const char *destProcess, const char *sourceProcess, bool _overwriteDfs, bool allowForeign)
     {
         cloneFilesEnabled = true;
@@ -2059,6 +2192,7 @@ private:
     Owned<IReferencedFileList> wufiles;
     Owned<const IHpccPackageMap> pm;
     StringBuffer dfsIP;
+    StringBuffer srcAddress;
     StringBuffer srcCluster;
     StringBuffer srcPrefix;
     StringAttr target;
@@ -2076,7 +2210,12 @@ bool CWsWorkunitsEx::onWUCopyQuerySet(IEspContext &context, IEspWUCopyQuerySetRe
     const char *source = req.getSource();
     if (!source || !*source)
         throw MakeStringException(ECLWATCH_MISSING_PARAMS, "No source target specified");
-    if (!isValidCluster(source))
+
+    StringBuffer srcAddress;
+    StringBuffer srcTarget;
+    if (!splitQueryPath(source, srcAddress, srcTarget, NULL))
+        throw MakeStringException(ECLWATCH_INVALID_INPUT, "Invalid source target");
+    if (!srcAddress.length() && !isValidCluster(srcTarget))
         throw MakeStringException(ECLWATCH_INVALID_CLUSTER_NAME, "Invalid source target name: %s", source);
 
     const char *target = req.getTarget();
@@ -2085,7 +2224,7 @@ bool CWsWorkunitsEx::onWUCopyQuerySet(IEspContext &context, IEspWUCopyQuerySetRe
     if (!isValidCluster(target))
         throw MakeStringException(ECLWATCH_INVALID_CLUSTER_NAME, "Invalid destination target name: %s", target);
 
-    QueryCloner cloner(&context, source, target);
+    QueryCloner cloner(&context, srcAddress, srcTarget, target);
 
     SCMStringBuffer process;
     if (req.getCopyFiles())
@@ -2130,11 +2269,12 @@ bool CWsWorkunitsEx::onWUQuerysetCopyQuery(IEspContext &context, IEspWUQuerySetC
         throw MakeStringException(ECLWATCH_INVALID_CLUSTER_NAME, "Invalid target name: %s", target);
 
     StringBuffer srcAddress, srcQuerySet, srcQuery;
-    if (!splitQueryPath(source, srcAddress, srcQuerySet, srcQuery))
+    if (!splitQueryPath(source, srcAddress, srcQuerySet, &srcQuery))
         throw MakeStringException(ECLWATCH_INVALID_INPUT, "Invalid source query path");
 
     StringAttr targetQueryName(req.getDestName());
     Owned<IClientWUQuerySetDetailsResponse> sourceQueryInfoResp;
+    IConstQuerySetQuery *srcInfo=NULL;
 
     StringBuffer remoteIP;
     StringBuffer wuid;
@@ -2144,7 +2284,11 @@ bool CWsWorkunitsEx::onWUQuerysetCopyQuery(IEspContext &context, IEspWUQuerySetC
         MemoryBuffer dll;
         StringBuffer dllname;
         StringBuffer queryName;
-        fetchRemoteWorkunit(context, srcAddress.str(), srcQuerySet.str(), srcQuery.str(), NULL, queryName, xml, dllname, dll, remoteIP, sourceQueryInfoResp);
+        fetchRemoteWorkunitAndQueryDetails(NULL, &context, srcAddress.str(), srcQuerySet.str(), srcQuery.str(), NULL, queryName, xml, dllname, dll, remoteIP, sourceQueryInfoResp);
+        if (sourceQueryInfoResp && sourceQueryInfoResp->getQuerysetQueries().ordinality())
+            srcInfo = &sourceQueryInfoResp->getQuerysetQueries().item(0);
+        if (srcInfo)
+            wuid.set(srcInfo->getWuid());
         if (targetQueryName.isEmpty())
             targetQueryName.set(queryName);
         deploySharedObject(context, wuid, dllname.str(), target, targetQueryName.get(), dll, queryDirectory.str(), xml.str());
@@ -2153,7 +2297,9 @@ bool CWsWorkunitsEx::onWUQuerysetCopyQuery(IEspContext &context, IEspWUQuerySetC
     {
         //Could get the atributes without soap call, but this creates a common data structure shared with fetching remote query info
         //Get query attributes before resolveQueryAlias, to avoid deadlock
-        sourceQueryInfoResp.setown(fetchQueryDetails(context, NULL, srcQuerySet, srcQuery));
+        sourceQueryInfoResp.setown(fetchQueryDetails(NULL, &context, NULL, srcQuerySet, srcQuery));
+        if (sourceQueryInfoResp && sourceQueryInfoResp->getQuerysetQueries().ordinality())
+            srcInfo = &sourceQueryInfoResp->getQuerysetQueries().item(0);
 
         Owned<IPropertyTree> queryset = getQueryRegistry(srcQuerySet.str(), true);
         if (!queryset)
@@ -2190,9 +2336,6 @@ bool CWsWorkunitsEx::onWUQuerysetCopyQuery(IEspContext &context, IEspWUQuerySetC
     Owned<IPropertyTree> queryTree = getQueryById(target, targetQueryId, false);
     if (queryTree)
     {
-        IConstQuerySetQuery *srcInfo=NULL;
-        if (sourceQueryInfoResp && sourceQueryInfoResp->getQuerysetQueries().ordinality())
-            srcInfo = &sourceQueryInfoResp->getQuerysetQueries().item(0);
         updateMemoryLimitSetting(queryTree, req.getMemoryLimit(), srcInfo);
         updateQueryPriority(queryTree, req.getPriority(), srcInfo);
         updateTimeLimitSetting(queryTree, req.getTimeLimit_isNull(), req.getTimeLimit(), srcInfo);

esp/services/ws_workunits/ws_workunitsService.cpp

@@ -3736,7 +3736,7 @@ inline StringBuffer &buildFullDllPath(StringBuffer &dllpath, StringBuffer &dllna
     return addPathSepChar(dllpath.set(dir)).append(sharedObjectFileName(dllname, name, ext, copy));
 }
 
-void writeSharedObject(const char *srcpath, const MemoryBuffer &obj, const char *dir, StringBuffer &dllpath, StringBuffer &dllname)
+void writeSharedObject(const char *srcpath, const MemoryBuffer &obj, const char *dir, StringBuffer &dllpath, StringBuffer &dllname, unsigned crc)
 {
     StringBuffer name, ext;
     if (srcpath && *srcpath)
@@ -3745,23 +3745,56 @@ void writeSharedObject(const char *srcpath, const MemoryBuffer &obj, const char
     unsigned copy=0;
     buildFullDllPath(dllpath.clear(), dllname.clear(), dir, name.str(), ext.str(), copy);
     while (checkFileExists(dllpath.str()))
+    {
+        if (crc && crc == crc_file(dllpath.str()))
+        {
+            DBGLOG("Workunit dll already exists: %s", dllpath.str());
+            return;
+        }
         buildFullDllPath(dllpath.clear(), dllname.clear(), dir, name.str(), ext.str(), ++copy);
-
+    }
     DBGLOG("Writing workunit dll: %s", dllpath.str());
     Owned<IFile> f = createIFile(dllpath.str());
     Owned<IFileIO> io = f->open(IFOcreate);
     io->write(0, obj.length(), obj.toByteArray());
 }
 
-void CWsWorkunitsEx::deploySharedObject(IEspContext &context, StringBuffer &wuid, const char *filename, const char *cluster, const char *name, const MemoryBuffer &obj, const char *dir, const char *xml)
+void deploySharedObject(IEspContext &context, StringBuffer &wuid, const char *filename, const char *cluster, const char *name, const MemoryBuffer &obj, const char *dir, const char *xml)
 {
     StringBuffer dllpath, dllname;
     StringBuffer srcname(filename);
+
+    unsigned crc = 0;
+    Owned<IPropertyTree> srcxml;
+    if (xml && *xml)
+    {
+        srcxml.setown(createPTreeFromXMLString(xml));
+        if (srcxml && wuid.length())
+        {
+            crc = srcxml->getPropInt("Query[1]/Associated[1]/File[@type='dll'][1]/@crc", 0);
+            if (crc)
+            {
+                Owned<IWorkUnitFactory> factory = getWorkUnitFactory(context.querySecManager(), context.queryUser());
+                Owned<IConstWorkUnit> cw = factory->openWorkUnit(wuid, false);
+                if (cw)
+                {
+                    //is this a previous copy of same query, or a WUID collision?
+                    if (cw->getHash() == (unsigned) srcxml->getPropInt64("@hash", 0))
+                    {
+                        Owned<IConstWUQuery> query = cw->getQuery();
+                        if (query && crc == query->getQueryDllCrc())
+                            return;
+                    }
+                }
+            }
+        }
+    }
+
     if (!srcname.length())
         srcname.append(name).append(SharedObjectExtension);
-    writeSharedObject(srcname.str(), obj, dir, dllpath, dllname);
+    writeSharedObject(srcname.str(), obj, dir, dllpath, dllname, crc);
 
-    NewWsWorkunit wu(context);
+    NewWsWorkunit wu(context, wuid); //duplicate wuid made unique
 
     StringBufferAdaptor isvWuid(wuid);
     wu->getWuid(isvWuid);
@@ -3782,9 +3815,8 @@ void CWsWorkunitsEx::deploySharedObject(IEspContext &context, StringBuffer &wuid
         wu->setJobName(name);
 
     //clean slate, copy only select items from processed workunit xml
-    if (xml && *xml)
+    if (srcxml)
     {
-        Owned<IPropertyTree> srcxml = createPTreeFromXMLString(xml);
         if (srcxml->hasProp("@jobName"))
             wu->setJobName(srcxml->queryProp("@jobName"));
         if (srcxml->hasProp("@token"))
@@ -3800,7 +3832,7 @@ void CWsWorkunitsEx::deploySharedObject(IEspContext &context, StringBuffer &wuid
     AuditSystemAccess(context.queryUserId(), true, "Updated %s", wuid.str());
 }
 
-void CWsWorkunitsEx::deploySharedObject(IEspContext &context, IEspWUDeployWorkunitRequest & req, IEspWUDeployWorkunitResponse & resp, const char *dir, const char *xml)
+void CWsWorkunitsEx::deploySharedObjectReq(IEspContext &context, IEspWUDeployWorkunitRequest & req, IEspWUDeployWorkunitResponse & resp, const char *dir, const char *xml)
 {
     if (isEmpty(req.getFileName()))
        throw MakeStringException(ECLWATCH_INVALID_INPUT, "File name required when deploying a shared object.");
@@ -3841,7 +3873,7 @@ bool CWsWorkunitsEx::onWUDeployWorkunit(IEspContext &context, IEspWUDeployWorkun
         if (strieq(type, "archive")|| strieq(type, "ecl_text"))
             deployEclOrArchive(context, req, resp);
         else if (strieq(type, "shared_object"))
-            deploySharedObject(context, req, resp, queryDirectory.str());
+            deploySharedObjectReq(context, req, resp, queryDirectory.str());
         else
             throw MakeStringException(ECLWATCH_INVALID_INPUT, "WUDeployWorkunit '%s' unknown object type.", type);
     }

esp/services/ws_workunits/ws_workunitsService.hpp

@@ -174,8 +174,7 @@ public:
     }
     void refreshValidClusters();
     bool isValidCluster(const char *cluster);
-    void deploySharedObject(IEspContext &context, IEspWUDeployWorkunitRequest & req, IEspWUDeployWorkunitResponse & resp, const char *dir, const char *xml=NULL);
-    void deploySharedObject(IEspContext &context, StringBuffer &wuid, const char *filename, const char *cluster, const char *name, const MemoryBuffer &obj, const char *dir, const char *xml=NULL);
+    void deploySharedObjectReq(IEspContext &context, IEspWUDeployWorkunitRequest & req, IEspWUDeployWorkunitResponse & resp, const char *dir, const char *xml=NULL);
     unsigned getGraphIdsByQueryId(const char *target, const char *queryId, StringArray& graphIds);
     bool getQueryFiles(const char* query, const char* target, StringArray& logicalFiles, IArrayOf<IEspQuerySuperFile> *superFiles);
     void getGraphsByQueryId(const char *target, const char *queryId, const char *graphName, const char *subGraphId, IArrayOf<IEspECLGraphEx>& ECLGraphs);
@@ -318,6 +317,8 @@ private:
     CWsWorkunitsEx *wswService;
 };
 
+void deploySharedObject(IEspContext &context, StringBuffer &wuid, const char *filename, const char *cluster, const char *name, const MemoryBuffer &obj, const char *dir, const char *xml=NULL);
+
 class CClusterQueryStateParam : public CInterface
 {
     Linked<CWsWorkunitsEx>          wsWorkunitsService;