HPCC-17491 Add ecl commands to import/export entire querysets

ecl queries export <target> --output=file
ecl queries import <target> <file> [--clone-active-state][--replace]

Signed-off-by: Anthony Fishbeck <anthony.fishbeck@lexisnexis.com>

ecl/eclcmd/eclcmd_common.hpp

@@ -43,6 +43,9 @@ typedef IEclCommand *(*EclCommandFactory)(const char *cmdname);
 #define ECLOPT_HELP "--help"
 #define ECLARG_HELP "help"
 
+#define ECLOPT_OUTPUT "--output"
+#define ECLOPT_OUTPUT_S "-O"
+
 #define ECLOPT_SERVER "--server"
 #define ECLOPT_SERVER_S "-s"
 #define ECLOPT_SERVER_INI "eclWatchIP"
@@ -78,6 +81,7 @@ typedef IEclCommand *(*EclCommandFactory)(const char *cmdname);
 #define ECLOPT_ALLOW_FOREIGN "--allow-foreign"
 
 #define ECLOPT_ACTIVE "--active"
+#define ECLOPT_ACTIVE_ONLY "--active-only"
 #define ECLOPT_ALL "--all"
 #define ECLOPT_INACTIVE "--inactive"
 #define ECLOPT_NO_ACTIVATE "--no-activate"
@@ -108,6 +112,8 @@ typedef IEclCommand *(*EclCommandFactory)(const char *cmdname);
 #define ECLOPT_UPDATE_CLONE_FROM "--update-clone-from"
 #define ECLOPT_DONT_APPEND_CLUSTER "--dont-append-cluster"
 #define ECLOPT_PART_NAME "--part-name"
+#define ECLOPT_PROTECT "--protect"
+#define ECLOPT_USE_EXISTING "--use-existing"
 
 #define ECLOPT_MAIN "--main"
 #define ECLOPT_MAIN_S "-main"  //eclcc compatible format
@@ -154,6 +160,7 @@ typedef IEclCommand *(*EclCommandFactory)(const char *cmdname);
 #define ECLOPT_PMID "--pmid"
 #define ECLOPT_PMID_S "-pm"
 #define ECLOPT_QUERYID "--queryid"
+#define ECLOPT_QUERIES "--queries"
 
 #define ECLOPT_DALIIP "--daliip"
 #define ECLOPT_PROCESS "--process"

ecl/eclcmd/queries/ecl-queries.cpp

@@ -18,6 +18,7 @@
 #include "jlog.hpp"
 #include "jfile.hpp"
 #include "jargv.hpp"
+#include "jflz.hpp"
 
 #include "build-config.h"
 
@@ -1177,6 +1178,317 @@ private:
     bool optDontAppendCluster = false; //Undesirable but here temporarily because DALI may have locking issues
 };
 
+class EclCmdQueriesExport : public EclCmdCommon
+{
+public:
+    EclCmdQueriesExport()
+    {
+    }
+    virtual eclCmdOptionMatchIndicator parseCommandLineOptions(ArgvIterator &iter)
+    {
+        if (iter.done())
+            return EclCmdOptionNoMatch;
+
+        for (; !iter.done(); iter.next())
+        {
+            const char *arg = iter.query();
+            if (*arg!='-')
+            {
+                if (optTarget.isEmpty())
+                    optTarget.set(arg);
+                else
+                {
+                    fprintf(stderr, "\nunrecognized argument %s\n", arg);
+                    return EclCmdOptionNoMatch;
+                }
+                continue;
+            }
+            if (iter.matchOption(optFilename, ECLOPT_OUTPUT) || iter.matchOption(optFilename, ECLOPT_OUTPUT_S))
+                continue;
+            if (iter.matchFlag(optActiveOnly, ECLOPT_ACTIVE_ONLY))
+                continue;
+            if (iter.matchFlag(optProtect, ECLOPT_PROTECT))
+                continue;
+            eclCmdOptionMatchIndicator ind = EclCmdCommon::matchCommandLineOption(iter, true);
+            if (ind != EclCmdOptionMatch)
+                return ind;
+        }
+        return EclCmdOptionMatch;
+    }
+    virtual bool finalizeOptions(IProperties *globals)
+    {
+        if (optTarget.isEmpty())
+        {
+            fputs("Target must be specified.\n", stderr);
+            return false;
+        }
+        if (optFilename.isEmpty())
+        {
+            StringBuffer name("./backup_queryset_");
+            name.append(optTarget);
+            if (optActiveOnly)
+                name.append("_activeonly_");
+            CDateTime dt;
+            dt.setNow();
+            dt.getString(name, true);
+        }
+        if (!EclCmdCommon::finalizeOptions(globals))
+            return false;
+        return true;
+    }
+
+    void saveAsFile(const MemoryBuffer &mb, const char *filepath)
+    {
+        Owned<IFile> file = createIFile(filepath);
+        Owned<IFileIO> io = file->open(IFOcreaterw);
+
+        fprintf(stdout, "\nWriting to file %s\n", file->queryFilename());
+
+        if (io.get())
+            io->write(0, mb.length(), mb.toByteArray());
+        else
+            fprintf(stderr, "\nFailed to create file %s\n", file->queryFilename());
+    }
+
+    virtual int processCMD()
+    {
+        Owned<IClientWsWorkunits> client = createCmdClient(WsWorkunits, *this);
+        Owned<IClientWUQuerysetExportRequest> req = client->createWUQuerysetExportRequest();
+        req->setTarget(optTarget);
+        req->setActiveOnly(optActiveOnly);
+        req->setCompress(true);
+        req->setProtect(optProtect);
+
+        Owned<IClientWUQuerysetExportResponse> resp = client->WUQuerysetExport(req);
+        int ret = outputMultiExceptionsEx(resp->getExceptions());
+        if (ret == 0)
+        {
+            if (!resp->getData().length())
+            {
+                fprintf(stderr, "\nEmpty Queryset returned\n");
+                return 1;
+            }
+            MemoryBuffer decompressed;
+            fastLZDecompressToBuffer(decompressed, const_cast<MemoryBuffer &>(resp->getData())); //unfortunate need for const_cast
+            if (!decompressed.length())
+            {
+                fprintf(stderr, "\nError decompressing response\n");
+                return 1;
+            }
+            decompressed.append('\0');
+            if (optFilename.length())
+                saveAsFile(decompressed, optFilename);
+            else
+            {
+                fputs(decompressed.toByteArray(), stdout); //for piping
+                fputs("\n", stdout); //for piping
+            }
+        }
+        return ret;
+    }
+
+    virtual void usage()
+    {
+        fputs("\nUsage:\n"
+            "\n"
+            "The 'queries export' command saves backup information about a given queryset.\n"
+            "\n"
+            "ecl queries export <target> [options]\n\n"
+            " Options:\n"
+            "   <target>               Name of target cluster to export from\n"
+            "   -O,--output=<file>     Filename to save exported backup information to (optional)\n"
+            "   --active-only          Only include active queries in the exported queryset\n"
+            "   --protect              Protect the workunits for the included queries\n"
+            " Common Options:\n",
+            stdout);
+        EclCmdCommon::usage();
+    }
+private:
+    StringAttr optTarget;
+    StringAttr optFilename;
+    bool optActiveOnly = false;
+    bool optProtect = false;
+};
+
+class EclCmdQueriesImport : public EclCmdCommon
+{
+public:
+    EclCmdQueriesImport()
+    {
+    }
+    virtual eclCmdOptionMatchIndicator parseCommandLineOptions(ArgvIterator &iter)
+    {
+        if (iter.done())
+            return EclCmdOptionNoMatch;
+
+        for (; !iter.done(); iter.next())
+        {
+            const char *arg = iter.query();
+            if (*arg!='-')
+            {
+                if (optDestQuerySet.isEmpty())
+                    optDestQuerySet.set(arg);
+                else if (optFilename.isEmpty())
+                        optFilename.set(arg);
+                else
+                {
+                    fprintf(stderr, "\nunrecognized argument %s\n", arg);
+                    return EclCmdOptionNoMatch;
+                }
+                continue;
+            }
+            if (iter.matchOption(optQueries, ECLOPT_QUERIES))
+                continue;
+            if (iter.matchOption(optDaliIP, ECLOPT_DALIIP))
+                continue;
+            if (iter.matchOption(optSourceProcess, ECLOPT_SOURCE_PROCESS))
+                continue;
+            if (iter.matchFlag(optCloneActiveState, ECLOPT_CLONE_ACTIVE_STATE))
+                continue;
+            if (iter.matchFlag(optDontCopyFiles, ECLOPT_DONT_COPY_FILES))
+                continue;
+            if (iter.matchFlag(optAllQueries, ECLOPT_ALL))
+                continue;
+            if (iter.matchFlag(optReplace, ECLOPT_REPLACE))
+                continue;
+            if (iter.matchFlag(optAllowForeign, ECLOPT_ALLOW_FOREIGN))
+                continue;
+            if (iter.matchFlag(optOverwrite, ECLOPT_OVERWRITE)||iter.matchFlag(optOverwrite, ECLOPT_OVERWRITE_S))
+                continue;
+            if (iter.matchFlag(optUpdateSuperfiles, ECLOPT_UPDATE_SUPER_FILES))
+                continue;
+            if (iter.matchFlag(optUpdateCloneFrom, ECLOPT_UPDATE_CLONE_FROM))
+                continue;
+            if (iter.matchFlag(optDontAppendCluster, ECLOPT_DONT_APPEND_CLUSTER))
+                continue;
+            eclCmdOptionMatchIndicator ind = EclCmdCommon::matchCommandLineOption(iter, true);
+            if (ind != EclCmdOptionMatch)
+                return ind;
+        }
+        return EclCmdOptionMatch;
+    }
+    virtual bool finalizeOptions(IProperties *globals)
+    {
+        if (!EclCmdCommon::finalizeOptions(globals))
+            return false;
+        if (optFilename.isEmpty() || optDestQuerySet.isEmpty())
+        {
+            fputs("Target and file name must both be specified.\n", stderr);
+            return false;
+        }
+        content.loadFile(optFilename, false);
+        return true;
+    }
+
+    virtual int processCMD()
+    {
+        Owned<IClientWsWorkunits> client = createCmdClient(WsWorkunits, *this);
+        Owned<IClientWUQuerysetImportRequest> req = client->createWUQuerysetImportRequest();
+
+        MemoryBuffer compressed;
+        fastLZCompressToBuffer(compressed, content.length()+1, content.str());
+        req->setCompressed(true);
+        req->setData(compressed);
+
+        req->setReplace(optReplace);
+        req->setActiveOnly(!optAllQueries);
+        req->setQueryMask(optQueries);
+        req->setTarget(optDestQuerySet);
+        req->setDfsServer(optDaliIP);
+        req->setSourceProcess(optSourceProcess);
+        req->setActivation(optCloneActiveState ? CQuerysetImportActivation_ImportedActive : CQuerysetImportActivation_None);
+        req->setOverwriteDfs(optOverwrite);
+        req->setUpdateSuperFiles(optUpdateSuperfiles);
+        req->setUpdateCloneFrom(optUpdateCloneFrom);
+        req->setAppendCluster(!optDontAppendCluster);
+        req->setCopyFiles(!optDontCopyFiles);
+        req->setAllowForeignFiles(optAllowForeign);
+        req->setIncludeFileErrors(true);
+
+        Owned<IClientWUQuerysetImportResponse> resp = client->WUQuerysetImport(req);
+        int ret = outputMultiExceptionsEx(resp->getExceptions());
+        if (outputQueryFileCopyErrors(resp->getFileErrors()))
+            ret = 1;
+
+        StringArray &imported = resp->getImportedQueries();
+        fputs("Queries Imported:\n", stdout);
+        if (!imported.length())
+            fputs("  none\n\n", stdout);
+        else
+        {
+            ForEachItemIn(i, imported)
+                fprintf(stdout, "  %s\n", imported.item(i));
+            fputs("\n", stdout);
+        }
+        StringArray &existing = resp->getExistingQueries();
+        fputs("Queries already on destination target:\n", stdout);
+        if (!existing.length())
+            fputs("  none\n\n", stdout);
+        else
+        {
+            ForEachItemIn(i, existing)
+                fprintf(stdout, "  %s\n", existing.item(i));
+            fputs("\n", stdout);
+        }
+        StringArray &missing = resp->getMissingWuids();
+        fputs("Missing workunits:\n", stdout);
+        if (!missing.length())
+            fputs("  none\n\n", stdout);
+        else
+        {
+            ForEachItemIn(i, missing)
+                fprintf(stdout, "  %s\n", missing.item(i));
+            fputs("\n", stdout);
+        }
+        return ret;
+    }
+    virtual void usage()
+    {
+        fputs("\nUsage:\n"
+            "\n"
+            "The 'queries import' command imports the contents of a queryset exported to disk.\n"
+            "\n"
+            "By default only active queries will be imported.  Use --all to import all queries.\n"
+            "\n"
+            "ecl queries import <target> <file> [--clone-active-state][--replace]\n"
+
+            "ecl queries import roxie1 queryset.xml\n"
+            "\n"
+            " Options:\n"
+            "   <target>               Target cluster to import queries to\n"
+            "   --all                  Copy both active and inactive queries\n"
+            "   --replace              Replace entire existing queryset\n"
+            "   --queries              Filter query ids to select for import\n"
+            "   --no-files             Do not copy DFS file information for referenced files\n"
+            "   --daliip=<ip>          Remote Dali DFS to use for copying file information\n"
+            "   --source-process       Process cluster to copy files from\n"
+            "   --clone-active-state   Make copied queries active if active on source\n"
+            "   -O, --overwrite        Completely replace existing DFS file information (dangerous)\n"
+            "   --update-super-files   Update local DFS super-files if remote DALI has changed\n"
+            "   --update-clone-from    Update local clone from location if remote DALI has changed\n"
+            "   --dont-append-cluster  Only use to avoid locking issues due to adding cluster to file\n"
+            "   --allow-foreign        Do not fail if foreign files are used in query (roxie)\n"
+            " Common Options:\n",
+            stdout);
+        EclCmdCommon::usage();
+    }
+private:
+    StringBuffer content;
+    StringAttr optFilename;
+    StringAttr optQueries;
+    StringAttr optDestQuerySet;
+    StringAttr optDaliIP;
+    StringAttr optSourceProcess;
+    bool optReplace = false;
+    bool optCloneActiveState = false;
+    bool optOverwrite = false;
+    bool optUpdateSuperfiles = false;
+    bool optUpdateCloneFrom = false;
+    bool optDontAppendCluster = false; //Undesirable but here temporarily because DALI may have locking issues
+    bool optDontCopyFiles = false;
+    bool optAllowForeign = false;
+    bool optAllQueries = false;
+};
 IEclCommand *createEclQueriesCommand(const char *cmdname)
 {
     if (!cmdname || !*cmdname)
@@ -1193,6 +1505,10 @@ IEclCommand *createEclQueriesCommand(const char *cmdname)
         return new EclCmdQueriesCopyQueryset();
     if (strieq(cmdname, "recreate"))
         return new EclCmdQueriesRecreate();
+    if (strieq(cmdname, "export"))
+        return new EclCmdQueriesExport();
+    if (strieq(cmdname, "import"))
+        return new EclCmdQueriesImport();
     return NULL;
 }
 
@@ -1217,6 +1533,8 @@ public:
             "      copy         copy a query from one target cluster to another\n"
             "      copy-set     copy queries from one target cluster to another\n"
             "      recreate     recompiles query into a new workunit\n"
+            "      export       export queryset information for backup\n"
+            "      import       import queryset information from backup file\n"
         );
     }
 };

esp/scm/ws_workunits.ecm

@@ -1626,6 +1626,62 @@ ESPresponse [exceptions_inline] WUMultiQuerySetDetailsResponse
     ESParray<ESPstruct WUQuerySetDetail> Querysets;
 };
 
+ESPrequest WUQuerysetExportRequest
+{
+    string  Target;
+    bool Compress(true);
+    bool ActiveOnly(false);
+    bool Protect(false);
+};
+
+ESPresponse [exceptions_inline] WUQuerysetExportResponse
+{
+    string Target;
+    bool Compressed;
+    binary Data;
+};
+
+ESPenum QuerysetImportActivation : string
+{
+    None("None"),
+    ImportedActive("ActivateImportedActive")
+};
+
+
+ESPrequest WUQuerysetImportRequest
+{
+    string Target;
+    string QueryMask;
+    bool Replace(false);
+    bool ActiveOnly(false);
+    ESPenum QuerysetImportActivation Activation;
+    bool Compressed(true);
+    binary Data;
+
+    bool AllowForeignFiles(true);
+
+    string DfsServer;
+    bool CopyFiles(true);
+    bool OverwriteDfs(false);
+    string SourceProcess;
+    bool UpdateSuperFiles(false); //usually wouldn't be needed, packagemap referencing superfiles?
+    bool UpdateCloneFrom(false); //explicity wan't to change where roxie will grab from
+    bool AppendCluster(true); //file exists on other local cluster, add new one, make optional in case of locking issues, but should be made to work
+    bool IncludeFileErrors(false);
+};
+
+ESPresponse [exceptions_inline] WUQuerysetImportResponse
+{
+    string  Target;
+    bool ClearedExisting(true);
+    bool Success(false);
+
+    ESParray<string, QueryId> ImportedQueries;
+    ESParray<string, QueryId> ExistingQueries;
+    ESParray<string, QueryId> MissingWuids;
+    ESParray<ESPStruct LogicalFileError, File> FileErrors;
+};
+
 ESPrequest [nil_remove] WUUpdateQueryEntryRequest
 {
     string QuerySet;
@@ -1997,6 +2053,8 @@ ESPservice [
     ESPmethod [cache_seconds(60), resp_xsl_default("/esp/xslt/WUQuerysetQueries.xslt")] WUQuerysetDetails(WUQuerySetDetailsRequest, WUQuerySetDetailsResponse);
     ESPmethod [cache_seconds(60), resp_xsl_default("/esp/xslt/WUQueryDetails.xslt")] WUQueryDetails(WUQueryDetailsRequest, WUQueryDetailsResponse);
     ESPmethod [cache_seconds(60)] WUMultiQuerysetDetails(WUMultiQuerySetDetailsRequest, WUMultiQuerySetDetailsResponse);
+    ESPmethod WUQuerysetImport(WUQuerysetImportRequest, WUQuerysetImportResponse);
+    ESPmethod WUQuerysetExport(WUQuerysetExportRequest, WUQuerysetExportResponse);
     ESPmethod WUQuerysetQueryAction(WUQuerySetQueryActionRequest, WUQuerySetQueryActionResponse);
     ESPmethod WUQuerysetAliasAction(WUQuerySetAliasActionRequest, WUQuerySetAliasActionResponse);
     ESPmethod WUQuerysetCopyQuery(WUQuerySetCopyQueryRequest, WUQuerySetCopyQueryResponse);

esp/services/ws_workunits/ws_workunitsQuerySets.cpp

@@ -18,6 +18,7 @@
 #include "ws_workunitsService.hpp"
 #include "ws_fs.hpp"
 #include "jlib.hpp"
+#include "jflz.hpp"
 #include "daclient.hpp"
 #include "dalienv.hpp"
 #include "dadfs.hpp"
@@ -2439,7 +2440,7 @@ class QueryCloner
 {
 public:
     QueryCloner(IEspContext *_context, const char *address, const char *source, const char *_target) :
-        context(_context), cloneFilesEnabled(false), target(_target), updateFlags(0), srcAddress(address)
+        context(_context), target(_target), srcAddress(address)
     {
         if (srcAddress.length())
             srcQuerySet.setown(fetchRemoteQuerySetInfo(context, srcAddress, source));
@@ -2455,6 +2456,17 @@ public:
         factory.setown(getWorkUnitFactory(context->querySecManager(), context->queryUser()));
     }
 
+    QueryCloner(IEspContext *_context, IPropertyTree *srcTree, const char *_target) :
+        context(_context), target(_target)
+    {
+        srcQuerySet.set(srcTree);
+        destQuerySet.setown(getQueryRegistry(target, false));
+        if (!destQuerySet) // getQueryRegistry should have created if not found
+            throw MakeStringException(ECLWATCH_QUERYSET_NOT_FOUND, "Destination Queryset %s could not be created, or found", target.str());
+
+        factory.setown(getWorkUnitFactory(context->querySecManager(), context->queryUser()));
+    }
+
     void setQueryDirectory(const char *dir)
     {
         queryDirectory.set(dir);
@@ -2523,7 +2535,16 @@ public:
         }
         StringBuffer newQueryId;
         Owned<IWorkUnit> workunit = factory->updateWorkUnit(wuid);
-        addQueryToQuerySet(workunit, destQuerySet, queryName, makeActive ? ACTIVATE_SUSPEND_PREVIOUS : DO_NOT_ACTIVATE, newQueryId, context->queryUserId());
+        if (!workunit)
+        {
+            StringBuffer msg(wuid);
+            msg.append(": ").append(query->queryProp("@id"));
+            missingWuids.append(msg);
+            return;
+        }
+
+        if (!newQueryId.length())
+            addQueryToQuerySet(workunit, destQuerySet, queryName, makeActive ? ACTIVATE_SUSPEND_PREVIOUS : DO_NOT_ACTIVATE, newQueryId, context->queryUserId());
         copiedQueryIds.append(newQueryId);
         Owned<IPropertyTree> destQuery = getQueryById(destQuerySet, newQueryId);
         if (destQuery)
@@ -2560,9 +2581,12 @@ public:
         }
     }
 
-    void cloneActiveLocal(bool makeActive)
+    void cloneActiveLocal(bool makeActive, const char *mask)
     {
-        Owned<IPropertyTreeIterator> activeQueries = srcQuerySet->getElements("Alias");
+        StringBuffer xpath("Alias");
+        if (mask && *mask)
+            xpath.appendf("[@id='%s']", mask);
+        Owned<IPropertyTreeIterator> activeQueries = srcQuerySet->getElements(xpath);
         ForEach(*activeQueries)
         {
             IPropertyTree &alias = activeQueries->query();
@@ -2578,7 +2602,7 @@ public:
         if (srcAddress.length())
             cloneActiveRemote(makeActive);
         else
-            cloneActiveLocal(makeActive);
+            cloneActiveLocal(makeActive, nullptr);
     }
 
     void cloneAllRemote(bool cloneActiveState)
@@ -2596,9 +2620,12 @@ public:
             cloneQueryRemote(&query, makeActive);
         }
     }
-    void cloneAllLocal(bool cloneActiveState)
+    void cloneAllLocal(bool cloneActiveState, const char *mask)
     {
-        Owned<IPropertyTreeIterator> allQueries = srcQuerySet->getElements("Query");
+        StringBuffer xpath("Query");
+        if (mask && *mask)
+            xpath.appendf("[@id='%s']", mask);
+        Owned<IPropertyTreeIterator> allQueries = srcQuerySet->getElements(xpath);
         ForEach(*allQueries)
         {
             IPropertyTree &query = allQueries->query();
@@ -2616,7 +2643,7 @@ public:
         if (srcAddress.length())
             cloneAllRemote(cloneActiveState);
         else
-            cloneAllLocal(cloneActiveState);
+            cloneAllLocal(cloneActiveState, nullptr);
     }
     void enableFileCloning(unsigned _updateFlags, const char *dfsServer, const char *destProcess, const char *sourceProcess, bool allowForeign)
     {
@@ -2664,12 +2691,13 @@ private:
     StringAttr target;
     StringAttr process;
     StringAttr queryDirectory;
-    bool cloneFilesEnabled;
-    unsigned updateFlags;
+    bool cloneFilesEnabled = false;
+    unsigned updateFlags = 0;
 
 public:
     StringArray existingQueryIds;
     StringArray copiedQueryIds;
+    StringArray missingWuids;
 };
 
 bool CWsWorkunitsEx::onWUCopyQuerySet(IEspContext &context, IEspWUCopyQuerySetRequest &req, IEspWUCopyQuerySetResponse &resp)
@@ -2859,6 +2887,159 @@ bool CWsWorkunitsEx::onWUQuerysetCopyQuery(IEspContext &context, IEspWUQuerySetC
     return true;
 }
 
+bool CWsWorkunitsEx::onWUQuerysetImport(IEspContext &context, IEspWUQuerysetImportRequest &req, IEspWUQuerysetImportResponse &resp)
+{
+    try
+    {
+        const char* target = req.getTarget();
+        if (!target || !*target)
+            throw MakeStringException(ECLWATCH_QUERYSET_NOT_FOUND, "Target not specified");
+
+        MemoryBuffer &mb = const_cast<MemoryBuffer &>(req.getData()); //for efficiency, content of request shouldn't matter after
+        if (req.getCompressed())
+        {
+            MemoryBuffer decompressed;
+            fastLZDecompressToBuffer(decompressed, mb);
+            mb.swapWith(decompressed);
+        }
+        mb.append('\0');
+
+        Owned<IPropertyTree> srcTree = createPTreeFromXMLString(mb.toByteArray());
+        const char *archivedTarget = srcTree->queryProp("@target");
+        if (archivedTarget && *archivedTarget) //support simple queryset or with archived (exported) root format
+        {
+            VStringBuffer xpath("QuerySet[@id='%s']", archivedTarget);
+            IPropertyTree *qsTree = srcTree->queryPropTree(xpath);
+            if (qsTree)
+                srcTree.setown(LINK(qsTree));
+        }
+        if (req.getReplace())
+        {
+            Owned<IPropertyTree> queryRegistry = getQueryRegistry(target, false);
+            queryRegistry->removeProp("*");
+            resp.setClearedExisting(true);
+        }
+
+        const bool activate = CQuerysetImportActivation_ImportedActive == req.getActivation(); //only two options now but may evolve
+
+        QueryCloner cloner(&context, srcTree, target);
+
+        SCMStringBuffer process;
+        if (req.getCopyFiles())
+        {
+            Owned <IConstWUClusterInfo> clusterInfo = getTargetClusterInfo(target);
+            if (clusterInfo && clusterInfo->getPlatform()==RoxieCluster)
+            {
+                clusterInfo->getRoxieProcess(process);
+                if (!process.length())
+                    throw MakeStringException(ECLWATCH_INVALID_CLUSTER_INFO, "DFS process cluster not found for destination target %s", target);
+                unsigned updateFlags = 0;
+                if (req.getOverwriteDfs())
+                    updateFlags |= (DALI_UPDATEF_REPLACE_FILE | DALI_UPDATEF_CLONE_FROM | DALI_UPDATEF_SUPERFILES);
+                if (req.getUpdateCloneFrom())
+                    updateFlags |= DALI_UPDATEF_CLONE_FROM;
+                if (req.getUpdateSuperFiles())
+                    updateFlags |= DALI_UPDATEF_SUPERFILES;
+                if (req.getAppendCluster())
+                    updateFlags |= DALI_UPDATEF_APPEND_CLUSTER;
+
+                cloner.enableFileCloning(updateFlags, req.getDfsServer(), process.str(), req.getSourceProcess(), req.getAllowForeignFiles());
+            }
+        }
+
+        if (req.getActiveOnly())
+            cloner.cloneActiveLocal(activate, req.getQueryMask());
+        else
+            cloner.cloneAllLocal(activate, req.getQueryMask());
+
+        cloner.cloneFiles();
+        if (req.getIncludeFileErrors())
+            cloner.gatherFileErrors(resp.getFileErrors());
+
+        resp.setImportedQueries(cloner.copiedQueryIds);
+        resp.setExistingQueries(cloner.existingQueryIds);
+        resp.setMissingWuids(cloner.missingWuids);
+    }
+    catch(IException* e)
+    {
+        FORWARDEXCEPTION(context, e,  ECLWATCH_INTERNAL_ERROR);
+    }
+    return true;
+}
+
+bool CWsWorkunitsEx::onWUQuerysetExport(IEspContext &context, IEspWUQuerysetExportRequest &req, IEspWUQuerysetExportResponse &resp)
+{
+    try
+    {
+        const char* target = req.getTarget();
+        if (!target || !*target)
+            throw MakeStringException(ECLWATCH_QUERYSET_NOT_FOUND, "Target not specified");
+
+        Owned<IPropertyTree> queryRegistry = getQueryRegistry(target, true);
+        if (req.getActiveOnly())
+        {
+            Owned<IPropertyTree> activeOnly = createPTree("QuerySet");
+            Owned<IAttributeIterator> attrs = queryRegistry->getAttributes();
+            ForEach(*attrs)
+                activeOnly->setProp(attrs->queryName(), attrs->queryValue());
+
+            Owned<IPropertyTreeIterator> aliases = queryRegistry->getElements("Alias");
+            ForEach(*aliases)
+            {
+                IPropertyTree &alias = aliases->query();
+                const char *id = alias.queryProp("@id");
+                if (id && *id)
+                {
+                    VStringBuffer xpath("Query[@id='%s']", id);
+                    IPropertyTree *query = queryRegistry->queryPropTree(xpath);
+                    if (query)
+                    {
+                        activeOnly->addPropTree("Query", LINK(query));
+                        activeOnly->addPropTree("Alias", LINK(&alias));
+                    }
+                }
+            }
+            queryRegistry.setown(activeOnly.getClear());
+        }
+
+        if (req.getProtect())
+        {
+            StringArray wuids;
+            Owned<IPropertyTreeIterator> queries = queryRegistry->getElements("Query");
+            ForEach(*queries)
+            {
+                IPropertyTree &query = queries->query();
+                const char *wuid = query.queryProp("@wuid");
+                if (wuid && *wuid)
+                    wuids.append(wuid);
+            }
+            if (wuids.length())
+                doProtectWorkunits(context, wuids, nullptr);
+        }
+        CDateTime dt;
+        dt.setNow();
+        StringBuffer dts;
+        VStringBuffer qs("<QuerySetArchive exported='%s' target='%s' activeOnly='%s'>\n", dt.getString(dts, true).str(), target, req.getActiveOnly() ? "true" : "false");
+        toXML(queryRegistry, qs);
+        qs.append("</QuerySetArchive>");
+
+        MemoryBuffer content;
+        if (req.getCompress())
+            fastLZCompressToBuffer(content, qs.length()+1, qs);
+        else
+            content.append(qs.str());
+
+        resp.setTarget(target);
+        resp.setCompressed(req.getCompress());
+        resp.setData(content);
+    }
+    catch(IException* e)
+    {
+        FORWARDEXCEPTION(context, e,  ECLWATCH_INTERNAL_ERROR);
+    }
+    return true;
+}
+
 void CWsWorkunitsEx::getGraphsByQueryId(const char *target, const char *queryId, const char *graphId, const char *subGraphId, IArrayOf<IEspECLGraphEx>& ECLGraphs)
 {
     if (!target || !*target)

esp/services/ws_workunits/ws_workunitsService.cpp

@@ -288,6 +288,22 @@ bool doAction(IEspContext& context, StringArray& wuids, CECLWUActions action, IP
     return bAllSuccess;
 }
 
+
+bool doProtectWorkunits(IEspContext& context, StringArray& wuids, IArrayOf<IConstWUActionResult>* results)
+{
+    Owned<IProperties> params(createProperties(true));
+    params->setProp("BlockTillFinishTimer", 0);
+
+    return doAction(context, wuids, CECLWUActions_Protect, params, results);
+}
+bool doUnProtectWorkunits(IEspContext& context, StringArray& wuids, IArrayOf<IConstWUActionResult>* results)
+{
+    Owned<IProperties> params(createProperties(true));
+    params->setProp("BlockTillFinishTimer", 0);
+
+    return doAction(context, wuids, CECLWUActions_Unprotect, params, results);
+}
+
 static void checkUpdateQuerysetLibraries()
 {
     Owned<IRemoteConnection> globalLock = querySDS().connect("/QuerySets/", myProcessSession(), RTM_LOCK_WRITE|RTM_CREATE_QUERY, SDS_LOCK_TIMEOUT);

esp/services/ws_workunits/ws_workunitsService.hpp

@@ -190,6 +190,8 @@ public:
     bool onWUPublishWorkunit(IEspContext &context, IEspWUPublishWorkunitRequest & req, IEspWUPublishWorkunitResponse & resp);
     bool onWUQuerysets(IEspContext &context, IEspWUQuerysetsRequest & req, IEspWUQuerysetsResponse & resp);
     bool onWUQuerysetDetails(IEspContext &context, IEspWUQuerySetDetailsRequest & req, IEspWUQuerySetDetailsResponse & resp);
+    bool onWUQuerysetExport(IEspContext &context, IEspWUQuerysetExportRequest &req, IEspWUQuerysetExportResponse &resp);
+    bool onWUQuerysetImport(IEspContext &context, IEspWUQuerysetImportRequest &req, IEspWUQuerysetImportResponse &resp);
     bool onWUMultiQuerysetDetails(IEspContext &context, IEspWUMultiQuerySetDetailsRequest &req, IEspWUMultiQuerySetDetailsResponse &resp);
     bool onWUQuerysetQueryAction(IEspContext &context, IEspWUQuerySetQueryActionRequest & req, IEspWUQuerySetQueryActionResponse & resp);
     bool onWUQuerysetAliasAction(IEspContext &context, IEspWUQuerySetAliasActionRequest &req, IEspWUQuerySetAliasActionResponse &resp);
@@ -409,5 +411,7 @@ public:
 };
 
 bool origValueChanged(const char *newValue, const char *origValue, StringBuffer &s, bool nillable=true);
+bool doProtectWorkunits(IEspContext& context, StringArray& wuids, IArrayOf<IConstWUActionResult>* results);
+bool doUnProtectWorkunits(IEspContext& context, StringArray& wuids, IArrayOf<IConstWUActionResult>* results);
 
 #endif