HPCC-15227 Using wutool to move workunits to/from dali also moves files

We don't want to archive off the generated dlls just because we are moving
from Dali to Cassandra.

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

cmake_modules/commonSetup.cmake

@@ -49,6 +49,7 @@ IF ("${COMMONSETUP_DONE}" STREQUAL "")
   option(DEVEL "Enable the building/inclusion of a Development component." OFF)
   option(CLIENTTOOLS_ONLY "Enable the building of Client Tools only." OFF)
   option(INCLUDE_PLUGINS "Enable the building of platform and all plugins for testing purposes" OFF)
+  option(INCLUDE_CASSANDRA "Include the Cassandra plugin in the base package" ON)
   option(PLUGIN "Enable building of a plugin" OFF)
   option(USE_SHLIBDEPS "Enable the use of dpkg-shlibdeps on ubuntu packaging" OFF)
 
@@ -239,6 +240,10 @@ IF ("${COMMONSETUP_DONE}" STREQUAL "")
         set(KAFKA ON)
     endif()
 
+  if (INCLUDE_CASSANDRA)
+    set(CASSANDRAEMBED ON)
+  endif()
+
   option(PORTALURL "Set url to hpccsystems portal download page")
 
   if ( NOT PORTALURL )

common/workunit/workunit.cpp

@@ -1389,8 +1389,8 @@ public:
             { return queryExtendedWU(c)->queryPTree(); }
     virtual IPropertyTree *getUnpackedTree(bool includeProgress) const
             { return queryExtendedWU(c)->getUnpackedTree(includeProgress); }
-    virtual bool archiveWorkUnit(const char *base,bool del,bool deldll,bool deleteOwned)
-            { return queryExtendedWU(c)->archiveWorkUnit(base,del,deldll,deleteOwned); }
+    virtual bool archiveWorkUnit(const char *base,bool del,bool deldll,bool deleteOwned,bool exportAssociatedFiles)
+            { return queryExtendedWU(c)->archiveWorkUnit(base,del,deldll,deleteOwned,exportAssociatedFiles); }
     virtual unsigned queryFileUsage(const char *filename) const
             { return c->queryFileUsage(filename); }
     virtual IJlibDateTime & getTimeScheduled(IJlibDateTime &val) const
@@ -2275,7 +2275,7 @@ IPropertyTree * pruneBranch(IPropertyTree * from, char const * xpath)
     return ret.getClear();
 }
 
-bool CWorkUnitFactory::restoreWorkUnit(const char *base, const char *wuid)
+bool CWorkUnitFactory::restoreWorkUnit(const char *base, const char *wuid, bool restoreAssociated)
 {
     StringBuffer path(base);
     addPathSepChar(path).append(wuid).append(".xml");
@@ -3199,9 +3199,9 @@ public:
         if (!secUser) secUser = defaultSecUser.get();
         return baseFactory->updateWorkUnit(wuid, secMgr, secUser);
     }
-    virtual bool restoreWorkUnit(const char *base, const char *wuid)
+    virtual bool restoreWorkUnit(const char *base, const char *wuid, bool restoreAssociated)
     {
-        return baseFactory->restoreWorkUnit(base, wuid);
+        return baseFactory->restoreWorkUnit(base, wuid, restoreAssociated);
     }
     virtual IWorkUnit * getGlobalWorkUnit(ISecManager *secMgr, ISecUser *secUser)
     {
@@ -3517,7 +3517,7 @@ bool modifyAndWriteWorkUnitXML(char const * wuid, StringBuffer & buf, StringBuff
     return (fileio->write(0,buf.length(),buf.str()) == buf.length());
 }
 
-bool CLocalWorkUnit::archiveWorkUnit(const char *base,bool del,bool ignoredllerrors,bool deleteOwned)
+bool CLocalWorkUnit::archiveWorkUnit(const char *base, bool del, bool ignoredllerrors, bool deleteOwned, bool exportAssociatedFiles)
 {
     CriticalBlock block(crit);
     StringBuffer path(base);
@@ -3558,7 +3558,6 @@ bool CLocalWorkUnit::archiveWorkUnit(const char *base,bool del,bool ignoredllerr
         }
         return false;
     }
-
     StringArray deleteExclusions; // associated files not to delete, added if failure to copy
     Owned<IConstWUAssociatedFileIterator> iter = &q->getAssociatedFiles();
     Owned<IPropertyTree> generatedDlls = createPTree("GeneratedDlls");
@@ -3587,46 +3586,49 @@ bool CLocalWorkUnit::archiveWorkUnit(const char *base,bool del,bool ignoredllerr
                 Owned<IPropertyTree> generatedDllBranch = createPTree();
                 generatedDllBranch->setProp("@name", entry->queryName());
                 generatedDllBranch->setProp("@kind", entry->queryKind());
-                try
-                {
-                    loc.setown(entry->getBestLocation()); //throws exception if no readable locations
-                }
-                catch(IException * e)
-                {
-                    exception.setown(e);
-                    loc.setown(entry->getBestLocationCandidate()); //this will be closest of the unreadable locations
-                }
-                RemoteFilename filename;
-                loc->getDllFilename(filename);
-                if (!exception)
+                if (exportAssociatedFiles)
                 {
-                    Owned<IFile> srcfile = createIFile(filename);
                     try
                     {
-                        if (dstFile->exists())
-                        {
-                            if (streq(srcfile->queryFilename(), dstFile->queryFilename()))
-                                deleteExclusions.append(name.str()); // restored workunit, referencing archive location for query dll (no longer true post HPCC-11191 fix)
-                            // still want to delete if already archived but there are source file copies
-                        }
-                        else
-                            copyFile(dstFile, srcfile);
+                        loc.setown(entry->getBestLocation()); //throws exception if no readable locations
                     }
                     catch(IException * e)
                     {
                         exception.setown(e);
+                        loc.setown(entry->getBestLocationCandidate()); //this will be closest of the unreadable locations
                     }
-                }
-                if (exception)
-                {
-                    if (ignoredllerrors)
+                    RemoteFilename filename;
+                    loc->getDllFilename(filename);
+                    if (!exception)
+                    {
+                        Owned<IFile> srcfile = createIFile(filename);
+                        try
+                        {
+                            if (dstFile->exists())
+                            {
+                                if (streq(srcfile->queryFilename(), dstFile->queryFilename()))
+                                    deleteExclusions.append(name.str()); // restored workunit, referencing archive location for query dll (no longer true post HPCC-11191 fix)
+                                // still want to delete if already archived but there are source file copies
+                            }
+                            else
+                                copyFile(dstFile, srcfile);
+                        }
+                        catch(IException * e)
+                        {
+                            exception.setown(e);
+                        }
+                    }
+                    if (exception)
                     {
-                        EXCLOG(exception.get(), "archiveWorkUnit (copying associated file)");
-                        //copy failed, so don't delete the registred dll files
-                        deleteExclusions.append(name.str());
+                        if (ignoredllerrors)
+                        {
+                            EXCLOG(exception.get(), "archiveWorkUnit (copying associated file)");
+                            //copy failed, so don't delete the registered dll files
+                            deleteExclusions.append(name.str());
+                        }
+                        else
+                            throw exception.getClear();
                     }
-                    else
-                        throw exception.getClear();
                 }
                 // Record Associated path to restore back to
                 StringBuffer restorePath;
@@ -3634,7 +3636,7 @@ bool CLocalWorkUnit::archiveWorkUnit(const char *base,bool del,bool ignoredllerr
                 generatedDllBranch->setProp("@location", restorePath.str());
                 generatedDlls->addPropTree("GeneratedDll", generatedDllBranch.getClear());
             }
-            else // no generated dll entry
+            else if (exportAssociatedFiles) // no generated dll entry
             {
                 Owned<IFile> srcFile = createIFile(curRfn);
                 try

common/workunit/workunit.hpp

@@ -1258,7 +1258,7 @@ interface IWorkUnitFactory : extends IPluggableFactory
     virtual IConstWorkUnit * openWorkUnit(const char *wuid, ISecManager *secmgr = NULL, ISecUser *secuser = NULL) = 0;
     virtual IConstWorkUnitIterator * getWorkUnitsByOwner(const char * owner, ISecManager *secmgr = NULL, ISecUser *secuser = NULL) = 0;
     virtual IWorkUnit * updateWorkUnit(const char * wuid, ISecManager *secmgr = NULL, ISecUser *secuser = NULL) = 0;
-    virtual bool restoreWorkUnit(const char *base, const char *wuid) = 0;
+    virtual bool restoreWorkUnit(const char *base, const char *wuid, bool restoreAssociatedFiles) = 0;
     virtual int setTracingLevel(int newlevel) = 0;
     virtual IWorkUnit * createNamedWorkUnit(const char * wuid, const char * app, const char * scope, ISecManager *secmgr = NULL, ISecUser *secuser = NULL) = 0;
     virtual IWorkUnit * getGlobalWorkUnit(ISecManager *secmgr = NULL, ISecUser *secuser = NULL) = 0;
@@ -1300,7 +1300,7 @@ interface IExtendedWUInterface
 {
     virtual unsigned calculateHash(unsigned prevHash) = 0;
     virtual void copyWorkUnit(IConstWorkUnit *cached, bool all) = 0;
-    virtual bool archiveWorkUnit(const char *base,bool del,bool ignoredllerrors,bool deleteOwned) = 0;
+    virtual bool archiveWorkUnit(const char *base,bool del,bool ignoredllerrors,bool deleteOwned,bool exportAssociatedFiles) = 0;
     virtual IPropertyTree *getUnpackedTree(bool includeProgress) const = 0;
     virtual IPropertyTree *queryPTree() const = 0;
     

common/workunit/workunit.ipp

@@ -313,7 +313,7 @@ public:
     virtual void getBuildVersion(IStringVal & buildVersion, IStringVal & eclVersion) const;
     virtual IPropertyTree * getDiskUsageStats();
     virtual IPropertyTreeIterator & getFileIterator() const;
-    virtual bool archiveWorkUnit(const char *base,bool del,bool ignoredllerrors,bool deleteOwned);
+    virtual bool archiveWorkUnit(const char *base,bool del,bool ignoredllerrors,bool deleteOwned,bool exportAssociatedFiles);
     virtual IJlibDateTime & getTimeScheduled(IJlibDateTime &val) const;
     virtual IPropertyTreeIterator & getFilesReadIterator() const;
     virtual void protect(bool protectMode);
@@ -595,7 +595,7 @@ public:
     virtual bool deleteWorkUnit(const char * wuid, ISecManager *secmgr, ISecUser *secuser);
     virtual IConstWorkUnit * openWorkUnit(const char * wuid, ISecManager *secmgr, ISecUser *secuser);
     virtual IWorkUnit * updateWorkUnit(const char * wuid, ISecManager *secmgr, ISecUser *secuser);
-    virtual bool restoreWorkUnit(const char *base, const char *wuid);
+    virtual bool restoreWorkUnit(const char *base, const char *wuid, bool restoreAssociated);
     virtual int setTracingLevel(int newlevel);
     virtual IWorkUnit * createNamedWorkUnit(const char * wuid, const char * app, const char *scope, ISecManager *secmgr, ISecUser *secuser);
     virtual IWorkUnit * getGlobalWorkUnit(ISecManager *secmgr, ISecUser *secuser) = 0;

dali/sasha/saarch.cpp

@@ -648,7 +648,7 @@ static bool doArchiveWorkUnit(IWorkUnitFactory *wufactory,const char *wuid, Stri
                         e->Release();
                     }
                 }
-                QUERYINTERFACE(wu.get(), IExtendedWUInterface)->archiveWorkUnit(path.str(),del,true,deleteOwned);
+                QUERYINTERFACE(wu.get(), IExtendedWUInterface)->archiveWorkUnit(path.str(),del,true,deleteOwned,true);
                 res.append("OK");
                 return true;
             }
@@ -674,7 +674,7 @@ static bool doRestoreWorkUnit(IWorkUnitFactory *wufactory,const char *wuid, Stri
         StringBuffer base;
         getLdsPath(ldspath.str(), base);
         try {
-            if (wufactory->restoreWorkUnit(base, wuid)) {
+            if (wufactory->restoreWorkUnit(base, wuid, true)) {
                 res.append("OK");
                 return true;
             }

tools/wutool/wutool.cpp

@@ -49,8 +49,8 @@ void usage()
            "   delete <workunits>  - Delete workunits\n"
            "   results <workunits> - Dump results from specified workunits\n"
            "\n"
-           "   archive <workunits> - Archive to xml files [TO=<directory>] [DEL=1] [KEEPFILERESULTS=1]\n"
-           "   restore <filenames> - Restore from xml files\n"
+           "   archive <workunits> - Archive to xml files [TO=<directory>] [DEL=1] [KEEPFILERESULTS=1] [INCLUDEFILES=1]\n"
+           "   restore <filenames> - Restore from xml files [INCLUDEFILES=1]\n"
             "\n"
            "   orphans             - Delete orphaned information from store\n"
            "   cleanup [days=NN]   - Delete workunits older than NN days\n"
@@ -107,7 +107,7 @@ void process(IConstWorkUnit &w, IProperties *globals)
         if (to.length()==0)
             to.append('.');
         StringAttr wuid(w.queryWuid());
-        if (QUERYINTERFACE(&w, IExtendedWUInterface)->archiveWorkUnit(to.str(), globals->getPropBool("DEL", false), true, !globals->getPropBool("KEEPFILERESULTS", false)))
+        if (QUERYINTERFACE(&w, IExtendedWUInterface)->archiveWorkUnit(to.str(), globals->getPropBool("DEL", false), true, !globals->getPropBool("KEEPFILERESULTS", true), globals->getPropBool("INCLUDEFILES", false)))
             printf("archived %s\n", wuid.str());
         else
             printf("archive of %s failed\n", wuid.str());
@@ -374,7 +374,7 @@ int main(int argc, const char *argv[])
                 {
                     if (base.length()==0)
                         base.append('.');
-                    if (factory->restoreWorkUnit(base.str(), wuid))
+                    if (factory->restoreWorkUnit(base.str(), wuid, globals->getPropBool("INCLUDEFILES", false)))
                         printf("restored %s\n", wuid.str());
                     else
                         printf("failed to restore %s\n", wuid.str());