HPCC-14111 To dfuplus, add capability to export and restore super files.

Improve savexml action to generate proper logical and super file xml blob.

Improve add action to load simple logical and superfile metainformation
from an XML file. If the restored metainformation belong to a superfile
then update all sub-files SuperOwner tag.

Fix missing '\n' at the end of some DFUPlus return messages.

Signed-off-by: Attila Vamos <attila.vamos@gmail.com>

dali/base/dadfs.cpp

@@ -1000,6 +1000,7 @@ public:
     IDistributedFile *createNew(IFileDescriptor * fdesc, bool includeports=false);
     IDistributedFile *createExternal(IFileDescriptor *desc, const char *name, bool includeports=false);
     IDistributedSuperFile *createSuperFile(const char *logicalname,IUserDescriptor *user,bool interleaved,bool ifdoesnotexist,IDistributedFileTransaction *transaction=NULL);
+    IDistributedSuperFile *createNewSuperFile(IPropertyTree *tree);
     void removeSuperFile(const char *_logicalname, bool delSubs, IUserDescriptor *user, IDistributedFileTransaction *transaction);
 
     IDistributedFileIterator *getIterator(const char *wildname, bool includesuper,IUserDescriptor *user);
@@ -1047,7 +1048,7 @@ public:
 
     GetFileClusterNamesType getFileClusterNames(const char *logicalname,StringArray &out); // returns 0 for normal file, 1 for
 
-    bool isSuperFile( const char *logicalname, IUserDescriptor *user, INode *foreigndali, unsigned timeout);
+    bool isSuperFile( const char *logicalname, IUserDescriptor *user=NULL, INode *foreigndali=NULL, unsigned timeout=0);
 
     void promoteSuperFiles(unsigned numsf,const char **sfnames,const char *addsubnames,bool delsub,bool createonlyonesuperfile,IUserDescriptor *user, unsigned timeout, StringArray &outunlinked);
     ISimpleSuperFileEnquiry * getSimpleSuperFileEnquiry(const char *logicalname,const char *title,IUserDescriptor *udesc,unsigned timeout);
@@ -3113,7 +3114,6 @@ public:
         return true;
     }
 
-
     virtual const char *queryDefaultDir() = 0;
     virtual unsigned numParts() = 0;
     virtual IDistributedFilePart &queryPart(unsigned idx) = 0;
@@ -4533,6 +4533,15 @@ public:
         dt.setNow();
         setAccessedTime(dt);
     }
+
+    virtual void validate()
+    {
+        if (!existsPhysicalPartFiles(0))
+        {
+            const char * logicalName = queryLogicalName();
+            throw MakeStringException(-1, "Some physical parts do not exists, for logical file : %s",(isEmptyString(logicalName) ? "[unattached]" : logicalName));
+        }
+    }
 };
 
 static unsigned findSubFileOrd(const char *name)
@@ -4952,7 +4961,7 @@ protected:
         return path.append("SubFile[@num=\"").append(idx+1).append("\"]");
     }
 
-    void loadSubFiles(IDistributedFileTransaction *transaction, unsigned timeout)
+    void loadSubFiles(IDistributedFileTransaction *transaction, unsigned timeout, bool link=false)
     {
         partscache.kill();
         StringBuffer path;
@@ -5011,6 +5020,8 @@ protected:
                         ThrowStringException(-1, "CDistributedSuperFile: SuperFile %s: corrupt subfile file '%s' cannot be found", logicalName.get(), subname.str());
                 }
                 subfiles.append(*subfile.getClear());
+                if (link)
+                    linkSubFile(f);
             }
             // This is *only* due to foreign files
             if (subfiles.ordinality() != n)
@@ -5190,11 +5201,8 @@ public:
 
     IMPLEMENT_IINTERFACE;
 
-    void init(CDistributedFileDirectory *_parent, IPropertyTree *_root, const CDfsLogicalFileName &_name, IUserDescriptor* user, IDistributedFileTransaction *transaction, unsigned timeout=INFINITE)
+    void commonInit(CDistributedFileDirectory *_parent, IPropertyTree *_root)
     {
-        assertex(_name.isSet());
-        setUserDescriptor(udesc,user);
-        logicalName.set(_name);
         parent = _parent;
         root.set(_root);
         const char *val = root->queryProp("@interleaved");
@@ -5202,6 +5210,14 @@ public:
             interleaved = atoi(val);
         else
             interleaved = strToBool(val)?1:0;
+    }
+
+    void init(CDistributedFileDirectory *_parent, IPropertyTree *_root, const CDfsLogicalFileName &_name, IUserDescriptor* user, IDistributedFileTransaction *transaction, unsigned timeout=INFINITE)
+    {
+        assertex(_name.isSet());
+        setUserDescriptor(udesc,user);
+        logicalName.set(_name);
+        commonInit(_parent, _root);
         loadSubFiles(transaction,timeout);
     }
 
@@ -5226,6 +5242,11 @@ public:
         init(_parent,tree,_name,user,transaction);
     }
 
+    CDistributedSuperFile(CDistributedFileDirectory *_parent, IPropertyTree *_root)
+    {
+        commonInit(_parent, _root);
+    }
+
     ~CDistributedSuperFile()
     {
         partscache.kill();
@@ -5491,6 +5512,7 @@ public:
         conn.setown(fcl.detach());
         assertex(conn.get()); // must have been attached
         root.setown(conn->getRoot());
+        loadSubFiles(NULL, 0, true);
     }
 
     void detach(unsigned timeoutMs=INFINITE)
@@ -5882,6 +5904,34 @@ public:
             throw MakeStringException(-1,"addSubFile: File %s is already a subfile of %s", sub->queryLogicalName(),queryLogicalName());
     }
 
+    void validate()
+    {
+        unsigned numSubfiles = root->getPropInt("@numsubfiles",0);
+        if (numSubfiles)
+        {
+            Owned<IPropertyTreeIterator> treeIter = root->getElements("SubFile");
+            unsigned subFileCount = 0;
+            ForEach(*treeIter)
+            {
+                IPropertyTree & st = treeIter->query();
+                StringBuffer subfilename;
+                st.getProp("@name", subfilename);
+                if (!parent->exists(subfilename.str(), NULL))
+                    throw MakeStringException(-1, "Logical subfile '%s' doesn't exists!", subfilename.str());
+
+                if (!parent->isSuperFile(subfilename.str()))
+                    if (!parent->existsPhysical(subfilename.str(), NULL))
+                    {
+                        const char * logicalName = queryLogicalName();
+                        throw MakeStringException(-1, "Some physical parts do not exists, for logical file : %s",(isEmptyString(logicalName) ? "[unattached]" : logicalName));
+                    }
+                subFileCount++;
+            }
+            if (numSubfiles != subFileCount)
+                throw MakeStringException(-1, "The value of @numsubfiles (%d) is not equal to the number of SubFile items (%d)!",numSubfiles, subFileCount);
+        }
+    }
+
 private:
     void doAddSubFile(IDistributedFile *_sub,bool before,const char *other,IDistributedFileTransactionExt *transaction) // takes ownership of sub
     {
@@ -7933,6 +7983,12 @@ IDistributedSuperFile *CDistributedFileDirectory::createSuperFile(const char *_l
 }
 
 // MORE: This should be implemented in DFSAccess later on
+IDistributedSuperFile *CDistributedFileDirectory::createNewSuperFile(IPropertyTree *tree)
+{
+    return new CDistributedSuperFile(this, tree);
+}
+
+// MORE: This should be implemented in DFSAccess later on
 void CDistributedFileDirectory::removeSuperFile(const char *_logicalname, bool delSubs, IUserDescriptor *user, IDistributedFileTransaction *transaction)
 {
     CDfsLogicalFileName logicalname;

dali/base/dadfs.hpp

@@ -395,6 +395,8 @@ interface IDistributedFile: extends IInterface
     
     virtual unsigned setDefaultTimeout(unsigned timems) = 0;                                // sets default timeout for SDS connections and locking
                                                                                             // returns previous value
+
+    virtual void validate() = 0;
 };
 
 
@@ -409,6 +411,7 @@ interface IDistributedFile: extends IInterface
  */
 interface IDistributedSuperFile: extends IDistributedFile
 {
+
     virtual IDistributedFileIterator *getSubFileIterator(bool supersub=false)=0;    // if supersub true recurse through sub-superfiles and only return normal files
     virtual void addSubFile(const char *subfile,
                             bool before=false,              // if true before other
@@ -432,7 +435,7 @@ interface IDistributedSuperFile: extends IDistributedFile
     virtual IDistributedFile *querySubFileNamed(const char *name,bool sub=true)=0;
     virtual unsigned numSubFiles(bool sub=false)=0;
 
-    // useful part funtions
+    // useful part functions
     virtual bool isInterleaved()=0;
     virtual IDistributedFile *querySubPart(unsigned partidx,                // superfile part index
                                            unsigned &subfilepartidx)=0;     // part index in subfile
@@ -580,6 +583,7 @@ interface IDistributedFileDirectory: extends IInterface
     virtual IFileDescriptor *getFileDescriptor(const char *lname,IUserDescriptor *user,const INode *foreigndali=NULL, unsigned foreigndalitimeout=FOREIGN_DALI_TIMEOUT) =0;
 
     virtual IDistributedSuperFile *createSuperFile(const char *logicalname,IUserDescriptor *user,bool interleaved,bool ifdoesnotexist=false,IDistributedFileTransaction *transaction=NULL) = 0;
+    virtual IDistributedSuperFile *createNewSuperFile(IPropertyTree *tree) = 0;
     virtual IDistributedSuperFile *lookupSuperFile(const char *logicalname,IUserDescriptor *user,
                                                     IDistributedFileTransaction *transaction=NULL, // transaction only used for looking up sub files
                                                     unsigned timeout=INFINITE) = 0;  // NB lookup will also return superfiles

dali/base/dafdesc.cpp

@@ -1975,7 +1975,6 @@ public:
         subfilecounts = NULL;
     }
 
-
     virtual ~CSuperFileDescriptor()
     {
         delete subfilecounts;
@@ -2331,7 +2330,6 @@ IFileDescriptor *deserializeFileDescriptorTree(IPropertyTree *tree, INamedGroupS
     return new CFileDescriptor(tree, resolver, flags);
 }
 
-
 inline bool validFNameChar(char c)
 {
     static const char *invalids = "*\"/:<>?\\|";

dali/dfu/dfuutil.cpp

@@ -838,15 +838,15 @@ public:
         }
     }
 
-    StringBuffer &getFileXML(const char *lfn,StringBuffer &out, IUserDescriptor *user)
+    StringBuffer &getFileXML(const char *lfn, StringBuffer &out, IUserDescriptor *user)
     {
-        Owned<IDistributedFile> file = queryDistributedFileDirectory().lookup(lfn,user);
+        Owned<IDistributedFile> file = queryDistributedFileDirectory().lookup(lfn, user);
         if (!file) {
             INamedGroupStore  &grpstore= queryNamedGroupStore();
             Owned<IGroup> grp = grpstore.lookup(lfn);
             if (grp) {
                 out.append("<Group name=\"").append(lfn).append("\">\n");
-                ForEachNodeInGroup(i,*grp) {
+                ForEachNodeInGroup(i, *grp) {
                     StringBuffer ip;
                     grp->getNode(i)->endpoint().getIpText(ip);
                     out.append("  <Node ip=\"").append(ip).append("\">\n");
@@ -857,21 +857,43 @@ public:
             else
                 throwError1(DFUERR_DFileNotFound, lfn);
         }
-        else {
-            Owned<IFileDescriptor> fdesc = file->getFileDescriptor();
-            Owned<IPropertyTree> t = fdesc->getFileTree();
+        else
+        {
+            Owned<IPropertyTree> t = queryDistributedFileDirectory().getFileTree(lfn, user);
             toXML(t, out, true);
         }
         return out;
     }
 
-    void addFileXML(const char *lfn,const StringBuffer &xml,IUserDescriptor *user)
+    void addFileXML(const char *lfn, const StringBuffer &xml, IUserDescriptor *user)
     {
         Owned<IPropertyTree> t = createPTreeFromXMLString(xml);
-        Owned<IFileDescriptor> fdesc = deserializeFileDescriptorTree(t,&queryNamedGroupStore(),0);
-        Owned<IDistributedFile> file = queryDistributedFileDirectory().createNew(fdesc,true);
-        if (file)
-            file->attach(lfn,user);
+        IDistributedFileDirectory &dfd = queryDistributedFileDirectory();
+
+        if (dfd.exists(lfn, user))
+            throw MakeStringException(-1, "Destination file '%s' already exists!", lfn);
+
+        // Check if this XML is a superfile map
+        Owned<IDistributedFile> file;
+        const char * nodeName = t->queryName();
+        if (0 == strcmp(nodeName, queryDfsXmlBranchName(DXB_SuperFile)))
+        {
+            // It seems XML is a super file
+            file.setown(dfd.createNewSuperFile(t));
+        }
+        else if (0 == strcmp(nodeName, queryDfsXmlBranchName(DXB_File)))
+        {
+            // Logical file map
+
+            Owned<IFileDescriptor> fdesc = deserializeFileDescriptorTree(t, &queryNamedGroupStore(), 0);
+            file.setown(dfd.createNew(fdesc, true));
+        }
+        else
+            throw MakeStringException(-1, "Unrecognised file XML root tag detected: '%s'", nodeName);
+
+        file->validate();
+        PROGLOG("Adding %s file %s.", file->querySuperFile()?"super":"logical", lfn);
+        file->attach(lfn,user);
     }
 
     void addFileRemote(const char *lfn,SocketEndpoint &srcdali,const char *srclfn,IUserDescriptor *user,IUserDescriptor *srcuser=NULL)

dali/dfuplus/dfuplus.cpp

@@ -1212,11 +1212,11 @@ int CDfuPlusHelper::superfile(const char* action)
 
         if(stricmp(action, "add") == 0)
         {
-            info("Addsuper successfully finished");
+            info("Addsuper successfully finished\n");
         }
         else if(stricmp(action, "remove") == 0)
         {
-            info("Removesuper successfully finished");
+            info("Removesuper successfully finished\n");
         }
     }
     else if(stricmp(action, "list") == 0)
@@ -1353,7 +1353,7 @@ int CDfuPlusHelper::add()
         }
     }
 
-    info("%s successfully added", lfn);
+    info("%s successfully added\n", lfn);
     return 0;
 }
 
@@ -1399,7 +1399,7 @@ int CDfuPlusHelper::status()
             return -1;
 
         case DFUstate_failed:
-            info("%s status: failed - %s", wuid, dfuwu.getSummaryMessage());
+            info("%s status: failed - %s\n", wuid, dfuwu.getSummaryMessage());
             return -1;
 
         case DFUstate_finished:

system/jlib/jsocket.cpp

@@ -5720,7 +5720,8 @@ void SocketEndpointArray::fromText(const char *text,unsigned defport)
     // this is quite complicated with (mixed) IPv4 and IPv6
     // only support 'full' IPv6 and no ranges
 
-
+    if (!text)
+        return;
     char *str = strdup(text);
     char *s = str;
     SocketEndpoint ep;