HPCC-16375 Add core functionality

Add core functionality to copy and extend file history in DFU
operation like: spray, copy.

It is the first subtask of "HPCC-16072 retain metadata (atleast
the owner) during file copy" issue.

Signed-off-by: Attila Vamos <attila.vamos@gmail.com>

dali/base/dadfs.hpp

@@ -377,27 +377,29 @@ interface IDistributedFile: extends IInterface
     virtual void setPreferredClusters(const char *clusters) = 0;
     virtual void setSingleClusterOnly() = 0;
 
-    virtual bool getFormatCrc(unsigned &crc) =0;   // CRC for record format 
-    virtual bool getRecordSize(size32_t &rsz) =0;   
+    virtual bool getFormatCrc(unsigned &crc) =0;   // CRC for record format
+    virtual bool getRecordSize(size32_t &rsz) =0;
     virtual bool getRecordLayout(MemoryBuffer &layout) =0;
 
 
     virtual void enqueueReplicate()=0;
 
-    virtual StringBuffer &getColumnMapping(StringBuffer &mapping) =0;   
-    virtual void setColumnMapping(const char *mapping) =0;   
+    virtual StringBuffer &getColumnMapping(StringBuffer &mapping) =0;
+    virtual void setColumnMapping(const char *mapping) =0;
 
     virtual bool canModify(StringBuffer &reason) = 0;
     virtual bool canRemove(StringBuffer &reason,bool ignoresub=false) = 0;
-    virtual void setProtect(const char *callerid, bool protect=true, unsigned timeoutms=INFINITE) = 0;                  
+    virtual void setProtect(const char *callerid, bool protect=true, unsigned timeoutms=INFINITE) = 0;
                                                                                             // sets or clears deletion protection
                                                                                             // returns true if locked (by anyone) after action
                                                                                             // if callerid NULL and protect=false, clears all
-    
+
     virtual unsigned setDefaultTimeout(unsigned timems) = 0;                                // sets default timeout for SDS connections and locking
                                                                                             // returns previous value
 
     virtual void validate() = 0;
+
+    virtual IPropertyTree &queryHistory() const = 0;                         // DFile History records
 };
 
 
@@ -494,6 +496,10 @@ public:
     {
         return file->queryAttributes();
     }
+    IDistributedFile *queryFile()
+    {
+        return file;
+    }
     bool needsReload()
     {
         return reload;
@@ -575,7 +581,7 @@ interface IDistributedFileDirectory: extends IInterface
     virtual bool removeEntry(const char *name, IUserDescriptor *user, IDistributedFileTransaction *transaction=NULL, unsigned timeoutms=INFINITE, bool throwException=false) = 0;
     virtual void renamePhysical(const char *oldname,const char *newname,IUserDescriptor *user,IDistributedFileTransaction *transaction) = 0;                         // renames the physical parts as well as entry
     virtual void removeEmptyScope(const char *scope) = 0;   // does nothing if called on non-empty scope
-    
+
 
     virtual bool exists(const char *logicalname,IUserDescriptor *user,bool notsuper=false,bool superonly=false) = 0;                           // logical name exists
     virtual bool existsPhysical(const char *logicalname,IUserDescriptor *user) = 0;                                                    // physical parts exists
@@ -646,11 +652,11 @@ interface IDistributedFileDirectory: extends IInterface
         const char *kind=S_LINK_RELATIONSHIP_KIND
     )=0;
 
-    virtual void removeAllFileRelationships(const char *filename)=0;        
+    virtual void removeAllFileRelationships(const char *filename)=0;
     virtual IFileRelationshipIterator *lookupAllFileRelationships(const char *filename)=0; // either primary or secondary
 
     virtual bool loadScopeContents(const char *scopelfn,
-                                   StringArray *scopes, 
+                                   StringArray *scopes,
                                    StringArray *supers,
                                    StringArray *files,
                                    bool includeemptyscopes=false
@@ -667,7 +673,7 @@ interface IDistributedFileDirectory: extends IInterface
     virtual unsigned queryProtectedCount(const CDfsLogicalFileName &logicalname,
                                   const char *callerid=NULL) = 0;                   // if NULL  then sums all
 
-    virtual bool getProtectedInfo (const CDfsLogicalFileName &logicalname, 
+    virtual bool getProtectedInfo (const CDfsLogicalFileName &logicalname,
                                            StringArray &names, UnsignedArray &counts) = 0;
 
     virtual IDFProtectedIterator *lookupProtectedFiles(const char *owner=NULL,bool notsuper=false,bool superonly=false)=0; // if owner = NULL then all
@@ -788,10 +794,10 @@ extern da_decl IDFAttributesIterator *createSubFileFilter(IDFAttributesIterator
 
 #define DFS_REPLICATE_QUEUE "dfs_replicate_queue"
 #define DRQ_STOP 0
-#define DRQ_REPLICATE 1 
+#define DRQ_REPLICATE 1
 
 
-// Useful property query functions 
+// Useful property query functions
 
 inline bool isFileKey(IPropertyTree &pt) { const char *kind = pt.queryProp("@kind"); return kind&&strieq(kind,"key"); }
 inline bool isFileKey(IDistributedFile *f) { return isFileKey(f->queryAttributes()); }

dali/base/dafdesc.cpp

@@ -37,9 +37,14 @@
 
 #define SDS_CONNECT_TIMEOUT  (1000*60*60*2)     // better than infinite
 
+// These are legacy and cannot be changed.
 #define SERIALIZATION_VERSION ((byte)0xd4)
 #define SERIALIZATION_VERSION2 ((byte)0xd5) // with trailing superfile info
 
+#define VERSION_MINHISTORY 1  // Add History to serialisation/de-serialisation
+// This version serialised in the extendedVersion of CFileDescriptor
+static const unsigned short serializationVersion = 1;
+
 bool isMulti(const char *str)
 {
     if (str&&!isSpecialPath(str))
@@ -589,6 +594,7 @@ public:
     IArrayOf<IClusterInfo> clusters;
 
     Owned<IPropertyTree> attr;
+    Owned<IPropertyTree> history;
     StringAttr directory;
     StringAttr partmask;
     virtual unsigned numParts() = 0;                                            // number of parts
@@ -937,7 +943,6 @@ class CFileDescriptor:  public CFileDescriptorBase, implements ISuperFileDescrip
     bool setupdone;
     byte version;
 
-
     IFileDescriptor &querySelf()
     {
         return *this;
@@ -1137,7 +1142,7 @@ class CFileDescriptor:  public CFileDescriptorBase, implements ISuperFileDescrip
                 if (pt.isMulti()) {
                     StringBuffer tmpon; // bit messy but need to ensure dir put back on before removing!
                     const char *on = pt.overridename.get();
-                    if (on&&*on&&!isAbsolutePath(on)&&!directory.isEmpty()) 
+                    if (on&&*on&&!isAbsolutePath(on)&&!directory.isEmpty())
                         on = addPathSepChar(tmpon.append(directory)).append(on).str();
                     StringBuffer tmp2;
                     splitDirMultiTail(on,tmp1,tmp2);
@@ -1266,41 +1271,47 @@ public:
         pending = NULL;
         setupdone = true;
         mb.read(version);
-        if ((version!=SERIALIZATION_VERSION)&&(version!=SERIALIZATION_VERSION2)) // check seialization matched
+        if ((version != SERIALIZATION_VERSION) && (version != SERIALIZATION_VERSION2)) // check serialization matched
             throw MakeStringException(-1,"FileDescriptor serialization version mismatch %d/%d",(int)SERIALIZATION_VERSION,(int)version);
         mb.read(tracename);
         mb.read(directory);
         mb.read(partmask);
         unsigned n;
         mb.read(n);
-        for (unsigned i1=0;i1<n;i1++)
+        for (unsigned i1 = 0; i1 < n; i1++)
             clusters.append(*deserializeClusterInfo(mb));
         unsigned partidx;
         mb.read(partidx);   // -1 if all parts, -2 if multiple parts
         mb.read(n); // numparts
         CPartDescriptor *part;
-        if (partidx==(unsigned)-2) {
+        if (partidx == (unsigned)-2)
+        {
             UnsignedArray pia;
             unsigned pi;
-            loop {
+            loop
+            {
                 mb.read(pi);
-                if (pi==(unsigned)-1)
+                if (pi == (unsigned)-1)
                     break;
                 pia.append(pi);
             }
-            for (unsigned i3=0;i3<n;i3++)
+            for (unsigned i3 = 0; i3 < n; i3++)
                 parts.append(NULL);
-            ForEachItemIn(i4,pia) {
+            ForEachItemIn(i4, pia)
+            {
                 unsigned p = pia.item(i4);
-                if (p<n) {
-                    part = new CPartDescriptor(*this,p,mb);
-                    parts.replace(part,p);
+                if (p < n) {
+                    part = new CPartDescriptor(*this, p, mb);
+                    parts.replace(part, p);
                 }
             }
-            if (partsret) {
-                ForEachItemIn(i5,pia) {
+            if (partsret)
+            {
+                ForEachItemIn(i5, pia)
+                {
                     unsigned p = pia.item(i5);
-                    if (p<parts.ordinality()) {
+                    if (p < parts.ordinality())
+                    {
                         CPartDescriptor *pt = (CPartDescriptor *)parts.item(p);
                         partsret->append(*LINK(pt));
                     }
@@ -1308,27 +1319,32 @@ public:
             }
 
         }
-        else {
-            for (unsigned i2=0;i2<n;i2++) {
-                if ((partidx==(unsigned)-1)||(partidx==i2)) {
-                    part = new CPartDescriptor(*this,i2,mb);
+        else
+        {
+            for (unsigned i2=0; i2 < n; i2++)
+            {
+                if ((partidx == (unsigned)-1) || (partidx == i2))
+                {
+                    part = new CPartDescriptor(*this, i2, mb);
                     if (partsret)
                         partsret->append(*LINK(part));
                 }
                 else
-                    part = NULL; // new CPartDescriptor(*this,i2,NULL);
+                    part = NULL; // new CPartDescriptor(*this, i2, NULL);
                 parts.append(part);
             }
         }
         attr.setown(createPTree(mb));
         if (!attr)
             attr.setown(createPTree("Attr")); // doubt can happen
-        if (version==SERIALIZATION_VERSION2) {
+        if (version == SERIALIZATION_VERSION2)
+        {
             if (subcounts)
                 *subcounts = new UnsignedArray;
             unsigned n;
             mb.read(n);
-            while (n) {
+            while (n)
+            {
                 unsigned np;
                 mb.read(np);
                 if (subcounts)
@@ -1340,17 +1356,35 @@ public:
             if (_interleaved)
                 *_interleaved = interleaved;
         }
-    }
 
+        // If we have more serialised data in MemoryBuffer
+        if (mb.remaining() >= sizeof(unsigned short))
+        {
+            unsigned short extendedVersion;
+            mb.read(extendedVersion);
 
+            if (extendedVersion >= VERSION_MINHISTORY)
+            {
+                history.setown(createPTree(mb));
+            }
+            else
+                throw MakeStringException(-1,"FileDescriptor extended serialization version shouldn't be zero!");
+        }
+    }
+
+    void ensureRequiredStructuresExist()
+    {
+        if (!attr) attr.setown(createPTree("Attr"));
+        if (!history) history.setown(createPTree("History"));
+    }
 
     CFileDescriptor(IPropertyTree *tree, INamedGroupStore *resolver, unsigned flags)
     {
         pending = NULL;
         if ((flags&IFDSF_ATTR_ONLY)||!tree) {
-            if (!tree)
-                tree = createPTree("Attr");
-            attr.setown(tree);
+            if (tree)
+                attr.setown(tree);
+            ensureRequiredStructuresExist();
             setupdone = false;
             return;
         }
@@ -1366,6 +1400,7 @@ public:
         Owned<IPropertyTreeIterator> piter;
         MemoryBuffer mb;
         IPropertyTree *at = pt.queryPropTree("Attr");
+        IPropertyTree *hist = pt.queryPropTree("History");
         getClusterInfo(pt,resolver,flags,clusters);
         offset_t totalsize = (offset_t)-1;
         if (flags&IFDSF_EXCLUDE_PARTS) {
@@ -1424,6 +1459,12 @@ public:
             attr.setown(createPTreeFromIPT(at));
         else
             attr.setown(createPTree("Attr"));
+
+        if (hist)
+            history.setown(createPTreeFromIPT(hist));
+        else
+            history.setown(createPTree("History"));
+
         if (totalsize!=(offset_t)-1)
             attr->setPropInt64("@size",totalsize);
     }
@@ -1505,6 +1546,12 @@ public:
         IPropertyTree *t = &queryProperties();
         if (!isEmptyPTree(t))
             pt.addPropTree("Attr",createPTreeFromIPT(t));
+
+        t = &queryHistory();
+        if (!isEmptyPTree(t))
+            pt.addPropTree("History",createPTreeFromIPT(t));
+        else
+            pt.addPropTree("History",createPTree("History"));
     }
 
     IPropertyTree *getFileTree(unsigned flags)
@@ -1702,6 +1749,12 @@ public:
         return *attr.get();
     }
 
+    IPropertyTree &queryHistory()
+    {
+        closePending();
+        return *history.get();
+    }
+
     bool isMulti(unsigned partidx=(unsigned)-1)
     {
         closePending();
@@ -1959,10 +2012,10 @@ public:
 class CSuperFileDescriptor:  public CFileDescriptor
 {
     UnsignedArray *subfilecounts;
-    bool interleaved; 
+    bool interleaved;
 public:
 
-    CSuperFileDescriptor(MemoryBuffer &mb, IArrayOf<IPartDescriptor> *partsret) 
+    CSuperFileDescriptor(MemoryBuffer &mb, IArrayOf<IPartDescriptor> *partsret)
         : CFileDescriptor(mb,partsret,&subfilecounts,&interleaved)
     {
     }
@@ -1989,7 +2042,7 @@ public:
         subfile = 0;
         if (!subfilecounts)  // its a file!
             return true;
-        if (interleaved) { 
+        if (interleaved) {
             unsigned p = 0;
             unsigned f = 0;
             bool found = false;
@@ -2015,7 +2068,7 @@ public:
         }
         else { // sequential
             while (subfile<subfilecounts->ordinality()) {
-                if (subpartnum<subfilecounts->item(subfile)) 
+                if (subpartnum<subfilecounts->item(subfile))
                     return true;
                 subpartnum -= subfilecounts->item(subfile);
                 subfile++;
@@ -2099,6 +2152,10 @@ void CFileDescriptor::serializeParts(MemoryBuffer &mb,unsigned *partlist, unsign
     queryProperties().serialize(mb);
     if (sdesc)
         sdesc->serializeSub(mb);
+
+    // Serialisation extensions like History.
+    mb.append(serializationVersion);
+    queryHistory().serialize(mb);
 }
 
 
@@ -2431,7 +2488,7 @@ void setBaseDirectory(const char * dir, unsigned replicateLevel, DFD_OS os)
 {
     // 2 possibilities
     // either its an absolute path
-    // or use /c$/thordata and /d$/thordata 
+    // or use /c$/thordata and /d$/thordata
     if (os==DFD_OSdefault)
 #ifdef _WIN32
         os = DFD_OSwindows;
@@ -2822,7 +2879,7 @@ IGroup *shrinkRepeatedGroup(IGroup *grp)
     unsigned w = grp->ordinality();
     for (unsigned i=1;i<w;i++) {
         unsigned j;
-        for (j=i;j<w;j++) 
+        for (j=i;j<w;j++)
             if (!grp->queryNode(j).equals(&grp->queryNode(j%i)))
                 break;
         if (j==w)
@@ -2846,19 +2903,19 @@ IFileDescriptor *createFileDescriptorFromRoxieXML(IPropertyTree *tree,const char
             id++;
         res->setTraceName(id);
     }
-    else 
+    else
         id = "";
     const char *dir = tree->queryProp("@directory");
     if (!dir||!*dir)
-        throw MakeStringException(-1,"createFileDescriptorFromRoxie: %s missing directory",id); 
+        throw MakeStringException(-1,"createFileDescriptorFromRoxie: %s missing directory",id);
     const char *mask = tree->queryProp("@partmask");
     if (!mask||!*mask)
-        throw MakeStringException(-1,"createFileDescriptorFromRoxie: %s missing part mask",id); 
+        throw MakeStringException(-1,"createFileDescriptorFromRoxie: %s missing part mask",id);
     unsigned np = tree->getPropInt("@numparts");
     IPropertyTree *part1 = tree->queryPropTree("Part_1");
     if (!part1)
-        throw MakeStringException(-1,"createFileDescriptorFromRoxie: %s missing part 1",id); 
-    
+        throw MakeStringException(-1,"createFileDescriptorFromRoxie: %s missing part 1",id);
+
     // assume same number of copies for all parts
     unsigned nc = 0;
     StringBuffer xpath;
@@ -2870,10 +2927,10 @@ IFileDescriptor *createFileDescriptorFromRoxieXML(IPropertyTree *tree,const char
             break;
         const char *path = loc->queryProp("@path");
         if (!path)
-            throw MakeStringException(-1,"createFileDescriptorFromRoxie: %s missing part 1 loc path",id); 
+            throw MakeStringException(-1,"createFileDescriptorFromRoxie: %s missing part 1 loc path",id);
         RemoteFilename rfn;
         rfn.setRemotePath(path);
-        if (rfn.queryEndpoint().isNull()) 
+        if (rfn.queryEndpoint().isNull())
             break;
         locdirs.append(rfn.getLocalPath(locpath.clear()).str());
         nc++;
@@ -2888,7 +2945,7 @@ IFileDescriptor *createFileDescriptorFromRoxieXML(IPropertyTree *tree,const char
     for (unsigned p=1;p<=np;p++) {
         IPropertyTree *part = tree->queryPropTree(xpath.clear().appendf("Part_%d",p));
         if (!part)
-            throw MakeStringException(-1,"createFileDescriptorFromRoxie: %s missing part %d",id,p); 
+            throw MakeStringException(-1,"createFileDescriptorFromRoxie: %s missing part %d",id,p);
         if (iskey&&(p==np)&&(np>1)) // leave off tlk
             continue;
         unsigned c;
@@ -2897,7 +2954,7 @@ IFileDescriptor *createFileDescriptorFromRoxieXML(IPropertyTree *tree,const char
             if (loc) {
                 const char *path = loc->queryProp("@path");
                 if (!path)
-                    throw MakeStringException(-1,"createFileDescriptorFromRoxie: %s missing part %d loc path",id,p); 
+                    throw MakeStringException(-1,"createFileDescriptorFromRoxie: %s missing part %d loc path",id,p);
                 RemoteFilename rfn;
                 rfn.setRemotePath(path);
                 bool found = false;
@@ -2913,7 +2970,7 @@ IFileDescriptor *createFileDescriptorFromRoxieXML(IPropertyTree *tree,const char
                 }
             }
             else
-                ERRLOG("createFileDescriptorFromRoxie: %s missing part %s",id,xpath.str()); 
+                ERRLOG("createFileDescriptorFromRoxie: %s missing part %s",id,xpath.str());
         }
     }
     res->setPartMask(mask);
@@ -2966,7 +3023,7 @@ IFileDescriptor *createFileDescriptorFromRoxieXML(IPropertyTree *tree,const char
     if (clustername) {
 #if 0
         Owned<IGroup> cgrp = queryNamedGroupStore().lookup(clustername);
-        if (!cgrp) 
+        if (!cgrp)
             throw MakeStringException(-1,"createFileDescriptorFromRoxieXML: Cluster %s not found",clustername);
         if (!cgrp->equals(grp))
             throw MakeStringException(-1,"createFileDescriptorFromRoxieXML: Cluster %s does not match XML",clustername);
@@ -2998,7 +3055,7 @@ IFileDescriptor *createFileDescriptorFromRoxieXML(IPropertyTree *tree,const char
             if (loc) {
                 const char *path = loc->queryProp("@path");
                 if (!path)
-                    throw MakeStringException(-1,"createFileDescriptorFromRoxie: %s missing part %d loc path",id,c+1); 
+                    throw MakeStringException(-1,"createFileDescriptorFromRoxie: %s missing part %d loc path",id,c+1);
                 StringBuffer fullpath(path);
                 addPathSepChar(fullpath).append(tree->queryProp("@directory"));
                 expandMask(addPathSepChar(fullpath),mask,p-1,np);

dali/base/dafdesc.hpp

@@ -62,9 +62,9 @@ enum GroupType { grp_thor, grp_thorspares, grp_roxie, grp_hthor, grp_unknown, __
 // repeatedPart flags
 #define CPDMSRP_lastRepeated     (0x20000000) // repeat last part
 #define CPDMSRP_onlyRepeated     (0x40000000) // or with repeatedPart for only repeated parts
-#define CPDMSRP_allRepeated      (0x80000000) // or with repeatedPart for all parts repeated 
+#define CPDMSRP_allRepeated      (0x80000000) // or with repeatedPart for all parts repeated
 #define CPDMSRP_notRepeated      ((unsigned)-1) // value of no repeated parts (i.e. only normal replication)
-#define CPDMSRP_partMask         (0xffffff) 
+#define CPDMSRP_partMask         (0xffffff)
 
 struct da_decl ClusterPartDiskMapSpec
 {
@@ -76,7 +76,7 @@ public:
     byte startDrv;          // normally 0 (c$)
     byte flags;             // CPDMSF_*
     unsigned interleave;    // for superfiles (1..)
-    unsigned repeatedPart;  // if part is repeated on every node (NB maxCopies does not include) 
+    unsigned repeatedPart;  // if part is repeated on every node (NB maxCopies does not include)
                             // ( | CPDMSRP_onlyRepeated for *only* repeats )
     StringAttr defaultBaseDir; // if set overrides *base* directory (i.e. /c$/dir/x/y becomes odir/x/y)
     StringAttr defaultReplicateDir;
@@ -101,7 +101,7 @@ public:
 #define CPDMSF_wrapToNextDrv    (0x01)      // whether should wrap to next drv
 #define CPDMSF_fillWidth        (0x02)      // replicate copies fill cluster serially (when num parts < clusterwidth/2)
 #define CPDMSF_packParts        (0x04)      // whether to save parts as binary
-#define CPDMSF_repeatedPart     (0x08)      // if repeated parts included 
+#define CPDMSF_repeatedPart     (0x08)      // if repeated parts included
 #define CPDMSF_defaultBaseDir   (0x10)      // set if defaultBaseDir present
 #define CPDMSF_defaultReplicateDir  (0x20)      // set if defaultBaseDir present
 #define CPDMSF_overloadedConfig  (0x40)      // set if overloaded mode
@@ -160,19 +160,19 @@ interface IFileDescriptor: extends IInterface
     for each cluster
         addCluster(grp,mapping) (or addCluster(grpname,mapping,resolver)
 
- or: 
+ or:
 
    for each cluster
         for each part
-            setPart(partnum,file,attr);     
+            setPart(partnum,file,attr);
         endCluster(mapping);
 
  or: (single cluster legacy)
 
     for each part
-        setPart(partnum,file,attr);  
- 
-   
+        setPart(partnum,file,attr);
+
+
 note it is an error to set different filenames for the same part on different clusters
 or to set the same part twice in the same cluster
 if endCluster is not called it will assume only one cluster and not replicated
@@ -241,6 +241,8 @@ if endCluster is not called it will assume only one cluster and not replicated
     virtual StringBuffer &getClusterLabel(unsigned clusternum,StringBuffer &ret) = 0; // node group name
 
     virtual void ensureReplicate() = 0;                                             // make sure a file can be replicated
+
+    virtual IPropertyTree &queryHistory() = 0;                                       // query file history records
 };
 
 interface ISuperFileDescriptor: extends IFileDescriptor
@@ -257,7 +259,7 @@ interface ISuperFileDescriptor: extends IFileDescriptor
 
 interface IClusterInfo: extends IInterface  // used by IFileDescriptor and IDistributedFile
 {
-    virtual StringBuffer &getGroupName(StringBuffer &name,IGroupResolver *resolver=NULL)=0; 
+    virtual StringBuffer &getGroupName(StringBuffer &name,IGroupResolver *resolver=NULL)=0;
     virtual const char *queryGroupName()=0;     // may be NULL
     virtual IGroup *queryGroup()=0;           // may be NULL
     virtual ClusterPartDiskMapSpec  &queryPartDiskMapping()=0;
@@ -306,11 +308,11 @@ extern da_decl StringBuffer &makePhysicalPartName(
                                 DFD_OS os=DFD_OSdefault,            // os must be specified if no dir specified
                                 const char *diroverride=NULL);      // override default directory
 extern da_decl StringBuffer &makeSinglePhysicalPartName(const char *lname, // single part file
-                                                        StringBuffer &result, 
-                                                        bool allowospath,   // allow an OS (absolute) file path 
+                                                        StringBuffer &result,
+                                                        bool allowospath,   // allow an OS (absolute) file path
                                                         bool &wasdfs,       // not OS path
                                                         const char *diroverride=NULL
-                                                        );    
+                                                        );
 
 // set/get defaults
 extern da_decl const char *queryBaseDirectory(GroupType groupType, unsigned replicateLevel=0, DFD_OS os=DFD_OSdefault);

dali/ft/daft.cpp

@@ -52,6 +52,7 @@ void CDistributedFileSystem::copy(IDistributedFile * from, IDistributedFile * to
 
     OwnedIFileSprayer sprayer = createFileSprayer(options, recovery, recoveryConnection, wuid);
 
+    sprayer->setOperation(dfu_copy);
     sprayer->setProgress(progress);
     sprayer->setAbort(abort);
     sprayer->setPartFilter(filter);
@@ -74,6 +75,7 @@ void CDistributedFileSystem::exportFile(IDistributedFile * from, IFileDescriptor
     LOG(MCdebugInfo, unknownJob, "DFS: export(%s,%s)", from->queryLogicalName(), to->getTraceName(temp).str());
 
     OwnedIFileSprayer sprayer = createFileSprayer(options, recovery, recoveryConnection, wuid);
+    sprayer->setOperation(dfu_export);
     sprayer->setProgress(progress);
     sprayer->setAbort(abort);
     sprayer->setPartFilter(filter);
@@ -92,6 +94,7 @@ void CDistributedFileSystem::import(IFileDescriptor * from, IDistributedFile * t
         LOG(MCdebugInfo, unknownJob, "DFS: import(%s)", from->getTraceName(temp).str());
 
     OwnedIFileSprayer sprayer = createFileSprayer(options, recovery, recoveryConnection, wuid);
+    sprayer->setOperation(dfu_import);
     sprayer->setProgress(progress);
     sprayer->setAbort(abort);
     sprayer->setPartFilter(filter);
@@ -109,6 +112,7 @@ void CDistributedFileSystem::move(IDistributedFile * from, IDistributedFile * to
         LOG(MCdebugInfo, unknownJob, "DFS: move(%s)", from->queryLogicalName());
 
     OwnedIFileSprayer sprayer = createFileSprayer(options, recovery, recoveryConnection, wuid);
+    sprayer->setOperation(dfu_move);
     sprayer->setProgress(progress);
     sprayer->setAbort(abort);
     sprayer->setPartFilter(filter);
@@ -124,6 +128,7 @@ void CDistributedFileSystem::replicate(IDistributedFile * from, IGroup *destgrou
     LOG(MCdebugInfo, unknownJob, "DFS: replicate(%s)", from->queryLogicalName());
 
     FileSprayer sprayer(options, recovery, recoveryConnection, wuid);
+    sprayer.setOperation(dfu_replicate_distributed);
     sprayer.setProgress(progress);
     sprayer.setAbort(abort);
     sprayer.setReplicate(true);
@@ -139,6 +144,7 @@ void CDistributedFileSystem::replicate(IFileDescriptor * fd, DaftReplicateMode m
     LOG(MCdebugInfo, unknownJob, "DFS: replicate(%s, %x)", fd->getTraceName(s).str(), (unsigned)mode);
 
     FileSprayer sprayer(options, recovery, recoveryConnection, wuid);
+    sprayer.setOperation(dfu_replicate);
     sprayer.setProgress(progress);
     sprayer.setAbort(abort);
     sprayer.setReplicate(true);
@@ -153,6 +159,7 @@ void CDistributedFileSystem::transfer(IFileDescriptor * from, IFileDescriptor *
     LOG(MCdebugInfo, unknownJob, "DFS: transfer(%s,%s)", from->getTraceName(s1).str(), to->getTraceName(s1).str());
 
     OwnedIFileSprayer sprayer = createFileSprayer(options, recovery, recoveryConnection, wuid);
+    sprayer->setOperation(dfu_transfer);
     sprayer->setAbort(abort);
     sprayer->setProgress(progress);
     sprayer->setPartFilter(filter);

dali/ft/filecopy.cpp

@@ -2493,6 +2493,8 @@ void FileSprayer::setSource(IDistributedFile * source)
 {
     distributedSource.set(source);
     srcAttr.setown(createPTreeFromIPT(&source->queryAttributes()));
+    srcHistory.setown(createPTreeFromIPT(&source->queryHistory()));
+
     extractSourceFormat(srcAttr);
     unsigned numParts = source->numParts();
     for (unsigned idx=0; idx < numParts; idx++)
@@ -2529,6 +2531,7 @@ void FileSprayer::setSource(IFileDescriptor * source, unsigned copy, unsigned mi
     IPropertyTree *attr = &source->queryProperties();
     extractSourceFormat(attr);
     srcAttr.setown(createPTreeFromIPT(&source->queryProperties()));
+    srcHistory.setown(createPTreeFromIPT(&source->queryHistory()));
     extractSourceFormat(srcAttr);
 
     RemoteFilename filename;
@@ -3249,13 +3252,112 @@ void FileSprayer::updateTargetProperties()
                 if (stricmp(aname, "Protect") != 0)
                     curProps.addPropTree(aname, createPTreeFromIPT(&iter->query()));
             }
+
+            // Handle history
+            IPropertyTree &curHistory = lock.queryFile()->queryHistory();
+
+            // copy history records from source file
+            Owned<IPropertyTreeIterator> historyIter = srcHistory->getElements("*");
+            ForEach(*historyIter)
+            {
+                const char *aname = historyIter->query().queryName();
+                curHistory.addPropTree(aname, createPTreeFromIPT(&historyIter->query()));
+            }
+
+            // Add new record about this operation
+            Owned<IPropertyTree> newRecord = createPTree();
+
+            CDateTime temp;
+            temp.setNow();
+            unsigned hour, min, sec, nanosec;
+            temp.getTime(hour, min, sec, nanosec);
+            temp.setTime(hour, min, sec, 0);
+
+            StringBuffer timestr;
+            newRecord->setProp("@timestamp",temp.getString(timestr).str());
+
+            newRecord->setProp("@owner", srcAttr->queryProp("@owner"));
+
+            if (srcAttr->hasProp("@workunit"))
+                newRecord->setProp("@workunit", srcAttr->queryProp("@workunit"));
+
+            newRecord->setProp("@operation", getOperationTypeString());
+
+            // In Spray case there is not distributedSource
+            if (distributedSource)
+            {
+                // add original file name from a single distributed source (like Copy)
+                RemoteFilename remoteFile;
+                distributedSource->queryPart(0).getFilename(remoteFile, 0);
+                splitAndStoreFileInfo(newRecord, remoteFile);
+            }
+            else
+            {
+                // add original file names from multiple sources (like Spray)
+                ForEachItemIn(idx, sources)
+                {
+                    FilePartInfo & curSource = sources.item(idx);
+                    RemoteFilename &remoteFile = curSource.filename;
+                    splitAndStoreFileInfo(newRecord, remoteFile, idx, false);
+                }
+            }
+            curHistory.addPropTree("Origin",newRecord.getClear());
         }
     }
     if (error)
         throw error.getClear();
 }
 
+void FileSprayer::splitAndStoreFileInfo(IPropertyTree * newRecord, RemoteFilename &remoteFileName,
+                                        aindex_t idx, bool isDistributedSource)
+{
+    StringBuffer drive;
+    StringBuffer path;
+    StringBuffer fileName;
+    StringBuffer ext;
+    remoteFileName.split(&drive, &path, &fileName, &ext);
+    if (idx == 0)
+    {
+        if (drive.isEmpty())
+        {
+            remoteFileName.queryIP().getIpText(drive.clear());
+            newRecord->setProp("@ip", drive.str());
+        }
+        else
+            newRecord->setProp("@drive", drive.str());
+
+        newRecord->setProp("@path", path.str());
+    }
+    // We don't want to store distributed file parts name extension
+    if (!isDistributedSource && ext.length())
+        fileName.append(ext);
 
+    // In spray multiple source files case keep all original filenames
+    if (newRecord->hasProp("@name"))
+    {
+        StringBuffer currentName;
+        newRecord->getProp("@name", currentName);
+        currentName.append(",").append(fileName);
+        fileName = currentName;
+    }
+
+    newRecord->setProp("@name", fileName.str());
+}
+
+void FileSprayer::setOperation(dfu_operation op)
+{
+    operation = op;
+}
+
+dfu_operation FileSprayer::getOperation() const
+{
+    return operation;
+}
+
+const char * FileSprayer::getOperationTypeString() const
+{
+    return DfuOperatonStr[operation];
+}
 
 bool FileSprayer::usePullOperation()
 {

dali/ft/filecopy.hpp

@@ -110,6 +110,28 @@ public:
 UtfReader::UtfFormat getUtfFormatType(FileFormatType type);
 bool sameEncoding(const FileFormat & src, const FileFormat & tgt);
 
+typedef enum {
+        dfu_unknown,
+        dfu_copy,
+        dfu_export,
+        dfu_import,
+        dfu_move,
+        dfu_replicate_distributed,
+        dfu_replicate,
+        dfu_transfer,
+    } dfu_operation;
+
+static const char * DfuOperatonStr[] =
+    {
+        "DFUunknown",
+        "DFUcopy",
+        "DFUexport",
+        "DFUimport",
+        "DFUmove",
+        "DFUreplicate_distributed",
+        "DFUreplicate",
+        "DFUtransfer"
+    };
 interface IFileSprayer : public IInterface
 {
 public:
@@ -129,6 +151,10 @@ public:
     virtual void setTarget(INode * target) = 0;
     virtual void spray() = 0;
     virtual void checkSourceTarget(IFileDescriptor * file) = 0;
+    virtual void setOperation(dfu_operation op) = 0;
+    virtual dfu_operation getOperation() const = 0;
+    virtual const char * getOperationTypeString() const = 0;
+
 };
 
 extern DALIFT_API IFileSprayer * createFileSprayer(IPropertyTree * _options, IPropertyTree * _progress, IRemoteConnection * recoveryConnection, const char *wuid);

dali/ft/filecopy.ipp

@@ -203,6 +203,9 @@ public:
     void setError(const SocketEndpoint & ep, IException * e);
     bool canLocateSlaveForNode(const IpAddress &ip);
     void checkSourceTarget(IFileDescriptor * file);
+    void setOperation(dfu_operation op);
+    dfu_operation getOperation() const;
+    const char * getOperationTypeString() const;
 
 protected:
     void addEmptyFilesToPartition(unsigned from, unsigned to);
@@ -271,6 +274,8 @@ protected:
     
 private:
     bool calcUsePull();
+    // Get and store Remote File Name parts into the History record
+    void splitAndStoreFileInfo(IPropertyTree * newRecord, RemoteFilename &remoteFileName, aindex_t idx = 0, bool isDistributedSource = true);
 
 protected:
     CIArrayOf<FilePartInfo> sources;
@@ -326,6 +331,8 @@ protected:
     offset_t                headerSize;
     offset_t                footerSize;
     int                     fileUmask;
+    Owned<IPropertyTree>    srcHistory;
+    dfu_operation           operation = dfu_unknown;
 };