Merge branch 'candidate-7.6.x' into candidate-7.8.x

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

dali/ft/filecopy.cpp

@@ -344,7 +344,7 @@ bool FileTransferThread::performTransfer()
         serialize(partition, msg);
         msg.append(sprayer.numParallelSlaves());
         msg.append(slaveUpdateFrequency);
-        msg.append(sprayer.replicate);
+        msg.append(sprayer.replicate); // NB: controls whether FtSlave copies source timestamp
         msg.append(sprayer.mirroring);
         msg.append(sprayer.isSafeMode);
 
@@ -566,6 +566,7 @@ FileSprayer::FileSprayer(IPropertyTree * _options, IPropertyTree * _progress, IR
 {
     totalSize = 0;
     replicate = false;
+    copySource = false;
     unknownSourceFormat = true;
     unknownTargetFormat = true;
     progressTree.set(_progress);
@@ -644,11 +645,10 @@ public:
             }
 
             renameDfuTempToFinal(targetFilename);
-            if (sprayer.replicate)
+            if (sprayer.replicate && !sprayer.mirroring)
             {
                 OwnedIFile file = createIFile(targetFilename);
-                if (!sprayer.mirroring)
-                    file->setTime(NULL, &cur.modifiedTime, NULL);
+                file->setTime(NULL, &cur.modifiedTime, NULL);
             }
             else if (cur.modifiedTime.isNull())
             {
@@ -807,6 +807,8 @@ void FileSprayer::assignPartitionFilenames()
         }
         cur.outputName.set(targets.item(cur.whichOutput).filename);
         setCanAccessDirectly(cur.outputName);
+
+        // NB: partition (cur) is serialized to ftslave and it's this modifiedTime is used if present
         if (replicate)
             cur.modifiedTime.set(targets.item(cur.whichOutput).modifiedTime);
     }
@@ -1300,10 +1302,10 @@ void FileSprayer::checkFormats()
 
         //If format omitted, and number of parts are the same then okay to omit the format
         if (sources.ordinality() == targets.ordinality() && !disallowImplicitReplicate())
-            replicate = true;
+            copySource = true;
 
         bool noSplit = !allowSplit();
-        if (!replicate && !noSplit)
+        if (!replicate && !copySource && !noSplit)
         {
             //copy to a single target => assume same format concatenated.
             if (targets.ordinality() != 1)
@@ -2927,16 +2929,14 @@ void FileSprayer::spray()
     progressTree->setPropBool(ANpull, usePullOperation());
 
     const char * splitPrefix = querySplitPrefix();
-    bool pretendreplicate = false;
     if (!replicate && (sources.ordinality() == targets.ordinality()))
     {
-        if (srcFormat.equals(tgtFormat) && !disallowImplicitReplicate()) {
-            pretendreplicate = true;
-            replicate = true;
-        }
+        if (srcFormat.equals(tgtFormat) && !disallowImplicitReplicate())
+            copySource = true;
     }
 
-    if (compressOutput&&!replicate) {
+    if (compressOutput&&!replicate&&!copySource)
+    {
         PROGLOG("Compress output forcing pull");
         options->setPropBool(ANpull, true);
         allowRecovery = false;
@@ -2944,11 +2944,11 @@ void FileSprayer::spray()
 
 
     gatherFileSizes(true);
-    if (!replicate||pretendreplicate)
-        analyseFileHeaders(!pretendreplicate); // if pretending replicate don't want to remove headers
+    if (!replicate||copySource) // NB: When copySource=true, analyseFileHeaders mainly just sets srcFormat.type
+        analyseFileHeaders(!copySource); // if pretending replicate don't want to remove headers
     afterGatherFileSizes();
 
-    if (compressOutput && !usePullOperation() && !replicate)
+    if (compressOutput && !usePullOperation() && !replicate && !copySource)
         throwError(DFTERR_CannotPushAndCompress);
 
     if (restorePartition())
@@ -2958,7 +2958,7 @@ void FileSprayer::spray()
     else
     {
         LOG(MCdebugProgress, job, "Calculate partition information");
-        if (replicate)
+        if (replicate || copySource)
             calculateOne2OnePartition();
         else if (!allowSplit())
             calculateNoSplitPartition();
@@ -2973,7 +2973,7 @@ void FileSprayer::spray()
         savePartition();
     }
     assignPartitionFilenames();     // assign source filenames - used in insertHeaders..
-    if (!replicate)
+    if (!replicate && !copySource)
     {
         LOG(MCdebugProgress, job, "Insert headers");
         insertHeaders();
@@ -3119,7 +3119,6 @@ void FileSprayer::updateTargetProperties()
                                     failedParts.append("Output CRC failed to match expected: ");
                                 curSource.filename.getPath(failedParts);
                                 failedParts.appendf("(%x,%x)",partCRC.get(),curSource.crc);
-
                             }
                         }
                     }

dali/ft/filecopy.ipp

@@ -305,6 +305,7 @@ protected:
     offset_t                totalSize;
     unsigned __int64        sizeToBeRead;
     bool                    replicate;
+    bool                    copySource;
     bool                    unknownSourceFormat;
     bool                    unknownTargetFormat;
     Owned<IException>       error;

dali/ft/fttransform.cpp

@@ -592,7 +592,7 @@ void TransferServer::deserializeAction(MemoryBuffer & msg, unsigned action)
     deserialize(partition, msg);
     msg.read(numParallelSlaves);
     msg.read(updateFrequency);
-    msg.read(replicate);
+    msg.read(copySourceTimeStamp);
     msg.read(mirror);
     msg.read(isSafeMode);
 
@@ -612,7 +612,7 @@ void TransferServer::deserializeAction(MemoryBuffer & msg, unsigned action)
         LOG(MCdebugProgress, unknownJob, "Process Push Command: %s", localFilename.str());
     }
     LOG(MCdebugProgress, unknownJob, "Num Parallel Slaves=%d Adjust=%d/%d", numParallelSlaves, adjust, updateFrequency);
-    LOG(MCdebugProgress, unknownJob, "replicate(%d) mirror(%d) safe(%d) incrc(%d) outcrc(%d)", replicate, mirror, isSafeMode, calcInputCRC, calcOutputCRC);
+    LOG(MCdebugProgress, unknownJob, "copySourceTimeStamp(%d) mirror(%d) safe(%d) incrc(%d) outcrc(%d)", copySourceTimeStamp, mirror, isSafeMode, calcInputCRC, calcOutputCRC);
 
     displayPartition(partition);
 
@@ -905,7 +905,7 @@ processedProgress:
                     if (fileUmask != -1)
                         output->setFilePermissions(~fileUmask&0666);
 
-                    if (mirror || replicate)
+                    if (mirror || copySourceTimeStamp)
                     {
                         OwnedIFile input = createIFile(curPartition.inputName);
                         CDateTime modifiedTime;

dali/ft/fttransform.ipp

@@ -231,7 +231,7 @@ protected:
     offset_t                totalLengthToRead;
     bool                    calcInputCRC;
     bool                    calcOutputCRC;
-    bool                    replicate;
+    bool                    copySourceTimeStamp;
     bool                    mirror;
     bool                    isSafeMode;
     unsigned                throttleNicSpeed;