HPCC-15199 Setting useTreeCopy may cause corrupt files on Roxie

The useTreeCopy option has never been tested in anger as far as I know (it's
been in the system for years, but no-one knows how the code works or what it
is supposed to do).

Give that the one time someone DID try it it seems to have not handled error
cases well and ended up with corrupted indexes on Roxie. the commit removes
the option altogether. The associated code is removed in 6.0.0 via a separate
pull request.

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

roxie/ccd/ccd.hpp

@@ -392,7 +392,6 @@ extern bool probeAllRows;
 extern bool steppingEnabled;
 extern bool simpleLocalKeyedJoins;
 extern bool enableKeyDiff;
-extern bool useTreeCopy;
 extern PTreeReaderOptions defaultXmlReadFlags;
 extern bool mergeSlaveStatistics;
 extern bool roxieMulticastEnabled;   // enable use of multicast for sending requests to slaves

roxie/ccd/ccdfile.cpp

@@ -742,11 +742,7 @@ class CRoxieFileCache : public CInterface, implements ICopyFileProgress, impleme
             StringBuffer destPath;
             StringBuffer prevTempFile;
             splitFilename(targetFilename, &destPath, &destPath, &prevTempFile, &prevTempFile);
-
-            if (useTreeCopy)
-                prevTempFile.append("*.tmp");
-            else
-                prevTempFile.append("*.$$$");
+            prevTempFile.append("*.$$$");
 
             Owned<IFile> dirf = createIFile(destPath.str());
             Owned<IDirectoryIterator> iter = dirf->directoryFiles(prevTempFile.str(),false,false);
@@ -800,7 +796,7 @@ class CRoxieFileCache : public CInterface, implements ICopyFileProgress, impleme
         {
             IpSubNet subnet; // preferred set but not required
             IpAddress fromip; // returned
-            Owned<IFile> destFile = createIFile(useTreeCopy?targetFilename:tempFile);
+            Owned<IFile> destFile = createIFile(tempFile);
 
             bool hardLinkCreated = false;
             unsigned start = msTick();
@@ -819,17 +815,11 @@ class CRoxieFileCache : public CInterface, implements ICopyFileProgress, impleme
                         StringBuffer str;
                         str.appendf("doCopyFile %s", sourceFile->queryFilename());
                         TimeSection timing(str.str());
-                        if (useTreeCopy)
-                            sourceFile->treeCopyTo(destFile, subnet, fromip, true, copyFlags);
-                        else
-                            sourceFile->copyTo(destFile,DEFAULT_COPY_BLKSIZE,NULL,false,copyFlags);
+                        sourceFile->copyTo(destFile,DEFAULT_COPY_BLKSIZE,NULL,false,copyFlags);
                     }
                     else
                     {
-                        if (useTreeCopy)
-                            sourceFile->treeCopyTo(destFile, subnet, fromip, true, copyFlags);
-                        else
-                            sourceFile->copyTo(destFile,DEFAULT_COPY_BLKSIZE,NULL,false,copyFlags);
+                        sourceFile->copyTo(destFile,DEFAULT_COPY_BLKSIZE,NULL,false,copyFlags);
                     }
                 }
                 f->setCopying(false);
@@ -838,26 +828,20 @@ class CRoxieFileCache : public CInterface, implements ICopyFileProgress, impleme
             catch(IException *E)
             {
                 f->setCopying(false);
-                if (!useTreeCopy)
-                { // done by tree copy
-                    EXCLOG(E, "Copy exception - remove templocal");
-                    destFile->remove(); 
-                }
+                EXCLOG(E, "Copy exception - remove templocal");
+                destFile->remove();
                 deleteTempFiles(targetFilename);
                 throw;
             }
             catch(...)
             {
                 f->setCopying(false);
-                if (!useTreeCopy)
-                { // done by tree copy
-                    DBGLOG("%s exception - remove templocal", msg);
-                    destFile->remove(); 
-                }
+                DBGLOG("%s exception - remove templocal", msg);
+                destFile->remove();
                 deleteTempFiles(targetFilename);
                 throw;
             }
-            if (!hardLinkCreated && !useTreeCopy)  // for hardlinks / treeCopy - no rename needed
+            if (!hardLinkCreated)  // for hardlinks no rename needed
             {
                 try
                 {

roxie/ccd/ccdmain.cpp

@@ -78,7 +78,6 @@ unsigned restarts = 0;
 bool heapSort = false;
 bool insertionSort = false;
 bool fieldTranslationEnabled = false;
-bool useTreeCopy = true;
 bool mergeSlaveStatistics = true;
 PTreeReaderOptions defaultXmlReadFlags = ptr_ignoreWhiteSpace;
 bool runOnce = false;
@@ -785,7 +784,6 @@ int STARTQUERY_API start_query(int argc, const char *argv[])
         trapTooManyActiveQueries = topology->getPropBool("@trapTooManyActiveQueries", true);
         maxEmptyLoopIterations = topology->getPropInt("@maxEmptyLoopIterations", 1000);
         maxGraphLoopIterations = topology->getPropInt("@maxGraphLoopIterations", 1000);
-        useTreeCopy = topology->getPropBool("@useTreeCopy", true);
         mergeSlaveStatistics = topology->getPropBool("@mergeSlaveStatistics", true);
 
         enableKeyDiff = topology->getPropBool("@enableKeyDiff", true);

roxie/ccd/ccdstate.cpp

@@ -2782,11 +2782,6 @@ private:
                 else
                     throw MakeStringException(ROXIE_MISSING_PARAMS, "Metric name or regex missing");
             }
-            else if (stricmp(queryName, "control:useTreeCopy")==0)
-            {
-                useTreeCopy = control->getPropBool("@val", true);
-                topology->setPropInt("@useTreeCopy", useTreeCopy);
-            }
             else
                 unknown = true;
             break;