Merge branch 'candidate-5.6.0' into candidate-6.0.0

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

common/environment/environment.cpp

@@ -830,6 +830,12 @@ public:
         str.set(root->queryProp("@directory"));
         return str;
     }
+    virtual IStringVal&     getUMask(IStringVal &str) const
+    {
+        if (root->hasProp("@umask"))
+            str.set(root->queryProp("@umask"));
+        return str;
+    }
 };
 
 #if 0

common/environment/environment.hpp

@@ -108,6 +108,7 @@ interface IConstDropZoneInfo : extends IConstEnvBase
     virtual IStringVal & getComputerName(IStringVal & str) const = 0;
     virtual IStringVal & getDescription(IStringVal & str) const = 0;
     virtual IStringVal & getDirectory(IStringVal & str) const = 0;
+    virtual IStringVal & getUMask(IStringVal & str) const = 0;
 };
 
 interface IConstEnvironment : extends IConstEnvBase

common/remote/rmtssh.cpp

@@ -400,7 +400,7 @@ public:
             }
             if (cmdline.length()==0) {
                 // ssh
-                cmdline.appendf("%s -n -o LogLevel=QUIET -o StrictHostKeyChecking=%s ",usepssh?"pssh":"ssh",strict?"yes":"no");
+                cmdline.appendf("%s -n -o LogLevel=QUIET -o StrictHostKeyChecking=%s ",usepssh?"pssh":"ssh",strict?"yes":"no -o UserKnownHostsFile=/dev/null");
                 if (!usepssh)
                     cmdline.append("-o BatchMode=yes ");
                 if (!identityfile.isEmpty())

dali/dfu/dfuwu.cpp

@@ -2066,6 +2066,17 @@ public:
     {
         queryRoot()->setPropBool("@preserveCompression",val);
     }
+    StringBuffer &getUMask(StringBuffer &str)const
+    {
+        if (queryRoot()->hasProp("@umask"))
+            queryRoot()->getProp("@umask",str);
+        return str;
+    }
+    void setUMask(const char *val)
+    {
+        queryRoot()->setProp("@umask",val);
+
+    }
 };
 
 class CExceptionIterator: public CInterface, implements IExceptionIterator

dali/dfu/dfuwu.hpp

@@ -171,6 +171,7 @@ interface IConstDFUoptions : extends IInterface
     virtual bool getRecordStructurePresent() const = 0;
     virtual bool getQuotedTerminator() const = 0;
     virtual bool getPreserveCompression() const = 0;
+    virtual StringBuffer &getUMask(StringBuffer &str)const =0;
 };
 
 interface IDFUoptions : extends IConstDFUoptions
@@ -209,6 +210,7 @@ interface IDFUoptions : extends IConstDFUoptions
     virtual void setRecordStructurePresent(bool val=false) = 0;
     virtual void setQuotedTerminator(bool val=true) = 0;
     virtual void setPreserveCompression(bool val=true) = 0;
+    virtual void setUMask(const char *val) = 0;
 };
 
 interface IConstDFUfileSpec: extends IInterface

esp/services/ws_fs/ws_fsService.cpp

@@ -2176,15 +2176,15 @@ bool CFileSprayEx::onReplicate(IEspContext &context, IEspReplicate &req, IEspRep
     return true;
 }
 
-const char* CFileSprayEx::getDropZoneDirByIP(const char* ip, StringBuffer& dir)
+void CFileSprayEx::getDropZoneInfoByIP(const char* ip, const char* destFileIn, StringBuffer& destFileOut, StringBuffer& mask)
 {
     if (!ip || !*ip)
-        return NULL;
+        return;
 
     Owned<IEnvironmentFactory> factory = getEnvironmentFactory();
     Owned<IConstEnvironment> env = factory->openEnvironment();
     if (!env)
-        return NULL;
+        return;
 
     Owned<IConstMachineInfo> machine = env->getMachineByAddress(ip);
     if (!machine)
@@ -2192,21 +2192,38 @@ const char* CFileSprayEx::getDropZoneDirByIP(const char* ip, StringBuffer& dir)
         IpAddress ipAddr;
         ipAddr.ipset(ip);
         if (!ipAddr.isLocal())
-            return NULL;
+            return;
         machine.setown(env->getMachineForLocalHost());
         if (!machine)
-            return NULL;
+            return;
     }
-    SCMStringBuffer computer, directory;
+    SCMStringBuffer computer, directory, maskBuf;
     machine->getName(computer);
     if (!computer.length())
-        return NULL;
+        return;
 
     Owned<IConstDropZoneInfo> dropZone = env->getDropZoneByComputer(computer.str());
     if (!dropZone)
-        return NULL;
+        return;
     dropZone->getDirectory(directory);
-    return dir.set(directory.str()).str();
+
+    StringBuffer destFile;
+    if (isAbsolutePath(destFileIn))
+        destFile.set(destFileIn);
+    else
+    {
+        destFile.set(directory.str());
+        if (destFile.length())
+            addPathSepChar(destFile);
+        destFile.append(destFileIn);
+    }
+    destFileOut.set(destFile.str());
+    if ((destFile.length() >= directory.length()) && !strnicmp(destFile.str(), directory.str(), directory.length()))
+    {
+        dropZone->getUMask(maskBuf);
+        if (maskBuf.length())
+            mask.set(maskBuf.str());
+    }
 }
 
 bool CFileSprayEx::onDespray(IEspContext &context, IEspDespray &req, IEspDesprayResponse &resp)
@@ -2256,16 +2273,11 @@ bool CFileSprayEx::onDespray(IEspContext &context, IEspDespray &req, IEspDespray
         {
             RemoteFilename rfn;
             SocketEndpoint ep(destip);
-            if (isAbsolutePath(destfile))
-                rfn.setPath(ep, destfile);
-            else
-            {
-                StringBuffer buf;
-                getDropZoneDirByIP(destip, buf);
-                if (buf.length())
-                    addPathSepChar(buf);
-                rfn.setPath(ep, buf.append(destfile).str());
-            }
+            StringBuffer destfileWithPath, umask;
+            getDropZoneInfoByIP(destip, destfile, destfileWithPath, umask);
+            rfn.setPath(ep, destfileWithPath.str());
+            if (umask.length())
+                options->setUMask(umask.str());
             destination->setSingleFilename(rfn);
         }
         else

esp/services/ws_fs/ws_fsService.hpp

@@ -125,7 +125,7 @@ protected:
     StringBuffer& getAcceptLanguage(IEspContext& context, StringBuffer& acceptLanguage);
     void appendGroupNode(IArrayOf<IEspGroupNode>& groupNodes, const char* nodeName, const char* clusterType, bool replicateOutputs);
     bool getOneDFUWorkunit(IEspContext& context, const char* wuid, IEspGetDFUWorkunitsResponse& resp);
-    const char* getDropZoneDirByIP(const char* destIP, StringBuffer& dir);
+    void getDropZoneInfoByIP(const char* destIP, const char* destFile, StringBuffer& path, StringBuffer& mask);
     void queryDropZoneInfo(const char* dropZone, const char* pathReq, StringBuffer& path, EnvMachineOS& os, IpAddress& ip);
     void addDropZoneFile(IEspContext& context, IDirectoryIterator* di, const char* name, const char* path,
         IArrayOf<IEspPhysicalFileStruct>& filesInFolder, IArrayOf<IEspPhysicalFileStruct>&files);

initfiles/bash/etc/init.d/hpcc-init.in

@@ -235,6 +235,7 @@ while true ; do
                 if [ -z $comp_return ]
                 then
                     log "Unknown component: $comp"
+                    echo "Unknown component: $comp"
                     exit 1
                 fi
                 for (( i=0; i<=${compListLen}; i++ ));do

roxie/ccd/ccdstate.cpp

@@ -3036,6 +3036,11 @@ void mergeStats(IPropertyTree *s1, IPropertyTree *s2)
 
 void mergeQueries(IPropertyTree *dest, IPropertyTree *src)
 {
+    Owned<IPropertyTreeIterator> elems = src->getElements("Exception");
+    ForEach(*elems)
+    {
+        dest->addPropTree("Exception", LINK(&elems->query()));
+    }
     IPropertyTree *destQueries = ensurePTree(dest, "Queries");
     IPropertyTree *srcQueries = src->queryPropTree("Queries");
     if (!srcQueries)

system/jhtree/jhtree.cpp

@@ -531,6 +531,8 @@ public:
         numsegs = 0;
         keyBuffer = NULL;
         keyCursor = NULL;
+        keySize = 0;
+        keyedSize = 0;
         eclKeySize = _eclKeySize;
         started = false;
         setKey(_key);
@@ -609,11 +611,6 @@ public:
                 throw e;
             }
         }
-        else
-        {
-            keySize = 0;
-            keyedSize = 0;
-        }
     }
 
     virtual void reset(bool crappyHack)
@@ -2550,11 +2547,7 @@ public:
             numkeys = _keyset->numParts();
         }
         else
-        {
-            keySize = 0;
-            keyedSize = 0;
             numkeys = 0;
-        }
         killBuffers();
     }