Merge branch 'candidate-5.6.0' into candidate-6.0.0

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

common/remote/hooks/git/gitfile.cpp

@@ -238,6 +238,7 @@ public:
     virtual void rename(const char *newTail) { UNIMPLEMENTED; }
     virtual void move(const char *newName) { UNIMPLEMENTED; }
     virtual void setReadOnly(bool ro) { UNIMPLEMENTED; }
+    virtual void setFilePermissions(unsigned fPerms) { UNIMPLEMENTED; }
     virtual bool setCompression(bool set) { UNIMPLEMENTED; }
     virtual offset_t compressedSize() { UNIMPLEMENTED; }
     virtual unsigned getCRC() { UNIMPLEMENTED; }

common/remote/hooks/libarchive/archive.cpp

@@ -399,6 +399,7 @@ public:
     virtual void rename(const char *newTail) { UNIMPLEMENTED; }
     virtual void move(const char *newName) { UNIMPLEMENTED; }
     virtual void setReadOnly(bool ro) { UNIMPLEMENTED; }
+    virtual void setFilePermissions(unsigned fPerms) { UNIMPLEMENTED; }
     virtual bool setCompression(bool set) { UNIMPLEMENTED; }
     virtual offset_t compressedSize() { UNIMPLEMENTED; }
     virtual unsigned getCRC() { UNIMPLEMENTED; }

common/remote/sockfile.cpp

@@ -312,6 +312,7 @@ enum {
     RFCsetthrottle, // legacy version
 // 1.9
     RFCsetthrottle2,
+    RFCsetfileperms,
     RFCmax,
     RFCunknown = 255 // 0 would have been more sensible, but can't break backward compatibility
 };
@@ -359,6 +360,7 @@ const char *RFCStrings[] =
     RFCText(RFCtreecopytmp),
     RFCText(RFCsetthrottle), // legacy version
     RFCText(RFCsetthrottle2),
+    RFCText(RFCsetfileperms),
     RFCText(RFCunknown),
 };
 static const char *getRFCText(RemoteFileCommandType cmd) { return RFCStrings[cmd]; }
@@ -1743,6 +1745,14 @@ public:
         sendRemoteCommand(sendBuffer, replyBuffer);
     }
 
+    void setFilePermissions(unsigned fPerms)
+    {
+        MemoryBuffer sendBuffer;
+        initSendBuffer(sendBuffer);
+        MemoryBuffer replyBuffer;
+        sendBuffer.append((RemoteFileCommandType)RFCsetfileperms).append(filename).append(fPerms);
+        sendRemoteCommand(sendBuffer, replyBuffer);
+    }
 
     offset_t size()
     {
@@ -2981,6 +2991,10 @@ inline void appendErr3(MemoryBuffer &reply, RemoteFileCommandType e, int code, c
 {
     StringBuffer msg;
     msg.appendf("ERROR: %s(%d) '%s'", getRFCText(e), code, errMsg?errMsg:"");
+    // some errors are RemoteFileCommandType, some are RFSERR_*
+    // RFCOpenIO needs remapping to non-zero for client to know its an error
+    if ((RemoteFileCommandType)e == RFCopenIO)
+        e = (RemoteFileCommandType)RFSERR_OpenFailed;
     reply.append((unsigned)e);
     reply.append(msg.str());
 }
@@ -4339,6 +4353,20 @@ public:
         return true;
     }
 
+    bool cmdSetFilePerms(MemoryBuffer &msg, MemoryBuffer &reply, CRemoteClientHandler &client)
+    {
+        IMPERSONATE_USER(client);
+        StringAttr name;
+        unsigned fPerms;
+        msg.read(name).read(fPerms);
+        if (TF_TRACE)
+            PROGLOG("setFilePerms,  '%s' 0%o",name.get(),fPerms);
+        Owned<IFile> file=createIFile(name);
+        file->setFilePermissions(fPerms);
+        reply.append((unsigned)RFEnoerror);
+        return true;
+    }
+
     bool cmdGetTime(MemoryBuffer &msg, MemoryBuffer &reply, CRemoteClientHandler &client)
     {
         IMPERSONATE_USER(client);
@@ -4889,6 +4917,7 @@ public:
             case RFCisdirectory:
             case RFCisreadonly:
             case RFCsetreadonly:
+            case RFCsetfileperms:
             case RFCgettime:
             case RFCsettime:
             case RFCcreatedir:
@@ -4940,6 +4969,7 @@ public:
                 MAPCOMMANDCLIENT(RFCisdirectory, cmdIsDir, *client);
                 MAPCOMMANDCLIENT(RFCisreadonly, cmdIsReadOnly, *client);
                 MAPCOMMANDCLIENT(RFCsetreadonly, cmdSetReadOnly, *client);
+                MAPCOMMANDCLIENT(RFCsetfileperms, cmdSetFilePerms, *client);
                 MAPCOMMANDCLIENT(RFCgettime, cmdGetTime, *client);
                 MAPCOMMANDCLIENT(RFCsettime, cmdSetTime, *client);
                 MAPCOMMANDCLIENT(RFCcreatedir, cmdCreateDir, *client);

dali/ft/filecopy.cpp

@@ -816,6 +816,25 @@ void FileSprayer::beforeTransfer()
         checker.For(targets.ordinality(), 25, true, true);
     }
 
+    int umask = -1;
+    if (options->hasProp("@umask"))
+    {
+        StringBuffer umaskStr;
+        options->getProp("@umask", umaskStr);
+        errno = 0;
+        umask = (int)strtol(umaskStr.str(), NULL, 8);
+        if (errno)
+        {
+            LOG(MCdebugInfo, job, "Invalid umask value <%s> ignored", umaskStr.str());
+            umask = -1;
+        }
+        else
+        {
+            // never strip off owner
+            umask &= 077;
+        }
+    }
+
     if (!isRecovering && !usePullOperation())
     {
         try {
@@ -830,7 +849,12 @@ void FileSprayer::beforeTransfer()
                     splitUNCFilename(remoteFilename.str(), &remoteDirectory, &remoteDirectory, NULL, NULL);
 
                     Owned<IFile> dir = createIFile(remoteDirectory.str());
-                    dir->createDirectory();
+                    if (!dir->exists())
+                    {
+                        dir->createDirectory();
+                        if (umask != -1)
+                            dir->setFilePermissions(~umask&0777);
+                    }
                 }
             }
         }
@@ -868,6 +892,8 @@ void FileSprayer::beforeTransfer()
                     remote.getPath(name);
                     throwError1(DFTERR_CouldNotCreateOutput, name.str());
                 }
+                if (umask != -1)
+                    file->setFilePermissions(~umask&0666);
                 //Create the headers on the utf files.
                 unsigned headerSize = getHeaderSize(tgtFormat.type);
                 if (headerSize)

dali/rfs/rfs.cpp

@@ -163,6 +163,9 @@ enum {
     RFCcopysection,
     RFCtreecopy,
     RFCtreecopytmp,
+    RFCsetthrottle,
+    RFCsetthrottle2,
+    RFCsetfileperms,
 #endif
     RFCmax,
     RFCnone = 255

esp/services/ws_fs/ws_fsService.cpp

@@ -2178,6 +2178,9 @@ bool CFileSprayEx::onReplicate(IEspContext &context, IEspReplicate &req, IEspRep
 
 void CFileSprayEx::getDropZoneInfoByIP(const char* ip, const char* destFileIn, StringBuffer& destFileOut, StringBuffer& mask)
 {
+    if (destFileIn && *destFileIn)
+        destFileOut.set(destFileIn);
+
     if (!ip || !*ip)
         return;
 

initfiles/componentfiles/configxml/dropzone.xsd.in

@@ -134,6 +134,13 @@
                     </xs:appinfo>
                 </xs:annotation>
             </xs:attribute>
+            <xs:attribute name="umask" type="xs:string" use="optional" default="022">
+                <xs:annotation>
+                    <xs:appinfo>
+                        <tooltip>umask for desprayed files</tooltip>
+                    </xs:appinfo>
+                </xs:annotation>
+            </xs:attribute>
         </xs:complexType>
     </xs:element>
 </xs:schema>

system/jlib/jfile.cpp

@@ -921,6 +921,17 @@ void CFile::setReadOnly(bool ro)
 #endif
 }
 
+void CFile::setFilePermissions(unsigned fPerms)
+{
+#ifndef _WIN32
+    struct stat info;
+    if (stat(filename, &info) != 0)
+        throw makeErrnoExceptionV("CFile::setFilePermissions() %s", filename.get());
+    if (chmod(filename, fPerms&0777) != 0)
+        throw makeErrnoExceptionV("CFile::setFilePermissions() %s", filename.get());
+#endif
+}
+
 
 offset_t CFile::size()
 {
@@ -1388,7 +1399,12 @@ public:
     virtual void setReadOnly(bool ro)
     {
         connect();
-        ifile->setReadOnly(ro);             
+        ifile->setReadOnly(ro);
+    }
+    virtual void setFilePermissions(unsigned fPerms)
+    {
+        connect();
+        ifile->setFilePermissions(fPerms);
     }
     virtual offset_t size()
     {

system/jlib/jfile.hpp

@@ -96,6 +96,7 @@ interface IFile :extends IInterface
     virtual void rename(const char *newTail) = 0;       // tail only preferred but can have full path if exactly matches existing dir
     virtual void move(const char *newName) = 0;         // can move between directories on same node (NB currently not always supported on remote files!)
     virtual void setReadOnly(bool ro) = 0;
+    virtual void setFilePermissions(unsigned fPerms) = 0;
     virtual offset_t size() = 0;
     virtual bool setCompression(bool set) = 0;
     virtual offset_t compressedSize() = 0;

system/jlib/jfile.ipp

@@ -51,6 +51,7 @@ public:
     virtual void rename(const char *newTail);
     virtual void move(const char *newName);
     virtual void setReadOnly(bool ro);
+    virtual void setFilePermissions(unsigned fPerms);
     virtual offset_t size();
     virtual bool setCompression(bool set);
     virtual offset_t compressedSize();

testing/unittests/jlibtests.cpp

@@ -414,6 +414,15 @@ protected:
                 extraFlags = IFEnocache;
             ifileio = ifile->open(IFOcreate, extraFlags);
 
+            try
+            {
+                ifile->setFilePermissions(0666);
+            }
+            catch (...)
+            {
+                fprintf(stdout, "ifile->setFilePermissions() exception\n");
+            }
+
             unsigned iter = nr / 40;
 
             __int64 pos = 0;