Merge pull request #278 from smeda/bug85258

FIX: 85258 Improve SSH deploy performance

Reviewed-By: Richard Chapman <rchapman@hpccsystems.com>

deployment/deploy/DeployTask.cpp

@@ -25,6 +25,44 @@
 #include "jmutex.hpp"
 #include "xslprocessor.hpp"
 
+offset_t getDirSize(const char* path, bool subdirs = true)
+{
+  offset_t size = 0;
+
+  if (!path || !*path)
+    return size;
+
+  Owned<IFile> pFile = createIFile(path);
+
+  if (pFile->exists())
+  {
+    if (pFile->isDirectory() == foundYes)
+    {
+      Owned<IDirectoryIterator> it = pFile->directoryFiles(NULL, false, true);
+
+      ForEach(*it)
+      {
+        if (it->isDir())
+        {
+          if (subdirs)
+          {
+            StringBuffer subdir;
+            it->getName(subdir);
+            StringBuffer dir(path);
+            dir.append(PATHSEPCHAR).append(subdir);
+            size += getDirSize(dir.str(), subdirs);
+          }
+        }
+        else
+          size += it->getFileSize();
+      }
+    }
+    else
+      size = pFile->size();
+  }
+
+  return size;
+}
 
 class CDeployTask : public CInterface, implements IDeployTask
 {
@@ -583,10 +621,20 @@ public:
            StringBuffer destpath,ip;
            StringBuffer passphr;
            getKeyPassphrase(passphr);
+           StringBuffer sb(source);
+           bool flag = (sb.charAt(sb.length() - 1) == '*');
+
+           if (flag)
+             sb.setLength(sb.length() - 1);
+
+           offset_t dirsize = getDirSize(sb.str());
+
+           if (flag && m_updateProgress)
+             copyProgress.onProgress(0, dirsize);
            
            stripNetAddr(target, destpath, ip);
-           cmdline.appendf("pscp -p -noagent -q -i %s -l %s %s \"%s\" %s:%s", m_sshKeyFile.sget(), m_sshUser.sget(), 
-            passphr.str() , source, ip.str(), destpath.str());
+           cmdline.appendf("pscp -p -noagent -q %s -i %s -l %s %s \"%s\" %s:%s", flag?"-r":"",
+             m_sshKeyFile.sget(), m_sshUser.sget(), passphr.str(), source, ip.str(), destpath.str());
            retcode = pipeSSHCmd(cmdline.str(), outbuf, errbuf);
 
            if (retcode)
@@ -602,7 +650,9 @@ public:
            }
            else
            {
-             if (m_updateProgress)
+             if (flag && m_updateProgress)
+               copyProgress.onProgress(dirsize, dirsize);
+             else if (m_updateProgress)
                copyProgress.onProgress(pSrcFile->size(), pSrcFile->size());
 
              Owned<IDeployTask> task = createDeployTask(*m_pCallback, "Chmod", m_processType, m_compName, m_instanceName, 
@@ -1112,7 +1162,8 @@ public:
                             stripNetAddr(path, destpath, ip);
                             StringBuffer passphr;
                             getKeyPassphrase(passphr);
-                            cmdline.appendf("plink -i %s -l %s %s %s %s %s", m_sshKeyFile.sget(), m_sshUser.sget(), passphr.str(), ip.str(), "mkdir", destpath.str());
+                            cmdline.appendf("plink -i %s -l %s %s %s %s %s", m_sshKeyFile.sget(),
+                              m_sshUser.sget(), passphr.str(), ip.str(), "mkdir -p", destpath.str());
                             retcode = pipeSSHCmd(cmdline.str(), outbuf, errbuf);
 
                             if (retcode && retcode != 1)
@@ -1522,43 +1573,16 @@ public:
        errmsg.append("Invalid SSH params");
        return false;
      }
+
      int retcode;
      Owned<IPipeProcess> pipe = createPipeProcess();
-     StringBuffer cmdline, workdir;
+     StringBuffer cmdline;
      StringBuffer passphr;
      getKeyPassphrase(passphr);
      cmdline.appendf("plink -i %s -l %s %s %s %s", m_sshKeyFile.sget(), m_sshUser.sget(), passphr.str(), ip, cmd);
-     workdir.append("");
-     if (pipe->run("Config",cmdline.str(),workdir,FALSE,true,true)) {
-       byte buf[4096*2];
-       loop {
-         size32_t read = pipe->read(sizeof(buf), buf);
-         output.append(read,(const char *)buf);
-         if (strstr(output.str(), "Passphrase for key "))
-         {
-           m_errorCode = -1;
-           errmsg.clear().append("Invalid or missing key passphrase");
-           pipe->abort();
-           return false;
-         }
-         break;
-       }
-       retcode = pipe->wait();
-       bool firsterr=true;
-       loop {
-         size32_t read = pipe->readError(sizeof(buf), buf);
-         errmsg.append(read,(const char *)buf);
-
-         if (strstr(errmsg.str(), "Wrong passphrase"))
-         {
-           retcode = 5;
-           errmsg.clear().append("Invalid or missing key passphrase. ");
-         }
-         break;
-       }
-     }
-
+     retcode = pipeSSHCmd(cmdline.str(), output, errmsg);
      m_processed = true;
+
      if (retcode && retcode != 1)
      {
        m_errorCode = retcode;
@@ -1577,34 +1601,69 @@ public:
    {
      int retcode = 1;
      Owned<IPipeProcess> pipe = createPipeProcess();
-     StringBuffer workdir;
-     workdir.append("");
-     if (pipe->run("ConfigEnv", cmdline, workdir, FALSE, true, true)) {
+     StringBuffer workdir("");
+
+     if (pipe->run("ConfigEnv", cmdline, workdir, FALSE, true, true))
+     {
        byte buf[4096*2];
-       loop {
-         size32_t read = pipe->read(sizeof(buf), buf);
-         outbuf.append(read,(const char *)buf);
-         if (strstr(outbuf.str(), "Passphrase for key "))
-         {
-           retcode = 5;
-           errbuf.clear().append("Invalid or missing key passphrase. ");
-           pipe->abort();
-           return retcode;
-         }
-         break;
+       size32_t read = pipe->read(sizeof(buf), buf);
+       outbuf.append(read,(const char *)buf);
+
+       if (strstr(outbuf.str(), "Passphrase for key "))
+       {
+         retcode = 5;
+         errbuf.clear().append("Invalid or missing key passphrase. ");
+         pipe->abort();
+         return retcode;
        }
+
        retcode = pipe->wait();
-       bool firsterr=true;
-       loop {
-         size32_t read = pipe->readError(sizeof(buf), buf);
-         errbuf.append(read,(const char *)buf);
-         if (strstr(errbuf.str(), "Wrong passphrase"))
+       read = pipe->readError(sizeof(buf), buf);
+       errbuf.append(read,(const char *)buf);
+
+       if (strstr(errbuf.str(), "Wrong passphrase"))
+       {
+         retcode = -1;
+         errbuf.clear().append("Invalid or missing key passphrase");
+       }
+       else if (strstr(errbuf.str(), "The server's host key is not cached in the registry."))
+       {
+         Owned<IPipeProcess> pipe1 = createPipeProcess();
+         StringBuffer cmd;
+         cmd.appendf("cmd /c \" echo y | %s \"", cmdline);
+
+         if (pipe1->run("ConfigEnv",cmd.str(),workdir,FALSE,true,true))
          {
-           retcode = -1;
-           errbuf.clear().append("Invalid or missing key passphrase");
+           size32_t read = pipe1->read(sizeof(buf), buf);
+           outbuf.append(read,(const char *)buf);
+           retcode = pipe1->wait();
+           read = pipe->readError(sizeof(buf), buf);
+
+           if (read)
+           {
+             errbuf.append(" ").append(read,(const char *)buf);
+             retcode = 7;
+           }
+           else
+             errbuf.clear();
+           }
+         }
+         else if (errbuf.length())
+         {
+           const char* psz = strstr(errbuf.str(), "Authenticating with public key \"");
+
+           if (!psz || psz != errbuf.str())
+             retcode = 2;
+           else
+           {
+             const char* psz1 = psz + strlen("Authenticating with public key \"");
+             const char* psz2 = strstr(psz1, "\"\r\n");
+             psz1 = psz2 + strlen("\"\r\n");
+
+             if (strlen(psz1))
+               retcode = 2;
+           }
          }
-         break;
-       }
      }
 
      return retcode;

deployment/deploy/DeploymentEngine.cpp

@@ -155,14 +155,45 @@ int CDeploymentEngine::getInstallFileCount()
     //return getInstallFiles().getInstallFileList().size(); 
 
     // Only count these we can handle properly
-    int count = 0;
+    int count = 0, xslcount = 0, total = 0;
     for (CInstallFileList::const_iterator it=files.begin(); it!=files.end(); ++it)
     {
-        const StlLinked<CInstallFile>& f = *it;
-        if (isMethodTrackable(f->getMethod().c_str()))
-            count++;
+      const StlLinked<CInstallFile>& f = *it;
+      const char* method = f->getMethod().c_str();
+      if (strieq(method,"copy"))
+        count++;
+      else if (startsWith(method,"xsl"))
+        xslcount++;
     }
-    return count;
+
+    bool isCached = m_instances.length() > 1;
+    total = isCached ? count : 0;
+    const char* depToFolder = m_envDepEngine.getDeployToFolder();
+
+    ForEachItemIn(idx, m_instances)
+    {
+      IPropertyTree& instance = m_instances.item(idx);
+      StringAttr curSSHUser, curSSHKeyFile, curSSHKeyPassphrase;
+      m_envDepEngine.getSSHAccountInfo(instance.queryProp("@computer"),
+        curSSHUser, curSSHKeyFile, curSSHKeyPassphrase);
+      total += xslcount;
+
+      if (m_useSSHIfDefined && !curSSHKeyFile.isEmpty() &&
+        !curSSHUser.isEmpty() && !(depToFolder && *depToFolder))
+      {
+          total += 1;
+
+          if (!isCached)
+          {
+            isCached = true;
+            total += count;
+          }
+      }
+      else
+        total += count;
+    }
+
+  return total;
 }
 
 //---------------------------------------------------------------------------
@@ -172,12 +203,16 @@ int CDeploymentEngine::getInstallFileCount()
 offset_t CDeploymentEngine::getInstallFileSize()
 {
     const CInstallFileList& files = getInstallFiles().getInstallFileList();
-    offset_t totalSize = 0;
+    offset_t fileSize = 0, xslSize = 0, total = 0;
+
     for (CInstallFileList::const_iterator it=files.begin(); it!=files.end(); ++it)
     {
-        const StlLinked<CInstallFile>& f = *it;
-        if (isMethodTrackable(f->getMethod().c_str()))
-            totalSize += f->getSrcSize();
+      const StlLinked<CInstallFile>& f = *it;
+      const char* method = f->getMethod().c_str();
+      if (strieq(method,"copy"))
+        fileSize += f->getSrcSize();
+      else if (startsWith(method,"xsl"))
+        xslSize += f->getSrcSize();
         
         /* debug
         if (f->getSrcSize()==24331)
@@ -186,7 +221,32 @@ offset_t CDeploymentEngine::getInstallFileSize()
             ::MessageBox((HWND)m_pCallback->getWindowHandle(), s.str(), "UNCOPIED",MB_OK);
         }*/
     }
-    return totalSize;
+
+    bool isCached = m_instances.length() > 1;
+    total = isCached ? fileSize : 0;
+    const char* depToFolder = m_envDepEngine.getDeployToFolder();
+
+    ForEachItemIn(idx, m_instances)
+    {
+      total += fileSize + xslSize;
+
+      if (!isCached)
+      {
+        IPropertyTree& instance = m_instances.item(idx);
+        StringAttr curSSHUser, curSSHKeyFile, curSSHKeyPassphrase;
+        m_envDepEngine.getSSHAccountInfo(instance.queryProp("@computer"), curSSHUser,
+          curSSHKeyFile, curSSHKeyPassphrase);
+
+        if (!curSSHKeyFile.isEmpty() && !m_curSSHUser.isEmpty() &&
+          !(depToFolder && *depToFolder))
+        {
+          isCached = true;
+          total += fileSize;
+        }
+      }
+    }
+
+  return total;
 }
 
 //---------------------------------------------------------------------------
@@ -209,12 +269,7 @@ void CDeploymentEngine::start()
             checkAbort();
             IPropertyTree& instance = m_instances.item(idx);
             m_curInstance = instance.queryProp("@name");
-            m_curSSHUser.clear();
-            m_curSSHKeyFile.clear();
-            m_curSSHKeyPassphrase.clear();
-            m_envDepEngine.getSSHAccountInfo(instance.queryProp("@computer"), m_curSSHUser, m_curSSHKeyFile, m_curSSHKeyPassphrase);
-            if (m_useSSHIfDefined)
-                m_useSSHIfDefined = !m_curSSHKeyFile.isEmpty();
+            setSSHVars(instance);
 
             try
             {
@@ -254,7 +309,7 @@ void CDeploymentEngine::startInstance(IPropertyTree& node, const char* fileName/
         "Starting %s process on %s", m_name.get(), hostDir.get());
     
     Owned<IDeployTask> task;
-    if (m_useSSHIfDefined && os == MachineOsLinux && !m_curSSHUser.isEmpty() && !m_curSSHKeyFile.isEmpty())
+    if (m_useSSHIfDefined)
     {
         const char* computer = node.queryProp("@computer");
         if (!computer || !*computer)
@@ -317,13 +372,8 @@ void CDeploymentEngine::stop()
             
             IPropertyTree& instance = m_instances.item(idx);
             m_curInstance = instance.queryProp("@name");
-            m_curSSHUser.clear();
-            m_curSSHKeyFile.clear();
-            m_curSSHKeyPassphrase.clear();
+            setSSHVars(instance);
 
-            m_envDepEngine.getSSHAccountInfo(instance.queryProp("@computer"), m_curSSHUser, m_curSSHKeyFile, m_curSSHKeyPassphrase);
-            if (m_useSSHIfDefined)
-                m_useSSHIfDefined = !m_curSSHKeyFile.isEmpty();
             
             try
             {
@@ -362,7 +412,7 @@ void CDeploymentEngine::stopInstance(IPropertyTree& node, const char* fileName/*
         "Stopping %s process on %s", m_name.get(), hostDir.get());
     
     Owned<IDeployTask> task;
-    if (m_useSSHIfDefined && os == MachineOsLinux && !m_curSSHUser.isEmpty() && !m_curSSHKeyFile.isEmpty())
+    if (m_useSSHIfDefined)
     {
         const char* computer = node.queryProp("@computer");
         if (!computer || !*computer)
@@ -522,7 +572,20 @@ void CDeploymentEngine::beforeDeploy()
     getTempPath(tempPath, sizeof(tempPath), m_name);
 
     ensurePath(tempPath);
-    if (m_instances.ordinality() > 1)
+    bool cacheFiles = false;
+    const char* depToFolder = m_envDepEngine.getDeployToFolder();
+
+    if (!m_compare && m_instances.ordinality() == 1 && !(depToFolder && *depToFolder))
+    {
+      IPropertyTree& instanceNode = m_instances.item(0);
+      const char* curInstance = instanceNode.queryProp("@name");
+      StringAttr sbSSHUser, sbSSHKeyFile, sbKeyPassphrase;
+      m_envDepEngine.getSSHAccountInfo(instanceNode.queryProp("@computer"),
+        sbSSHUser, sbSSHKeyFile, sbKeyPassphrase);
+      cacheFiles = !sbSSHKeyFile.isEmpty();
+    }
+
+    if (m_instances.ordinality() > 1 || cacheFiles)
     {
         strcat(tempPath, "Cache");
         char* pszEnd = tempPath + strlen(tempPath);
@@ -564,9 +627,7 @@ void CDeploymentEngine::_deploy(bool useTempDir)
         
         IPropertyTree& instanceNode = m_instances.item(idx);
         m_curInstance = instanceNode.queryProp("@name");
-        m_envDepEngine.getSSHAccountInfo(instanceNode.queryProp("@computer"), m_curSSHUser, m_curSSHKeyFile, m_curSSHKeyPassphrase);
-        if (m_useSSHIfDefined)
-            m_useSSHIfDefined = !m_curSSHKeyFile.isEmpty();
+        setSSHVars(instanceNode);
         
         try
         {
@@ -672,11 +733,7 @@ void CDeploymentEngine::backupDirs()
         
         IPropertyTree& instance = m_instances.item(idx);
         m_curInstance = instance.queryProp("@name");
-        m_curSSHUser.clear();
-        m_curSSHKeyFile.clear();
-        m_curSSHKeyPassphrase.clear();
-        m_envDepEngine.getSSHAccountInfo(instance.queryProp("@computer"), m_curSSHUser, m_curSSHKeyFile, m_curSSHKeyPassphrase);
-        m_useSSHIfDefined = !m_curSSHKeyFile.isEmpty();
+        setSSHVars(instance);
         
         try
         {
@@ -1405,6 +1462,8 @@ void CDeploymentEngine::copyInstallFiles(const char* instanceName, int instanceI
         const CInstallFileList& fileList = m_installFiles.getInstallFileList();
         int nItems = fileList.size();
         int n;
+        bool recCopyDone = false;
+
         for (n=0; n<nItems; n++)
         {
             CInstallFile& installFile = *fileList[n];
@@ -1431,6 +1490,32 @@ void CDeploymentEngine::copyInstallFiles(const char* instanceName, int instanceI
                 if (params && !*params)
                     params = NULL;
 
+                if (m_useSSHIfDefined && !bCacheFiles && !m_compare &&
+                    !strcmp(method, "copy") && strcmp(instanceName, "Cache"))
+                {
+                  if (!recCopyDone)
+                  {
+                    StringBuffer sbsrc(source);
+
+                    if (strrchr(source, PATHSEPCHAR))
+                      sbsrc.setLength(strrchr(source, PATHSEPCHAR) - source + 1);
+
+                    sbsrc.append("*");
+                    StringBuffer sbdst(dest.c_str());
+
+                    if (strrchr(sbdst.str(), PATHSEPCHAR))
+                      sbdst.setLength(strrchr(sbdst.str(), PATHSEPCHAR) - sbdst.str() + 1);
+
+                    Owned<IDeployTask> task = createDeployTask(*m_pCallback, "Copy Directory", m_process.queryName(), m_name.get(),
+                      m_curInstance, sbsrc.str(), sbdst.str(), m_curSSHUser.sget(), m_curSSHKeyFile.sget(), m_curSSHKeyPassphrase.sget(), m_useSSHIfDefined, os);
+                    task->setFlags(m_deployFlags & DCFLAGS_ALL);
+                    m_threadPool->start(task);
+                    recCopyDone = true;
+                  }
+
+                  continue;
+                }
+
                 if (!processInstallFile(m_process, instanceName, method, source, dest.c_str(), os, bCacheable, params))
                     break;
 
@@ -2575,3 +2660,17 @@ bool CDeploymentEngine::checkSSHFileExists(const char* dir) const
 
   return flag;
 }
+
+void CDeploymentEngine::setSSHVars(IPropertyTree& instance)
+{
+  EnvMachineOS os = m_envDepEngine.lookupMachineOS(instance);
+  m_curSSHUser.clear();
+  m_curSSHKeyFile.clear();
+  m_curSSHKeyPassphrase.clear();
+  m_envDepEngine.getSSHAccountInfo(instance.queryProp("@computer"), m_curSSHUser, m_curSSHKeyFile, m_curSSHKeyPassphrase);
+  m_useSSHIfDefined = !m_curSSHKeyFile.isEmpty() && !m_curSSHUser.isEmpty() && os == MachineOsLinux;
+  const char* depToFolder = m_envDepEngine.getDeployToFolder();
+
+  if (depToFolder && *depToFolder)
+    m_useSSHIfDefined = false;
+}

deployment/deploy/DeploymentEngine.hpp

@@ -304,6 +304,7 @@ private:
     bool searchDeployMap(const char* fileName, const char* optionalFileExt) const;
     void getBackupDirName(const char* from, StringBuffer& to);
     bool checkSSHFileExists(const char* dir) const;
+    void setSSHVars(IPropertyTree& instance);
 
 protected:
    Owned<IThreadPool>      m_threadPool;