Merge pull request #9540 from AttilaVamos/HPCC-16946-improvement-6.4.0

HPCC-16946 Add server list handling to CLocalEnvironment

Reviewed-By: Jake Smith <jake.smith@lexisnexis.com>
Reviewed-By: Richard Chapman <rchapman@hpccsystems.com>

common/environment/environment.cpp

@@ -59,6 +59,24 @@ protected:
     unsigned maxIndex = 0;
 };
 
+class CConstDropZoneServerInfoIterator : public CSimpleInterfaceOf<IConstDropZoneServerInfoIterator>
+{
+public:
+    CConstDropZoneServerInfoIterator(const IConstDropZoneInfo * dropZone);
+
+    virtual bool first() override;
+    virtual bool next() override;
+    virtual bool isValid() override;
+    virtual IConstDropZoneServerInfo & query() override;
+    virtual unsigned count() const override;
+
+protected:
+    Owned<IConstDropZoneServerInfo> curr;
+    Owned<CLocalEnvironment> constEnv;
+    Owned<IPropertyTreeIterator> serverListIt;
+    unsigned maxIndex = 0;
+};
+
 class CConstDropZoneInfoIterator : public CSimpleInterfaceOf<IConstDropZoneInfoIterator>
 {
 public:
@@ -918,6 +936,28 @@ public:
 
 };
 
+class CConstDropZoneServerInfo : public CConstEnvBase, implements IConstDropZoneServerInfo
+{
+public:
+    IMPLEMENT_IINTERFACE;
+    IMPLEMENT_ICONSTENVBASE;
+    CConstDropZoneServerInfo(CLocalEnvironment *env, IPropertyTree *root) : CConstEnvBase(env, root), prop(root) {}
+
+    virtual StringBuffer & getName(StringBuffer & name) const
+    {
+        name.append(prop->queryProp("@name"));
+        return name;
+    }
+    virtual StringBuffer & getServer(StringBuffer & server) const
+    {
+        server.append(prop->queryProp("@server"));
+        return server;
+    }
+
+private:
+    IPropertyTree * prop;
+};
+
 class CConstDropZoneInfo : public CConstEnvBase, implements IConstDropZoneInfo
 {
 public:
@@ -950,6 +990,10 @@ public:
     {
         return root->getPropBool("@ECLWatchVisible", true);
     }
+    virtual IConstDropZoneServerInfoIterator * getServers() const
+    {
+        return new CConstDropZoneServerInfoIterator(this);
+    }
 };
 
 #if 0
@@ -1586,53 +1630,62 @@ void CLocalEnvironment::clearCache()
 
 IConstDropZoneInfo * CLocalEnvironment::getDropZoneByAddressPath(const char * netaddress, const char *targetFilePath) const
 {
-    IConstDropZoneInfo * retVal = nullptr;
-
-    Owned<IConstMachineInfo> machineInfo = getMachineByAddress(netaddress);
-    if (!machineInfo)
-        return retVal;
+    IConstDropZoneInfo * dropZone = nullptr;
+    IpAddress targetIp(netaddress);
+    unsigned dropzonePathLen = _MAX_PATH + 1;
 
-    SCMStringBuffer machineName;
-    machineInfo->getName(machineName);
-    const char * pmachineName = machineName.str();
-    Owned<IConstDropZoneInfoIterator> zoneIt= getDropZoneIteratorByComputer(pmachineName);
 #ifdef _DEBUG
-    LOG(MCdebugInfo, unknownJob, "Check remote drop zones (%d) by iterator on '%s':", zoneIt->count(), pmachineName);
+    LOG(MCdebugInfo, unknownJob, "Netaddress: '%s', targetFilePath: '%s'", netaddress, targetFilePath);
 #endif
+    // Check the directory path first
 
-    SCMStringBuffer dropzonePath;
-    const char * pdropzonePath = nullptr;
-    unsigned dropzonePathLen = _MAX_PATH + 1;
-
+    Owned<IConstDropZoneInfoIterator> zoneIt = getDropZoneIterator();
     ForEach(*zoneIt)
     {
         SCMStringBuffer dropZoneDir;
         zoneIt->query().getDirectory(dropZoneDir);
         StringBuffer fullDropZoneDir(dropZoneDir.str());
         addPathSepChar(fullDropZoneDir);
-        const char * pdropzoneDir = fullDropZoneDir.str();
-
-        // Check target file path starts with this Drop zone path
-        // the drop zone paths can be nested (nothing forbids it) like
-        // dz1: /a/b/c/d
-        // dz2: /a/b/c
-        // dz3: /a/b
-        // check all and select the shortest
+        IConstDropZoneInfo * candidateDropZone = nullptr;
 
-        if (strncmp(pdropzoneDir, targetFilePath, strlen(pdropzoneDir)) == 0)
+        if (strncmp(fullDropZoneDir, targetFilePath, fullDropZoneDir.length()) == 0)
         {
-            if (dropzonePathLen > strlen(pdropzoneDir))
+            candidateDropZone = &zoneIt->query();
+
+            // The backward compatibility built in IConstDropZoneServerInfoIterator
+            Owned<IConstDropZoneServerInfoIterator> dropzoneServerListIt = candidateDropZone->getServers();
+            ForEach(*dropzoneServerListIt)
             {
-                dropzonePathLen = strlen(pdropzoneDir);
-                dropzonePath.set(pdropzoneDir);
-                pdropzonePath = dropzonePath.str();
-                retVal = &zoneIt->query();
+                StringBuffer dropzoneServer;
+                dropzoneServerListIt->query().getServer(dropzoneServer);
+                // It can be a hostname or an IP -> get the IP
+                IpAddress serverIP(dropzoneServer.str());
+
+#ifdef _DEBUG
+                StringBuffer serverIpString;
+                serverIP.getIpText(serverIpString);
+                LOG(MCdebugInfo, unknownJob, "Listed server: '%s', IP: '%s'", dropzoneServer.str(), serverIpString.str());
+#endif
+                if (targetIp.ipequals(serverIP))
+                {
+                    // OK the target is a valid machine in the server list we have a right drop zone candidate
+                    // Keep this candidate drop zone if its directory path is shorter than we already have
+                    if (dropzonePathLen > fullDropZoneDir.length())
+                    {
+                       dropzonePathLen = fullDropZoneDir.length();
+                       dropZone = candidateDropZone;
+                    }
+                    break;
+                }
             }
         }
     }
-    return retVal;
+    // Both path and machine matched
+    return dropZone;
 }
 
+
+
 IConstDropZoneInfoIterator * CLocalEnvironment::getDropZoneIteratorByComputer(const char * computer) const
 {
     return new CConstDropZoneIteratorByComputer(computer);
@@ -1749,6 +1802,89 @@ unsigned CConstDropZoneIteratorByComputer::count() const
 
 //--------------------------------------------------
 
+
+CConstDropZoneServerInfoIterator::CConstDropZoneServerInfoIterator(const IConstDropZoneInfo * dropZone)
+{
+    Owned<IEnvironmentFactory> factory = getEnvironmentFactory();
+    constEnv.setown((CLocalEnvironment *)factory->openEnvironment());
+
+    // For backward compatibility
+    SCMStringBuffer dropZoneMachineName;
+    // Returns dropzone '@computer' value if it is exists
+    dropZone->getComputerName(dropZoneMachineName);
+
+    if (0 != dropZoneMachineName.length())
+    {
+        // Create a ServerList for legacy element.
+        Owned<IPropertyTree> legacyServerList = createPTree();
+
+        Owned<IConstMachineInfo> machineInfo = constEnv->getMachine(dropZoneMachineName.str());
+        if (machineInfo)
+        {
+            SCMStringBuffer dropZoneMachineNetAddress;
+            machineInfo->getNetAddress(dropZoneMachineNetAddress);
+
+            // Create a single ServerList record related to @computer
+            //<ServerList name="ServerList" server="<IP_of_@computer>"/>
+            Owned<IPropertyTree> newRecord = createPTree();
+            newRecord->setProp("@name", "ServerList");
+            newRecord->setProp("@server", dropZoneMachineNetAddress.str());
+            legacyServerList->addPropTree("ServerList",newRecord.getClear());
+
+            maxIndex = 1;
+        }
+        else
+        {
+            // Something is terrible wrong because there is no matching machine for DropZone @computer
+            maxIndex = 0;
+        }
+        serverListIt.setown(legacyServerList->getElements("ServerList"));
+    }
+    else
+    {
+        Owned<IPropertyTree> pSrc = &dropZone->getPTree();
+        serverListIt.setown(pSrc->getElements("ServerList"));
+        maxIndex = pSrc->getCount("ServerList");
+    }
+}
+
+bool CConstDropZoneServerInfoIterator::first()
+{
+    bool hasFirst = serverListIt->first();
+    if (hasFirst)
+        curr.setown(new CConstDropZoneServerInfo(constEnv, &serverListIt->query()));
+    else
+        curr.clear();
+    return hasFirst;
+}
+
+bool CConstDropZoneServerInfoIterator::next()
+{
+    bool hasNext = serverListIt->next();
+    if (hasNext)
+        curr.setown(new CConstDropZoneServerInfo(constEnv, &serverListIt->query()));
+    else
+        curr.clear();
+    return hasNext;
+}
+
+bool CConstDropZoneServerInfoIterator::isValid()
+{
+    return nullptr != curr;
+}
+
+IConstDropZoneServerInfo & CConstDropZoneServerInfoIterator::query()
+{
+    return *curr;
+}
+
+unsigned CConstDropZoneServerInfoIterator::count() const
+{
+    return maxIndex;
+}
+
+//--------------------------------------------------
+
 CConstDropZoneInfoIterator::CConstDropZoneInfoIterator()
 {
     Owned<IEnvironmentFactory> factory = getEnvironmentFactory();

common/environment/environment.hpp

@@ -104,6 +104,17 @@ interface IConstInstanceInfo : extends IConstEnvBase
     virtual bool getRunInfo(IStringVal & progpath, IStringVal & workdir, const char * defaultprogname) const = 0;
 };
 
+interface IConstDropZoneServerInfo : extends IConstEnvBase
+{
+    virtual StringBuffer & getName(StringBuffer & name) const = 0;
+    virtual StringBuffer & getServer(StringBuffer & server) const = 0;
+};
+
+interface IConstDropZoneServerInfoIterator : extends IIteratorOf<IConstDropZoneServerInfo>
+{
+    virtual unsigned count() const = 0;
+};
+
 interface IConstDropZoneInfo : extends IConstEnvBase
 {
     virtual IStringVal & getComputerName(IStringVal & str) const = 0;
@@ -111,6 +122,7 @@ interface IConstDropZoneInfo : extends IConstEnvBase
     virtual IStringVal & getDirectory(IStringVal & str) const = 0;
     virtual IStringVal & getUMask(IStringVal & str) const = 0;
     virtual bool isECLWatchVisible() const = 0;
+    virtual IConstDropZoneServerInfoIterator * getServers() const = 0;
 };
 
 interface  IConstDropZoneInfoIterator : extends IIteratorOf<IConstDropZoneInfo>