Merge remote-tracking branch 'origin/closedown-4.2.x'

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

common/thorhelper/enginecontext.hpp

@@ -24,6 +24,7 @@
 interface IEngineContext
 {
     virtual DALI_UID getGlobalUniqueIds(unsigned num, SocketEndpoint *_foreignNode) = 0;
+    virtual bool allowDaliAccess() const = 0;
 };
 
 #endif // ENGINECONTEXT_HPP

common/workunit/workunit.cpp

@@ -4366,6 +4366,7 @@ class CEnvironmentClusterInfo: public CInterface, implements IConstWUClusterInfo
     SocketEndpointArray roxieServers;
     StringAttr thorQueue;
     StringArray thorProcesses;
+    StringArray primaryThorProcesses;
     StringAttr prefix;
     ClusterType platform;
     unsigned clusterWidth;
@@ -4379,10 +4380,20 @@ public:
         {
             thorQueue.set(getClusterThorQueueName(queue.clear(), name));
             clusterWidth = 0;
+            bool isMultiThor = (thors.length() > 1);
             ForEachItemIn(i,thors) 
             {
                 IPropertyTree &thor = thors.item(i);
-                thorProcesses.append(thor.queryProp("@name"));
+                const char* thorName = thor.queryProp("@name");
+                thorProcesses.append(thorName);
+                if (!isMultiThor)
+                    primaryThorProcesses.append(thorName);
+                else
+                {
+                    const char *nodeGroup = thor.queryProp("@nodeGroup");
+                    if (!nodeGroup || strieq(nodeGroup, thorName))
+                        primaryThorProcesses.append(thorName);
+                }
                 unsigned nodes = thor.getCount("ThorSlaveProcess");
                 if (!nodes)
                     throw MakeStringException(WUERR_MismatchClusterSize,"CEnvironmentClusterInfo: Thor cluster can not have 0 slave processes");
@@ -4461,6 +4472,10 @@ public:
     {
         return thorProcesses;
     }
+    const StringArray & getPrimaryThorProcesses() const
+    {
+        return primaryThorProcesses;
+    }
 
     const SocketEndpointArray & getRoxieServers() const
     {

common/workunit/workunit.hpp

@@ -553,6 +553,7 @@ interface IConstWUClusterInfo : extends IInterface
     virtual IStringVal & getServerQueue(IStringVal & str) const = 0;
     virtual IStringVal & getRoxieProcess(IStringVal & str) const = 0;
     virtual const StringArray & getThorProcesses() const = 0;
+    virtual const StringArray & getPrimaryThorProcesses() const = 0;
     virtual const SocketEndpointArray & getRoxieServers() const = 0;
 };
 

dali/dfu/dfuutil.cpp

@@ -405,6 +405,38 @@ public:
             throw afor2.exc.getClear();
     }
 
+    void updateCloneFrom(IFileDescriptor *srcfdesc, IFileDescriptor *dstfdesc, INode *srcdali, const char *srcCluster)
+    {
+        if (!srcdali || srcdali->endpoint().isNull())
+            dstfdesc->queryProperties().setProp("@cloneFromPeerCluster", srcCluster);
+        else
+        {
+            StringBuffer s;
+            dstfdesc->queryProperties().setProp("@cloneFrom", srcdali->endpoint().getUrlStr(s).str());
+            dstfdesc->queryProperties().setProp("@cloneFromDir", srcfdesc->queryDefaultDir());
+            if (srcCluster && *srcCluster) //where to copy from has been explicity set to a remote location, don't copy from local sources
+                dstfdesc->queryProperties().setProp("@cloneFromPeerCluster", "-");
+            if (prefix.length())
+                dstfdesc->queryProperties().setProp("@cloneFromPrefix", prefix.get());
+
+            while(dstfdesc->queryProperties().removeProp("cloneFromGroup"));
+
+            unsigned numClusters = srcfdesc->numClusters();
+            for (unsigned clusterNum = 0; clusterNum < numClusters; clusterNum++)
+            {
+                StringBuffer sourceGroup;
+                srcfdesc->getClusterGroupName(clusterNum, sourceGroup, NULL);
+                if (srcCluster && *srcCluster && !streq(sourceGroup, srcCluster))
+                    continue;
+                Owned<IPropertyTree> groupInfo = createPTree("cloneFromGroup");
+                groupInfo->setProp("@groupName", sourceGroup);
+                ClusterPartDiskMapSpec &spec = srcfdesc->queryPartDiskMapping(clusterNum);
+                spec.toProp(groupInfo);
+                dstfdesc->queryProperties().addPropTree("cloneFromGroup", groupInfo.getClear());
+            }
+        }
+    }
+
     void cloneSubFile(IPropertyTree *ftree,const char *destfilename, INode *srcdali, const char *srcCluster)   // name already has prefix added
     {
         Owned<IFileDescriptor> srcfdesc = deserializeFileDescriptorTree(ftree, NULL, 0);
@@ -453,31 +485,7 @@ public:
             physicalReplicateFile(dstfdesc,destfilename);
         }
 
-        if (!srcdali || srcdali->endpoint().isNull())
-            dstfdesc->queryProperties().setProp("@cloneFromPeerCluster", srcCluster);
-        else
-        {
-            StringBuffer s;
-            dstfdesc->queryProperties().setProp("@cloneFrom", srcdali->endpoint().getUrlStr(s).str());
-            dstfdesc->queryProperties().setProp("@cloneFromDir", srcfdesc->queryDefaultDir());
-            if (srcCluster && *srcCluster) //where to copy from has been explicity set to a remote location, don't copy from local sources
-                dstfdesc->queryProperties().setProp("@cloneFromPeerCluster", "-");
-            if (prefix.length())
-                dstfdesc->queryProperties().setProp("@cloneFromPrefix", prefix.get());
-            unsigned numClusters = srcfdesc->numClusters();
-            for (unsigned clusterNum = 0; clusterNum < numClusters; clusterNum++)
-            {
-                StringBuffer sourceGroup;
-                srcfdesc->getClusterGroupName(clusterNum, sourceGroup, NULL);
-                if (srcCluster && *srcCluster && !streq(sourceGroup, srcCluster))
-                    continue;
-                Owned<IPropertyTree> groupInfo = createPTree("cloneFromGroup");
-                groupInfo->setProp("@groupName", sourceGroup);
-                ClusterPartDiskMapSpec &spec = srcfdesc->queryPartDiskMapping(clusterNum);
-                spec.toProp(groupInfo);
-                dstfdesc->queryProperties().addPropTree("cloneFromGroup", groupInfo.getClear());
-            }
-        }
+        updateCloneFrom(srcfdesc, dstfdesc, srcdali, srcCluster);
 
         Owned<IDistributedFile> dstfile = fdir->createNew(dstfdesc);
         dstfile->attach(destfilename,userdesc);
@@ -678,6 +686,11 @@ public:
             StringBuffer s;
             throw MakeStringException(-1,"Source file %s could not be found in Dali %s",slfn.get(),srcdali?srcdali->endpoint().getUrlStr(s).str():"(local)");
         }
+        IPropertyTree *attsrc = ftree->queryPropTree("Attr");
+        if (!attsrc) {
+            StringBuffer s;
+            throw MakeStringException(-1,"Attributes for source file %s could not be found in Dali %s",slfn.get(),srcdali?srcdali->endpoint().getUrlStr(s).str():"(local)");
+        }
         CDfsLogicalFileName dlfn;
         dlfn.set(destfilename);
         if (strcmp(ftree->queryName(),queryDfsXmlBranchName(DXB_File))!=0) {
@@ -698,6 +711,18 @@ public:
         if (dfile) {
             if (!overwrite)
                 throw MakeStringException(-1,"Destination file %s already exists",dlfn.get());
+
+            IPropertyTree &attloc = dfile->queryAttributes();
+            if (dfile->numParts() == (unsigned)ftree->getPropInt("@numparts") &&
+                attsrc->getPropInt("@eclCRC") == attloc.getPropInt("@eclCRC") &&
+                attsrc->getPropInt("@totalCRC") == attloc.getPropInt64("@totalCRC"))
+            {
+                Owned<IFileDescriptor> dstfdesc=dfile->getFileDescriptor();
+                Owned<IFileDescriptor> srcfdesc = deserializeFileDescriptorTree(ftree, NULL, 0);
+                updateCloneFrom(srcfdesc, dstfdesc, srcdali, srcCluster);
+                return;
+            }
+
             dfile->detach();
             dfile.clear();
         }
@@ -985,6 +1010,7 @@ public:
                          bool dophysicalcopy                // NB *not* using DFU server
                          )
     {
+        DBGLOG("createSingleFileClone src=%s@%s, dst=%s@%s, prefix=%s, ow=%d, docopy=%d", srcname, srcCluster, dstname, cluster1, prefix, overwrite, dophysicalcopy);
         CFileCloner cloner;
         cloner.init(cluster1,clustmap,repeattlk,cluster2,userdesc,foreigndali,foreignuserdesc,NULL,overwrite,dophysicalcopy);
         cloner.srcCluster.set(srcCluster);

esp/eclwatch/ws_XSLT/dfu.xslt

@@ -655,7 +655,7 @@
           <img id="mn{position()}" class="menu1" src="/esp/files/img/menu1.png" onclick="{$popup}"></img>
       </td>
             <td>
-              <xsl:if test="isZipfile=1">
+              <xsl:if test="IsCompressed=1">
                 <img border="0" src="/esp/files/img/zip.gif" title="Compressed" width="16" height="16"/>
               </xsl:if>
             </td>

esp/eclwatch/ws_XSLT/dfu_file.xslt

@@ -229,7 +229,7 @@
                     <tr><th>Job Name:</th><td><xsl:value-of select="JobName"/></td></tr>
                 </xsl:if>
                 <tr><th>Size:</th><td><xsl:value-of select="Filesize"/>
-                    <xsl:if test="number(ZipFile)">
+                    <xsl:if test="IsCompressed=1">
                         (This is a compressed file.)
                     </xsl:if>
                 </td></tr>

esp/services/ws_dfu/ws_dfuXRefService.cpp

@@ -455,6 +455,24 @@ bool CWsDfuXRefEx::onDFUXRefBuildCancel(IEspContext &context, IEspDFUXRefBuildCa
     return true;
 }
 
+void CWsDfuXRefEx::addXRefNode(const char* name, IPropertyTree* pXRefNodeTree)
+{
+    IPropertyTree* XRefTreeNode = pXRefNodeTree->addPropTree("XRefNode", createPTree(ipt_caseInsensitive));
+    XRefTreeNode->setProp("Name",name);
+    Owned<IXRefNode> xRefNode = XRefNodeManager->getXRefNode(name);
+    if (!xRefNode)
+    {
+        XRefTreeNode->setProp("Modified","");
+        XRefTreeNode->setProp("Status","Not Run");
+    }
+    else
+    {
+        StringBuffer modified, status;
+        XRefTreeNode->setProp("Modified",xRefNode->getLastModified(modified).str());
+        XRefTreeNode->setProp("Status",xRefNode->getStatus(status).str());
+    }
+}
+
 bool CWsDfuXRefEx::onDFUXRefList(IEspContext &context, IEspDFUXRefListRequest &req, IEspDFUXRefListResponse &resp)
 {
     try
@@ -467,39 +485,25 @@ bool CWsDfuXRefEx::onDFUXRefList(IEspContext &context, IEspDFUXRefListRequest &r
         DBGLOG("CWsDfuXRefEx::onDFUXRefList User=%s",username.str());
 
 
-        //Firstly we need to get a list of the available Thor Cluster....
-        IArrayOf<IEspTpCluster> clusters;
-        CTpWrapper _topology;
-        _topology.getClusterProcessList(eqThorCluster,clusters,false,true);
-        ///_topology.getClusterList(eqRoxieCluster,clusters,false,true);
+        CConstWUClusterInfoArray clusters;
+        getEnvironmentClusterInfo(clusters);
 
+        BoolHash uniqueProcesses;
         Owned<IPropertyTree> pXRefNodeTree = createPTree("XRefNodes");
-        //DBGLOG("CWsDfuXRefEx::onDFUXRefList1\n");
-
-        for (unsigned x=0;x<=clusters.ordinality();x++)
+        ForEachItemIn(c, clusters)
         {
-            IPropertyTree* XRefTreeNode = pXRefNodeTree->addPropTree("XRefNode", createPTree(ipt_caseInsensitive));
-            
-            IEspTpCluster* cluster = x<clusters.ordinality()?&clusters.item(x):NULL;        
-            const char *clustername = cluster?cluster->getName():"SuperFiles";
-
-            XRefTreeNode->setProp("Name",clustername);
-            //create the node if it doesn;t exist
-            Owned<IXRefNode> xRefNode = XRefNodeManager->getXRefNode(clustername);
-            if (xRefNode == 0)
-            {
-                XRefTreeNode->setProp("Modified","");
-                XRefTreeNode->setProp("Status","Not Run");
-            }
-            else
+            IConstWUClusterInfo &cluster = clusters.item(c);
+            const StringArray &primaryThorProcesses = cluster.getPrimaryThorProcesses();
+            ForEachItemIn(i,primaryThorProcesses)
             {
-                  StringBuffer buf;
-                XRefTreeNode->setProp("Modified",xRefNode->getLastModified(buf).str());
-                    buf.clear();
-                XRefTreeNode->setProp("Status",xRefNode->getStatus(buf).str());
+                const char *thorProcess = primaryThorProcesses.item(i);
+                if (uniqueProcesses.getValue(thorProcess))
+                    continue;
+                uniqueProcesses.setValue(thorProcess, true);
+                addXRefNode(thorProcess, pXRefNodeTree);
             }
         }
-        
+        addXRefNode("SuperFiles", pXRefNodeTree);
 
         StringBuffer buf;
         resp.setDFUXRefListResult(toXML(pXRefNodeTree, buf).str());

esp/services/ws_dfu/ws_dfuXRefService.hpp

@@ -160,6 +160,7 @@ private:
 
 private:
     IXRefFilesNode* getFileNodeInterface(IXRefNode& XRefNode,const char* nodeType);
+    void addXRefNode(const char* name, IPropertyTree* pXRefNodeTree);
 public:
    IMPLEMENT_IINTERFACE;
 

esp/services/ws_packageprocess/ws_packageprocessService.cpp

@@ -236,7 +236,10 @@ void addPackageMapInfo(StringArray &filesNotFound, IPropertyTree *pkgSetRegistry
                     IPropertyTree &subtree = sub_iter->query();
                     const char *subid = subtree.queryProp("@value");
                     if (subid && *subid == '~')
-                        subtree.setProp("@value", subid+1);
+                    {
+                        StringAttr value(subid+1);
+                        subtree.setProp("@value", value.get());
+                    }
                 }
             }
             mapTree->addPropTree("Package", LINK(&item));

plugins/fileservices/fileservices.cpp

@@ -1557,6 +1557,13 @@ FILESERVICES_API void FILESERVICES_CALL fsRemoteDirectory(size32_t & __lenResult
 
 FILESERVICES_API void FILESERVICES_CALL fsLogicalFileList(ICodeContext *ctx, size32_t & __lenResult,void * & __result, const char *mask, bool includenormal, bool includesuper, bool unknownszero, const char *foreigndali)
 {
+    IEngineContext *engineCtx = ctx->queryEngineContext();
+    if (engineCtx && !engineCtx->allowDaliAccess())
+    {
+        Owned<IException> e = MakeStringException(-1, "FileServices.LogicalFileList cannot access Dali in this context - this normally means it is being called from a thor slave");
+        EXCLOG(e, NULL);
+        throw e.getClear();
+    }
     MemoryBuffer mb;
     if (!mask||!*mask)
         mask ="*";

roxie/ccd/ccddali.cpp

@@ -435,6 +435,8 @@ public:
         }
         else // Legacy mode - recently cloned files should have the extra info
         {
+            if (traceLevel > 1)
+                DBGLOG("checkClonedFromRemote: Resolving %s in legacy mode", _lfn);
             SocketEndpoint cloneFrom;
             cloneFrom.set(fdesc->queryProperties().queryProp("@cloneFrom"));
             if (cloneFrom.isNull())
@@ -462,8 +464,7 @@ public:
     {
         if (isConnected)
         {
-            if (traceLevel > 1)
-                DBGLOG("Dali lookup %s", logicalName);
+            unsigned start = msTick();
             CDfsLogicalFileName lfn;
             lfn.set(logicalName);
             Owned<IDistributedFile> dfsFile = queryDistributedFileDirectory().lookup(lfn, userdesc.get(), writeAccess, cacheIt);
@@ -488,6 +489,8 @@ public:
                 xpath.append(lcname.append(logicalName).toLowerCase());
                 writeCache(xpath.str(), xpath.str(), pt);
             }
+            if (traceLevel > 1)
+                DBGLOG("Dali lookup %s returned %s in %u ms", logicalName, dfsFile != NULL ? "match" : "NO match", msTick()-start);
             return dfsFile.getClear();
         }
         else

roxie/ccd/ccdfile.cpp

@@ -1613,6 +1613,7 @@ protected:
     PointerIArrayOf<IFileDescriptor> remoteSubFiles; // note - on slaves, the file descriptors may have incomplete info. On originating server is always complete
     PointerIArrayOf<IDefRecordMeta> diskMeta;
     IArrayOf<IDistributedFile> subDFiles;  // To make sure subfiles get locked too
+    IArrayOf<IResolvedFile> subRFiles;  // To make sure subfiles get locked too
 
     Owned <IPropertyTree> properties;
 
@@ -1697,7 +1698,9 @@ public:
     virtual void beforeDispose()
     {
         if (cached)
+        {
             cached->removeCache(this);
+        }
     }
     virtual unsigned numSubFiles() const
     {
@@ -2039,8 +2042,7 @@ public:
             assertex(sub->fileType==fileType);
         else
             fileType = sub->fileType;
-        if (sub->dFile)
-            subDFiles.append(*LINK(sub->dFile));
+        subRFiles.append((IResolvedFile &) *LINK(_sub));
         ForEachItemIn(idx, sub->subFiles)
         {
             addFile(sub->subNames.item(idx), LINK(sub->subFiles.item(idx)), LINK(sub->remoteSubFiles.item(idx)));
@@ -2067,6 +2069,8 @@ public:
     {
         if (cached)
         {
+            if (traceLevel > 9)
+                DBGLOG("setCache removing from prior cache %s", queryFileName());
             if (cache==NULL)
                 cached->removeCache(this);
             else

roxie/ccd/ccdstate.cpp

@@ -206,6 +206,8 @@ public:
     virtual void removeCache(const IResolvedFile *file)
     {
         CriticalBlock b(cacheLock);
+        if (traceLevel > 9)
+            DBGLOG("removeCache %s", file->queryFileName());
         // NOTE: it's theoretically possible for the final release to happen after a replacement has been inserted into hash table. 
         // So only remove from hash table if what we find there matches the item that is being deleted.
         IResolvedFile *goer = files.getValue(file->queryFileName());
@@ -223,6 +225,8 @@ public:
             LINK(cache);
             if (cache->isAlive())
                 return cache;
+            if (traceLevel)
+                DBGLOG("Not returning %s from cache as isAlive() returned false", filename);
         }
         return NULL;
     }
@@ -239,18 +243,25 @@ static Owned<KeptLowerCaseAtomTable> daliMisses;
 static void noteDaliMiss(const char *filename)
 {
     CriticalBlock b(daliMissesCrit);
+    if (traceLevel > 9)
+        DBGLOG("noteDaliMiss %s", filename);
     daliMisses->addAtom(filename);
 }
 
 static bool checkCachedDaliMiss(const char *filename)
 {
     CriticalBlock b(daliMissesCrit);
-    return daliMisses->find(filename) != NULL;
+    bool ret = daliMisses->find(filename) != NULL;
+    if (traceLevel > 9)
+        DBGLOG("checkCachedDaliMiss %s returns %d", filename, ret);
+    return ret;
 }
 
 static void clearDaliMisses()
 {
     CriticalBlock b(daliMissesCrit);
+    if (traceLevel)
+        DBGLOG("Clearing dali misses cache");
     daliMisses.setown(new KeptLowerCaseAtomTable);
 }
 
@@ -287,9 +298,15 @@ protected:
     static IResolvedFile *resolveLFNusingDaliOrLocal(const char *fileName, bool cacheIt, bool writeAccess, bool alwaysCreate)
     {
         // MORE - look at alwaysCreate... This may be useful to implement earlier locking semantics.
+        if (traceLevel > 9)
+            DBGLOG("resolveLFNusingDaliOrLocal %s %d %d %d", fileName, cacheIt, writeAccess, alwaysCreate);
         IResolvedFile* result = daliFiles.lookupCache(fileName);
         if (result)
+        {
+            if (traceLevel > 9)
+                DBGLOG("resolveLFNusingDaliOrLocal %s - cache hit", fileName);
             return result;
+        }
         if (!checkCachedDaliMiss(fileName))
         {
             Owned<IRoxieDaliHelper> daliHelper = connectToDali();
@@ -336,6 +353,8 @@ protected:
         }
         if (cacheIt)
         {
+            if (traceLevel > 9)
+                DBGLOG("resolveLFNusingDaliOrLocal %s - cache add %d", fileName, result != NULL);
             if (result)
                 daliFiles.addCache(fileName, result);
             else
@@ -381,6 +400,8 @@ protected:
                         // implies that a package file had ~ in subfile names - shouldn;t really, but we allow it (and just strip the ~
                         subFileName.remove(0,1);
                     }
+                    if (traceLevel > 9)
+                        DBGLOG("Looking up subfile %s", subFileName.str());
                     Owned<const IResolvedFile> subFileInfo = lookupExpandedFileName(subFileName, cache, false, false);  // NOTE - overwriting a superfile does NOT require write access to subfiles
                     if (subFileInfo)
                     {

thorlcr/activities/thdiskbase.cpp

@@ -74,7 +74,7 @@ void CDiskReadMasterBase::init()
                 }
             }
         }
-        validateFile(file);
+        validateFile();
         void *ekey;
         size32_t ekeylen;
         helper->getEncryptKey(ekeylen,ekey);

thorlcr/activities/thdiskbase.ipp

@@ -40,7 +40,7 @@ public:
     void init();
     void serializeSlaveData(MemoryBuffer &dst, unsigned slave);
     void done();
-    virtual void validateFile(IDistributedFile *file) { }
+    virtual void validateFile() { }
     void deserializeStats(unsigned node, MemoryBuffer &mb);
     void getXGMML(unsigned idx, IPropertyTree *edge);
 };

thorlcr/graph/thgraphslave.cpp

@@ -1020,6 +1020,11 @@ public:
             foreignNode.set(globals->queryProp("@DALISERVERS"));
         return ::getGlobalUniqueIds(num, &foreignNode);
     }
+    virtual bool allowDaliAccess() const
+    {
+        // NB. includes access to foreign Dalis.
+        return globals->getPropBool("Debug/@slaveDaliClient");
+    }
 };
 
 class CSlaveGraphTempHandler : public CGraphTempHandler