Merge remote-tracking branch 'origin/closedown-4.2.x'

Conflicts:
	esp/services/ws_workunits/ws_workunitsQuerySets.cpp

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

common/workunit/referencedfilelist.cpp

@@ -65,13 +65,51 @@ const char *skipForeign(const char *name, StringBuffer *ip)
     return name;
 }
 
+void splitDfsLocation(const char *address, StringBuffer &cluster, StringBuffer &ip, StringBuffer &prefix, const char *defaultCluster)
+{
+    if (!address || !*address)
+    {
+        cluster.append(defaultCluster);
+        return;
+    }
+
+    const char *s=strchr(address, '@');
+    if (s)
+    {
+        cluster.append(s - address, address);
+        address = s + 1;
+    }
+    else if (defaultCluster && *defaultCluster)
+        cluster.append(defaultCluster);
+    s=strchr(address, '/');
+    if (!s)
+        ip.append(address);
+    else
+    {
+        ip.append(s - address, address);
+        prefix.append(s+1);
+    }
+}
+
+void splitDerivedDfsLocation(const char *address, StringBuffer &cluster, StringBuffer &ip, StringBuffer &prefix, const char *defaultCluster, const char *baseCluster, const char *baseIP, const char *basePrefix)
+{
+    if (address && *address)
+    {
+        splitDfsLocation(address, cluster, ip, prefix, defaultCluster);
+        return;
+    }
+    ip.append(baseIP);
+    cluster.append(baseCluster);
+    prefix.append(basePrefix);
+}
+
 class ReferencedFileList;
 
 class ReferencedFile : public CInterface, implements IReferencedFile
 {
 public:
     IMPLEMENT_IINTERFACE;
-    ReferencedFile(const char *lfn, const char *sourceIP, const char *srcCluster, bool isSubFile, unsigned _flags, const char *_pkgid) : flags(_flags), pkgid(_pkgid)
+    ReferencedFile(const char *lfn, const char *sourceIP, const char *srcCluster, const char *prefix, bool isSubFile, unsigned _flags, const char *_pkgid) : flags(_flags), pkgid(_pkgid)
     {
         logicalName.set(skipForeign(lfn, &daliip)).toLowerCase();
         if (daliip.length())
@@ -79,6 +117,7 @@ public:
         else
             daliip.set(sourceIP);
         fileSrcCluster.set(srcCluster);
+        filePrefix.set(prefix);
         if (isSubFile)
             flags |= RefSubFile;
     }
@@ -87,14 +126,16 @@ public:
     {
         flags &= RefSubFile;
     }
-    IPropertyTree *getForeignOrRemoteFileTree(IUserDescriptor *user, INode *remote);
+
+    IPropertyTree *getRemoteFileTree(IUserDescriptor *user, INode *remote, const char *remotePrefix);
+    IPropertyTree *getSpecifiedOrRemoteFileTree(IUserDescriptor *user, INode *remote, const char *remotePrefix);
 
     void processLocalFileInfo(IDistributedFile *df, const char *dstCluster, const char *srcCluster, StringArray *subfiles);
     void processRemoteFileTree(IPropertyTree *tree, const char *srcCluster, StringArray *subfiles);
 
     void resolveLocal(const char *dstCluster, const char *srcCluster, IUserDescriptor *user, StringArray *subfiles);
-    void resolveRemote(IUserDescriptor *user, INode *remote, const char *dstCluster, const char *srcCluster, bool checkLocalFirst, StringArray *subfiles, bool resolveForeign=false);
-    void resolve(const char *dstCluster, const char *srcCluster, IUserDescriptor *user, INode *remote, bool checkLocalFirst, StringArray *subfiles, bool resolveForeign=false);
+    void resolveRemote(IUserDescriptor *user, INode *remote, const char *remotePrefix, const char *dstCluster, const char *srcCluster, bool checkLocalFirst, StringArray *subfiles, bool resolveForeign=false);
+    void resolve(const char *dstCluster, const char *srcCluster, IUserDescriptor *user, INode *remote, const char *remotePrefix, bool checkLocalFirst, StringArray *subfiles, bool resolveForeign=false);
 
     virtual const char *getLogicalName() const {return logicalName.str();}
     virtual unsigned getFlags() const {return flags;}
@@ -106,7 +147,7 @@ public:
             ep.set(NULL);
         return ep;
     }
-    virtual void cloneInfo(IDFUhelper *helper, IUserDescriptor *user, INode *remote, const char *dstCluster, const char *srcCluster, bool overwrite=false, bool cloneForeign=false);
+    virtual void cloneInfo(IDFUhelper *helper, IUserDescriptor *user, const char *dstCluster, const char *srcCluster, bool overwrite=false, bool cloneForeign=false);
     void cloneSuperInfo(ReferencedFileList *list, IUserDescriptor *user, INode *remote, bool overwrite);
     virtual const char *queryPackageId() const {return pkgid.get();}
 
@@ -114,6 +155,7 @@ public:
     StringBuffer logicalName;
     StringAttr pkgid;
     StringBuffer daliip;
+    StringBuffer filePrefix;
     StringAttr fileSrcCluster;
     unsigned flags;
 };
@@ -131,18 +173,18 @@ public:
         }
     }
 
-    void ensureFile(const char *ln, unsigned flags, const char *pkgid, const char *daliip=NULL, const char *srcCluster=NULL);
+    void ensureFile(const char *ln, unsigned flags, const char *pkgid, const char *daliip=NULL, const char *srcCluster=NULL, const char *remotePrefix=NULL);
 
-    virtual void addFile(const char *ln, const char *daliip=NULL, const char *srcCluster=NULL);
+    virtual void addFile(const char *ln, const char *daliip=NULL, const char *srcCluster=NULL, const char *remotePrefix=NULL);
     virtual void addFiles(StringArray &files);
     virtual void addFilesFromWorkUnit(IConstWorkUnit *cw);
     virtual void addFilesFromQuery(IConstWorkUnit *cw, const IHpccPackageMap *pm, const char *queryid);
     virtual void addFilesFromQuery(IConstWorkUnit *cw, const IHpccPackage *pkg);
     virtual void addFilesFromPackageMap(IPropertyTree *pm);
 
-    void addFileFromSubFile(IPropertyTree &subFile, const char *_daliip, const char *srcCluster);
-    void addFilesFromSuperFile(IPropertyTree &superFile, const char *_daliip, const char *srcCluster);
-    void addFilesFromPackage(IPropertyTree &package, const char *_daliip, const char *srcCluster);
+    void addFileFromSubFile(IPropertyTree &subFile, const char *_daliip, const char *srcCluster, const char *_remotePrefix);
+    void addFilesFromSuperFile(IPropertyTree &superFile, const char *_daliip, const char *srcCluster, const char *_remotePrefix);
+    void addFilesFromPackage(IPropertyTree &package, const char *_daliip, const char *srcCluster, const char *_remotePrefix);
 
     virtual IReferencedFileIterator *getFiles();
     virtual void cloneFileInfo(IDFUhelper *helper, bool overwrite, bool cloneSuperInfo, bool cloneForeign=false);
@@ -152,7 +194,7 @@ public:
         cloneFileInfo(helper, overwrite, cloneSuperInfo, cloneForeign);
         cloneRelationships();
     }
-    virtual void resolveFiles(const char *process, const char *remoteIP, const char *srcCluster, bool checkLocalFirst, bool addSubFiles, bool resolveForeign=false);
+    virtual void resolveFiles(const char *process, const char *remoteIP, const char *_remotePrefix, const char *srcCluster, bool checkLocalFirst, bool addSubFiles, bool resolveForeign=false);
     void resolveSubFiles(StringArray &subfiles, bool checkLocalFirst, bool resolveForeign);
 
 public:
@@ -161,6 +203,7 @@ public:
     MapStringToMyClass<ReferencedFile> map;
     StringAttr process;
     StringAttr srcCluster;
+    StringAttr remotePrefix;
 };
 
 void ReferencedFile::processLocalFileInfo(IDistributedFile *df, const char *dstCluster, const char *srcCluster, StringArray *subfiles)
@@ -232,20 +275,36 @@ void ReferencedFile::resolveLocal(const char *dstCluster, const char *srcCluster
         flags |= RefFileNotFound;
 }
 
-IPropertyTree *ReferencedFile::getForeignOrRemoteFileTree(IUserDescriptor *user, INode *remote)
+IPropertyTree *ReferencedFile::getRemoteFileTree(IUserDescriptor *user, INode *remote, const char *remotePrefix)
+{
+    if (!remote)
+        return NULL;
+    StringBuffer remoteLFN;
+    if (remotePrefix && *remotePrefix)
+        remoteLFN.append(remotePrefix).append("::").append(logicalName);
+    return queryDistributedFileDirectory().getFileTree(remoteLFN.length() ? remoteLFN : logicalName, user, remote, WF_LOOKUP_TIMEOUT, false, false);
+}
+
+IPropertyTree *ReferencedFile::getSpecifiedOrRemoteFileTree(IUserDescriptor *user, INode *remote, const char *remotePrefix)
 {
-    Owned<INode> daliNode;
     if (daliip.length())
     {
+        Owned<INode> daliNode;
         daliNode.setown(createINode(daliip));
-        remote = daliNode;
+        return getRemoteFileTree(user, daliNode, filePrefix);
     }
     if (!remote)
         return NULL;
-    return queryDistributedFileDirectory().getFileTree(logicalName.str(), user, remote, WF_LOOKUP_TIMEOUT, false, false);
+    StringBuffer remoteLFN;
+    Owned<IPropertyTree> fileTree = getRemoteFileTree(user, remote, remotePrefix);
+    if (!fileTree)
+        return NULL;
+    remote->endpoint().getUrlStr(daliip);
+    filePrefix.set(remotePrefix);
+    return fileTree.getClear();
 }
 
-void ReferencedFile::resolveRemote(IUserDescriptor *user, INode *remote, const char *dstCluster, const char *srcCluster, bool checkLocalFirst, StringArray *subfiles, bool resolveForeign)
+void ReferencedFile::resolveRemote(IUserDescriptor *user, INode *remote, const char *remotePrefix, const char *dstCluster, const char *srcCluster, bool checkLocalFirst, StringArray *subfiles, bool resolveForeign)
 {
     if ((flags & RefFileForeign) && !resolveForeign)
         return;
@@ -261,22 +320,22 @@ void ReferencedFile::resolveRemote(IUserDescriptor *user, INode *remote, const c
             return;
         }
     }
-    Owned<IPropertyTree> tree = getForeignOrRemoteFileTree(user, remote);
+    Owned<IPropertyTree> tree = getSpecifiedOrRemoteFileTree(user, remote, remotePrefix);
     if (tree)
         processRemoteFileTree(tree, srcCluster, subfiles);
     else
         flags |= RefFileNotFound;
 }
 
-void ReferencedFile::resolve(const char *dstCluster, const char *srcCluster, IUserDescriptor *user, INode *remote, bool checkLocalFirst, StringArray *subfiles, bool resolveForeign)
+void ReferencedFile::resolve(const char *dstCluster, const char *srcCluster, IUserDescriptor *user, INode *remote, const char *remotePrefix, bool checkLocalFirst, StringArray *subfiles, bool resolveForeign)
 {
     if (daliip.length() || remote)
-        resolveRemote(user, remote, dstCluster, srcCluster, checkLocalFirst, subfiles, resolveForeign);
+        resolveRemote(user, remote, remotePrefix, dstCluster, srcCluster, checkLocalFirst, subfiles, resolveForeign);
     else
         resolveLocal(dstCluster, srcCluster, user, subfiles);
 }
 
-void ReferencedFile::cloneInfo(IDFUhelper *helper, IUserDescriptor *user, INode *remote, const char *dstCluster, const char *srcCluster, bool overwrite, bool cloneForeign)
+void ReferencedFile::cloneInfo(IDFUhelper *helper, IUserDescriptor *user, const char *dstCluster, const char *srcCluster, bool overwrite, bool cloneForeign)
 {
     if ((flags & RefFileCloned) || (flags & RefFileSuper) || (flags & RefFileInPackage))
         return;
@@ -284,20 +343,20 @@ void ReferencedFile::cloneInfo(IDFUhelper *helper, IUserDescriptor *user, INode
         return;
     if (!(flags & (RefFileRemote | RefFileForeign | RefFileNotOnCluster)))
         return;
-
-    StringBuffer addr;
-    if (flags & RefFileForeign)
-        addr.set(daliip);
-    else if (remote)
-        remote->endpoint().getUrlStr(addr);
-
+    if (!daliip.length())
+        return;
     if (fileSrcCluster.length())
         srcCluster = fileSrcCluster;
 
     try
     {
-        helper->createSingleFileClone(logicalName.str(), srcCluster, logicalName.str(), dstCluster,
-            DFUcpdm_c_replicated_by_d, true, NULL, user, addr.str(), NULL, overwrite, false);
+        StringBuffer srcLFN;
+        if (filePrefix.length())
+            srcLFN.append(filePrefix.str()).append("::");
+        srcLFN.append(logicalName.str());
+
+        helper->createSingleFileClone(srcLFN, srcCluster, logicalName, dstCluster, filePrefix,
+            DFUcpdm_c_replicated_by_d, true, NULL, user, daliip, NULL, overwrite, false);
         flags |= RefFileCloned;
     }
     catch (IException *e)
@@ -309,7 +368,7 @@ void ReferencedFile::cloneInfo(IDFUhelper *helper, IUserDescriptor *user, INode
     catch (...)
     {
         flags |= RefFileCopyInfoFailed;
-        DBGLOG("Unknown error copying file info for %s, from %s", logicalName.str(), addr.str());
+        DBGLOG("Unknown error copying file info for [%s::] %s, from %s on dfs-dali %s", filePrefix.str(), logicalName.str(), fileSrcCluster.length() ? fileSrcCluster.get() : "*", daliip.str());
     }
 }
 
@@ -320,7 +379,7 @@ void ReferencedFile::cloneSuperInfo(ReferencedFileList *list, IUserDescriptor *u
 
     try
     {
-        Owned<IPropertyTree> tree = getForeignOrRemoteFileTree(user, remote);
+        Owned<IPropertyTree> tree = getSpecifiedOrRemoteFileTree(user, remote, NULL);
         if (!tree)
             return;
 
@@ -402,9 +461,9 @@ public:
     Owned<HashIterator> iter;
 };
 
-void ReferencedFileList::ensureFile(const char *ln, unsigned flags, const char *pkgid, const char *daliip, const char *srcCluster)
+void ReferencedFileList::ensureFile(const char *ln, unsigned flags, const char *pkgid, const char *daliip, const char *srcCluster, const char *prefix)
 {
-    Owned<ReferencedFile> file = new ReferencedFile(ln, daliip, srcCluster, false, flags, pkgid);
+    Owned<ReferencedFile> file = new ReferencedFile(ln, daliip, srcCluster, prefix, false, flags, pkgid);
     if (!file->logicalName.length())
         return;
     ReferencedFile *existing = map.getValue(file->getLogicalName());
@@ -417,9 +476,9 @@ void ReferencedFileList::ensureFile(const char *ln, unsigned flags, const char *
     }
 }
 
-void ReferencedFileList::addFile(const char *ln, const char *daliip, const char *srcCluster)
+void ReferencedFileList::addFile(const char *ln, const char *daliip, const char *srcCluster, const char *prefix)
 {
-    ensureFile(ln, 0, NULL, daliip, srcCluster);
+    ensureFile(ln, 0, NULL, daliip, srcCluster, prefix);
 }
 
 void ReferencedFileList::addFiles(StringArray &files)
@@ -428,42 +487,49 @@ void ReferencedFileList::addFiles(StringArray &files)
         addFile(files.item(i));
 }
 
-void ReferencedFileList::addFileFromSubFile(IPropertyTree &subFile, const char *daliip, const char *srcCluster)
+void ReferencedFileList::addFileFromSubFile(IPropertyTree &subFile, const char *ip, const char *cluster, const char *prefix)
 {
-    addFile(subFile.queryProp("@value"), daliip, srcCluster);
+    addFile(subFile.queryProp("@value"), ip, cluster, prefix);
 }
 
-void ReferencedFileList::addFilesFromSuperFile(IPropertyTree &superFile, const char *daliip, const char *srcCluster)
+void ReferencedFileList::addFilesFromSuperFile(IPropertyTree &superFile, const char *_ip, const char *_cluster, const char *_prefix)
 {
-    if (superFile.hasProp("@daliip"))
-        daliip = superFile.queryProp("@daliip");
+    StringBuffer ip;
+    StringBuffer cluster;
+    StringBuffer prefix;
+    splitDerivedDfsLocation(superFile.queryProp("@daliip"), cluster, ip, prefix, NULL, _cluster, _ip, _prefix);
     if (superFile.hasProp("@sourceCluster"))
-        srcCluster = superFile.queryProp("@sourceCluster");
+        cluster.set(superFile.queryProp("@sourceCluster"));
 
     Owned<IPropertyTreeIterator> subFiles = superFile.getElements("SubFile[@value]");
     ForEach(*subFiles)
-        addFileFromSubFile(subFiles->query(), daliip, srcCluster);
+        addFileFromSubFile(subFiles->query(), ip, cluster, prefix);
 }
 
-void ReferencedFileList::addFilesFromPackage(IPropertyTree &package, const char *daliip, const char *srcCluster)
+void ReferencedFileList::addFilesFromPackage(IPropertyTree &package, const char *_ip, const char *_cluster, const char *_prefix)
 {
-    if (package.hasProp("@daliip"))
-        daliip = package.queryProp("@daliip");
+    StringBuffer ip;
+    StringBuffer cluster;
+    StringBuffer prefix;
+    splitDerivedDfsLocation(package.queryProp("@daliip"), cluster, ip, prefix, NULL, _cluster, _ip, _prefix);
     if (package.hasProp("@sourceCluster"))
-        srcCluster = package.queryProp("@sourceCluster");
+        cluster.set(package.queryProp("@sourceCluster"));
 
     Owned<IPropertyTreeIterator> supers = package.getElements("SuperFile");
     ForEach(*supers)
-        addFilesFromSuperFile(supers->query(), daliip, srcCluster);
+        addFilesFromSuperFile(supers->query(), ip, cluster, prefix);
 }
 
 void ReferencedFileList::addFilesFromPackageMap(IPropertyTree *pm)
 {
-    const char *daliip = pm->queryProp("@daliip");
+    StringBuffer ip;
+    StringBuffer cluster;
+    StringBuffer prefix;
     const char *srcCluster = pm->queryProp("@sourceCluster");
+    splitDerivedDfsLocation(pm->queryProp("@daliip"), cluster, ip, prefix, srcCluster, srcCluster, NULL, NULL);
     Owned<IPropertyTreeIterator> packages = pm->getElements("Package");
     ForEach(*packages)
-        addFilesFromPackage(packages->query(), daliip, srcCluster);
+        addFilesFromPackage(packages->query(), ip, cluster, prefix);
 }
 
 void ReferencedFileList::addFilesFromQuery(IConstWorkUnit *cw, const IHpccPackage *pkg)
@@ -533,10 +599,10 @@ void ReferencedFileList::resolveSubFiles(StringArray &subfiles, bool checkLocalF
     StringArray childSubFiles;
     ForEachItemIn(i, subfiles)
     {
-        Owned<ReferencedFile> file = new ReferencedFile(subfiles.item(i), NULL, NULL, true, 0, NULL);
+        Owned<ReferencedFile> file = new ReferencedFile(subfiles.item(i), NULL, NULL, NULL, true, 0, NULL);
         if (file->logicalName.length() && !map.getValue(file->getLogicalName()))
         {
-            file->resolve(process.get(), srcCluster, user, remote, checkLocalFirst, &childSubFiles, resolveForeign);
+            file->resolve(process.get(), srcCluster, user, remote, remotePrefix, checkLocalFirst, &childSubFiles, resolveForeign);
             const char *ln = file->getLogicalName();
             map.setValue(ln, file.getClear());
         }
@@ -545,17 +611,18 @@ void ReferencedFileList::resolveSubFiles(StringArray &subfiles, bool checkLocalF
         resolveSubFiles(childSubFiles, checkLocalFirst, resolveForeign);
 }
 
-void ReferencedFileList::resolveFiles(const char *_process, const char *remoteIP, const char *_srcCluster, bool checkLocalFirst, bool expandSuperFiles, bool resolveForeign)
+void ReferencedFileList::resolveFiles(const char *_process, const char *remoteIP, const char *_remotePrefix, const char *_srcCluster, bool checkLocalFirst, bool expandSuperFiles, bool resolveForeign)
 {
     process.set(_process);
     remote.setown((remoteIP && *remoteIP) ? createINode(remoteIP, 7070) : NULL);
     srcCluster.set(_srcCluster);
+    remotePrefix.set(_remotePrefix);
 
     StringArray subfiles;
     {
         ReferencedFileIterator files(this);
         ForEach(files)
-            files.queryObject().resolve(process, srcCluster, user, remote, checkLocalFirst, expandSuperFiles ? &subfiles : NULL, resolveForeign);
+            files.queryObject().resolve(process, srcCluster, user, remote, remotePrefix, checkLocalFirst, expandSuperFiles ? &subfiles : NULL, resolveForeign);
     }
     if (expandSuperFiles)
         resolveSubFiles(subfiles, checkLocalFirst, resolveForeign);
@@ -565,7 +632,7 @@ void ReferencedFileList::cloneFileInfo(IDFUhelper *helper, bool overwrite, bool
 {
     ReferencedFileIterator files(this);
     ForEach(files)
-        files.queryObject().cloneInfo(helper, user, remote, process, srcCluster, overwrite, cloneForeign);
+        files.queryObject().cloneInfo(helper, user, process, srcCluster, overwrite, cloneForeign);
     if (cloneSuperInfo)
         ForEach(files)
             files.queryObject().cloneSuperInfo(this, user, remote, overwrite);

common/workunit/referencedfilelist.hpp

@@ -54,11 +54,11 @@ interface IReferencedFileList : extends IInterface
     virtual void addFilesFromQuery(IConstWorkUnit *cw, const IHpccPackage *pkg)=0;
     virtual void addFilesFromPackageMap(IPropertyTree *pm)=0;
 
-    virtual void addFile(const char *ln, const char *daliip=NULL, const char *sourceProcessCluster=NULL)=0;
+    virtual void addFile(const char *ln, const char *daliip=NULL, const char *sourceProcessCluster=NULL, const char *remotePrefix=NULL)=0;
     virtual void addFiles(StringArray &files)=0;
 
     virtual IReferencedFileIterator *getFiles()=0;
-    virtual void resolveFiles(const char *process, const char *remoteIP, const char *srcCluster, bool checkLocalFirst, bool addSubFiles, bool resolveForeign=false)=0;
+    virtual void resolveFiles(const char *process, const char *remoteIP, const char * remotePrefix, const char *srcCluster, bool checkLocalFirst, bool addSubFiles, bool resolveForeign=false)=0;
     virtual void cloneAllInfo(IDFUhelper *helper, bool overwrite, bool cloneSuperInfo, bool cloneForeign=false)=0;
     virtual void cloneFileInfo(IDFUhelper *helper, bool overwrite, bool cloneSuperInfo, bool cloneForeign=false)=0;
     virtual void cloneRelationships()=0;
@@ -68,4 +68,7 @@ extern WORKUNIT_API const char *skipForeign(const char *name, StringBuffer *ip=N
 
 extern WORKUNIT_API IReferencedFileList *createReferencedFileList(const char *user, const char *pw);
 
+extern WORKUNIT_API void splitDfsLocation(const char *address, StringBuffer &cluster, StringBuffer &ip, StringBuffer &prefix, const char *defaultCluster);
+extern WORKUNIT_API void splitDerivedDfsLocation(const char *address, StringBuffer &cluster, StringBuffer &ip, StringBuffer &prefix, const char *defaultCluster, const char *baseCluster, const char *baseIP, const char *basePrefix);
+
 #endif //REFFILE_LIST_HPP

common/workunit/workunit.cpp

@@ -4612,9 +4612,21 @@ extern WORKUNIT_API bool isProcessCluster(const char *remoteDali, const char *pr
     Owned<INode> remote = createINode(remoteDali, 7070);
     if (!remote)
         return false;
+
     VStringBuffer xpath("Environment/Software/*Cluster[@name=\"%s\"]/@name", process);
-    Owned<IPropertyTreeIterator> clusters = querySDS().getElementsRaw(xpath, remote, 1000*60*1);
-    return clusters->first();
+    try
+    {
+        Owned<IPropertyTreeIterator> clusters = querySDS().getElementsRaw(xpath, remote, 1000*60*1);
+        return clusters->first();
+    }
+    catch (IException *E)
+    {
+        StringBuffer msg;
+        E->errorMessage(msg);
+        DBGLOG("Exception validating cluster %s/%s: %s", remoteDali, xpath.str(), msg.str());
+        E->Release();
+    }
+    return true;
 }
 
 IConstWUClusterInfo* getTargetClusterInfo(IPropertyTree *environment, IPropertyTree *cluster)

dali/dfu/dfuutil.cpp

@@ -133,6 +133,7 @@ public:
     Linked<IUserDescriptor> foreignuserdesc;
     StringAttr srcCluster;
     StringAttr cluster1;
+    StringAttr prefix;
     Owned<IGroup> grp1;
     ClusterPartDiskMapSpec spec1;
     StringAttr cluster2;
@@ -461,6 +462,8 @@ public:
             dstfdesc->queryProperties().setProp("@cloneFromDir", srcfdesc->queryDefaultDir());
             if (srcCluster && *srcCluster) //where to copy from has been explicity set to a remote location, don't copy from local sources
                 dstfdesc->queryProperties().setProp("@cloneFromPeerCluster", "-");
+            if (prefix.length())
+                dstfdesc->queryProperties().setProp("@cloneFromPrefix", prefix.get());
             unsigned numClusters = srcfdesc->numClusters();
             for (unsigned clusterNum = 0; clusterNum < numClusters; clusterNum++)
             {
@@ -971,6 +974,7 @@ public:
                          const char *srcCluster,            // optional specific cluster to copy data from
                          const char *dstname,               // dst LFN
                          const char *cluster1,              // group name of roxie cluster
+                         const char *prefix,
                          DFUclusterPartDiskMapping clustmap, // how the nodes are mapped
                          bool repeattlk,                    // repeat last part on all nodes if key
                          const char *cluster2,              // servers cluster (for just tlk)
@@ -984,6 +988,7 @@ public:
         CFileCloner cloner;
         cloner.init(cluster1,clustmap,repeattlk,cluster2,userdesc,foreigndali,foreignuserdesc,NULL,overwrite,dophysicalcopy);
         cloner.srcCluster.set(srcCluster);
+        cloner.prefix.set(prefix);
         cloner.cloneFile(srcname,dstname);
     }
 

dali/dfu/dfuutil.hpp

@@ -59,6 +59,7 @@ interface IDFUhelper: extends IInterface
                          const char *srcCluster,
                          const char *dstname,               // dst LFN
                          const char *cluster1,              // group name of roxie cluster
+                         const char *prefix,
                          DFUclusterPartDiskMapping clustmap, // how the nodes are mapped
                          bool repeattlk,                    // repeat last part on all nodes if key
                          const char *cluster2,              // servers cluster (for just tlk)

ecl/eclagent/eclagent.cpp

@@ -70,6 +70,7 @@ static const char XMLHEADER[] = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>";
 
 //#define DEFAULT_REALTHOR_HOST "localhost"
 #define PERSIST_LOCK_TIMEOUT 10000
+#define PERSIST_LOCK_SLEEP 5000
 
 #define ABORT_CHECK_INTERVAL 30     // seconds
 #define ABORT_DEADMAN_INTERVAL (60*5)  // seconds
@@ -2507,11 +2508,15 @@ bool EclAgent::checkPersistUptoDate(const char * logicalName, unsigned eclCRC, u
 bool EclAgent::changePersistLockMode(IRemoteConnection *persistLock, unsigned mode, const char * name, bool repeat)
 {
     LOG(MCrunlock, unknownJob, "Waiting to change persist lock to %s for %s", (mode == RTM_LOCK_WRITE) ? "write" : "read", name);
+    //When converting a read lock to a write lock so the persist can be rebuilt hold onto the lock as short as
+    //possible.  Otherwise lots of workunits each trying to convert read locks to write locks will mean
+    //that the read lock is never released by all the workunits at the same time, so no workunit can progress.
+    unsigned timeout = repeat ? PERSIST_LOCK_TIMEOUT : 0;
     loop
     {
         try
         {
-            persistLock->changeMode(mode, PERSIST_LOCK_TIMEOUT);
+            persistLock->changeMode(mode, timeout);
             reportProgress("Changed persist lock");
             return true;
         }
@@ -2526,6 +2531,8 @@ bool EclAgent::changePersistLockMode(IRemoteConnection *persistLock, unsigned mo
             reportProgress("Failed to convert persist lock"); // gives a chance to abort
             return false;
         }
+        //This is only executed when converting write->read.  There is significant doubt whether the changeMode()
+        //can ever fail - and whether the execution can ever get here.
         reportProgress("Waiting to convert persist lock"); // gives a chance to abort
     }
 }
@@ -2605,7 +2612,7 @@ bool EclAgent::isPersistUptoDate(Owned<IRemoteConnection> &persistLock, const ch
 
         //failed to get a write lock, so release our read lock
         persistLock.clear();
-        MilliSleep(getRandom()%2000);
+        MilliSleep(PERSIST_LOCK_SLEEP + (getRandom()%PERSIST_LOCK_SLEEP));
         persistLock.setown(getPersistReadLock(logicalName));
     }
     setRunning();
@@ -2747,7 +2754,7 @@ void EclAgent::deleteLRUPersists(const char * logicalName, int keep)
                 while (!changePersistLockMode(persistLock, RTM_LOCK_WRITE, goer, false))
                 {
                     persistLock.clear();
-                    MilliSleep(getRandom()%2000);
+                    MilliSleep(PERSIST_LOCK_SLEEP + (getRandom()%PERSIST_LOCK_SLEEP));
                     persistLock.setown(getPersistReadLock(goer));
                 }
                 Owned<IDistributedFile> f = queryDistributedFileDirectory().lookup(goer, queryUserDescriptor(), true);

esp/files/ecl/EmbedJava.ecl

@@ -32,8 +32,8 @@ SET OF UNICODE testStringArray7(SET OF UNICODE s)
 add1(10);
 add2('Hello');
 add3('World');
-add4(U'Leovenaðes');
-add5(U'?????');
+add4(U'Leovenaðes');
+add5(U'你好世界');
 addChar('A');
 
 cat('Hello', ' world');

esp/services/ws_packageprocess/packageprocess_errors.h

@@ -35,5 +35,6 @@
 #define PKG_INVALID_CLUSTER_TYPE   PKG_PROCESS_ERROR_START+12
 #define PKG_INVALID_QUERY_NAME   PKG_PROCESS_ERROR_START+13
 #define PKG_INFO_NOT_DEFINED   PKG_PROCESS_ERROR_START+14
+#define PKG_INVALID_PARAMETER   PKG_PROCESS_ERROR_START+15
 
 #endif

esp/services/ws_packageprocess/ws_packageprocessService.cpp

@@ -123,7 +123,7 @@ bool isFileKnownOnCluster(const char *logicalname, const char *target, IUserDesc
     return isFileKnownOnCluster(logicalname, clusterInfo, userdesc);
 }
 
-void cloneFileInfoToDali(StringArray &notFound, IPropertyTree *packageMap, const char *lookupDaliIp, IConstWUClusterInfo *dstInfo, const char *srcCluster, bool overWrite, IUserDescriptor* userdesc)
+void cloneFileInfoToDali(StringArray &notFound, IPropertyTree *packageMap, const char *lookupDaliIp, IConstWUClusterInfo *dstInfo, const char *srcCluster, const char *remotePrefix, bool overWrite, IUserDescriptor* userdesc)
 {
     StringBuffer user;
     StringBuffer password;
@@ -138,7 +138,7 @@ void cloneFileInfoToDali(StringArray &notFound, IPropertyTree *packageMap, const
     wufiles->addFilesFromPackageMap(packageMap);
     SCMStringBuffer processName;
     dstInfo->getRoxieProcess(processName);
-    wufiles->resolveFiles(processName.str(), lookupDaliIp, srcCluster, !overWrite, false);
+    wufiles->resolveFiles(processName.str(), lookupDaliIp, remotePrefix, srcCluster, !overWrite, false);
     Owned<IDFUhelper> helper = createIDFUhelper();
     wufiles->cloneAllInfo(helper, overWrite, true);
 
@@ -151,13 +151,13 @@ void cloneFileInfoToDali(StringArray &notFound, IPropertyTree *packageMap, const
     }
 }
 
-void cloneFileInfoToDali(StringArray &notFound, IPropertyTree *packageMap, const char *lookupDaliIp, const char *dstCluster, const char *srcCluster, bool overWrite, IUserDescriptor* userdesc)
+void cloneFileInfoToDali(StringArray &notFound, IPropertyTree *packageMap, const char *lookupDaliIp, const char *dstCluster, const char *srcCluster, const char *prefix, bool overWrite, IUserDescriptor* userdesc)
 {
     Owned<IConstWUClusterInfo> clusterInfo = getTargetClusterInfo(dstCluster);
     if (!clusterInfo)
         throw MakeStringException(PKG_TARGET_NOT_DEFINED, "Could not find information about target cluster %s ", dstCluster);
 
-    cloneFileInfoToDali(notFound, packageMap, lookupDaliIp, clusterInfo, srcCluster, overWrite, userdesc);
+    cloneFileInfoToDali(notFound, packageMap, lookupDaliIp, clusterInfo, srcCluster, prefix, overWrite, userdesc);
 }
 
 
@@ -175,7 +175,7 @@ void makePackageActive(IPropertyTree *pkgSetRegistry, IPropertyTree *pkgSetTree,
 
 //////////////////////////////////////////////////////////
 
-void addPackageMapInfo(StringArray &filesNotFound, IPropertyTree *pkgSetRegistry, const char *target, const char *pmid, const char *packageSetName, const char *lookupDaliIp, const char *srcCluster, IPropertyTree *packageInfo, bool activate, bool overWrite, IUserDescriptor* userdesc)
+void addPackageMapInfo(StringArray &filesNotFound, IPropertyTree *pkgSetRegistry, const char *target, const char *pmid, const char *packageSetName, const char *lookupDaliIp, const char *srcCluster, const char *prefix, IPropertyTree *packageInfo, bool activate, bool overWrite, IUserDescriptor* userdesc)
 {
     if (srcCluster && *srcCluster)
     {
@@ -248,7 +248,7 @@ void addPackageMapInfo(StringArray &filesNotFound, IPropertyTree *pkgSetRegistry
     }
 
     mergePTree(mapTree, baseInfo);
-    cloneFileInfoToDali(filesNotFound, mapTree, lookupDaliIp, clusterInfo, srcCluster, overWrite, userdesc);
+    cloneFileInfoToDali(filesNotFound, mapTree, lookupDaliIp, clusterInfo, srcCluster, prefix, overWrite, userdesc);
 
     globalLock->commit();
 
@@ -534,12 +534,17 @@ bool CWsPackageProcessEx::onAddPackage(IEspContext &context, IEspAddPackageReque
         userdesc->set(user, password);
     }
 
+    StringBuffer srcCluster;
+    StringBuffer daliip;
+    StringBuffer prefix;
+    splitDerivedDfsLocation(req.getDaliIp(), srcCluster, daliip, prefix, req.getSourceProcess(), req.getSourceProcess(), NULL, NULL);
+
     Owned<IPropertyTree> packageTree = createPTreeFromXMLString(info.str());
     Owned<IPropertyTree> pkgSetRegistry = getPkgSetRegistry(processName.get(), false);
     StringArray filesNotFound;
     StringBuffer pkgSetId;
     buildPkgSetId(pkgSetId, processName.get());
-    addPackageMapInfo(filesNotFound, pkgSetRegistry, target.get(), pmid.str(), pkgSetId.str(), req.getDaliIp(), req.getSourceProcess(), LINK(packageTree), activate, overWrite, userdesc);
+    addPackageMapInfo(filesNotFound, pkgSetRegistry, target.get(), pmid.str(), pkgSetId.str(), daliip.str(), srcCluster.str(), prefix.str(), LINK(packageTree), activate, overWrite, userdesc);
     resp.setFilesNotFound(filesNotFound);
 
     StringBuffer msg;
@@ -778,7 +783,7 @@ bool CWsPackageProcessEx::onValidatePackage(IEspContext &context, IEspValidatePa
     {
         Owned<IReferencedFileList> pmfiles = createReferencedFileList(context.queryUserId(), context.queryPassword());
         pmfiles->addFilesFromPackageMap(mapTree);
-        pmfiles->resolveFiles(process.str(), NULL, NULL, true, false);
+        pmfiles->resolveFiles(process.str(), NULL, NULL, NULL, true, false);
         Owned<IReferencedFileIterator> files = pmfiles->getFiles();
         StringArray notInDFS;
         ForEach(*files)

esp/services/ws_workunits/ws_workunitsQuerySets.cpp

@@ -433,7 +433,7 @@ static inline void updateQueryPriority(IPropertyTree *queryTree, const char *val
     }
 }
 
-void copyQueryFilesToCluster(IEspContext &context, IConstWorkUnit *cw, const char *remoteIP, const char *target, const char *srcCluster, const char *queryname, bool overwrite)
+void copyQueryFilesToCluster(IEspContext &context, IConstWorkUnit *cw, const char *remoteIP, const char *remotePrefix, const char *target, const char *srcCluster, const char *queryname, bool overwrite)
 {
     if (!target || !*target)
         return;
@@ -451,7 +451,7 @@ void copyQueryFilesToCluster(IEspContext &context, IConstWorkUnit *cw, const cha
         if (queryname && *queryname)
             queryname = queryid.append(queryname).append(".0").str(); //prepublish dummy version number to support fuzzy match like queries="myquery.*" in package
         wufiles->addFilesFromQuery(cw, (ps) ? ps->queryActiveMap(target) : NULL, queryname);
-        wufiles->resolveFiles(process.str(), remoteIP, srcCluster, !overwrite, true);
+        wufiles->resolveFiles(process.str(), remoteIP, remotePrefix, srcCluster, !overwrite, true);
         Owned<IDFUhelper> helper = createIDFUhelper();
         wufiles->cloneAllInfo(helper, overwrite, true);
     }
@@ -527,16 +527,19 @@ bool CWsWorkunitsEx::onWUPublishWorkunit(IEspContext &context, IEspWUPublishWork
         throw MakeStringException(ECLWATCH_MISSING_PARAMS, "Cluster name not defined for publishing workunit %s", wuid.str());
     if (!isValidCluster(target.str()))
         throw MakeStringException(ECLWATCH_INVALID_CLUSTER_NAME, "Invalid cluster name: %s", target.str());
-    const char *srcCluster = req.getSourceProcess();
-    const char *daliIP = req.getRemoteDali();
-    if (srcCluster && *srcCluster)
+    StringBuffer daliIP;
+    StringBuffer srcCluster;
+    StringBuffer srcPrefix;
+    splitDerivedDfsLocation(req.getRemoteDali(), srcCluster, daliIP, srcPrefix, req.getSourceProcess(),req.getSourceProcess(), NULL, NULL);
+
+    if (srcCluster.length())
     {
         if (!isProcessCluster(daliIP, srcCluster))
-            throw MakeStringException(ECLWATCH_INVALID_CLUSTER_NAME, "Process cluster %s not found on %s DALI", srcCluster, daliIP ? daliIP : "local");
+            throw MakeStringException(ECLWATCH_INVALID_CLUSTER_NAME, "Process cluster %s not found on %s DALI", srcCluster.str(), daliIP.length() ? daliIP.str() : "local");
     }
 
     if (!req.getDontCopyFiles())
-        copyQueryFilesToCluster(context, cw, req.getRemoteDali(), target.str(), req.getSourceProcess(), queryName.str(), false);
+        copyQueryFilesToCluster(context, cw, daliIP, srcPrefix, target.str(), srcCluster, queryName.str(), false);
 
     WorkunitUpdate wu(&cw->lock());
     if (req.getUpdateWorkUnitName() && notEmpty(req.getJobName()))
@@ -1577,8 +1580,11 @@ bool CWsWorkunitsEx::onWUQuerysetCopyQuery(IEspContext &context, IEspWUQuerySetC
 
     if (!req.getDontCopyFiles())
     {
-        const char *reqDali = req.getDaliServer();
-        copyQueryFilesToCluster(context, cw, (reqDali && *reqDali) ? reqDali : remoteIP.str(), target, req.getSourceProcess(), targetQueryName.get(), req.getOverwrite());
+        StringBuffer daliIP;
+        StringBuffer srcCluster;
+        StringBuffer srcPrefix;
+        splitDerivedDfsLocation(req.getDaliServer(), srcCluster, daliIP, srcPrefix, req.getSourceProcess(), req.getSourceProcess(), remoteIP.str(), NULL);
+        copyQueryFilesToCluster(context, cw, daliIP.str(), srcPrefix, target, srcCluster, targetQueryName.get(), req.getOverwrite());
     }
 
     WorkunitUpdate wu(&cw->lock());

roxie/ccd/ccdstate.cpp

@@ -374,12 +374,20 @@ protected:
             {
                 StringBuffer subFileName;
                 subFileInfo->getSubFileName(idx, subFileName);
-                Owned<const IResolvedFile> subFileInfo = lookupExpandedFileName(subFileName, cache, false, false);  // NOTE - overwriting a superfile does NOT require write access to subfiles
-                if (subFileInfo)
+                if (subFileName.length())  // Empty subfile names can come from package file - just ignore
                 {
-                    if (!super)
-                        super.setown(createResolvedFile(fileName, NULL, true));
-                    super->addSubFile(subFileInfo);
+                    if (subFileName.charAt(0)=='~')
+                    {
+                        // implies that a package file had ~ in subfile names - shouldn;t really, but we allow it (and just strip the ~
+                        subFileName.remove(0,1);
+                    }
+                    Owned<const IResolvedFile> subFileInfo = lookupExpandedFileName(subFileName, cache, false, false);  // NOTE - overwriting a superfile does NOT require write access to subfiles
+                    if (subFileInfo)
+                    {
+                        if (!super)
+                            super.setown(createResolvedFile(fileName, NULL, true));
+                        super->addSubFile(subFileInfo);
+                    }
                 }
             }
             if (super && cache)