HPCC-14106 Read query's xref info using IReferencedFile

In the existing WsWorkunits.WUQueryDetails, query's dfu
info is read through control:getQueryXrefInfo. That does
not work for queries that failed to load. In this fix,
query's xref info is read through IReferencedFile.

Signed-off-by: wangkx <kevin.wang@lexisnexis.com>

common/workunit/referencedfilelist.cpp

@@ -125,7 +125,7 @@ class ReferencedFile : public CInterface, implements IReferencedFile
 public:
     IMPLEMENT_IINTERFACE;
     ReferencedFile(const char *lfn, const char *sourceIP, const char *srcCluster, const char *prefix, bool isSubFile, unsigned _flags, const char *_pkgid, bool noDfs, bool calcSize)
-    : flags(_flags), pkgid(_pkgid), noDfsResolution(noDfs), calcFileSize(calcSize), fileSize(0), numParts(0)
+    : flags(_flags), pkgid(_pkgid), noDfsResolution(noDfs), calcFileSize(calcSize), fileSize(0), numParts(0), trackSubFiles(false)
     {
         {
             //Scope ensures strings are assigned
@@ -155,7 +155,7 @@ public:
 
     void resolveLocal(const char *dstCluster, const char *srcCluster, IUserDescriptor *user, StringArray *subfiles);
     void resolveRemote(IUserDescriptor *user, INode *remote, const char *remotePrefix, const char *dstCluster, const char *srcCluster, bool checkLocalFirst, StringArray *subfiles, bool resolveForeign=false);
-    void resolve(const char *dstCluster, const char *srcCluster, IUserDescriptor *user, INode *remote, const char *remotePrefix, bool checkLocalFirst, StringArray *subfiles, bool resolveForeign=false);
+    void resolve(const char *dstCluster, const char *srcCluster, IUserDescriptor *user, INode *remote, const char *remotePrefix, bool checkLocalFirst, StringArray *subfiles, bool trackSubFiles, bool resolveForeign=false);
 
     virtual const char *getLogicalName() const {return logicalName.str();}
     virtual unsigned getFlags() const {return flags;}
@@ -178,8 +178,9 @@ public:
     {
         return numParts;
     }
-
+    virtual const StringArray &getSubFileNames() const { return subFileNames; };
 public:
+    StringArray subFileNames;
     StringAttr logicalName;
     StringAttr pkgid;
     StringAttr daliip;
@@ -190,6 +191,7 @@ public:
     unsigned flags;
     bool noDfsResolution;
     bool calcFileSize;
+    bool trackSubFiles;
 };
 
 class ReferencedFileList : public CInterface, implements IReferencedFileList
@@ -235,8 +237,8 @@ public:
         cloneFileInfo(helper, overwrite, cloneSuperInfo, cloneForeign, redundancy, channelsPerNode, replicateOffset, defReplicateFolder);
         cloneRelationships();
     }
-    virtual void resolveFiles(const char *process, const char *remoteIP, const char *_remotePrefix, const char *srcCluster, bool checkLocalFirst, bool addSubFiles, bool resolveForeign=false);
-    void resolveSubFiles(StringArray &subfiles, bool checkLocalFirst, bool resolveForeign);
+    virtual void resolveFiles(const char *process, const char *remoteIP, const char *_remotePrefix, const char *srcCluster, bool checkLocalFirst, bool addSubFiles, bool trackSubFiles, bool resolveForeign=false);
+    void resolveSubFiles(StringArray &subfiles, bool checkLocalFirst, bool trackSubFiles, bool resolveForeign);
 
 public:
     Owned<IUserDescriptor> user;
@@ -264,6 +266,8 @@ void ReferencedFile::processLocalFileInfo(IDistributedFile *df, const char *dstC
                 StringBuffer name;
                 sub.getLogicalName(name);
                 subfiles->append(name.str());
+                if (trackSubFiles)
+                    subFileNames.append(name.str());
             }
         }
     }
@@ -302,6 +306,8 @@ void ReferencedFile::processRemoteFileTree(IPropertyTree *tree, const char *srcC
                 if (flags & RefFileForeign)
                     lfn = foreignLfn.append("foreign::").append(this->daliip).append("::").append(lfn).str();
                 subfiles->append(lfn);
+                if (trackSubFiles)
+                    subFileNames.append(lfn);
             }
         }
     }
@@ -366,7 +372,7 @@ IPropertyTree *ReferencedFile::getSpecifiedOrRemoteFileTree(IUserDescriptor *use
 
 void ReferencedFile::resolveRemote(IUserDescriptor *user, INode *remote, const char *remotePrefix, const char *dstCluster, const char *srcCluster, bool checkLocalFirst, StringArray *subfiles, bool resolveForeign)
 {
-    if ((flags & RefFileForeign) && !resolveForeign)
+    if ((flags & RefFileForeign) && !resolveForeign && !trackSubFiles)
         return;
     if (flags & RefFileInPackage)
         return;
@@ -400,8 +406,9 @@ void ReferencedFile::resolveRemote(IUserDescriptor *user, INode *remote, const c
     flags |= RefFileNotFound;
 }
 
-void ReferencedFile::resolve(const char *dstCluster, const char *srcCluster, IUserDescriptor *user, INode *remote, const char *remotePrefix, bool checkLocalFirst, StringArray *subfiles, bool resolveForeign)
+void ReferencedFile::resolve(const char *dstCluster, const char *srcCluster, IUserDescriptor *user, INode *remote, const char *remotePrefix, bool checkLocalFirst, StringArray *subfiles, bool _trackSubFiles, bool resolveForeign)
 {
+    trackSubFiles = _trackSubFiles;
     if (daliip.length() || remote)
         resolveRemote(user, remote, remotePrefix, dstCluster, srcCluster, checkLocalFirst, subfiles, resolveForeign);
     else
@@ -667,7 +674,7 @@ void ReferencedFileList::addFilesFromWorkUnit(IConstWorkUnit *cw)
     addFilesFromQuery(cw, NULL, NULL);
 }
 
-void ReferencedFileList::resolveSubFiles(StringArray &subfiles, bool checkLocalFirst, bool resolveForeign)
+void ReferencedFileList::resolveSubFiles(StringArray &subfiles, bool checkLocalFirst, bool trackSubFiles, bool resolveForeign)
 {
     StringArray childSubFiles;
     ForEachItemIn(i, subfiles)
@@ -679,17 +686,17 @@ void ReferencedFileList::resolveSubFiles(StringArray &subfiles, bool checkLocalF
         Owned<ReferencedFile> file = new ReferencedFile(lfn, NULL, NULL, NULL, true, 0, NULL, false, allowSizeCalc);
         if (file->logicalName.length() && !map.getValue(file->getLogicalName()))
         {
-            file->resolve(process.get(), srcCluster, user, remote, remotePrefix, checkLocalFirst, &childSubFiles, resolveForeign);
+            file->resolve(process.get(), srcCluster, user, remote, remotePrefix, checkLocalFirst, &childSubFiles, trackSubFiles, resolveForeign);
             const char *ln = file->getLogicalName();
             // NOTE: setValue links its parameter
             map.setValue(ln, file);
         }
     }
     if (childSubFiles.length())
-        resolveSubFiles(childSubFiles, checkLocalFirst, resolveForeign);
+        resolveSubFiles(childSubFiles, checkLocalFirst, trackSubFiles, resolveForeign);
 }
 
-void ReferencedFileList::resolveFiles(const char *_process, const char *remoteIP, const char *_remotePrefix, const char *_srcCluster, bool checkLocalFirst, bool expandSuperFiles, bool resolveForeign)
+void ReferencedFileList::resolveFiles(const char *_process, const char *remoteIP, const char *_remotePrefix, const char *_srcCluster, bool checkLocalFirst, bool expandSuperFiles, bool trackSubFiles, bool resolveForeign)
 {
     process.set(_process);
     remote.setown((remoteIP && *remoteIP) ? createINode(remoteIP, 7070) : NULL);
@@ -700,10 +707,10 @@ void ReferencedFileList::resolveFiles(const char *_process, const char *remoteIP
     {
         ReferencedFileIterator files(this);
         ForEach(files)
-            files.queryObject().resolve(process, srcCluster, user, remote, remotePrefix, checkLocalFirst, expandSuperFiles ? &subfiles : NULL, resolveForeign);
+            files.queryObject().resolve(process, srcCluster, user, remote, remotePrefix, checkLocalFirst, expandSuperFiles ? &subfiles : NULL, trackSubFiles, resolveForeign);
     }
     if (expandSuperFiles)
-        resolveSubFiles(subfiles, checkLocalFirst, resolveForeign);
+        resolveSubFiles(subfiles, checkLocalFirst, trackSubFiles, resolveForeign);
 }
 
 void ReferencedFileList::cloneFileInfo(IDFUhelper *helper, bool overwrite, bool cloneSuperInfo, bool cloneForeign, unsigned redundancy, unsigned channelsPerNode, int replicateOffset, const char *defReplicateFolder)

common/workunit/referencedfilelist.hpp

@@ -44,6 +44,7 @@ interface IReferencedFile : extends IInterface
     virtual const char *queryPackageId() const =0;
     virtual __int64 getFileSize()=0;
     virtual unsigned getNumParts()=0;
+    virtual const StringArray &getSubFileNames() const =0;
 };
 
 interface IReferencedFileIterator : extends IIteratorOf<IReferencedFile> { };
@@ -59,7 +60,7 @@ interface IReferencedFileList : extends IInterface
     virtual void addFiles(StringArray &files)=0;
 
     virtual IReferencedFileIterator *getFiles()=0;
-    virtual void resolveFiles(const char *process, const char *remoteIP, const char * remotePrefix, const char *srcCluster, bool checkLocalFirst, bool addSubFiles, bool resolveForeign=false)=0;
+    virtual void resolveFiles(const char *process, const char *remoteIP, const char * remotePrefix, const char *srcCluster, bool checkLocalFirst, bool addSubFiles, bool trackSubFiles, bool resolveForeign=false)=0;
     virtual void cloneAllInfo(IDFUhelper *helper, bool overwrite, bool cloneSuperInfo, bool cloneForeign, unsigned redundancy, unsigned channelsPerNode, int replicateOffset, const char *defRepFolder)=0;
     virtual void cloneFileInfo(IDFUhelper *helper, bool overwrite, bool cloneSuperInfo, bool cloneForeign, unsigned redundancy, unsigned channelsPerNode, int replicateOffset, const char *defRepFolder)=0;
     virtual void cloneRelationships()=0;

esp/scm/ws_workunits.ecm

@@ -1427,6 +1427,7 @@ ESPStruct QuerySuperFile
 {
     string Name;
     ESParray<string, File> SubFiles;
+    [min_ver("1.57")] ESParray<ESPstruct QuerySuperFile, SuperFile> SuperFiles;
 };
 
 ESPresponse [exceptions_inline] WUQueryDetailsResponse
@@ -1738,7 +1739,7 @@ ESPresponse [exceptions_inline, nil_remove] WUGetArchiveFileResponse
 };
 
 ESPservice [
-    version("1.56"), default_client_version("1.56"),
+    version("1.57"), default_client_version("1.57"),
     noforms,exceptions_inline("./smc_xslt/exceptions.xslt"),use_method_name] WsWorkunits
 {
     ESPmethod [resp_xsl_default("/esp/xslt/workunits.xslt")]     WUQuery(WUQueryRequest, WUQueryResponse);

esp/services/ws_packageprocess/ws_packageprocessService.cpp

@@ -139,7 +139,7 @@ void cloneFileInfoToDali(StringArray &notFound, IPropertyTree *packageMap, const
     wufiles->addFilesFromPackageMap(packageMap);
     SCMStringBuffer processName;
     dstInfo->getRoxieProcess(processName);
-    wufiles->resolveFiles(processName.str(), lookupDaliIp, remotePrefix, srcCluster, !overWrite, false);
+    wufiles->resolveFiles(processName.str(), lookupDaliIp, remotePrefix, srcCluster, !overWrite, false, false);
 
     StringBuffer defReplicateFolder;
     getConfigurationDirectory(NULL, "data2", "roxie", processName.str(), defReplicateFolder);
@@ -813,7 +813,7 @@ bool CWsPackageProcessEx::onValidatePackage(IEspContext &context, IEspValidatePa
     {
         Owned<IReferencedFileList> pmfiles = createReferencedFileList(context.queryUserId(), context.queryPassword(), true, false);
         pmfiles->addFilesFromPackageMap(mapTree);
-        pmfiles->resolveFiles(process.str(), NULL, NULL, NULL, true, false);
+        pmfiles->resolveFiles(process.str(), NULL, NULL, NULL, true, false, false);
         Owned<IReferencedFileIterator> files = pmfiles->getFiles();
         StringArray notInDFS;
         ForEach(*files)

esp/services/ws_workunits/ws_workunitsQuerySets.cpp

@@ -26,7 +26,6 @@
 #include "roxiecontrol.hpp"
 #include "dfuutil.hpp"
 #include "dautils.hpp"
-#include "referencedfilelist.hpp"
 #include "httpclient.hpp"
 
 #define DALI_FILE_LOOKUP_TIMEOUT (1000*15*1)  // 15 seconds
@@ -363,7 +362,7 @@ void QueryFilesInUse::loadTarget(IPropertyTree *t, const char *target, unsigned
         wufiles->addFilesFromQuery(cw, pm, queryid);
         if (aborting)
             return;
-        wufiles->resolveFiles(process.str(), NULL, NULL, NULL, true, true, false);
+        wufiles->resolveFiles(process.str(), NULL, NULL, NULL, true, true, false, false);
 
         Owned<IReferencedFileIterator> files = wufiles->getFiles();
         ForEach(*files)
@@ -690,7 +689,7 @@ void copyQueryFilesToCluster(IEspContext &context, IConstWorkUnit *cw, const cha
         if (queryname && *queryname)
             queryname = queryid.append(queryname).append(".0").str(); //prepublish dummy version number to support fuzzy match like queries="myquery.*" in package
         wufiles->addFilesFromQuery(cw, (ps) ? ps->queryActiveMap(target) : NULL, queryname);
-        wufiles->resolveFiles(process.str(), remoteIP, remotePrefix, srcCluster, !overwrite, true, true);
+        wufiles->resolveFiles(process.str(), remoteIP, remotePrefix, srcCluster, !overwrite, true, false, true);
         StringBuffer defReplicateFolder;
         getConfigurationDirectory(NULL, "data2", "roxie", process.str(), defReplicateFolder);
         Owned<IDFUhelper> helper = createIDFUhelper();
@@ -1588,7 +1587,7 @@ bool CWsWorkunitsEx::onWUQueryDetails(IEspContext &context, IEspWUQueryDetailsRe
 
     StringArray logicalFiles;
     IArrayOf<IEspQuerySuperFile> superFiles;
-    getQueryFiles(queryId, querySet, logicalFiles, req.getIncludeSuperFiles() ? &superFiles : NULL);
+    getQueryFiles(context, wuid, queryId, querySet, logicalFiles, req.getIncludeSuperFiles() ? &superFiles : NULL);
     if (logicalFiles.length())
         resp.setLogicalFiles(logicalFiles);
     if (superFiles.length())
@@ -1692,7 +1691,59 @@ int EspQuerySuperFileCompareFunc(IInterface * const *i1, IInterface * const *i2)
     return strcmp(name1, name2);
 }
 
-bool CWsWorkunitsEx::getQueryFiles(const char* query, const char* target, StringArray& logicalFiles, IArrayOf<IEspQuerySuperFile> *respSuperFiles)
+IReferencedFile* CWsWorkunitsEx::getReferencedFileByName(const char* name, IReferencedFileList* wufiles)
+{
+    Owned<IReferencedFileIterator> refFileItr = wufiles->getFiles();
+    ForEach(*refFileItr)
+    {
+        IReferencedFile& rf = refFileItr->query();
+        const char* lfn = rf.getLogicalName();
+        if (lfn && strieq(lfn, name))
+            return &rf;
+    }
+    return NULL;
+}
+
+void CWsWorkunitsEx::readSuperFiles(IEspContext &context, IReferencedFile* rf, const char* fileName, IReferencedFileList* wufiles, IArrayOf<IEspQuerySuperFile>* files)
+{
+    double version = context.getClientVersion();
+    StringArray subFiles;
+    IArrayOf<IEspQuerySuperFile> superFiles;
+    const StringArray& subFileNames = rf->getSubFileNames();
+    ForEachItemIn(i, subFileNames)
+    {
+        const char* name = subFileNames.item(i);
+        if (!name || !*name)
+            continue;
+        IReferencedFile* pRF = getReferencedFileByName(name, wufiles);
+        if (!pRF)
+            continue;
+        if (!(pRF->getFlags() & RefFileSuper))
+        {
+            subFiles.append(name);
+        }
+        else if (version >= 1.57)
+        {
+            readSuperFiles(context, pRF, name, wufiles, &superFiles);
+        }
+    }
+
+    Owned<IEspQuerySuperFile> newSuperFile = createQuerySuperFile();
+    newSuperFile->setName(fileName);
+    if (subFiles.length())
+    {
+        subFiles.sortAscii();
+        newSuperFile->setSubFiles(subFiles);
+    }
+    if ((version >= 1.57) && superFiles.length())
+    {
+        superFiles.sort(EspQuerySuperFileCompareFunc);
+        newSuperFile->setSuperFiles(superFiles);
+    }
+    files->append(*newSuperFile.getClear());
+}
+
+bool CWsWorkunitsEx::getQueryFiles(IEspContext &context, const char* wuid, const char* query, const char* target, StringArray& logicalFiles, IArrayOf<IEspQuerySuperFile> *respSuperFiles)
 {
     try
     {
@@ -1700,55 +1751,38 @@ bool CWsWorkunitsEx::getQueryFiles(const char* query, const char* target, String
         if (!info || (info->getPlatform()!=RoxieCluster))
             return false;
 
-        const SocketEndpointArray &eps = info->getRoxieServers();
-        if (eps.empty())
+        SCMStringBuffer process;
+        info->getRoxieProcess(process);
+        if (!process.length())
             return false;
 
-        StringBuffer control;
-        control.appendf("<control:getQueryXrefInfo full='1'><Query id='%s'/></control:getQueryXrefInfo>",  query);
-        Owned<ISocket> sock = ISocket::connect_timeout(eps.item(0), ROXIECONNECTIONTIMEOUT);
-        Owned<IPropertyTree> result = sendRoxieControlQuery(sock, control.str(), ROXIECONTROLQUERYTIMEOUT);
-        if (!result)
+        Owned<IWorkUnitFactory> factory = getWorkUnitFactory();
+        Owned<IConstWorkUnit> cw = factory->openWorkUnit(wuid);
+        if (!cw)
             return false;
 
-        StringBuffer xpath("Endpoint/Queries/Query/");
-        if (!respSuperFiles)
-            xpath.append('/');
-        xpath.append("File");
-        Owned<IPropertyTreeIterator> files = result->getElements(xpath);
-        ForEach (*files)
+        StringArray superFileNames;
+        Owned<IHpccPackageSet> ps = createPackageSet(process.str());
+        Owned<IReferencedFileList> wufiles = createReferencedFileList(context.queryUserId(),
+            context.queryPassword(), true, true);
+        wufiles->addFilesFromQuery(cw, (ps) ? ps->queryActiveMap(target) : NULL, query);
+        wufiles->resolveFiles(process.str(), NULL, NULL, NULL, true, true, true, true);
+        Owned<IReferencedFileIterator> refFileItr = wufiles->getFiles();
+        ForEach(*refFileItr)
         {
-            IPropertyTree &file = files->query();
-            const char* fileName = file.queryProp("@name");
-            if (fileName && *fileName)
-                logicalFiles.append(fileName);
+            IReferencedFile &rf = refFileItr->query();
+            const char *lfn = rf.getLogicalName();
+            if (lfn && *lfn)
+            {
+                logicalFiles.append(lfn);
+                if (respSuperFiles && (rf.getFlags() & RefFileSuper))
+                    readSuperFiles(context, &rf, lfn, wufiles, respSuperFiles);
+            }
         }
-        logicalFiles.sortAscii();
 
+        logicalFiles.sortAscii();
         if (respSuperFiles)
-        {
-            Owned<IPropertyTreeIterator> superFiles = result->getElements("Endpoint/Queries/Query/SuperFile");
-            ForEach (*superFiles)
-            {
-                IPropertyTree &super = superFiles->query();
-                Owned<IEspQuerySuperFile> respSuperFile = createQuerySuperFile();
-                respSuperFile->setName(super.queryProp("@name"));
-                Owned<IPropertyTreeIterator> fileIter = super.getElements("File");
-                StringArray respSubFiles;
-                ForEach (*fileIter)
-                {
-                    IPropertyTree &fileItem = fileIter->query();
-                    const char* fileName = fileItem.queryProp("@name");
-                    if (fileName && *fileName)
-                        respSubFiles.append(fileName);
-                }
-                respSubFiles.sortAscii();
-
-                respSuperFile->setSubFiles(respSubFiles);
-                respSuperFiles->append(*respSuperFile.getClear());
-            }
             respSuperFiles->sort(EspQuerySuperFileCompareFunc);
-        }
         return true;
     }
     catch(IMultiException *me)
@@ -2253,7 +2287,7 @@ public:
     {
         if (cloneFilesEnabled)
         {
-            wufiles->resolveFiles(process, dfsIP, srcPrefix, srcCluster, !overwriteDfs, true, true);
+            wufiles->resolveFiles(process, dfsIP, srcPrefix, srcCluster, !overwriteDfs, true, false, true);
             Owned<IDFUhelper> helper = createIDFUhelper();
             Owned <IConstWUClusterInfo> cl = getTargetClusterInfo(target);
             if (cl)

esp/services/ws_workunits/ws_workunitsService.hpp

@@ -25,6 +25,7 @@
 #ifdef _USE_ZLIB
 #include "zcrypt.hpp"
 #endif
+#include "referencedfilelist.hpp"
 
 #define UFO_DIRTY                                0x01
 #define UFO_RELOAD_TARGETS_CHANGED_PMID          0x02
@@ -176,7 +177,7 @@ public:
     bool isValidCluster(const char *cluster);
     void deploySharedObjectReq(IEspContext &context, IEspWUDeployWorkunitRequest & req, IEspWUDeployWorkunitResponse & resp, const char *dir, const char *xml=NULL);
     unsigned getGraphIdsByQueryId(const char *target, const char *queryId, StringArray& graphIds);
-    bool getQueryFiles(const char* query, const char* target, StringArray& logicalFiles, IArrayOf<IEspQuerySuperFile> *superFiles);
+    bool getQueryFiles(IEspContext &context, const char* wuid, const char* query, const char* target, StringArray& logicalFiles, IArrayOf<IEspQuerySuperFile> *superFiles);
     void getGraphsByQueryId(const char *target, const char *queryId, const char *graphName, const char *subGraphId, IArrayOf<IEspECLGraphEx>& ECLGraphs);
     void checkAndSetClusterQueryState(IEspContext &context, const char* cluster, const char* querySetId, IArrayOf<IEspQuerySetQuery>& queries, bool checkAllNodes);
     void checkAndSetClusterQueryState(IEspContext &context, const char* cluster, StringArray& querySetIds, IArrayOf<IEspQuerySetQuery>& queries, bool checkAllNodes);
@@ -272,6 +273,8 @@ private:
     void readGraph(IEspContext& context, const char* subGraphId, WUGraphIDType& id, bool running,
         IConstWUGraph* graph, IArrayOf<IEspECLGraphEx>& graphs);
     IPropertyTree* getWorkunitArchive(IEspContext &context, WsWuInfo& winfo, const char* wuid, unsigned cacheMinutes);
+    void readSuperFiles(IEspContext &context, IReferencedFile* rf, const char* fileName, IReferencedFileList* wufiles, IArrayOf<IEspQuerySuperFile>* files);
+    IReferencedFile* getReferencedFileByName(const char* name, IReferencedFileList* wufiles);
 
     unsigned awusCacheMinutes;
     StringBuffer queryDirectory;