HPCC-11253 Do not lock superfiles in roxie

Don't include spill files or write activities.

Handle opt flag on indexes properly.

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

roxie/ccd/ccdactivities.cpp

@@ -5348,20 +5348,25 @@ public:
 
         try  // operations does not want any missing file errors to be fatal, or throw traps - just log it
         {
-            bool isOpt = _graphNode.getPropBool("att[@name='_isOpt']/@value") || pretendAllOpt;
-            const char *fileName = queryNodeFileName(_graphNode);
-            const char *indexName = queryNodeIndexName(_graphNode);
-            if (indexName && (!fileName || !streq(indexName, fileName)))
+            ThorActivityKind kind = getActivityKind(_graphNode);
+            if (kind != TAKdiskwrite && kind != TAKindexwrite && kind != TAKpiperead && kind != TAKpipewrite)
             {
-                indexfile.setown(_queryFactory.queryPackage().lookupFileName(indexName, isOpt, true, true, _queryFactory.queryWorkUnit()));
-                if (indexfile)
-                    keyArray.setown(indexfile->getKeyArray(NULL, &layoutTranslators, isOpt, queryFactory.queryChannel(), queryFactory.getEnableFieldTranslation()));
-            }
-            if (fileName)
-            {
-                datafile.setown(_queryFactory.queryPackage().lookupFileName(fileName, isOpt, true, true, _queryFactory.queryWorkUnit()));
-                if (datafile)
-                    fileArray.setown(datafile->getIFileIOArray(isOpt, queryFactory.queryChannel()));
+                const char *fileName = queryNodeFileName(_graphNode, kind);
+                const char *indexName = queryNodeIndexName(_graphNode, kind);
+                if (indexName)
+                {
+                    bool isOpt = pretendAllOpt || _graphNode.getPropBool("att[@name='_isIndexOpt']/@value");
+                    indexfile.setown(_queryFactory.queryPackage().lookupFileName(indexName, isOpt, true, true, _queryFactory.queryWorkUnit()));
+                    if (indexfile)
+                        keyArray.setown(indexfile->getKeyArray(NULL, &layoutTranslators, isOpt, queryFactory.queryChannel(), queryFactory.getEnableFieldTranslation()));
+                }
+                if (fileName)
+                {
+                    bool isOpt = pretendAllOpt || _graphNode.getPropBool("att[@name='_isOpt']/@value");
+                    datafile.setown(_queryFactory.queryPackage().lookupFileName(fileName, isOpt, true, true, _queryFactory.queryWorkUnit()));
+                    if (datafile)
+                        fileArray.setown(datafile->getIFileIOArray(isOpt, queryFactory.queryChannel()));
+                }
             }
         }
         catch(IException *E)

roxie/ccd/ccdquery.cpp

@@ -921,27 +921,34 @@ public:
                             ForEach(*nodes)
                             {
                                 IPropertyTree &node = nodes->query();
-                                const char *fileName = queryNodeFileName(node);
-                                const char *indexName = queryNodeIndexName(node);
-                                // MORE - what about write? What about packages that resolve everything without dali?
-                                if (indexName && (!fileName || !streq(indexName, fileName)))
+                                ThorActivityKind kind = getActivityKind(node);
+                                if (kind != TAKdiskwrite && kind != TAKindexwrite && kind != TAKpiperead && kind != TAKpipewrite)
                                 {
-                                    bool isOpt = node.getPropBool("att[@name='_isIndexOpt']/@value") || pretendAllOpt;
-                                    const IResolvedFile *indexFile = package.lookupFileName(indexName, isOpt, true, true, wu);
-                                    if (indexFile)
+                                    const char *fileName = queryNodeFileName(node, kind);
+                                    const char *indexName = queryNodeIndexName(node, kind);
+                                    // What about packages that resolve everything without dali?
+                                    if (indexName)
                                     {
-                                        hashValue = indexFile->addHash64(hashValue);
-                                        files.append(*const_cast<IResolvedFile *>(indexFile));
+                                        bool isOpt = pretendAllOpt || node.getPropBool("att[@name='_isIndexOpt']/@value");
+                                        const IResolvedFile *indexFile = package.lookupFileName(indexName, isOpt, true, true, wu);
+                                        if (indexFile)
+                                        {
+                                            hashValue = indexFile->addHash64(hashValue);
+                                            files.append(*const_cast<IResolvedFile *>(indexFile));
+                                        }
                                     }
-                                }
-                                if (fileName)
-                                {
-                                    bool isOpt = node.getPropBool("att[@name='_isOpt']/@value") || pretendAllOpt;
-                                    const IResolvedFile *dataFile = package.lookupFileName(fileName, isOpt, true, true, wu);
-                                    if (dataFile)
+                                    if (fileName)
                                     {
-                                        hashValue = dataFile->addHash64(hashValue);
-                                        files.append(*const_cast<IResolvedFile *>(dataFile));
+                                        if (!node.getPropBool("att[@name='_isSpill']/@value") && !node.getPropBool("att[@name='_isSpillGlobal']/@value"))
+                                        {
+                                            bool isOpt = pretendAllOpt || node.getPropBool("att[@name='_isOpt']/@value");
+                                            const IResolvedFile *dataFile = package.lookupFileName(fileName, isOpt, true, true, wu);
+                                            if (dataFile)
+                                            {
+                                                hashValue = dataFile->addHash64(hashValue);
+                                                files.append(*const_cast<IResolvedFile *>(dataFile));
+                                            }
+                                        }
                                     }
                                 }
                             }

roxie/ccd/ccdquery.hpp

@@ -277,7 +277,7 @@ inline unsigned getGraphId(IPropertyTree & node)
     return node.getPropInt("att[@name=\"_graphId\"]/@value", 0);
 }
 
-inline ThorActivityKind getActivityKind(IPropertyTree & node)
+inline ThorActivityKind getActivityKind(const IPropertyTree & node)
 {
     return (ThorActivityKind) node.getPropInt("att[@name=\"_kind\"]/@value", TAKnone);
 }

roxie/ccd/ccdserver.cpp

@@ -23388,24 +23388,29 @@ public:
             if (_graphNode.getPropBool("att[@name='_isSpill']/@value", false) || _graphNode.getPropBool("att[@name='_isSpillGlobal']/@value", false))
                 return;  // ignore 'spills'
             bool isLocal = _graphNode.getPropBool("att[@name='local']/@value") && queryFactory.queryChannel()!=0;
-            bool isOpt = _graphNode.getPropBool("att[@name='_isOpt']/@value") || pretendAllOpt;
-            const char *fileName = queryNodeFileName(_graphNode);
-            const char *indexName = queryNodeIndexName(_graphNode);
-            if (indexName && (!fileName || !streq(indexName, fileName)))
+            ThorActivityKind kind = getActivityKind(_graphNode);
+            if (kind != TAKdiskwrite && kind != TAKindexwrite && kind != TAKpiperead && kind != TAKpipewrite)
             {
-                indexfile.setown(queryFactory.queryPackage().lookupFileName(indexName, isOpt, true, true, queryFactory.queryWorkUnit()));
-                if (indexfile)
-                    keySet.setown(indexfile->getKeyArray(NULL, &layoutTranslators, isOpt, isLocal ? queryFactory.queryChannel() : 0, false));
-            }
-            if (fileName)
-            {
-                datafile.setown(_queryFactory.queryPackage().lookupFileName(fileName, isOpt, true, true, queryFactory.queryWorkUnit()));
-                if (datafile)
+                const char *fileName = queryNodeFileName(_graphNode, kind);
+                const char *indexName = queryNodeIndexName(_graphNode, kind);
+                if (indexName)
                 {
-                    if (isLocal)
-                        files.setown(datafile->getIFileIOArray(isOpt, queryFactory.queryChannel()));
-                    else
-                        map.setown(datafile->getFileMap());
+                    bool isOpt = pretendAllOpt || _graphNode.getPropBool("att[@name='_isIndexOpt']/@value");
+                    indexfile.setown(queryFactory.queryPackage().lookupFileName(indexName, isOpt, true, true, queryFactory.queryWorkUnit()));
+                    if (indexfile)
+                        keySet.setown(indexfile->getKeyArray(NULL, &layoutTranslators, isOpt, isLocal ? queryFactory.queryChannel() : 0, false));
+                }
+                if (fileName)
+                {
+                    bool isOpt = pretendAllOpt || _graphNode.getPropBool("att[@name='_isOpt']/@value");
+                    datafile.setown(_queryFactory.queryPackage().lookupFileName(fileName, isOpt, true, true, queryFactory.queryWorkUnit()));
+                    if (datafile)
+                    {
+                        if (isLocal)
+                            files.setown(datafile->getIFileIOArray(isOpt, queryFactory.queryChannel()));
+                        else
+                            map.setown(datafile->getFileMap());
+                    }
                 }
             }
         }
@@ -24953,7 +24958,7 @@ public:
         if (!isHalfKeyed && !variableFetchFileName)
         {
             bool isFetchOpt = (helper->getFetchFlags() & FFdatafileoptional) != 0;
-            datafile.setown(_queryFactory.queryPackage().lookupFileName(queryNodeFileName(_graphNode), isFetchOpt, true, true, _queryFactory.queryWorkUnit()));
+            datafile.setown(_queryFactory.queryPackage().lookupFileName(queryNodeFileName(_graphNode, _kind), isFetchOpt, true, true, _queryFactory.queryWorkUnit()));
             if (datafile)
             {
                 if (isLocal)

roxie/ccd/ccdstate.cpp

@@ -72,7 +72,7 @@ SafePluginMap *plugins;
 
 // These two helper functions will return the original filenames placed in the XGMML by the codegen, regardless of how/if roxieconfig resolved them
 
-const char *queryNodeFileName(const IPropertyTree &graphNode)
+static const char *_queryNodeFileName(const IPropertyTree &graphNode)
 {
     if (graphNode.hasProp("att[@name='_file_dynamic']"))
         return NULL;
@@ -80,19 +80,48 @@ const char *queryNodeFileName(const IPropertyTree &graphNode)
         return graphNode.queryProp("att[@name='_fileName']/@value");
 }
 
-const char *queryNodeIndexName(const IPropertyTree &graphNode)
+static const char *_queryNodeIndexName(const IPropertyTree &graphNode)
 {
     if (graphNode.hasProp("att[@name='_indexFile_dynamic']"))
         return NULL;
     else
-    {
-        const char * id = graphNode.queryProp("att[@name='_indexFileName']/@value");
-        if (!id)
-            id = queryNodeFileName(graphNode);
-        return id;
+        return graphNode.queryProp("att[@name='_indexFileName']/@value");
+}
+
+static bool isSimpleIndexActivity(ThorActivityKind kind)
+{
+    switch (kind)
+    {
+    case TAKindexaggregate:
+    case TAKindexcount:
+    case TAKindexexists:
+    case TAKindexgroupaggregate:
+    case TAKindexgroupcount:
+    case TAKindexgroupexists:
+    case TAKindexnormalize:
+    case TAKindexread:
+        return true;
+    default:
+        return false;
     }
 }
 
+const char *queryNodeFileName(const IPropertyTree &graphNode, ThorActivityKind kind)
+{
+    if (isSimpleIndexActivity(kind))
+        return false;
+    else
+        return _queryNodeFileName(graphNode);
+}
+
+const char *queryNodeIndexName(const IPropertyTree &graphNode, ThorActivityKind kind)
+{
+    if (isSimpleIndexActivity(kind))
+        return _queryNodeFileName(graphNode);
+    else
+        return _queryNodeIndexName(graphNode);
+}
+
 // DelayedReleaser mechanism hangs on to a link to an object for a while...
 
 class DelayedReleaseQueueItem : public CInterfaceOf<IInterface>
@@ -583,7 +612,7 @@ public:
     virtual IPropertyTreeIterator *getInMemoryIndexInfo(const IPropertyTree &graphNode) const 
     {
         StringBuffer xpath;
-        xpath.append("SuperFile[@id='").append(queryNodeFileName(graphNode)).append("']");
+        xpath.append("SuperFile[@id='").append(queryNodeFileName(graphNode, getActivityKind(graphNode))).append("']");
         return lookupElements(xpath.str(), "MemIndex");
     }
 

roxie/ccd/ccdstate.hpp

@@ -140,8 +140,8 @@ extern void mergeStats(IPropertyTree *s1, IPropertyTree *s2, unsigned level);
 extern void mergeStats(IPropertyTree *s1, IPropertyTree *s2);
 extern void mergeQueries(IPropertyTree *s1, IPropertyTree *s2);
 
-extern const char *queryNodeFileName(const IPropertyTree &graphNode);
-extern const char *queryNodeIndexName(const IPropertyTree &graphNode);
+extern const char *queryNodeFileName(const IPropertyTree &graphNode, ThorActivityKind kind);
+extern const char *queryNodeIndexName(const IPropertyTree &graphNode, ThorActivityKind kind);
 
 extern void createDelayedReleaser();
 extern void stopDelayedReleaser();