HPCC-11786 Roxie cored when processing a file notification

When a CResolvedFile was released (because no query uses it any
more), the subscription to dali for the superfile remained active.
This would result in a core if the superfile was subsequently updated

Also added the ability to disable the superfile subscription system.

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

initfiles/componentfiles/configxml/roxie.xsd.in

@@ -609,6 +609,13 @@
         </xs:appinfo>
       </xs:annotation>
     </xs:attribute>
+    <xs:attribute name="lockSuperFiles" type="xs:boolean" use="optional" default="false">
+      <xs:annotation>
+        <xs:appinfo>
+          <tooltip>If enabled, superfiles will be locked while queries that use them are loaded</tooltip>
+        </xs:appinfo>
+      </xs:annotation>
+    </xs:attribute>
     <xs:attribute name="lowTimeout" type="xs:nonNegativeInteger" use="optional" default="10000">
       <xs:annotation>
         <xs:appinfo>

initfiles/etc/DIR_NAME/environment.xml.in

@@ -846,6 +846,7 @@
                 linuxYield="false"
                 localFilesExpire="-1"
                 localSlave="true"
+                lockSuperFiles="false"
                 logFullQueries="false"
                 logQueueDrop="32"
                 logQueueLen="512"

roxie/ccd/ccd.hpp

@@ -330,6 +330,7 @@ extern MapStringTo<int> *preferredClusters;
 extern StringArray allQuerySetNames;
 
 extern bool allFilesDynamic;
+extern bool lockSuperFiles;
 extern bool crcResources;
 extern bool logFullQueries;
 extern bool blindLogging;

roxie/ccd/ccddali.cpp

@@ -52,6 +52,8 @@ public:
     }
     ~CDaliPackageWatcher()
     {
+        if (change)
+            unsubscribe();
     }
     virtual void subscribe(bool exact)
     {

roxie/ccd/ccdfile.cpp

@@ -1744,7 +1744,7 @@ public:
                 }
                 // We have to clone the properties since we don't want to keep the superfile locked
                 properties.setown(createPTreeFromIPT(&dFile->queryAttributes()));
-                if (!isDynamic)
+                if (!isDynamic && !lockSuperFiles)
                 {
                     notifier.setown(daliHelper->getSuperFileSubscription(lfn, this));
                     dFile.clear();  // We don't lock superfiles, except dynamic ones

roxie/ccd/ccdmain.cpp

@@ -94,6 +94,7 @@ IPropertyTree* ccdChannels;
 StringArray allQuerySetNames;
 
 bool allFilesDynamic;
+bool lockSuperFiles;
 bool crcResources;
 bool useRemoteResources;
 bool checkFileDate;
@@ -669,6 +670,7 @@ int STARTQUERY_API start_query(int argc, const char *argv[])
         minIbytiDelay = topology->getPropInt("@minIbytiDelay", 2);
         initIbytiDelay = topology->getPropInt("@initIbytiDelay", 50);
         allFilesDynamic = topology->getPropBool("@allFilesDynamic", false);
+        lockSuperFiles = topology->getPropBool("@lockSuperFiles", false);
         crcResources = topology->getPropBool("@crcResources", false);
         ignoreOrphans = topology->getPropBool("@ignoreOrphans", true);
         chunkingHeap = topology->getPropBool("@chunkingHeap", true);

roxie/ccd/ccdquery.cpp

@@ -908,7 +908,7 @@ public:
             hashValue = rtlHash64VStr(dll->queryDll()->queryName(), hashValue);
             if (traceLevel > 8)
                 DBGLOG("getQueryHash: %s %"I64F"u from dll", id, hashValue);
-            if (!allFilesDynamic && !isDynamic && !package.isCompulsory())
+            if (!lockSuperFiles && !allFilesDynamic && !isDynamic && !package.isCompulsory())
             {
                 IConstWorkUnit *wu = dll->queryWorkUnit();
                 if (wu) // wu may be null in some unit test cases