Merge remote-tracking branch 'origin/candidate-3.10.x'

Conflicts:
	common/workunit/CMakeLists.txt
	initfiles/componentfiles/configxml/roxie.xsd.in
	roxie/ccd/ccdserver.cpp

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

common/fileview2/fvdisksource.cpp

@@ -251,7 +251,11 @@ bool DirectDiskDataSource::fetchRowData(MemoryBuffer & out, __int64 offset)
     physical.readData(out, offset, returnedMeta->getMaxRecordSize());
     if (out.length() == 0)
         return false;
-    out.setLength(returnedMeta->getRecordSize(out.toByteArray()));
+    size32_t actualLength = returnedMeta->getRecordSize(out.toByteArray());
+    if (actualLength > readBlockSize)
+        throwError(FVERR_RowTooLarge);
+
+    out.setLength(actualLength);
     return true;
 }
 

common/fileview2/fverror.hpp

@@ -52,6 +52,7 @@
 #define FVERR_BadStringTermination              2026
 #define FVERR_CannotBrowseFile                  2027
 #define FVERR_PluginMismatch                    2028
+#define FVERR_RowTooLarge                       2029
 
 #define FVERR_CouldNotResolveX_Text             "Could not resolve file '%s' in DFS"
 #define FVERR_NoRecordDescription_Text          "DFS did not contain record description for '%s'"
@@ -81,5 +82,6 @@
 #define FVERR_UnrecognisedMappingFunctionXY_Text    "Unrecognised field mapping function %s.%s"
 #define FVERR_BadStringTermination_Text          "String not terminated correctly %.*s"
 #define FVERR_CannotBrowseFile_Text              "Cannot browse file '%s'"
+#define FVERR_RowTooLarge_Text                   "Row too large"
 
 #endif

common/fileview2/fvsource.cpp

@@ -24,6 +24,7 @@
 
 #include "fileview.hpp"
 #include "fvsource.ipp"
+#include "fverror.hpp"
 #include "hqlerror.hpp"
 #include "eclhelper.hpp"
 #include "hqlattr.hpp"
@@ -707,6 +708,8 @@ VariableRowBlock::VariableRowBlock(MemoryBuffer & _buffer, __int64 _start, __int
     {
         rowIndex.append(cur);
         cur += recordSize->getRecordSize(max-cur, buff + cur);
+        if (cur > max)
+            throwError(FVERR_RowTooLarge);
     }
     buffer.setLength(cur);
     rowIndex.append(cur);

common/fileview2/fvsource.ipp

@@ -25,7 +25,7 @@
 
 //Following constants configure different sizes etc.
 
-#define DISK_BLOCK_SIZE     8096            // Size of chunks read directly from file.
+#define DISK_BLOCK_SIZE     0x10000         // Size of chunks read directly from file.
 #define PAGED_WU_LIMIT      0x20000         // Page load work unit results >= this size.
 #define WU_BLOCK_SIZE       0x4000          // Size of chunks read from Work unit
 #define DISKREAD_PAGE_SIZE  200             // Number of rows to read in each chunk from file.

common/workunit/CMakeLists.txt

@@ -29,6 +29,7 @@ set (    SRCS
          wujobq.cpp
          package.cpp
          workflow.cpp
+         referencedfilelist.cpp
                  
          workunit.hpp 
          wuerror.hpp
@@ -36,6 +37,7 @@ set (    SRCS
          package.h
          workflow.hpp
          pkgimpl.hpp
+         referencedfilelist.hpp
     )
 
 include_directories ( 
@@ -45,6 +47,7 @@ include_directories (
          ./../../common/deftype 
          ./../../system/include 
          ./../../dali/base 
+         ./../../dali/dfu
          ./../../rtl/include 
          ./../../common/dllserver 
          ./../../system/jlib

common/workunit/package.h

@@ -45,7 +45,7 @@ interface IHpccPackageMap : extends IInterface
     virtual const IHpccPackage *matchPackage(const char *name) const = 0;
     virtual const char *queryPackageId() const = 0;
     virtual bool isActive() const = 0;
-    virtual bool validate(StringArray &warn, StringArray &err, StringArray &unmatchedQueries, StringArray &unusedPackages) const = 0;
+    virtual bool validate(const char *queryid, StringArray &warn, StringArray &err, StringArray &unmatchedQueries, StringArray &unusedPackages, StringArray &unmatchedFiles) const = 0;
 };
 
 interface IHpccPackageSet : extends IInterface

common/workunit/pkgimpl.hpp

@@ -23,6 +23,7 @@
 #include "jptree.hpp"
 #include "jregexp.hpp"
 #include "package.h"
+#include "referencedfilelist.hpp"
 
 class CPackageSuperFileArray : public CInterface, implements ISimpleSuperFileEnquiry
 {
@@ -373,14 +374,14 @@ public:
         load(getPackageMapById(id, true));
     }
 
-    virtual bool validate(StringArray &warn, StringArray &err, StringArray &unmatchedQueries, StringArray &unusedPackages) const
+    virtual bool validate(const char *queryToCheck, StringArray &warn, StringArray &err, 
+        StringArray &unmatchedQueries, StringArray &unusedPackages, StringArray &unmatchedFiles) const
     {
         bool isValid = true;
         MapStringTo<bool> referencedPackages;
         Owned<IPropertyTree> qs = getQueryRegistry(querySet, true);
         if (!qs)
             throw MakeStringException(PACKAGE_TARGET_NOT_FOUND, "Target %s not found", querySet.sget());
-        Owned<IPropertyTreeIterator> queries = qs->getElements("Query");
         HashIterator it(packages);
         ForEach (it)
         {
@@ -404,11 +405,32 @@ public:
                     referencedPackages.setValue(baseId, true);
             }
         }
+        StringBuffer xpath("Query");
+        if (queryToCheck && *queryToCheck)
+            xpath.appendf("[@id='%s']", queryToCheck);
+        Owned<IPropertyTreeIterator> queries = qs->getElements(xpath);
+        if (!queries->first())
+        {
+            warn.append("No Queries found");
+            return isValid;
+        }
+
+        Owned<IWorkUnitFactory> wufactory = getWorkUnitFactory(NULL, NULL);
         ForEach(*queries)
         {
             const char *queryid = queries->query().queryProp("@id");
             if (queryid && *queryid)
             {
+                Owned<IReferencedFileList> filelist = createReferencedFileList(NULL, NULL);
+                Owned<IConstWorkUnit> cw = wufactory->openWorkUnit(queries->query().queryProp("@wuid"), false);
+                filelist->addFilesFromQuery(cw, this, queryid);
+                Owned<IReferencedFileIterator> refFiles = filelist->getFiles();
+                ForEach(*refFiles)
+                {
+                    VStringBuffer fullname("%s/%s", queryid, refFiles->query().getLogicalName());
+                    unmatchedFiles.append(fullname);
+                }
+
                 const IHpccPackage *matched = matchPackage(queryid);
                 if (matched)
                 {

common/roxiemanager/referencedfilelist.cpp

@@ -128,11 +128,11 @@ public:
     virtual void addFilesFromQuery(IConstWorkUnit *cw, const IHpccPackageMap *pm, const char *queryid);
 
     virtual IReferencedFileIterator *getFiles();
-    virtual void cloneFileInfo(bool overwrite, bool cloneSuperInfo);
+    virtual void cloneFileInfo(IDFUhelper *helper, bool overwrite, bool cloneSuperInfo);
     virtual void cloneRelationships();
-    virtual void cloneAllInfo(bool overwrite, bool cloneSuperInfo)
+    virtual void cloneAllInfo(IDFUhelper *helper, bool overwrite, bool cloneSuperInfo)
     {
-        cloneFileInfo(overwrite, cloneSuperInfo);
+        cloneFileInfo(helper, overwrite, cloneSuperInfo);
         cloneRelationships();
     }
     virtual void resolveFiles(const char *process, const char *remoteIP, bool checkLocalFirst, bool addSubFiles);
@@ -472,9 +472,8 @@ void ReferencedFileList::resolveFiles(const char *_process, const char *remoteIP
         resolveSubFiles(subfiles, checkLocalFirst);
 }
 
-void ReferencedFileList::cloneFileInfo(bool overwrite, bool cloneSuperInfo)
+void ReferencedFileList::cloneFileInfo(IDFUhelper *helper, bool overwrite, bool cloneSuperInfo)
 {
-    Owned<IDFUhelper> helper = createIDFUhelper();
     ReferencedFileIterator files(this);
     ForEach(files)
         files.queryObject().cloneInfo(helper, user, remote, process, overwrite);

common/roxiemanager/referencedfilelist.hpp

@@ -22,7 +22,6 @@
 #include "jlib.hpp"
 #include "workunit.hpp"
 #include "package.h"
-#include "dadfs.hpp"
 #include "dfuutil.hpp"
 
 #define RefFileNone           0x000
@@ -55,11 +54,11 @@ interface IReferencedFileList : extends IInterface
 
     virtual IReferencedFileIterator *getFiles()=0;
     virtual void resolveFiles(const char *process, const char *remoteIP, bool checkLocalFirst, bool addSubFiles)=0;
-    virtual void cloneAllInfo(bool overwrite, bool cloneSuperInfo)=0;
-    virtual void cloneFileInfo(bool overwrite, bool cloneSuperInfo)=0;
+    virtual void cloneAllInfo(IDFUhelper *helper, bool overwrite, bool cloneSuperInfo)=0;
+    virtual void cloneFileInfo(IDFUhelper *helper, bool overwrite, bool cloneSuperInfo)=0;
     virtual void cloneRelationships()=0;
 };
 
-IReferencedFileList *createReferencedFileList(const char *user, const char *pw);
+extern WORKUNIT_API IReferencedFileList *createReferencedFileList(const char *user, const char *pw);
 
 #endif //REFFILE_LIST_HPP

dali/base/dadfs.cpp

@@ -965,17 +965,25 @@ protected:
     Linked<IDistributedFileTransaction> transaction;
     IArrayOf<IDistributedFile> lockedFiles;
     DFTransactionState state;
-    void addFileLock(IDistributedFile* file) {
+    void addFileLock(IDistributedFile* file)
+    {
         // derived's prepare must call this before locking
         lockedFiles.append(*LINK(file));
         // Make sure this is in transaction's cache
         transaction->addFile(file);
     }
-    bool lock() {
+    bool lock(bool *dirty=NULL)
+    {
         // Files most have been acquired already by derived's class prepare
-        ForEachItemIn(i,lockedFiles) {
-            try {
-                lockedFiles.item(i).lockProperties(SDS_SUB_LOCK_TIMEOUT);
+        ForEachItemIn(i,lockedFiles)
+        {
+            try
+            {
+                if (lockedFiles.item(i).lockProperties(SDS_SUB_LOCK_TIMEOUT)) // returns true if needs reload
+                {
+                    if (dirty)
+                        *dirty = true;
+                }
             }
             catch (ISDSException *e)
             {
@@ -989,7 +997,8 @@ protected:
         }
         return true;
     }
-    void unlock() {
+    void unlock()
+    {
         for(unsigned i=0; i<locked; i++)
             lockedFiles.item(i).unlockProperties(state);
         locked = 0;
@@ -4113,14 +4122,17 @@ class CDistributedSuperFile: public CDistributedFileBase<IDistributedSuperFile>
             parent.setown(transaction->lookupSuperFile(parentlname));   
             if (!parent)
                 throw MakeStringException(-1,"addSubFile: SuperFile %s cannot be found",parentlname.get());
-            if (!subfile.isEmpty()) {
-                try {
+            if (!subfile.isEmpty())
+            {
+                try
+                {
                     sub.setown(transaction->lookupFile(subfile,SDS_SUB_LOCK_TIMEOUT));
                     // Must validate before locking for update below, to check sub is not already in parent (and therefore locked already)
-                    CDistributedSuperFile *sf = QUERYINTERFACE(parent.get(),CDistributedSuperFile);
+                    CDistributedSuperFile *sf = dynamic_cast<CDistributedSuperFile *>(parent.get());;
                     sf->validateAddSubFile(sub);
                 }
-                catch (IDFS_Exception *e) {
+                catch (IDFS_Exception *e)
+                {
                     if (e->errorCode()!=DFSERR_LookupConnectionTimout)
                         throw;
                     return false;
@@ -4131,8 +4143,19 @@ class CDistributedSuperFile: public CDistributedFileBase<IDistributedSuperFile>
             // Try to lock all files
             addFileLock(parent);
             addFileLock(sub);
-            if (lock())
+            bool dirty=false;
+            if (lock(&dirty))
+            {
+                if (dirty)
+                {
+                    // in the process of previous attempt to lock for exclusive access, locks were released
+                    // need to reload to ensure position and # of files is correct
+                    CDistributedSuperFile *sf = dynamic_cast<CDistributedSuperFile *>(parent.get());
+                    if (sf)
+                        sf->loadSubFiles(transaction, SDS_TRANSACTION_RETRY);
+                }
                 return true;
+            }
             unlock();
             return false;
         }
@@ -4174,11 +4197,14 @@ class CDistributedSuperFile: public CDistributedFileBase<IDistributedSuperFile>
             parent.setown(transaction->lookupSuperFile(parentlname,true));
             if (!parent)
                 throw MakeStringException(-1,"removeSubFile: SuperFile %s cannot be found",parentlname.get());
-            if (!subfile.isEmpty()) {
-                try {
+            if (!subfile.isEmpty())
+            {
+                try
+                {
                     sub.setown(transaction->lookupFile(subfile,SDS_SUB_LOCK_TIMEOUT));
                 }
-                catch (IDFS_Exception *e) {
+                catch (IDFS_Exception *e)
+                {
                     if (e->errorCode()!=DFSERR_LookupConnectionTimout)
                         throw;
                     return false;
@@ -4190,8 +4216,19 @@ class CDistributedSuperFile: public CDistributedFileBase<IDistributedSuperFile>
             addFileLock(parent);
             if (sub)
                 addFileLock(sub);
-            if (lock())
+            bool dirty=false;
+            if (lock(&dirty))
+            {
+                if (dirty)
+                {
+                    // in the process of previous attempt to lock for exclusive access, locks were released
+                    // need to reload to ensure position and # of files is correct
+                    CDistributedSuperFile *sf = dynamic_cast<CDistributedSuperFile *>(parent.get());
+                    if (sf)
+                        sf->loadSubFiles(transaction, SDS_TRANSACTION_RETRY);
+                }
                 return true;
+            }
             unlock();
             return false;
         }
@@ -4230,6 +4267,31 @@ class CDistributedSuperFile: public CDistributedFileBase<IDistributedSuperFile>
         Linked<IDistributedSuperFile> file;
         StringAttr parentlname;
         StringAttr filelname;
+
+        bool refresh(IDistributedSuperFile *super) // returns true if any changes
+        {
+            if (!super)
+                return false;
+            IArrayOf<IDistributedFile> copyOfSubFiles;
+            unsigned s=0;
+            for (; s<super->numSubFiles(); s++)
+               copyOfSubFiles.append(*LINK(&super->querySubFile(s)));
+            CDistributedSuperFile *_super = dynamic_cast<CDistributedSuperFile *>(super);
+            _super->loadSubFiles(transaction, SDS_TRANSACTION_RETRY);
+            if (copyOfSubFiles.ordinality() != super->numSubFiles())
+                return true;
+            for (s=0; s<super->numSubFiles(); s++)
+            {
+                IDistributedFile *file = &(super->querySubFile(s));
+                if (file != &copyOfSubFiles.item(s))
+                    return true;
+            }
+            return false;
+        }
+        bool refresh() // returns true if any changes
+        {
+            return (refresh(parent.get()) || refresh(file.get()));
+        }
     public:
         cSwapFileAction(IDistributedFileTransaction *_transaction,const char *_parentlname,const char *_filelname)
             : CDFAction(_transaction), parentlname(_parentlname), filelname(_filelname)
@@ -4242,7 +4304,8 @@ class CDistributedSuperFile: public CDistributedFileBase<IDistributedSuperFile>
             if (!parent)
                 throw MakeStringException(-1,"swapSuperFile: SuperFile %s cannot be found",parentlname.get());
             file.setown(transaction->lookupSuperFile(filelname));
-            if (!file) {
+            if (!file)
+            {
                 parent.clear();
                 throw MakeStringException(-1,"swapSuperFile: SuperFile %s cannot be found",filelname.get());
             }
@@ -4253,8 +4316,16 @@ class CDistributedSuperFile: public CDistributedFileBase<IDistributedSuperFile>
             addFileLock(file);
             for (unsigned i=0; i<file->numSubFiles(); i++)
                 addFileLock(&file->querySubFile(i));
-            if (lock())
-                return true;
+            bool dirty=false;
+            if (lock(&dirty))
+            {
+                if (!dirty)
+                    return true;
+                // in the process of previous attempt to lock for exclusive access, locks were released
+                // need to reload to ensure position and # of files is correct
+                if (!refresh()) // refreshes the supers and checks if any changes, if there are, transaction will unlock and retry
+                    return true;
+            }
             unlock();
             return false;
         }

docs/HPCCClientTools/CT_Mods/CT_ECL_CLI.xml

@@ -1971,12 +1971,13 @@ ecl packagemap add roxie mypackagemap.pkg --daliip=192.168.11.11
       <sect2 id="eclpackagevalidate" role="brk">
         <title>ecl packagemap validate</title>
 
-        <para><emphasis role="bold">ecl packagemap validate &lt;filename&gt;
-        </emphasis></para>
+        <para><emphasis role="bold">ecl packagemap validate &lt;target&gt;
+        [&lt;filename&gt;] </emphasis></para>
 
         <para>Examples:</para>
 
-        <programlisting>ecl packagemap validate mypackagemap.pkg</programlisting>
+        <programlisting>ecl packagemap validate roxie mypackagemap.pkg
+ecl packagemap validate roxie --active</programlisting>
 
         <para>The packagemap validate command verifies that :</para>
 
@@ -1992,13 +1993,21 @@ ecl packagemap add roxie mypackagemap.pkg --daliip=192.168.11.11
           </listitem>
 
           <listitem>
-            <para>All Roxie queries are defined in the package </para>
+            <para>All Roxie queries are defined in the package</para>
           </listitem>
         </itemizedlist>
 
-        <para></para>
+        <para>The result will also list any files that are used by queries but
+        not mapped in the packagemap.</para>
 
-        <para><informaltable colsep="0" frame="none" rowsep="0">
+        <para>Filename, --active, and --pmid are mutually exclusive. The
+        --active or --pmid options validate a packagemap that has already been
+        added instead of a local file. </para>
+
+        <para>The --queryid option checks the files in a query instead of all
+        the queries in the target queryset. This is quicker when you only need
+        to validate the files for a single query. <informaltable colsep="0"
+            frame="none" rowsep="0">
             <tgroup cols="2">
               <colspec align="left" colwidth="175.55pt" />
 
@@ -2008,7 +2017,7 @@ ecl packagemap add roxie mypackagemap.pkg --daliip=192.168.11.11
                 <row>
                   <entry>ecl packagemap validate</entry>
 
-                  <entry>Calls the packagemap validate command. </entry>
+                  <entry>Calls the packagemap validate command.</entry>
                 </row>
 
                 <row>
@@ -2018,7 +2027,7 @@ ecl packagemap add roxie mypackagemap.pkg --daliip=192.168.11.11
                 <row>
                   <entry>validate</entry>
 
-                  <entry>validates packagemap info</entry>
+                  <entry>Validates packagemap info</entry>
                 </row>
 
                 <row>
@@ -2033,6 +2042,13 @@ ecl packagemap add roxie mypackagemap.pkg --daliip=192.168.11.11
                 </row>
 
                 <row>
+                  <entry>target</entry>
+
+                  <entry>The target containing the packagemap to
+                  validate</entry>
+                </row>
+
+                <row>
                   <entry><emphasis role="bold">Options</emphasis></entry>
                 </row>
 
@@ -2050,6 +2066,26 @@ ecl packagemap add roxie mypackagemap.pkg --daliip=192.168.11.11
                 </row>
 
                 <row>
+                  <entry>--active</entry>
+
+                  <entry>Validates the packagemap that is active for the given
+                  target</entry>
+                </row>
+
+                <row>
+                  <entry>--pmid=&lt;packagemapid&gt;</entry>
+
+                  <entry>Validates the given packagemap</entry>
+                </row>
+
+                <row>
+                  <entry>--queryid</entry>
+
+                  <entry>Validate the files for the given queryid if they are
+                  mapped in the packagemap</entry>
+                </row>
+
+                <row>
                   <entry>--port</entry>
 
                   <entry>The eclwatch services port (Default is 8010)</entry>

ecl/ecl-package/ecl-package.cpp

@@ -598,7 +598,7 @@ private:
 class EclCmdPackageValidate : public EclCmdCommon
 {
 public:
-    EclCmdPackageValidate()
+    EclCmdPackageValidate() : optValidateActive(false)
     {
     }
     virtual bool parseCommandLineOptions(ArgvIterator &iter)
@@ -625,6 +625,12 @@ public:
                 }
                 continue;
             }
+            if (iter.matchFlag(optValidateActive, ECLOPT_ACTIVE))
+                continue;
+            if (iter.matchOption(optPMID, ECLOPT_PMID) || iter.matchOption(optPMID, ECLOPT_PMID_S))
+                continue;
+            if (iter.matchOption(optQueryId, ECLOPT_QUERYID))
+                continue;
             if (EclCmdCommon::matchCommandLineOption(iter, true)!=EclCmdOptionMatch)
                 return false;
         }
@@ -638,9 +644,18 @@ public:
             return false;
         }
         StringBuffer err;
-        if (optFileName.isEmpty())
-            err.append("\n ... Missing package file name\n\n");
-        else if (optTarget.isEmpty())
+        int pcount=0;
+        if (optFileName.length())
+            pcount++;
+        if (optPMID.length())
+            pcount++;
+        if (optValidateActive)
+            pcount++;
+        if (pcount==0)
+            err.append("\n ... Package file name, --pmid, or --active required\n\n");
+        else if (pcount > 1)
+            err.append("\n ... Package file name, --pmid, and --active are mutually exclusive\n\n");
+        if (optTarget.isEmpty())
             err.append("\n ... Specify a cluster name\n\n");
 
         if (err.length())
@@ -654,14 +669,20 @@ public:
     virtual int processCMD()
     {
         Owned<IClientWsPackageProcess> packageProcessClient = getWsPackageSoapService(optServer, optPort, optUsername, optPassword);
-        StringBuffer pkgInfo;
-        pkgInfo.loadFile(optFileName);
+        Owned<IClientValidatePackageRequest> request = packageProcessClient->createValidatePackageRequest();
 
-        fprintf(stdout, "\nvalidating packagemap file %s\n\n", optFileName.sget());
+        if (optFileName.length())
+        {
+            StringBuffer pkgInfo;
+            pkgInfo.loadFile(optFileName);
+            fprintf(stdout, "\nvalidating packagemap file %s\n\n", optFileName.sget());
+            request->setInfo(pkgInfo);
+        }
 
-        Owned<IClientValidatePackageRequest> request = packageProcessClient->createValidatePackageRequest();
-        request->setInfo(pkgInfo);
+        request->setActive(optValidateActive);
+        request->setPMID(optPMID);
         request->setTarget(optTarget);
+        request->setQueryIdToVerify(optQueryId);
 
         Owned<IClientValidatePackageResponse> resp = packageProcessClient->ValidatePackage(request);
         if (resp->getExceptions().ordinality()>0)
@@ -698,6 +719,13 @@ public:
             ForEachItemIn(i, unusedPackages)
                 fprintf(stderr, "      %s\n", unusedPackages.item(i));
         }
+        StringArray &unusedFiles = resp->getFiles().getUnmatched();
+        if (unusedFiles.ordinality()>0)
+        {
+            fputs("\n   Files without matching package definitions:\n", stderr);
+            ForEachItemIn(i, unusedFiles)
+                fprintf(stderr, "      %s\n", unusedFiles.item(i));
+        }
 
         return 0;
     }
@@ -710,8 +738,10 @@ public:
                     "\n"
                     "ecl packagemap validate <target> <filename>\n"
                     " Options:\n"
-                    "   <target>                    name of target to use when adding package map information\n"
-                    "   <filename>                  name of file containing package map information\n",
+                    "   <target>                    name of target to use when validating package map information\n"
+                    "   <filename>                  name of file containing package map information\n"
+                    "   --active                    validate the active packagemap\n"
+                    "   -pm, --pmid                 id of packagemap to validate\n",
                     stdout);
 
         EclCmdCommon::usage();
@@ -719,6 +749,9 @@ public:
 private:
     StringAttr optFileName;
     StringAttr optTarget;
+    StringAttr optPMID;
+    StringAttr optQueryId;
+    bool optValidateActive;
 };
 
 IEclCommand *createPackageSubCommand(const char *cmdname)

ecl/eclcmd/eclcmd_common.hpp

@@ -66,6 +66,8 @@ typedef IEclCommand *(*EclCommandFactory)(const char *cmdname);
 
 #define ECLOPT_DONT_COPY_FILES "--no-files"
 
+#define ECLOPT_ACTIVE "--active"
+#define ECLOPT_ALL "--all"
 #define ECLOPT_INACTIVE "--inactive"
 #define ECLOPT_NO_ACTIVATE "--no-activate"
 #define ECLOPT_ACTIVATE "--activate"
@@ -110,6 +112,10 @@ typedef IEclCommand *(*EclCommandFactory)(const char *cmdname);
 #define ECLOPT_QUERYSET_S "-qs"
 #define ECLOPT_VERSION "--version"
 #define ECLOPT_SHOW "--show"
+#define ECLOPT_PMID "--pmid"
+#define ECLOPT_PMID_S "-pm"
+#define ECLOPT_QUERYID "--queryid"
+
 
 #define ECLOPT_DALIIP "--daliip"
 #define ECLOPT_PROCESS "--process"

ecl/hqlcpp/hqlttcpp.cpp

@@ -658,7 +658,6 @@ IHqlExpression * HqlThorBoundaryTransformer::createTransformed(IHqlExpression *
     case no_field:
     case no_constant:
     case no_attr:
-    case no_attr_expr:
     case no_attr_link:
     case no_getresult:
     case no_left:
@@ -992,7 +991,8 @@ void HqlCppTranslator::markThorBoundaries(WorkflowArray & array)
     HqlThorBoundaryTransformer thorTransformer(wu(), targetRoxie(), options.maxRootMaybeThorActions, options.resourceConditionalActions, options.resourceSequential);
     ForEachItemIn(idx, array)
     {
-        HqlExprArray & exprs = array.item(idx).queryExprs();
+        WorkflowItem & cur = array.item(idx);
+        HqlExprArray & exprs = cur.queryExprs();
         HqlExprArray bounded;
 
         thorTransformer.transformRoot(exprs, bounded);

esp/eclwatch/ws_XSLT/WUQueryDetails.xslt

@@ -51,7 +51,9 @@
                 <script language="JavaScript1.2">
                     var querySet = '<xsl:value-of select="QuerySet"/>';
                     var queryId = '<xsl:value-of select="QueryId"/>';
+                    var queryName = '<xsl:value-of select="QueryName"/>';
                     var suspended = '<xsl:value-of select="Suspended"/>';
+                    var activated = '<xsl:value-of select="Activated"/>';
                     <xsl:text disable-output-escaping="yes"><![CDATA[
                       function deleteQuery() {
                         actionWorkunits('Delete');
@@ -60,8 +62,12 @@
                       function toggleQuery() {
                         actionWorkunits('ToggleSuspend');
                       }
-                      function activateQuery() {
-                        actionWorkunits('Activate');
+
+                      function toggleActivated() {
+                        if (activated == 1)
+                          actionAliases('Deactivate');
+                        else
+                          actionWorkunits('Activate');
                       }
 
                       function getQueryActions(Action) {
@@ -75,8 +81,11 @@
                           var connectionCallback = {
                               success: function(o) {
                                   var xmlDoc = o.responseXML;
-                                  document.location.replace( document.location.href );
-
+                                  if (Action == 'Delete') {
+                                    document.location.replace( "/WsWorkunits/WUQuerysetDetails?QuerySetName=" + querySet);
+                                  } else {
+                                    document.location.replace(document.location.href);
+                                  }
                               },
                               failure: function(o) {
                                   alert('Failure:' + o.statusText);
@@ -96,6 +105,37 @@
                           return;
                       }
 
+                      function getAliasActions(Action) {
+                          var soapXML = '<WUQuerysetAliasAction><QuerySetName>' + querySet + '</QuerySetName><Action>' + Action + '</Action><Aliases>';
+                          soapXML += '<Alias><Name>' + queryName + '</Name></Alias>';
+                          soapXML += '</Aliases></WUQuerysetAliasAction>';
+                          return soapXML;
+                      }
+
+                      function actionAliases(Action) {
+                          var connectionCallback = {
+                              success: function(o) {
+                                  var xmlDoc = o.responseXML;
+                                  document.location.replace( document.location.href );
+                              },
+                              failure: function(o) {
+                                  alert('Failure:' + o.statusText);
+                              }
+                          };
+
+                          var postBody = '<soap:Envelope xmlns:soap="http://schemas.xmlsoap.org/soap/envelope/" xmlns:SOAP-ENC="http://schemas.xmlsoap.org/soap/encoding/" xmlns="http://webservices.seisint.com/ws_roxieconfig"><soap:Body>' + getAliasActions(Action) + '</soap:Body></soap:Envelope>';
+
+                          YAHOO.util.Connect.initHeader("SOAPAction", "/WsWorkunits/WUQuerysetActionAliases?");
+                          YAHOO.util.Connect.initHeader("Content-Type", "text/xml");
+                          YAHOO.util.Connect._use_default_post_header = false;
+
+                          var getXML = YAHOO.util.Connect.asyncRequest("POST",
+                                  "/WsWorkunits/WUQuerysetActionAliases",
+                                  connectionCallback, postBody);
+                          return;
+
+                      }
+
                       function DFUFileDetails(fileName) {
                           document.location.href='/WsDfu/DFUInfo?Name='+escape(fileName);
                       }
@@ -155,6 +195,18 @@
                                 <td><xsl:value-of select="SuspendedBy"/></td>
                             </tr>
                         </xsl:if>
+                        <tr>
+                            <th>
+                                Activated:
+                            </th>
+                            <td>
+                                <input type="checkbox" onclick="toggleActivated();">
+                                    <xsl:if test="Activated=1">
+                                        <xsl:attribute name="checked"/>
+                                    </xsl:if>
+                                </input>
+                            </td>
+                        </tr>
                         <xsl:if test="string-length(Label)">
                             <tr>
                                 <th>Label:</th>
@@ -194,7 +246,6 @@
                     </table>
                 </form>
                 <input id="deleteBtn" type="button" value="Delete" onclick="deleteQuery();"> </input>
-                <input id="activateBtn" type="button" value="Activate" onclick="activateQuery();"> </input>
             </body>
         </html>
     </xsl:template>

esp/scm/ws_packageprocess.ecm

@@ -114,6 +114,9 @@ ESPrequest ValidatePackageRequest
 {
     string Info;
     string Target;
+    bool Active;
+    string PMID;
+    string QueryIdToVerify;
 };
 
 ESPstruct ValidatePackageInfo
@@ -126,13 +129,20 @@ ESPstruct ValidatePackageQueries
     ESParray<string> Unmatched;
 };
 
+ESPstruct ValidatePackageFiles
+{
+    ESParray<string> Unmatched;
+};
+
 ESPresponse [exceptions_inline] ValidatePackageResponse
 {
+    string PMID;
     ESPstruct BasePackageStatus status;
     ESParray<string> Warnings;
     ESParray<string> Errors;
     ESPstruct ValidatePackageInfo packages;
     ESPstruct ValidatePackageQueries queries;
+    ESPstruct ValidatePackageFiles files;
 };
 
 ESPservice [version("1.00"), default_client_version("1.00"), exceptions_inline("./smc_xslt/exceptions.xslt")] WsPackageProcess

esp/scm/ws_workunits.ecm

@@ -1207,6 +1207,7 @@ ESPresponse [exceptions_inline] WUQueryDetailsResponse
     string Wuid;
     string Dll;
     bool Suspended;
+    [min_ver("1.42")] bool Activated;
     string SuspendedBy;
     string PublishedBy;
     string Comment;
@@ -1337,7 +1338,7 @@ ESPresponse [exceptions_inline] WUQuerySetCopyQueryResponse
 };
 
 ESPservice [
-    version("1.41"), default_client_version("1.41"),
+    version("1.42"), default_client_version("1.42"),
     noforms,exceptions_inline("./smc_xslt/exceptions.xslt"),use_method_name] WsWorkunits
 {
     ESPmethod [resp_xsl_default("/esp/xslt/workunits.xslt")]     WUQuery(WUQueryRequest, WUQueryResponse);

esp/services/ws_packageprocess/CMakeLists.txt

@@ -22,8 +22,6 @@ set (    SRCS
          ws_packageprocessService.cpp
          ws_packageprocessService.hpp
          packageprocess_errors.h
-         ${HPCC_SOURCE_DIR}/common/roxiemanager/referencedfilelist.cpp
-         ${HPCC_SOURCE_DIR}/common/roxiemanager/referencedfilelist.hpp
     )
 
 include_directories (

esp/services/ws_packageprocess/packageprocess_errors.h

@@ -29,5 +29,8 @@
 #define PKG_DEACTIVATE_NOT_FOUND   PKG_PROCESS_ERROR_START+6
 #define PKG_DELETE_NOT_FOUND   PKG_PROCESS_ERROR_START+7
 #define PKG_NONE_DEFINED   PKG_PROCESS_ERROR_START+8
+#define PKG_CREATE_PACKAGESET_FAILED   PKG_PROCESS_ERROR_START+9
+#define PKG_PACKAGEMAP_NOT_FOUND   PKG_PROCESS_ERROR_START+10
+#define PKG_LOAD_PACKAGEMAP_FAILED   PKG_PROCESS_ERROR_START+11
 
 #endif

esp/services/ws_packageprocess/ws_packageprocessService.cpp

@@ -138,7 +138,8 @@ void cloneFileInfoToDali(StringArray &fileNames, const char *lookupDaliIp, ICons
     SCMStringBuffer processName;
     clusterInfo->getRoxieProcess(processName);
     wufiles->resolveFiles(processName.str(), lookupDaliIp, !overWrite, false);
-    wufiles->cloneAllInfo(overWrite, true);
+    Owned<IDFUhelper> helper = createIDFUhelper();
+    wufiles->cloneAllInfo(helper, overWrite, true);
 }
 
 void cloneFileInfoToDali(StringArray &fileNames, const char *lookupDaliIp, const char *target, bool overWrite, IUserDescriptor* userdesc)
@@ -515,13 +516,43 @@ bool CWsPackageProcessEx::onValidatePackage(IEspContext &context, IEspValidatePa
     StringArray errors;
     StringArray unmatchedQueries;
     StringArray unusedPackages;
+    StringArray unmatchedFiles;
 
-    Owned<IHpccPackageMap> map = createPackageMapFromXml(req.getInfo(), req.getTarget(), NULL);
-    map->validate(warnings, errors, unmatchedQueries, unusedPackages);
+    Owned<IHpccPackageSet> set;
+    Owned<IHpccPackageMap> ownedmap;
+    const IHpccPackageMap *map = NULL;
 
+    if (req.getActive()) //validate active map
+    {
+        set.setown(createPackageSet("*"));
+        if (!set)
+            throw MakeStringException(PKG_CREATE_PACKAGESET_FAILED, "Unable to create PackageSet");
+        map = set->queryActiveMap(req.getTarget());
+        if (!map)
+            throw MakeStringException(PKG_PACKAGEMAP_NOT_FOUND, "Active package map not found");
+    }
+    else if (req.getPMID())
+    {
+        ownedmap.setown(createPackageMapFromPtree(getPackageMapById(req.getPMID(), true), req.getTarget(), req.getPMID()));
+        if (!ownedmap)
+            throw MakeStringException(PKG_LOAD_PACKAGEMAP_FAILED, "Error loading package map %s", req.getPMID());
+        map = ownedmap;
+    }
+    else
+    {
+        ownedmap.setown(createPackageMapFromXml(req.getInfo(), req.getTarget(), NULL));
+        if (!ownedmap)
+            throw MakeStringException(PKG_LOAD_PACKAGEMAP_FAILED, "Error processing package file content");
+        map = ownedmap;
+    }
+
+    map->validate(req.getQueryIdToVerify(), warnings, errors, unmatchedQueries, unusedPackages, unmatchedFiles);
+
+    resp.setPMID(map->queryPackageId());
     resp.setWarnings(warnings);
     resp.setErrors(errors);
     resp.updateQueries().setUnmatched(unmatchedQueries);
     resp.updatePackages().setUnmatched(unusedPackages);
+    resp.updateFiles().setUnmatched(unmatchedFiles);
     return true;
 }

esp/services/ws_workunits/CMakeLists.txt

@@ -34,8 +34,6 @@ set (    SRCS
          ${ESPSCM_GENERATED_DIR}/ws_fs_esp.cpp
          ${HPCC_SOURCE_DIR}/esp/scm/ws_workunits.ecm
          ${HPCC_SOURCE_DIR}/esp/clients/roxiecontrol.cpp
-         ${HPCC_SOURCE_DIR}/common/roxiemanager/referencedfilelist.cpp
-         ${HPCC_SOURCE_DIR}/common/roxiemanager/referencedfilelist.hpp
          ws_workunitsPlugin.cpp
          ws_workunitsService.cpp
          ws_workunitsService.hpp
@@ -76,6 +74,7 @@ include_directories (
          ./../../bindings
          ./../../smc/SMCLib
          ./../../bindings/SOAP/xpp
+         ${HPCC_SOURCE_DIR}/dali/dfu
     )
 
 ADD_DEFINITIONS( -D_USRDLL -DWS_WORKUNITS_EXPORTS )

esp/services/ws_workunits/ws_workunitsQuerySets.cpp

@@ -447,7 +447,8 @@ void copyQueryFilesToCluster(IEspContext &context, IConstWorkUnit *cw, const cha
         Owned<IHpccPackageSet> ps = createPackageSet(process.str());
         wufiles->addFilesFromQuery(cw, (ps) ? ps->queryActiveMap(target) : NULL, queryid);
         wufiles->resolveFiles(process.str(), remoteIP, !overwrite, true);
-        wufiles->cloneAllInfo(overwrite, true);
+        Owned<IDFUhelper> helper = createIDFUhelper();
+        wufiles->cloneAllInfo(helper, overwrite, true);
     }
 }
 
@@ -878,14 +879,15 @@ bool CWsWorkunitsEx::onWUQueryDetails(IEspContext &context, IEspWUQueryDetailsRe
 
     StringBuffer xpath;
     xpath.clear().append("Query[@id='").append(queryId).append("']");
-    IPropertyTree *query = queryRegistry->queryPropTree(xpath);
+    IPropertyTree *query = queryRegistry->queryPropTree(xpath.str());
     if (!query)
     {
         DBGLOG("No matching Query");
         return false;
     }
 
-    resp.setQueryName(query->queryProp("@name"));
+    const char* queryName = query->queryProp("@name");
+    resp.setQueryName(queryName);
     resp.setWuid(query->queryProp("@wuid"));
     resp.setDll(query->queryProp("@dll"));
     resp.setPublishedBy(query->queryProp("@publishedBy"));
@@ -898,6 +900,17 @@ bool CWsWorkunitsEx::onWUQueryDetails(IEspContext &context, IEspWUQueryDetailsRe
     if (logicalFiles.length())
         resp.setLogicalFiles(logicalFiles);
 
+    double version = context.getClientVersion();
+    if (version >= 1.42)
+    {
+        xpath.clear().appendf("Alias[@name='%s']", queryName);
+        IPropertyTree *alias = queryRegistry->queryPropTree(xpath.str());
+        if (!alias)
+            resp.setActivated(false);
+        else
+            resp.setActivated(true);
+    }
+
     return true;
 }
 

initfiles/componentfiles/configxml/roxie.xsd.in

@@ -425,10 +425,17 @@
     </xs:attribute>
  </xs:attributeGroup>
  <xs:attributeGroup name="Options">
+    <xs:attribute name="allFilesDynamic" type="xs:boolean" use="optional" default="false">
+      <xs:annotation>
+        <xs:appinfo>
+          <tooltip>If enabled, files will be resolved per-query and not locked between queries</tooltip>
+        </xs:appinfo>
+      </xs:annotation>
+    </xs:attribute>
     <xs:attribute name="allowRoxieOnDemand" type="xs:boolean" use="optional" default="false">
       <xs:annotation>
         <xs:appinfo>
-          <tooltip>allow roxie to load, run, add queries meant to be run 1 time (no file data copies for these queries</tooltip>
+          <tooltip>Allow roxie to load, run, add queries meant to be run 1 time (no file data copies for these queries</tooltip>
         </xs:appinfo>
       </xs:annotation>
     </xs:attribute>

initfiles/etc/DIR_NAME/environment.xml.in

@@ -753,7 +753,8 @@
              netAddress="."
              program="${EXEC_PATH}/ftslave"/>
   </FTSlaveProcess>
-  <RoxieCluster allowRoxieOnDemand="false"
+  <RoxieCluster allFilesDynamic="true"
+                allowRoxieOnDemand="false"
                 baseDataDir="${RUNTIME_PATH}/hpcc-data/roxie"
                 blindLogging="false"
                 blobCacheMem="0"

roxie/ccd/CMakeLists.txt

@@ -72,6 +72,7 @@ include_directories (
          ./../../common/deftype 
          ./../../system/include 
          ./../../dali/base 
+         ./../../dali/dfu 
          ./../../roxie/roxiemem 
          ./../../common/dllserver 
          ./../../system/jlib 

roxie/ccd/ccd.hpp

@@ -325,6 +325,7 @@ extern IPropertyTree* ccdChannels;
 extern IPropertyTree* topology;
 extern StringArray allQuerySetNames;
 
+extern bool allFilesDynamic;
 extern bool crcResources;
 extern bool logFullQueries;
 extern bool blindLogging;

roxie/ccd/ccdactivities.cpp

@@ -833,7 +833,7 @@ public:
         forceUnkeyed(_forceUnkeyed)
     {
         helper = (IHThorDiskReadBaseArg *) basehelper;
-        variableFileName = (helper->getFlags() & (TDXvarfilename|TDXdynamicfilename)) != 0;
+        variableFileName = allFilesDynamic || ((helper->getFlags() & (TDXvarfilename|TDXdynamicfilename)) != 0);
         isOpt = (helper->getFlags() & TDRoptional) != 0;
         diskSize.set(helper->queryDiskRecordSize());
         processed = 0;
@@ -976,7 +976,7 @@ public:
         : CSlaveActivityFactory(_graphNode, _subgraphId, _queryFactory, _helperFactory)
     {
         Owned<IHThorDiskReadBaseArg> helper = (IHThorDiskReadBaseArg *) helperFactory();
-        bool variableFileName = (helper->getFlags() & (TDXvarfilename|TDXdynamicfilename)) != 0;
+        bool variableFileName = allFilesDynamic || ((helper->getFlags() & (TDXvarfilename|TDXdynamicfilename)) != 0);
         if (!variableFileName)
         {
             bool isOpt = (helper->getFlags() & TDRoptional) != 0;
@@ -3071,7 +3071,7 @@ public:
         m.setBuffer(indexLayoutSize, indexLayoutMeta.getdata());
         activityMeta.setown(deserializeRecordMeta(m, true));
         layoutTranslators.setown(new TranslatorArray);
-        bool variableFileName = (helper->getFlags() & (TIRvarfilename|TIRdynamicfilename)) != 0;
+        bool variableFileName = allFilesDynamic || ((helper->getFlags() & (TIRvarfilename|TIRdynamicfilename)) != 0);
         if (!variableFileName)
         {
             bool isOpt = (helper->getFlags() & TIRoptional) != 0;
@@ -3226,7 +3226,7 @@ public:
         stepExtra(SSEFreadAhead, NULL)
     {
         indexHelper = (IHThorIndexReadBaseArg *) basehelper;
-        variableFileName = (indexHelper->getFlags() & (TIRvarfilename|TIRdynamicfilename)) != 0;
+        variableFileName = allFilesDynamic || ((indexHelper->getFlags() & (TIRvarfilename|TIRdynamicfilename)) != 0);
         isOpt = (indexHelper->getFlags() & TDRoptional) != 0;
         inputData = NULL;
         inputCount = 0;
@@ -4268,7 +4268,7 @@ public:
     {
         Owned<IHThorFetchBaseArg> helper = (IHThorFetchBaseArg *) helperFactory();
         IHThorFetchContext * fetchContext = static_cast<IHThorFetchContext *>(helper->selectInterface(TAIfetchcontext_1));
-        bool variableFileName = (fetchContext->getFetchFlags() & (FFvarfilename|FFdynamicfilename)) != 0;
+        bool variableFileName = allFilesDynamic || ((fetchContext->getFetchFlags() & (FFvarfilename|FFdynamicfilename)) != 0);
         if (!variableFileName)
         {
             bool isOpt = (fetchContext->getFetchFlags() & FFdatafileoptional) != 0;
@@ -4317,7 +4317,7 @@ public:
         helper = (IHThorFetchBaseArg *) basehelper;
         fetchContext = static_cast<IHThorFetchContext *>(helper->selectInterface(TAIfetchcontext_1));
         base = 0;
-        variableFileName = (fetchContext->getFetchFlags() & (FFvarfilename|FFdynamicfilename)) != 0;
+        variableFileName = allFilesDynamic || ((fetchContext->getFetchFlags() & (FFvarfilename|FFdynamicfilename)) != 0);
         isOpt = (fetchContext->getFetchFlags() & FFdatafileoptional) != 0;
         onCreate();
         inputData = (char *) serializedCreate.readDirect(0);
@@ -4617,7 +4617,7 @@ public:
         m.setBuffer(indexLayoutSize, indexLayoutMeta.getdata());
         activityMeta.setown(deserializeRecordMeta(m, true));
         layoutTranslators.setown(new TranslatorArray);
-        bool variableFileName = (helper->getJoinFlags() & (JFvarindexfilename|JFdynamicindexfilename)) != 0;
+        bool variableFileName = allFilesDynamic || ((helper->getJoinFlags() & (JFvarindexfilename|JFdynamicindexfilename)) != 0);
         if (!variableFileName)
         {
             bool isOpt = (helper->getJoinFlags() & JFindexoptional) != 0;
@@ -4659,7 +4659,7 @@ public:
         : factory(_aFactory), CRoxieKeyedActivity(_logctx, _packet, _hFactory, _aFactory)
     {
         helper = (IHThorKeyedJoinArg *) basehelper;
-        variableFileName = (helper->getJoinFlags() & (JFvarindexfilename|JFdynamicindexfilename)) != 0;
+        variableFileName = allFilesDynamic || ((helper->getJoinFlags() & (JFvarindexfilename|JFdynamicindexfilename)) != 0);
         inputDone = 0;
         processed = 0;
         candidateCount = 0;
@@ -4963,7 +4963,7 @@ public:
     {
         Owned<IHThorKeyedJoinArg> helper = (IHThorKeyedJoinArg *) helperFactory();
         assertex(helper->diskAccessRequired());
-        bool variableFileName = (helper->getFetchFlags() & (FFvarfilename|FFdynamicfilename)) != 0;
+        bool variableFileName = allFilesDynamic || ((helper->getFetchFlags() & (FFvarfilename|FFdynamicfilename)) != 0);
         if (!variableFileName)
         {
             bool isOpt = (helper->getFetchFlags() & FFdatafileoptional) != 0;
@@ -5016,7 +5016,7 @@ public:
         // MORE - no continuation row support?
         base = 0;
         helper = (IHThorKeyedJoinArg *) basehelper;
-        variableFileName = (helper->getFetchFlags() & (FFvarfilename|FFdynamicfilename)) != 0;
+        variableFileName = allFilesDynamic || ((helper->getFetchFlags() & (FFvarfilename|FFdynamicfilename)) != 0);
         onCreate();
         inputData = (const char *) serializedCreate.readDirect(0);
         inputLimit = inputData + (serializedCreate.length() - serializedCreate.getPos());

roxie/ccd/ccdfile.cpp

@@ -122,6 +122,7 @@ public:
     
     ~CLazyFileIO()
     {
+        setFailure(); // ensures the open file count properly maintained
     }
 
     virtual void beforeDispose()
@@ -1150,7 +1151,7 @@ public:
         try
         {
             CriticalBlock b(crit);
-            ILazyFileIO *f = files.getValue(localLocation);
+            Linked<ILazyFileIO> f = files.getValue(localLocation);
             if (f && f->isAlive())
             {
                 if ((size != -1 && size != f->getSize()) ||
@@ -1177,7 +1178,7 @@ public:
                     throw MakeStringException(ROXIE_MISMATCH, "Different version of %s already loaded: sizes = %"I64F"d %"I64F"d  Date = %s  %s", id, size, f->getSize(), modifiedDt.str(), fileDt.str());
                 }
                 else
-                    return LINK(f);
+                    return f.getClear();
             }
 
             ret.setown(openFile(id, partNo, fileType, localLocation, peerRoxieCopiedLocationInfo, deployedLocationInfo, size, modified, memFile, crc, isCompressed));  // for now don't check crcs
@@ -1319,25 +1320,22 @@ public:
         ForEach(h)
         {
             ILazyFileIO *f = files.mapToValue(&h.query());
+            const char *fname = remote ? f->querySource()->queryFilename() : f->queryFilename();
             if (f->isAlive() && f->isOpen() && f->isRemote()==remote && !f->isCopying())
             {
                 unsigned age = msTick() - f->getLastAccessed();
                 if (age > maxFileAge[remote])
                 {
                     if (traceLevel > 5)
-                    {
-                        const char *fname;
-                        if (remote)
-                            fname = f->querySource()->queryFilename();
-                        else
-                            fname = f->queryFilename();
                         DBGLOG("Closing inactive %s file %s (last accessed %u ms ago)", remote ? "remote" : "local",  fname, age);
-                    }
                     f->close();
                 }
                 else
                     goers.append(*f);
             }
+            else if (traceLevel > 8)
+                DBGLOG("Ignoring idle %s file %s", remote ? "remote" : "local",  fname);
+
         }
         unsigned numFilesLeft = goers.ordinality(); 
         if (numFilesLeft > maxFilesOpen[remote])

roxie/ccd/ccdmain.cpp

@@ -96,6 +96,7 @@ CriticalSection ccdChannelsCrit;
 IPropertyTree* ccdChannels;
 StringArray allQuerySetNames;
 
+bool allFilesDynamic;
 bool crcResources;
 bool useRemoteResources;
 bool checkFileDate;
@@ -645,6 +646,7 @@ int STARTQUERY_API start_query(int argc, const char *argv[])
         doIbytiDelay = topology->getPropBool("@doIbytiDelay", true);
         minIbytiDelay = topology->getPropInt("@minIbytiDelay", 2);
         initIbytiDelay = topology->getPropInt("@initIbytiDelay", 50);
+        allFilesDynamic = topology->getPropBool("@allFilesDynamic", false);
         crcResources = topology->getPropBool("@crcResources", false);
         chunkingHeap = topology->getPropBool("@chunkingHeap", true);
         readTimeout = topology->getPropInt("@readTimeout", 300);

roxie/ccd/ccdserver.cpp

@@ -11189,11 +11189,14 @@ public:
         properties.setPropInt64("@recordCount", reccount);
         SCMStringBuffer info;
         WorkunitUpdate workUnit = ctx->updateWorkUnit();
-        properties.setProp("@owner", workUnit->getUser(info).str());
-        info.clear();
-        properties.setProp("@workunit", workUnit->getWuid(info).str());
-        info.clear();
-        properties.setProp("@job", workUnit->getJobName(info).str());
+        if (workUnit)
+        {
+            properties.setProp("@owner", workUnit->getUser(info).str());
+            info.clear();
+            properties.setProp("@workunit", workUnit->getWuid(info).str());
+            info.clear();
+            properties.setProp("@job", workUnit->getJobName(info).str());
+        }
         char const * rececl = helper.queryRecordECL();
         if(rececl && *rececl)
             properties.setProp("ECL", rececl);
@@ -19528,7 +19531,7 @@ public:
         isKeyed = false;
         stopAfter = I64C(0x7FFFFFFFFFFFFFFF);
         diskSize.set(helper.queryDiskRecordSize());
-        variableFileName = (helper.getFlags() & (TDXvarfilename|TDXdynamicfilename)) != 0;
+        variableFileName = allFilesDynamic || ((helper.getFlags() & (TDXvarfilename|TDXdynamicfilename)) != 0);
         isOpt = (helper.getFlags() & TDRoptional) != 0;
     }
 
@@ -19558,15 +19561,17 @@ public:
             eof = true;
         else
         {
+            numParts = 0;
             if (variableFileName)
             {
                 OwnedRoxieString fileName(helper.getFileName());
                 varFileInfo.setown(resolveLFN(fileName, isOpt));
-                Owned<IFilePartMap> map = varFileInfo->getFileMap();
-                if (map)
-                    numParts = map->getNumParts();
-                else
-                    numParts = 0;
+                if (varFileInfo)
+                {
+                    Owned<IFilePartMap> map = varFileInfo->getFileMap();
+                    if (map)
+                        numParts = map->getNumParts();
+                }
             }
             if (!numParts)
             {
@@ -20550,7 +20555,7 @@ public:
         isLocal = _graphNode.getPropBool("att[@name='local']/@value") && queryFactory.queryChannel()!=0;
         Owned<IHThorDiskReadBaseArg> helper = (IHThorDiskReadBaseArg *) helperFactory();
         sorted = (helper->getFlags() & TDRunsorted) == 0;
-        variableFileName = (helper->getFlags() & (TDXvarfilename|TDXdynamicfilename)) != 0;
+        variableFileName = allFilesDynamic || ((helper->getFlags() & (TDXvarfilename|TDXdynamicfilename)) != 0);
         maySkip = (helper->getFlags() & (TDRkeyedlimitskips|TDRkeyedlimitcreates|TDRlimitskips|TDRlimitcreates)) != 0;
         quotes = separators = terminators = escapes = NULL;
         if (!variableFileName)
@@ -20654,8 +20659,11 @@ protected:
     {
         OwnedRoxieString indexName(indexHelper.getFileName());
         varFileInfo.setown(resolveLFN(indexName, isOpt));
-        translators.setown(new TranslatorArray) ;
-        keySet.setown(varFileInfo->getKeyArray(factory->queryActivityMeta(), translators, isOpt, isLocal ? factory->queryQueryFactory().queryChannel() : 0, factory->queryQueryFactory().getEnableFieldTranslation()));
+        if (varFileInfo)
+        {
+            translators.setown(new TranslatorArray) ;
+            keySet.setown(varFileInfo->getKeyArray(factory->queryActivityMeta(), translators, isOpt, isLocal ? factory->queryQueryFactory().queryChannel() : 0, factory->queryQueryFactory().getEnableFieldTranslation()));
+        }
         variableInfoPending = false;
     }
 
@@ -20673,7 +20681,7 @@ public:
     {
         indexHelper.setCallback(&callback);
         steppedExtra = static_cast<IHThorSteppedSourceExtra *>(indexHelper.selectInterface(TAIsteppedsourceextra_1));
-        variableFileName = (indexHelper.getFlags() & (TIRvarfilename|TIRdynamicfilename)) != 0;
+        variableFileName = allFilesDynamic || ((indexHelper.getFlags() & (TIRvarfilename|TIRdynamicfilename)) != 0);
         variableInfoPending = false;
         isOpt = (indexHelper.getFlags() & TIRoptional) != 0;
         seekGEOffset = 0;
@@ -21313,7 +21321,7 @@ public:
         steppedExtra = static_cast<IHThorSteppedSourceExtra *>(indexHelper.selectInterface(TAIsteppedsourceextra_1));
         limitTransformExtra = static_cast<IHThorSourceLimitTransformExtra *>(indexHelper.selectInterface(TAIsourcelimittransformextra_1));
         unsigned flags = indexHelper.getFlags();
-        variableFileName = (flags & (TIRvarfilename|TIRdynamicfilename)) != 0;
+        variableFileName = allFilesDynamic || ((flags & (TIRvarfilename|TIRdynamicfilename)) != 0);
         variableInfoPending = false;
         isOpt = (flags & TIRoptional) != 0;
         optimizeSteppedPostFilter = (flags & TIRunfilteredtransform) != 0;
@@ -21655,7 +21663,7 @@ public:
         activityMeta.setown(deserializeRecordMeta(m, true));
         enableFieldTranslation = queryFactory.getEnableFieldTranslation();
         translatorArray.setown(new TranslatorArray);
-        variableFileName = (flags & (TIRvarfilename|TIRdynamicfilename)) != 0;
+        variableFileName = allFilesDynamic || ((flags & (TIRvarfilename|TIRdynamicfilename)) != 0);
         if (!variableFileName)
         {
             bool isOpt = (flags & TIRoptional) != 0;
@@ -22570,7 +22578,7 @@ public:
         : CRoxieServerActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind)
     {
         Owned<IHThorCountFileArg> helper = (IHThorCountFileArg *) helperFactory();
-        variableFileName = (helper->getFlags() & (TDXvarfilename|TDXdynamicfilename)) != 0;
+        variableFileName = allFilesDynamic || ((helper->getFlags() & (TDXvarfilename|TDXdynamicfilename)) != 0);
         assertex(helper->queryRecordSize()->isFixedSize());
         if (!variableFileName)
         {
@@ -22644,7 +22652,7 @@ public:
     {
         fetchContext = static_cast<IHThorFetchContext *>(helper.selectInterface(TAIfetchcontext_1));
         needsRHS = helper.transformNeedsRhs();
-        variableFileName = (fetchContext->getFetchFlags() & (FFvarfilename|FFdynamicfilename)) != 0;
+        variableFileName = allFilesDynamic || ((fetchContext->getFetchFlags() & (FFvarfilename|FFdynamicfilename)) != 0);
         isOpt = (fetchContext->getFetchFlags() & FFdatafileoptional) != 0;
     }
 
@@ -22675,7 +22683,8 @@ public:
         {
             OwnedRoxieString fname(fetchContext->getFileName());
             varFileInfo.setown(resolveLFN(fname, isOpt));
-            map.setown(varFileInfo->getFileMap());
+            if (varFileInfo)
+                map.setown(varFileInfo->getFileMap());
         }
         puller.start(parentExtractSize, parentExtract, paused, ctx->fetchPreload(), false, ctx);
     }
@@ -22790,7 +22799,7 @@ public:
     {
         Owned<IHThorFetchBaseArg> helper = (IHThorFetchBaseArg *) helperFactory();
         IHThorFetchContext *fetchContext = static_cast<IHThorFetchContext *>(helper->selectInterface(TAIfetchcontext_1));
-        variableFileName = (fetchContext->getFetchFlags() & (FFvarfilename|FFdynamicfilename)) != 0;
+        variableFileName = allFilesDynamic || ((fetchContext->getFetchFlags() & (FFvarfilename|FFdynamicfilename)) != 0);
         if (!variableFileName)
         {
             OwnedRoxieString fname(fetchContext->getFileName());
@@ -23287,7 +23296,7 @@ public:
           puller(false),
           isLocal(_isLocal)
     {
-        variableIndexFileName = (helper.getJoinFlags() & (JFvarindexfilename|JFdynamicindexfilename)) != 0;
+        variableIndexFileName = allFilesDynamic || ((helper.getJoinFlags() & (JFvarindexfilename|JFdynamicindexfilename)) != 0);
         indexReadInputRecordVariable = indexReadMeta->isVariableSize();
         indexReadInput = NULL;
         rootIndex = NULL;
@@ -23357,9 +23366,12 @@ public:
         else if (variableIndexFileName)
         {
             OwnedRoxieString indexFileName(helper.getIndexFileName());
-            varFileInfo.setown(resolveLFN(indexFileName, false));
-            translators.setown(new TranslatorArray);
-            keySet.setown(varFileInfo->getKeyArray(factory->queryActivityMeta(), translators, false, isLocal ? factory->queryQueryFactory().queryChannel() : 0, factory->queryQueryFactory().getEnableFieldTranslation())); // MORE - isLocal?
+            varFileInfo.setown(resolveLFN(indexFileName, (helper.getJoinFlags() & JFindexoptional) != 0));
+            if (varFileInfo)
+            {
+                translators.setown(new TranslatorArray);
+                keySet.setown(varFileInfo->getKeyArray(factory->queryActivityMeta(), translators, false, isLocal ? factory->queryQueryFactory().queryChannel() : 0, factory->queryQueryFactory().getEnableFieldTranslation())); // MORE - isLocal?
+            }
         }
         puller.start(parentExtractSize, parentExtract, paused, ctx->fullKeyedJoinPreload(), false, ctx);
     }
@@ -23991,7 +24003,7 @@ public:
           map(_map)
     {
         CRoxieServerKeyedJoinBase::setInput(0, head.queryOutput(0));
-        variableFetchFileName = (helper.getFetchFlags() & (FFvarfilename|FFdynamicfilename)) != 0;
+        variableFetchFileName = allFilesDynamic || ((helper.getFetchFlags() & (FFvarfilename|FFdynamicfilename)) != 0);
     }
     
     virtual const IResolvedFile *queryVarFileInfo() const
@@ -24012,9 +24024,11 @@ public:
         CRoxieServerKeyedJoinBase::start(parentExtractSize, parentExtract, paused);
         if (variableFetchFileName)
         {
+            bool isFetchOpt = (helper.getFetchFlags() & FFdatafileoptional) != 0;
             OwnedRoxieString fname(helper.getFileName());
-            varFetchFileInfo.setown(resolveLFN(fname, false));
-            map.setown(varFetchFileInfo->getFileMap());
+            varFetchFileInfo.setown(resolveLFN(fname, isFetchOpt));
+            if (varFetchFileInfo)
+                map.setown(varFetchFileInfo->getFileMap());
         }
         puller.start(parentExtractSize, parentExtract, paused, ctx->keyedJoinPreload(), false, ctx);
     }
@@ -24101,7 +24115,7 @@ public:
           keySet(_keySet),
           translators(_translators)
     {
-        variableIndexFileName = (helper.getJoinFlags() & (JFvarindexfilename|JFdynamicindexfilename)) != 0;
+        variableIndexFileName = allFilesDynamic || ((helper.getJoinFlags() & (JFvarindexfilename|JFdynamicindexfilename)) != 0);
         indexReadInputRecordVariable = indexReadMeta->isVariableSize();
     }
 
@@ -24147,9 +24161,12 @@ public:
         else if (variableIndexFileName)
         {
             OwnedRoxieString indexFileName(helper.getIndexFileName());
-            varFileInfo.setown(resolveLFN(indexFileName, false));
-            translators.setown(new TranslatorArray);
-            keySet.setown(varFileInfo->getKeyArray(factory->queryActivityMeta(), translators, false, isLocal ? factory->queryQueryFactory().queryChannel() : 0, factory->queryQueryFactory().getEnableFieldTranslation())); 
+            varFileInfo.setown(resolveLFN(indexFileName, (helper.getJoinFlags() & JFindexoptional) != 0));
+            if (varFileInfo)
+            {
+                translators.setown(new TranslatorArray);
+                keySet.setown(varFileInfo->getKeyArray(factory->queryActivityMeta(), translators, false, isLocal ? factory->queryQueryFactory().queryChannel() : 0, factory->queryQueryFactory().getEnableFieldTranslation()));
+            }
         }
         puller.start(parentExtractSize, parentExtract, paused, ctx->keyedJoinPreload(), isSimple, ctx);
 
@@ -24363,8 +24380,8 @@ public:
         enableFieldTranslation = queryFactory.getEnableFieldTranslation();
         translatorArray.setown(new TranslatorArray);
         joinFlags = helper->getJoinFlags();
-        variableIndexFileName = (joinFlags & (JFvarindexfilename|JFdynamicindexfilename)) != 0;
-        variableFetchFileName = (helper->getFetchFlags() & (FFvarfilename|FFdynamicfilename)) != 0;
+        variableIndexFileName = allFilesDynamic || ((joinFlags & (JFvarindexfilename|JFdynamicindexfilename)) != 0);
+        variableFetchFileName = allFilesDynamic || ((helper->getFetchFlags() & (FFvarfilename|FFdynamicfilename)) != 0);
         if (!variableIndexFileName)
         {
             bool isOpt = (joinFlags & JFindexoptional) != 0;

roxie/ccd/ccdstate.cpp

@@ -513,9 +513,9 @@ public:
     {
         return BASE::isActive();
     }
-    virtual bool validate(StringArray &wrn, StringArray &err, StringArray &unmatchedQueries, StringArray &unusedPackages) const
+    virtual bool validate(const char *queryid, StringArray &wrn, StringArray &err, StringArray &unmatchedQueries, StringArray &unusedPackages, StringArray &unmatchedFiles) const
     {
-        return BASE::validate(wrn, err, unmatchedQueries, unusedPackages);
+        return BASE::validate(queryid, wrn, err, unmatchedQueries, unusedPackages, unmatchedFiles);
     }
 
     virtual const IRoxiePackage *queryRoxiePackage(const char *name) const

testing/ecl/roxie/dynamicoptflag.ecl

@@ -0,0 +1,49 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+d := dataset(DYNAMIC('regress::no::such::file'), {string10 f}, FLAT, OPT);
+output(d);
+count(d);
+output(d(f='NOT'));
+count(d(f='NOT'));
+
+p := PRELOAD(dataset(DYNAMIC('regress::no::such::file::either'), {string10 f}, FLAT, OPT));
+output(p);
+count(p);
+output(p(f='NOT'));
+count(p(f='NOT'));
+
+p2 := PRELOAD(dataset(DYNAMIC('regress::no::such::file::again'), {string10 f}, FLAT, OPT), 2);
+output(p2);
+count(p2);
+output(p2(f='NOT'));
+count(p2(f='NOT'));
+
+i := INDEX(d,{f},{},DYNAMIC('regress::nor::this'), OPT);
+output(i);
+count(i);
+output(i(f='NOT'));
+count(i(f='NOT'));
+MAX(i(f>'NOT'), f);
+
+j := JOIN(d, i, KEYED(LEFT.f = right.f));
+output(j);
+
+j1 := JOIN(d(f='NOT'), d, KEYED(LEFT.f = right.f), KEYED(i));
+output(j1);
+
+output(FETCH(d, i(f='not'), 0));

testing/ecl/roxie/key/dynamicoptflag.xml

@@ -0,0 +1,49 @@
+<Dataset name='Result 1'>
+</Dataset>
+<Dataset name='Result 2'>
+ <Row><Result_2>0</Result_2></Row>
+</Dataset>
+<Dataset name='Result 3'>
+</Dataset>
+<Dataset name='Result 4'>
+ <Row><Result_4>0</Result_4></Row>
+</Dataset>
+<Dataset name='Result 5'>
+</Dataset>
+<Dataset name='Result 6'>
+ <Row><Result_6>0</Result_6></Row>
+</Dataset>
+<Dataset name='Result 7'>
+</Dataset>
+<Dataset name='Result 8'>
+ <Row><Result_8>0</Result_8></Row>
+</Dataset>
+<Dataset name='Result 9'>
+</Dataset>
+<Dataset name='Result 10'>
+ <Row><Result_10>0</Result_10></Row>
+</Dataset>
+<Dataset name='Result 11'>
+</Dataset>
+<Dataset name='Result 12'>
+ <Row><Result_12>0</Result_12></Row>
+</Dataset>
+<Dataset name='Result 13'>
+</Dataset>
+<Dataset name='Result 14'>
+ <Row><Result_14>0</Result_14></Row>
+</Dataset>
+<Dataset name='Result 15'>
+</Dataset>
+<Dataset name='Result 16'>
+ <Row><Result_16>0</Result_16></Row>
+</Dataset>
+<Dataset name='Result 17'>
+ <Row><Result_17>          </Result_17></Row>
+</Dataset>
+<Dataset name='Result 18'>
+</Dataset>
+<Dataset name='Result 19'>
+</Dataset>
+<Dataset name='Result 20'>
+</Dataset>