Merge remote-tracking branch 'origin/candidate-3.10.0' into candidate-3.10.x

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

common/workunit/workunit.cpp

@@ -657,7 +657,7 @@ public:
     virtual void requestAbort();
     virtual void subscribe(WUSubscribeOptions options);
     virtual unsigned calculateHash(unsigned prevHash);
-    virtual void copyWorkUnit(IConstWorkUnit *cached);
+    virtual void copyWorkUnit(IConstWorkUnit *cached, bool all);
     virtual unsigned queryFileUsage(const char *filename) const;
     virtual bool getCloneable() const;
     virtual IUserDescriptor * queryUserDescriptor() const;
@@ -705,7 +705,7 @@ public:
     void setAgentSession(__int64 sessionId);
     void setAgentPID(unsigned pid);
     void setSecurityToken(const char *value);
-    void setTimerInfo(const char * name, const char * instance, unsigned ms, unsigned count, unsigned max);
+    void setTimerInfo(const char * name, const char * instance, unsigned ms, unsigned count, unsigned __int64 max);
     void setTracingValue(const char * propname, const char * value);
     void setTracingValueInt(const char * propname, int value);
     void setUser(const char * value);
@@ -1076,8 +1076,8 @@ public:
             { c->requestAbort(); }
     virtual unsigned calculateHash(unsigned prevHash)
             { return c->calculateHash(prevHash); }
-    virtual void copyWorkUnit(IConstWorkUnit *cached)
-            { c->copyWorkUnit(cached); }
+    virtual void copyWorkUnit(IConstWorkUnit *cached, bool all)
+            { c->copyWorkUnit(cached, all); }
     virtual bool archiveWorkUnit(const char *base,bool del,bool deldll,bool deleteOwned)
             { return c->archiveWorkUnit(base,del,deldll,deleteOwned); }
     virtual void packWorkUnit(bool pack)
@@ -1165,7 +1165,7 @@ public:
             { c->setAgentSession(sessionId); }
     virtual void setAgentPID(unsigned pid)
             { c->setAgentPID(pid); }
-    virtual void setTimerInfo(const char * name, const char * instance, unsigned ms, unsigned count, unsigned max)
+    virtual void setTimerInfo(const char * name, const char * instance, unsigned ms, unsigned count, unsigned __int64 max)
             { c->setTimerInfo(name, instance, ms, count, max); }
     virtual void setTracingValue(const char * propname, const char * value)
             { c->setTracingValue(propname, value); }
@@ -4526,7 +4526,7 @@ static void copyTree(IPropertyTree * to, const IPropertyTree * from, const char
         to->setPropTree(xpath, match);
 }
 
-void CLocalWorkUnit::copyWorkUnit(IConstWorkUnit *cached)
+void CLocalWorkUnit::copyWorkUnit(IConstWorkUnit *cached, bool all)
 {
     CLocalWorkUnit *from = QUERYINTERFACE(cached, CLocalWorkUnit);
     if (!from)
@@ -4577,6 +4577,17 @@ void CLocalWorkUnit::copyWorkUnit(IConstWorkUnit *cached)
     copyTree(p, fromP, "Results");
     copyTree(p, fromP, "Graphs");
     copyTree(p, fromP, "Workflow");
+    if (all)
+    {
+        // Merge timing info from both branches
+        pt = fromP->getBranch("Timings");
+        if (pt)
+        {
+            IPropertyTree *tgtTimings = ensurePTree(p, "Timings");
+            mergePTree(tgtTimings, pt);
+            pt->Release();
+        }
+    }
 
     updateProp(p, fromP, "@clusterName");
     updateProp(p, fromP, "allowedclusters");
@@ -4946,7 +4957,7 @@ bool parseGraphTimerLabel(const char *label, StringBuffer &graphName, unsigned &
     return true;
 }
 
-void CLocalWorkUnit::setTimerInfo(const char *name, const char *subname, unsigned ms, unsigned count, unsigned max)
+void CLocalWorkUnit::setTimerInfo(const char *name, const char *subname, unsigned ms, unsigned count, unsigned __int64 max)
 {
     CriticalBlock block(crit);
     IPropertyTree *timings = p->queryPropTree("Timings");
@@ -4966,9 +4977,9 @@ void CLocalWorkUnit::setTimerInfo(const char *name, const char *subname, unsigne
     }
     timing->setPropInt("@count", count);
     timing->setPropInt("@duration", ms);
-    if (!max && 1==count) max = ms;
+    if (!max && 1==count) max = ms * 1000000; // max is in nanoseconds
     if (max)
-        timing->setPropInt("@max", max);
+        timing->setPropInt64("@max", max);
 }
 
 void CLocalWorkUnit::setTimeStamp(const char *application, const char *instance, const char *event, bool add)

common/workunit/workunit.hpp

@@ -955,7 +955,7 @@ interface IWorkUnit : extends IConstWorkUnit
     virtual void setStateEx(const char * text) = 0;
     virtual void setAgentSession(__int64 sessionId) = 0;
     virtual void setAgentPID(unsigned pid) = 0;
-    virtual void setTimerInfo(const char * name, const char * instance, unsigned ms, unsigned count, unsigned max) = 0;
+    virtual void setTimerInfo(const char * name, const char * instance, unsigned ms, unsigned count, unsigned __int64 max) = 0;
     virtual void setTracingValue(const char * propname, const char * value) = 0;
     virtual void setTracingValueInt(const char * propname, int value) = 0;
     virtual void setUser(const char * value) = 0;
@@ -1120,7 +1120,7 @@ interface IWorkflowScheduleConnection : extends IInterface
 interface IExtendedWUInterface
 {
     virtual unsigned calculateHash(unsigned prevHash) = 0;
-    virtual void copyWorkUnit(IConstWorkUnit *cached) = 0;
+    virtual void copyWorkUnit(IConstWorkUnit *cached, bool all) = 0;
     virtual bool archiveWorkUnit(const char *base,bool del,bool ignoredllerrors,bool deleteOwned) = 0;
     virtual void packWorkUnit(bool pack=true) = 0;
     

ecl/eclagent/eclagent.cpp

@@ -3262,7 +3262,7 @@ extern int HTHOR_API eclagent_main(int argc, const char *argv[], StringBuffer *
                 Owned<IWorkUnitFactory> factory = getWorkUnitFactory();
                 Owned<IWorkUnit> daliWu = factory->createWorkUnit(NULL, "eclagent", "eclagent");
                 IExtendedWUInterface * extendedWu = queryExtendedWU(daliWu);
-                extendedWu->copyWorkUnit(standAloneWorkUnit);
+                extendedWu->copyWorkUnit(standAloneWorkUnit, true);
                 daliWu->getWuid(wuid);
                 globals->setProp("WUID", wuid.str());
                 standAloneWorkUnit.clear();

ecl/eclcc/eclcc.cpp

@@ -194,6 +194,7 @@ public:
         argc = _argc;
         argv = _argv;
         logVerbose = false;
+        logTimings = false;
         optArchive = false;
         optGenerateMeta = false;
         optGenerateDepend = false;
@@ -205,6 +206,7 @@ public:
         optOnlyCompile = false;
         optBatchMode = false;
         optSaveQueryText = false;
+        optGenerateHeader = false;
         optTargetClusterType = HThorCluster;
         optTargetCompiler = DEFAULT_COMPILER;
         optThreads = 0;
@@ -226,7 +228,8 @@ protected:
     void applyDebugOptions(IWorkUnit * wu);
     bool checkWithinRepository(StringBuffer & attributePath, const char * sourcePathname);
     IFileIO * createArchiveOutputFile(EclCompileInstance & instance);
-    ICppCompiler * createCompiler(const char * coreName);
+    ICppCompiler *createCompiler(const char * coreName, const char * sourceDir = NULL, const char * targetDir = NULL);
+    bool generatePrecompiledHeader();
     void generateOutput(EclCompileInstance & instance);
     void instantECL(EclCompileInstance & instance, IWorkUnit *wu, const char * queryFullName, IErrorReceiver *errs, const char * outputFile);
     bool isWithinPath(const char * sourcePathname, const char * searchPath);
@@ -252,6 +255,7 @@ protected:
     Owned<IEclRepository> includeRepository;
     const char * programName;
 
+    StringBuffer cppIncludePath;
     StringBuffer pluginsPath;
     StringBuffer hooksPath;
     StringBuffer templatePath;
@@ -281,6 +285,7 @@ protected:
     unsigned batchSplit;
     unsigned optLogDetail;
     bool logVerbose;
+    bool logTimings;
     bool optArchive;
     bool optGenerateMeta;
     bool optGenerateDepend;
@@ -292,6 +297,7 @@ protected:
     bool optOnlyCompile;
     bool optSaveQueryText;
     bool optLegacy;
+    bool optGenerateHeader;
     int argc;
     const char **argv;
 };
@@ -419,7 +425,7 @@ void EclCC::loadOptions()
 
     globals.setown(createProperties(optIniFilename, true));
 
-    StringBuffer compilerPath, includePath, libraryPath;
+    StringBuffer compilerPath, libraryPath;
 
     if (globals->hasProp("targetGcc"))
         optTargetCompiler = globals->getPropBool("targetGcc") ? GccCppCompiler : Vs6CppCompiler;
@@ -433,7 +439,7 @@ void EclCC::loadOptions()
         extractOption(compilerPath, globals, "CL_PATH", "compilerPath", "/usr", NULL);
 #endif
         extractOption(libraryPath, globals, "ECLCC_LIBRARY_PATH", "libraryPath", syspath, "lib");
-        extractOption(includePath, globals, "ECLCC_INCLUDE_PATH", "includePath", syspath, "componentfiles/cl/include");
+        extractOption(cppIncludePath, globals, "ECLCC_INCLUDE_PATH", "includePath", syspath, "componentfiles/cl/include");
         extractOption(pluginsPath, globals, "ECLCC_PLUGIN_PATH", "plugins", syspath, "plugins");
         extractOption(hooksPath, globals, "HPCC_FILEHOOKS_PATH", "filehooks", syspath, "filehooks");
         extractOption(templatePath, globals, "ECLCC_TPL_PATH", "templatePath", syspath, "componentfiles");
@@ -458,7 +464,7 @@ void EclCC::loadOptions()
         installFileHooks(hooksPath.str());
 
     if (!optNoCompile)
-        setCompilerPath(compilerPath.str(), includePath.str(), libraryPath.str(), NULL, optTargetCompiler, logVerbose);
+        setCompilerPath(compilerPath.str(), cppIncludePath.str(), libraryPath.str(), NULL, optTargetCompiler, logVerbose);
 }
 
 //=========================================================================================
@@ -496,10 +502,8 @@ void EclCC::applyDebugOptions(IWorkUnit * wu)
 
 //=========================================================================================
 
-ICppCompiler * EclCC::createCompiler(const char * coreName)
+ICppCompiler * EclCC::createCompiler(const char * coreName, const char * sourceDir, const char * targetDir)
 {
-    const char * sourceDir = NULL;
-    const char * targetDir = NULL;
     Owned<ICppCompiler> compiler = ::createCompiler(coreName, sourceDir, targetDir, optTargetCompiler, logVerbose);
     compiler->setOnlyCompile(optOnlyCompile);
     compiler->setCCLogPath(cclogFilename);
@@ -1257,6 +1261,55 @@ void EclCC::processReference(EclCompileInstance & instance, const char * queryAt
     generateOutput(instance);
 }
 
+bool EclCC::generatePrecompiledHeader()
+{
+    if (inputFiles.ordinality() != 0)
+    {
+        ERRLOG("No input files should be specified when generating precompiled header");
+        return false;
+    }
+    StringArray paths;
+    paths.appendList(cppIncludePath, ENVSEPSTR);
+    const char *foundPath = NULL;
+    ForEachItemIn(idx, paths)
+    {
+        StringBuffer fullpath;
+        fullpath.append(paths.item(idx));
+        addPathSepChar(fullpath).append("eclinclude.hpp");
+        if (checkFileExists(fullpath))
+        {
+            foundPath = paths.item(idx);
+            break;
+        }
+    }
+    if (!foundPath)
+    {
+        ERRLOG("Cannot find eclinclude.hpp");
+        return false;
+    }
+    Owned<ICppCompiler> compiler = createCompiler("eclinclude.hpp", foundPath, NULL);
+    compiler->setDebug(true);  // a precompiled header with debug can be used for no-debug, but not vice versa
+    compiler->setPrecompileHeader(true);
+    if (compiler->compile())
+    {
+        try
+        {
+            Owned<IFile> log = createIFile(cclogFilename);
+            log->remove();
+        }
+        catch (IException * e)
+        {
+            e->Release();
+        }
+        return true;
+    }
+    else
+    {
+        ERRLOG("Compilation failed - see %s for details", cclogFilename.str());
+        return false;
+    }
+}
+
 
 bool EclCC::processFiles()
 {
@@ -1265,7 +1318,11 @@ bool EclCC::processFiles()
     {
         processArgvFilename(inputFiles, inputFileNames.item(idx));
     }
-    if (inputFiles.ordinality() == 0)
+    if (optGenerateHeader)
+    {
+        return generatePrecompiledHeader();
+    }
+    else if (inputFiles.ordinality() == 0)
     {
         if (optBatchMode || !optQueryRepositoryReference)
         {
@@ -1305,8 +1362,11 @@ bool EclCC::processFiles()
         ok = (errs->errCount() == 0);
     }
 
-    if (logVerbose)
-        defaultTimer->printTimings();
+    if (logTimings)
+    {
+        StringBuffer s;
+        fprintf(stderr, "%s", defaultTimer->getTimings(s).str());
+    }
     return ok;
 }
 
@@ -1431,6 +1491,9 @@ bool EclCC::parseCommandLineOptions(int argc, const char* argv[])
         else if (iter.matchFlag(optOutputDirectory, "-P"))
         {
         }
+        else if (iter.matchFlag(optGenerateHeader, "-pch"))
+        {
+        }
         else if (iter.matchFlag(optSaveQueryText, "-q"))
         {
         }
@@ -1472,6 +1535,9 @@ bool EclCC::parseCommandLineOptions(int argc, const char* argv[])
             if (batchPart >= batchSplit)
                 batchPart = 0;
         }
+        else if (iter.matchFlag(logTimings, "--timings"))
+        {
+        }
         else if (iter.matchOption(tempArg, "-platform") || /*deprecated*/ iter.matchOption(tempArg, "-target"))
         {
             if (!setTargetPlatformOption(tempArg.get(), optTargetClusterType))
@@ -1531,7 +1597,7 @@ bool EclCC::parseCommandLineOptions(int argc, const char* argv[])
 
     if (inputFileNames.ordinality() == 0)
     {
-        if (!optBatchMode && optQueryRepositoryReference)
+        if (optGenerateHeader || (!optBatchMode && optQueryRepositoryReference))
             return true;
         ERRLOG("No input filenames supplied");
         return false;
@@ -1602,11 +1668,15 @@ const char * const helpText[] = {
 #ifdef _WIN32
     "!   -m            Enable leak checking",
 #endif
+#ifndef _WIN32
+    "!   -pch          Generate precompiled header for eclinclude.hpp",
+#endif
     "!   -P <path>     Specify the path of the output files (only with -b option)",
     "    -specs file   Read eclcc configuration from specified file",
     "!   -split m:n    Process a subset m of n input files (only with -b option)",
     "    -v --verbose  Output additional tracing information while compiling",
     "    --version     Output version information",
+    "!   --timings     Output additional timing information",
     "!",
     "!#options",
     "! -factivitiesPerCpp      Number of activities in each c++ file",

ecl/eclccserver/eclccserver.cpp

@@ -52,41 +52,59 @@ class EclccCompileThread : public CInterface, implements IPooledThread
 
     void reportError(const char *errStr, unsigned retcode)
     {
-        // MORE - if copyWorkUnit copied errors this would not be needed... should it?
         // A typical error looks like this: stdin:(385,29): warning C1041: Record doesn't have an explicit maximum record size
         // we will also see (and want to skip) nn error(s), nn warning(s)
-        RegExpr errCount, errParse;
+        RegExpr errCount, errParse, timings;
+        timings.init("Timing: {.+} total={[0-9]+}ms max={[0-9]+}us count={[0-9]+} ave={[0-9]+}us");
         errCount.init("[0-9]+ errors?, [0-9]+ warnings?.*");
         errParse.init("^{.+}\\({[0-9]+},{[0-9]+}\\): {[a-z]+} [A-Za-z]*{[0-9]+}:{.*$}");
         if (!errCount.find(errStr))
         {
-            Owned<IWUException> err = workunit->createException();
-            err->setExceptionSource("eclcc");
-            if (errParse.find(errStr))
+            if (timings.find(errStr))
             {
-                StringBuffer file, line, col, errClass, errCode, errText;
-                errParse.findstr(file, 1);
-                errParse.findstr(line, 2);
-                errParse.findstr(col, 3);
-                errParse.findstr(errClass, 4);
-                errParse.findstr(errCode, 5);
-                errParse.findstr(errText, 6);
-                err->setExceptionFileName(file);
-                err->setExceptionLineNo(atoi(line));
-                err->setExceptionColumn(atoi(col));
-                if (stricmp(errClass, "warning")==0)
-                    err->setSeverity(ExceptionSeverityWarning);
-                else
-                    err->setSeverity(ExceptionSeverityError);
-                err->setExceptionCode(atoi(errCode));
-                err->setExceptionMessage(errText);
-                err->setExceptionFileName(file); // any point if it just says stdin?
+                StringBuffer section, total, max, count, ave;
+                timings.findstr(section, 1);
+                timings.findstr(total, 2);
+                timings.findstr(max, 3);
+                timings.findstr(count, 4);
+                timings.findstr(ave, 5);
+                if (workunit->getDebugValueBool("addTimingToWorkunit", true))
+                {
+                    section.insert(0, "eclcc: ");
+                    unsigned __int64 umax = atoi(max); // in microseconds
+                    workunit->setTimerInfo(section.str(), NULL, atoi(total), atoi(count), umax*1000); // max is stored in nanoseconds
+                }
             }
             else
             {
-                err->setSeverity(retcode ? ExceptionSeverityError : ExceptionSeverityWarning);
-                err->setExceptionMessage(errStr);
-                DBGLOG("%s", errStr);
+                Owned<IWUException> err = workunit->createException();
+                err->setExceptionSource("eclcc");
+                if (errParse.find(errStr))
+                {
+                    StringBuffer file, line, col, errClass, errCode, errText;
+                    errParse.findstr(file, 1);
+                    errParse.findstr(line, 2);
+                    errParse.findstr(col, 3);
+                    errParse.findstr(errClass, 4);
+                    errParse.findstr(errCode, 5);
+                    errParse.findstr(errText, 6);
+                    err->setExceptionFileName(file);
+                    err->setExceptionLineNo(atoi(line));
+                    err->setExceptionColumn(atoi(col));
+                    if (stricmp(errClass, "warning")==0)
+                        err->setSeverity(ExceptionSeverityWarning);
+                    else
+                        err->setSeverity(ExceptionSeverityError);
+                    err->setExceptionCode(atoi(errCode));
+                    err->setExceptionMessage(errText);
+                    err->setExceptionFileName(file); // any point if it just says stdin?
+                }
+                else
+                {
+                    err->setSeverity(retcode ? ExceptionSeverityError : ExceptionSeverityWarning);
+                    err->setExceptionMessage(errStr);
+                    DBGLOG("%s", errStr);
+                }
             }
         }
     }
@@ -110,6 +128,8 @@ class EclccCompileThread : public CInterface, implements IPooledThread
             eclccCmd.append(" -");
         if (mainDefinition.length())
             eclccCmd.append(" -main ").append(mainDefinition);
+        if (workunit->getDebugValueBool("addTimingToWorkunit", true))
+            eclccCmd.append(" --timings");
 
         Owned<IPropertyTreeIterator> options = globals->getElements("./Option");
         ForEach(*options)
@@ -169,6 +189,7 @@ class EclccCompileThread : public CInterface, implements IPooledThread
         }
         try
         {
+            unsigned time = msTick();
             Owned<IPipeProcess> pipe = createPipeProcess();
             Owned<ErrorReader> errorReader = new ErrorReader(pipe, this);
             pipe->run("eclcc", eclccCmd, ".", true, false, true, 0);
@@ -198,7 +219,7 @@ class EclccCompileThread : public CInterface, implements IPooledThread
                 {
                     Owned<ILocalWorkUnit> embeddedWU = createLocalWorkUnit();
                     embeddedWU->loadXML(wuXML);
-                    queryExtendedWU(workunit)->copyWorkUnit(embeddedWU);
+                    queryExtendedWU(workunit)->copyWorkUnit(embeddedWU, true);
                     SCMStringBuffer jobname;
                     if (embeddedWU->getJobName(jobname).length()) //let ECL win naming job during initial compile
                         workunit->setJobName(jobname.str());
@@ -212,6 +233,9 @@ class EclccCompileThread : public CInterface, implements IPooledThread
                 Owned<IWUQuery> query = workunit->updateQuery();
                 associateLocalFile(query, FileTypeDll, realdllfilename, "Workunit DLL", crc);
                 queryDllServer().registerDll(realdllname.str(), "Workunit DLL", dllurl.str());
+                time = msTick()-time;
+                if (workunit->getDebugValueBool("addTimingToWorkunit", true))
+                    workunit->setTimerInfo("eclccserver: create workunit", NULL, time, 1, 0);
 
                 workunit->commit();
                 return true;

ecl/eclcmd/roxie/ecl-roxie.cpp

@@ -384,7 +384,7 @@ public:
     virtual void usage()
     {
         fprintf(stdout,"\nUsage:\n\n"
-            "ecl roixe <command> [command options]\n\n"
+            "ecl roxie <command> [command options]\n\n"
             "   Queries Commands:\n"
             "      attach         (re)attach a roxie cluster from dali\n"
             "      detach         detach a roxie cluster from dali\n"

ecl/scheduleadmin/scheduleadmin.cpp

@@ -262,6 +262,7 @@ private:
 
 int main(int argc, char * const * argv)
 {
+    InitModuleObjects();
     if((argc==2) && (stricmp(argv[1], "help")==0))
         usage(0);
     if(argc<3) usage();

esp/files/ECLPlayground.css

@@ -137,3 +137,6 @@ h1 {
 	max-height: 60px;
 	background-color: rgb(238, 238, 238);
 }
+.resultGridCell {
+	font-family: monospace;
+}

esp/files/dojox/xml/DomParser.js

@@ -0,0 +1,319 @@
+//>>built
+define("dojox/xml/DomParser",["dojo/_base/kernel","dojo/_base/array"],function(_1){
+_1.getObject("xml",true,dojox);
+dojox.xml.DomParser=new (function(){
+var _2={ELEMENT:1,ATTRIBUTE:2,TEXT:3,CDATA_SECTION:4,PROCESSING_INSTRUCTION:7,COMMENT:8,DOCUMENT:9};
+var _3=/<([^>\/\s+]*)([^>]*)>([^<]*)/g;
+var _4=/([^=]*)=(("([^"]*)")|('([^']*)'))/g;
+var _5=/<!ENTITY\s+([^"]*)\s+"([^"]*)">/g;
+var _6=/<!\[CDATA\[([\u0001-\uFFFF]*?)\]\]>/g;
+var _7=/<!--([\u0001-\uFFFF]*?)-->/g;
+var _8=/^\s+|\s+$/g;
+var _9=/\s+/g;
+var _a=/\&gt;/g;
+var _b=/\&lt;/g;
+var _c=/\&quot;/g;
+var _d=/\&apos;/g;
+var _e=/\&amp;/g;
+var _f="_def_";
+function _10(){
+return new (function(){
+var all={};
+this.nodeType=_2.DOCUMENT;
+this.nodeName="#document";
+this.namespaces={};
+this._nsPaths={};
+this.childNodes=[];
+this.documentElement=null;
+this._add=function(obj){
+if(typeof (obj.id)!="undefined"){
+all[obj.id]=obj;
+}
+};
+this._remove=function(id){
+if(all[id]){
+delete all[id];
+}
+};
+this.byId=this.getElementById=function(id){
+return all[id];
+};
+this.byName=this.getElementsByTagName=_11;
+this.byNameNS=this.getElementsByTagNameNS=_12;
+this.childrenByName=_13;
+this.childrenByNameNS=_14;
+})();
+};
+function _11(_15){
+function _16(_17,_18,arr){
+_1.forEach(_17.childNodes,function(c){
+if(c.nodeType==_2.ELEMENT){
+if(_18=="*"){
+arr.push(c);
+}else{
+if(c.nodeName==_18){
+arr.push(c);
+}
+}
+_16(c,_18,arr);
+}
+});
+};
+var a=[];
+_16(this,_15,a);
+return a;
+};
+function _12(_19,ns){
+function _1a(_1b,_1c,ns,arr){
+_1.forEach(_1b.childNodes,function(c){
+if(c.nodeType==_2.ELEMENT){
+if(_1c=="*"&&c.ownerDocument._nsPaths[ns]==c.namespace){
+arr.push(c);
+}else{
+if(c.localName==_1c&&c.ownerDocument._nsPaths[ns]==c.namespace){
+arr.push(c);
+}
+}
+_1a(c,_1c,ns,arr);
+}
+});
+};
+if(!ns){
+ns=_f;
+}
+var a=[];
+_1a(this,_19,ns,a);
+return a;
+};
+function _13(_1d){
+var a=[];
+_1.forEach(this.childNodes,function(c){
+if(c.nodeType==_2.ELEMENT){
+if(_1d=="*"){
+a.push(c);
+}else{
+if(c.nodeName==_1d){
+a.push(c);
+}
+}
+}
+});
+return a;
+};
+function _14(_1e,ns){
+var a=[];
+_1.forEach(this.childNodes,function(c){
+if(c.nodeType==_2.ELEMENT){
+if(_1e=="*"&&c.ownerDocument._nsPaths[ns]==c.namespace){
+a.push(c);
+}else{
+if(c.localName==_1e&&c.ownerDocument._nsPaths[ns]==c.namespace){
+a.push(c);
+}
+}
+}
+});
+return a;
+};
+function _1f(v){
+return {nodeType:_2.TEXT,nodeName:"#text",nodeValue:v.replace(_9," ").replace(_a,">").replace(_b,"<").replace(_d,"'").replace(_c,"\"").replace(_e,"&")};
+};
+function _20(_21){
+for(var i=0;i<this.attributes.length;i++){
+if(this.attributes[i].nodeName==_21){
+return this.attributes[i].nodeValue;
+}
+}
+return null;
+};
+function _22(_23,ns){
+for(var i=0;i<this.attributes.length;i++){
+if(this.ownerDocument._nsPaths[ns]==this.attributes[i].namespace&&this.attributes[i].localName==_23){
+return this.attributes[i].nodeValue;
+}
+}
+return null;
+};
+function _24(_25,val){
+var old=null;
+for(var i=0;i<this.attributes.length;i++){
+if(this.attributes[i].nodeName==_25){
+old=this.attributes[i].nodeValue;
+this.attributes[i].nodeValue=val;
+break;
+}
+}
+if(_25=="id"){
+if(old!=null){
+this.ownerDocument._remove(old);
+}
+this.ownerDocument._add(this);
+}
+};
+function _26(_27,val,ns){
+for(var i=0;i<this.attributes.length;i++){
+if(this.ownerDocument._nsPaths[ns]==this.attributes[i].namespace&&this.attributes[i].localName==_27){
+this.attributes[i].nodeValue=val;
+return;
+}
+}
+};
+function _28(){
+var p=this.parentNode;
+if(p){
+for(var i=0;i<p.childNodes.length;i++){
+if(p.childNodes[i]==this&&i>0){
+return p.childNodes[i-1];
+}
+}
+}
+return null;
+};
+function _29(){
+var p=this.parentNode;
+if(p){
+for(var i=0;i<p.childNodes.length;i++){
+if(p.childNodes[i]==this&&(i+1)<p.childNodes.length){
+return p.childNodes[i+1];
+}
+}
+}
+return null;
+};
+this.parse=function(str){
+var _2a=_10();
+if(str==null){
+return _2a;
+}
+if(str.length==0){
+return _2a;
+}
+if(str.indexOf("<!ENTITY")>0){
+var _2b,eRe=[];
+if(_5.test(str)){
+_5.lastIndex=0;
+while((_2b=_5.exec(str))!=null){
+eRe.push({entity:"&"+_2b[1].replace(_8,"")+";",expression:_2b[2]});
+}
+for(var i=0;i<eRe.length;i++){
+str=str.replace(new RegExp(eRe[i].entity,"g"),eRe[i].expression);
+}
+}
+}
+var _2c=[],_2d;
+while((_2d=_6.exec(str))!=null){
+_2c.push(_2d[1]);
+}
+for(var i=0;i<_2c.length;i++){
+str=str.replace(_2c[i],i);
+}
+var _2e=[],_2f;
+while((_2f=_7.exec(str))!=null){
+_2e.push(_2f[1]);
+}
+for(i=0;i<_2e.length;i++){
+str=str.replace(_2e[i],i);
+}
+var res,obj=_2a;
+while((res=_3.exec(str))!=null){
+if(res[2].charAt(0)=="/"&&res[2].replace(_8,"").length>1){
+if(obj.parentNode){
+obj=obj.parentNode;
+}
+var _30=(res[3]||"").replace(_8,"");
+if(_30.length>0){
+obj.childNodes.push(_1f(_30));
+}
+}else{
+if(res[1].length>0){
+if(res[1].charAt(0)=="?"){
+var _31=res[1].substr(1);
+var _32=res[2].substr(0,res[2].length-2);
+obj.childNodes.push({nodeType:_2.PROCESSING_INSTRUCTION,nodeName:_31,nodeValue:_32});
+}else{
+if(res[1].charAt(0)=="!"){
+if(res[1].indexOf("![CDATA[")==0){
+var val=parseInt(res[1].replace("![CDATA[","").replace("]]",""));
+obj.childNodes.push({nodeType:_2.CDATA_SECTION,nodeName:"#cdata-section",nodeValue:_2c[val]});
+}else{
+if(res[1].substr(0,3)=="!--"){
+var val=parseInt(res[1].replace("!--","").replace("--",""));
+obj.childNodes.push({nodeType:_2.COMMENT,nodeName:"#comment",nodeValue:_2e[val]});
+}
+}
+}else{
+var _31=res[1].replace(_8,"");
+var o={nodeType:_2.ELEMENT,nodeName:_31,localName:_31,namespace:_f,ownerDocument:_2a,attributes:[],parentNode:null,childNodes:[]};
+if(_31.indexOf(":")>-1){
+var t=_31.split(":");
+o.namespace=t[0];
+o.localName=t[1];
+}
+o.byName=o.getElementsByTagName=_11;
+o.byNameNS=o.getElementsByTagNameNS=_12;
+o.childrenByName=_13;
+o.childrenByNameNS=_14;
+o.getAttribute=_20;
+o.getAttributeNS=_22;
+o.setAttribute=_24;
+o.setAttributeNS=_26;
+o.previous=o.previousSibling=_28;
+o.next=o.nextSibling=_29;
+var _33;
+while((_33=_4.exec(res[2]))!=null){
+if(_33.length>0){
+var _31=_33[1].replace(_8,"");
+var val=(_33[4]||_33[6]||"").replace(_9," ").replace(_a,">").replace(_b,"<").replace(_d,"'").replace(_c,"\"").replace(_e,"&");
+if(_31.indexOf("xmlns")==0){
+if(_31.indexOf(":")>0){
+var ns=_31.split(":");
+_2a.namespaces[ns[1]]=val;
+_2a._nsPaths[val]=ns[1];
+}else{
+_2a.namespaces[_f]=val;
+_2a._nsPaths[val]=_f;
+}
+}else{
+var ln=_31;
+var ns=_f;
+if(_31.indexOf(":")>0){
+var t=_31.split(":");
+ln=t[1];
+ns=t[0];
+}
+o.attributes.push({nodeType:_2.ATTRIBUTE,nodeName:_31,localName:ln,namespace:ns,nodeValue:val});
+if(ln=="id"){
+o.id=val;
+}
+}
+}
+}
+_2a._add(o);
+if(obj){
+obj.childNodes.push(o);
+o.parentNode=obj;
+if(res[2].charAt(res[2].length-1)!="/"){
+obj=o;
+}
+}
+var _30=res[3];
+if(_30.length>0){
+obj.childNodes.push(_1f(_30));
+}
+}
+}
+}
+}
+}
+for(var i=0;i<_2a.childNodes.length;i++){
+var e=_2a.childNodes[i];
+if(e.nodeType==_2.ELEMENT){
+_2a.documentElement=e;
+break;
+}
+}
+return _2a;
+};
+})();
+return dojox.xml.DomParser;
+});

esp/files/dojox/xml/README

@@ -0,0 +1,40 @@
+-------------------------------------------------------------------------------
+DojoX XML Utilities
+-------------------------------------------------------------------------------
+Version 0.1
+Release date: 05/30/2007	
+-------------------------------------------------------------------------------
+Project state:
+experimental
+-------------------------------------------------------------------------------
+Credits 
+	Tom Trenka (ttrenka@gmail.com): DomParser
+	
+-------------------------------------------------------------------------------
+Project description
+
+The goal of DojoX XML Utilities is provide differing XML utilities for use
+in various places.  Currently this includes a native JS DomParser, but will
+most likely be expanded to include things as dealing with x-browser forks
+(like the Sarissa project), various DOM utilites, and more.
+-------------------------------------------------------------------------------
+Dependencies:
+
+DojoX XML relies only on the Dojo Base package system.
+-------------------------------------------------------------------------------
+Documentation
+
+None at the time of writing.  The only object is dojox.xml.DomParser (a singleton),
+which has one method: parse:
+
+dojox.xml.DomParser.parse(xmlString)
+-------------------------------------------------------------------------------
+Installation instructions
+
+Grab the following from the Dojo SVN Repository:
+http://svn.dojotoolkit.org/var/src/dojo/dojox/trunk/xml/*
+
+Install into the following directory structure:
+/dojox/xml/
+
+...which should be at the same level as your Dojo checkout.

esp/files/dojox/xml/Script.js

@@ -0,0 +1,8 @@
+//>>built
+define("dojox/xml/Script",["dojo/_base/kernel","dojo/_base/declare","dojo/parser","./widgetParser"],function(_1,_2,_3){
+dojo.getObject("xml",true,dojox);
+_1("dojox.xml.Script",null,{constructor:function(_4,_5){
+_2.instantiate(_3._processScript(_5));
+}});
+return dojox.xml.Script;
+});

esp/files/dojox/xml/parser.js

@@ -0,0 +1,131 @@
+//>>built
+define("dojox/xml/parser",["dojo/_base/kernel","dojo/_base/lang","dojo/_base/array","dojo/_base/window","dojo/_base/sniff"],function(_1){
+_1.getObject("xml.parser",true,dojox);
+dojox.xml.parser.parse=function(_2,_3){
+var _4=_1.doc;
+var _5;
+_3=_3||"text/xml";
+if(_2&&_1.trim(_2)&&"DOMParser" in _1.global){
+var _6=new DOMParser();
+_5=_6.parseFromString(_2,_3);
+var de=_5.documentElement;
+var _7="http://www.mozilla.org/newlayout/xml/parsererror.xml";
+if(de.nodeName=="parsererror"&&de.namespaceURI==_7){
+var _8=de.getElementsByTagNameNS(_7,"sourcetext")[0];
+if(_8){
+_8=_8.firstChild.data;
+}
+throw new Error("Error parsing text "+de.firstChild.data+" \n"+_8);
+}
+return _5;
+}else{
+if("ActiveXObject" in _1.global){
+var ms=function(n){
+return "MSXML"+n+".DOMDocument";
+};
+var dp=["Microsoft.XMLDOM",ms(6),ms(4),ms(3),ms(2)];
+_1.some(dp,function(p){
+try{
+_5=new ActiveXObject(p);
+}
+catch(e){
+return false;
+}
+return true;
+});
+if(_2&&_5){
+_5.async=false;
+_5.loadXML(_2);
+var pe=_5.parseError;
+if(pe.errorCode!==0){
+throw new Error("Line: "+pe.line+"\n"+"Col: "+pe.linepos+"\n"+"Reason: "+pe.reason+"\n"+"Error Code: "+pe.errorCode+"\n"+"Source: "+pe.srcText);
+}
+}
+if(_5){
+return _5;
+}
+}else{
+if(_4.implementation&&_4.implementation.createDocument){
+if(_2&&_1.trim(_2)&&_4.createElement){
+var _9=_4.createElement("xml");
+_9.innerHTML=_2;
+var _a=_4.implementation.createDocument("foo","",null);
+_1.forEach(_9.childNodes,function(_b){
+_a.importNode(_b,true);
+});
+return _a;
+}else{
+return _4.implementation.createDocument("","",null);
+}
+}
+}
+}
+return null;
+};
+dojox.xml.parser.textContent=function(_c,_d){
+if(arguments.length>1){
+var _e=_c.ownerDocument||_1.doc;
+dojox.xml.parser.replaceChildren(_c,_e.createTextNode(_d));
+return _d;
+}else{
+if(_c.textContent!==undefined){
+return _c.textContent;
+}
+var _f="";
+if(_c){
+_1.forEach(_c.childNodes,function(_10){
+switch(_10.nodeType){
+case 1:
+case 5:
+_f+=dojox.xml.parser.textContent(_10);
+break;
+case 3:
+case 2:
+case 4:
+_f+=_10.nodeValue;
+}
+});
+}
+return _f;
+}
+};
+dojox.xml.parser.replaceChildren=function(_11,_12){
+var _13=[];
+if(_1.isIE){
+_1.forEach(_11.childNodes,function(_14){
+_13.push(_14);
+});
+}
+dojox.xml.parser.removeChildren(_11);
+_1.forEach(_13,_1.destroy);
+if(!_1.isArray(_12)){
+_11.appendChild(_12);
+}else{
+_1.forEach(_12,function(_15){
+_11.appendChild(_15);
+});
+}
+};
+dojox.xml.parser.removeChildren=function(_16){
+var _17=_16.childNodes.length;
+while(_16.hasChildNodes()){
+_16.removeChild(_16.firstChild);
+}
+return _17;
+};
+dojox.xml.parser.innerXML=function(_18){
+if(_18.innerXML){
+return _18.innerXML;
+}else{
+if(_18.xml){
+return _18.xml;
+}else{
+if(typeof XMLSerializer!="undefined"){
+return (new XMLSerializer()).serializeToString(_18);
+}
+}
+}
+return null;
+};
+return dojox.xml.parser;
+});

esp/files/dojox/xml/widgetParser.js

@@ -0,0 +1,61 @@
+//>>built
+define("dojox/xml/widgetParser",["dojo/_base/lang","dojo/_base/window","dojo/_base/sniff","dojo/query","dojo/parser","dojox/xml/parser"],function(_1,_2,_3,_4,_5,_6){
+var _7=lang.getObject("dojox.xml",true);
+xXml.widgetParser=new function(){
+var d=_1;
+this.parseNode=function(_8){
+var _9=[];
+d.query("script[type='text/xml']",_8).forEach(function(_a){
+_9.push.apply(_9,this._processScript(_a));
+},this).orphan();
+return d.parser.instantiate(_9);
+};
+this._processScript=function(_b){
+var _c=_b.src?d._getText(_b.src):_b.innerHTML||_b.firstChild.nodeValue;
+var _d=this.toHTML(dojox.xml.parser.parse(_c).firstChild);
+var _e=d.query("[dojoType]",_d);
+_4(">",_d).place(_b,"before");
+_b.parentNode.removeChild(_b);
+return _e;
+};
+this.toHTML=function(_f){
+var _10;
+var _11=_f.nodeName;
+var dd=_2.doc;
+var _12=_f.nodeType;
+if(_12>=3){
+return dd.createTextNode((_12==3||_12==4)?_f.nodeValue:"");
+}
+var _13=_f.localName||_11.split(":").pop();
+var _14=_f.namespaceURI||(_f.getNamespaceUri?_f.getNamespaceUri():"");
+if(_14=="html"){
+_10=dd.createElement(_13);
+}else{
+var _15=_14+"."+_13;
+_10=_10||dd.createElement((_15=="dijit.form.ComboBox")?"select":"div");
+_10.setAttribute("dojoType",_15);
+}
+d.forEach(_f.attributes,function(_16){
+var _17=_16.name||_16.nodeName;
+var _18=_16.value||_16.nodeValue;
+if(_17.indexOf("xmlns")!=0){
+if(_3("ie")&&_17=="style"){
+_10.style.setAttribute("cssText",_18);
+}else{
+_10.setAttribute(_17,_18);
+}
+}
+});
+d.forEach(_f.childNodes,function(cn){
+var _19=this.toHTML(cn);
+if(_13=="script"){
+_10.text+=_19.nodeValue;
+}else{
+_10.appendChild(_19);
+}
+},this);
+return _10;
+};
+}();
+return _7.widgetParser;
+});

esp/files/scripts/ESPResult.js

@@ -14,13 +14,20 @@
 #    limitations under the License.
 ############################################################################## */
 define([
-	"dojo/_base/declare",
-	"dojo/data/ObjectStore",
-	"hpcc/WsWorkunits",
-	"hpcc/ESPBase"
-], function (declare, ObjectStore, WsWorkunits, ESPBase) {
+    "dojo/_base/declare",
+    "dojo/data/ObjectStore",
+    "dojo/dom-construct",
+
+    "dojox/xml/parser",
+    "dojox/xml/DomParser",
+    "hpcc/WsWorkunits",
+    "hpcc/ESPBase"
+], function (declare, ObjectStore, domConstruct,
+            parser, DomParser,
+            WsWorkunits, ESPBase) {
 	return declare(ESPBase, {
 		store: null,
+		hasChildDataset: false,
 		Total: "-1",
 
 		constructor: function (args) {
@@ -40,38 +47,177 @@ define([
 			return this.Total != "-1";
 		},
 
-		getStructure: function () {
-			var retVal = [];
-			retVal.push({
-				name: "##",
-				field: this.store.idProperty,
-				width: "40px"
-			});
-			for (var i = 0; i < this.ECLSchemas.length; ++i) {
-				retVal.push({
-					name: this.ECLSchemas[i].ColumnName,
-					field: this.ECLSchemas[i].ColumnName,
-					width: this.extractWidth(this.ECLSchemas[i].ColumnType, this.ECLSchemas[i].ColumnName)
-				});
-			}
-			return retVal;
-		},
+        getFirstSchemaNode: function (node, name) {
+            if (node && node.attributes) {
+                if ((node.localName && node.localName == name) || (node.hasAttributes() && node.getAttribute("name") == name)) {
+                    return node;
+                }
+            }
+            for (var i = 0; i < node.childNodes.length; ++i) {
+                var retVal = this.getFirstSchemaNode(node.childNodes[i], name);
+                if (retVal) {
+                    return retVal;
+                }
+            }
+            return null;
+        },
+
+        getFirstSequenceNode: function (schemaNode) {
+            var row = this.getFirstSchemaNode(schemaNode, "Row");
+            if (!row)
+                return null;
+            var complexType = this.getFirstSchemaNode(row, "complexType");
+            if (!complexType)
+                return null;
+            return this.getFirstSchemaNode(complexType, "sequence");
+        },
+
+        rowToTable: function (cell) {
+            var table = domConstruct.create("table", { border: 1, cellspacing: 0, width: "100%" });
+            if (cell && cell.Row) {
+                if (!cell.Row.length) {
+                    cell.Row = [cell.Row];
+                }
+
+                for (i = 0; i < cell.Row.length; ++i) {
+                    if (i == 0) {
+                        var tr = domConstruct.create("tr", null, table);
+                        for (key in cell.Row[i]) {
+                            var th = domConstruct.create("th", { innerHTML: key }, tr);
+                        }
+                    }
+                    var tr = domConstruct.create("tr", null, table);
+                    for (key in cell.Row[i]) {
+                        if (cell.Row[i][key].Row) {
+                            var td = domConstruct.create("td", null, tr);
+                            td.appendChild(this.rowToTable(cell.Row[i][key]));
+                        } else {
+                            var td = domConstruct.create("td", { innerHTML: cell.Row[i][key] }, tr);
+                        }
+                    }
+                }
+            }
+            return table;
+        },
+
+        getRowStructure: function (parentNode) {
+            var retVal = [];
+            var sequence = this.getFirstSequenceNode(parentNode, "sequence");
+            if (!sequence)
+                return retVal;
+
+            for (var i = 0; i < sequence.childNodes.length; ++i) {
+                var node = sequence.childNodes[i];
+                if (node.hasAttributes()) {
+                    var name = node.getAttribute("name");
+                    var type = node.getAttribute("type");
+                    if (name && type) {
+                        retVal.push({
+                            name: name,
+                            field: name,
+                            width: this.extractWidth(type, name),
+                            classes: "resultGridCell"
+                        });
+                    }
+                    if (node.hasChildNodes()) {
+                        this.hasChildDataset = true;
+                        var context = this;
+                        retVal.push({
+                            name: name,
+                            field: name,
+                            formatter: function (cell, row, grid) {
+                                var div = document.createElement("div");
+                                div.appendChild(context.rowToTable(cell));
+                                return div.innerHTML;
+                            },
+                            width: this.getRowWidth(node),
+                            classes: "resultGridCell"
+                        });
+                    }
+                }
+            }
+            return retVal;
+        },
+
+        getStructure: function () {
+            var structure = [
+                {
+                    cells: [
+                           [
+                            {
+                                name: "##", field: this.store.idProperty, width: "40px", classes: "resultGridCell"
+                            }
+                         ]
+                    ]
+                }
+            ];
+
+            var dom = parser.parse(this.XmlSchema);
+            var dataset = this.getFirstSchemaNode(dom, "Dataset");
+            var innerStruct = this.getRowStructure(dataset);
+            for (var i = 0; i < innerStruct.length; ++i) {
+                structure[0].cells[structure[0].cells.length - 1].push(innerStruct[i]);
+            }
+            return structure;
+        },
+
+        getRowWidth: function (parentNode) {
+            var retVal = 0;
+            var sequence = this.getFirstSequenceNode(parentNode, "sequence");
+            if (!sequence)
+                return retVal;
+
+            for (var i = 0; i < sequence.childNodes.length; ++i) {
+                var node = sequence.childNodes[i];
+                if (node.hasAttributes()) {
+                    var name = node.getAttribute("name");
+                    var type = node.getAttribute("type");
+                    if (name && type) {
+                        retVal += this.extractWidth(type, name);
+                    } else if (node.hasChildNodes()) {
+                        retVal += this.getRowWidth(node);
+                    }
+                }
+            }
+            return retVal;
+        },
 
 		extractWidth: function (type, name) {
-			var numStr = "0123456789";
 			var retVal = -1;
-			var i = type.length;
-			while (i >= 0) {
-				if (numStr.indexOf(type.charAt(--i)) == -1)
+
+			switch (type) {
+				case "xs:boolean":
+					retVal = 5;
+					break;
+				case "xs:integer":
+					retVal = 8;
+					break;
+				case "xs:nonNegativeInteger":
+					retVal = 8;
+					break;
+				case "xs:double":
+					retVal = 8;
+					break;
+				default:
+					var numStr = "0123456789";
+					var underbarPos = type.lastIndexOf("_");
+					var i = underbarPos > 0 ? underbarPos : type.length;
+					while (i >= 0) {
+						if (numStr.indexOf(type.charAt(--i)) == -1)
+							break;
+					}
+					if (i > 0 && i + 1 < type.length) {
+						retVal = parseInt(type.substring(i + 1, type.length));
+					}
+					if (type.indexOf("data") == 0) {
+						retVal *= 2;
+					}
 					break;
 			}
-			if (i > 0)
-				retVal = parseInt(type.substring(i + 1, type.length));
-
 			if (retVal < name.length)
 				retVal = name.length;
 
-			return Math.round(retVal * 2 / 3);
+			return retVal;
 		},
 
 		getObjectStore: function () {

esp/files/scripts/ESPWorkunit.js

@@ -158,6 +158,7 @@ define([
 				IncludeDebugValues: false,
 				IncludeApplicationValues: false,
 				IncludeWorkflows: false,
+				IncludeXmlSchemas: args.onGetResults ? true : false,
 				SuppressResultSchemas: args.onGetResults ? false : true,
 				rawxml_: true
 			};

esp/files/scripts/ResultsControl.js

@@ -17,14 +17,19 @@ define([
 	"dojo/_base/declare",
 	"dojo/store/Memory",
 	"dojo/data/ObjectStore",
+
+	"dijit/registry",
+	"dijit/layout/ContentPane",
+
 	"dojox/grid/DataGrid",
 	"dojox/grid/EnhancedGrid",
 	"dojox/grid/enhanced/plugins/Pagination",
 	"dojox/grid/enhanced/plugins/Filter",
-	"dojox/grid/enhanced/plugins/NestedSorting",
-	"dijit/registry",
-	"dijit/layout/ContentPane"
-], function (declare, Memory, ObjectStore, DataGrid, EnhancedGrid, Pagination, Filter, NestedSorting, registry, ContentPane) {
+	"dojox/grid/enhanced/plugins/NestedSorting"
+
+], function (declare, Memory, ObjectStore,
+    registry, ContentPane,
+    DataGrid, EnhancedGrid, Pagination, Filter, NestedSorting) {
 	return declare(null, {
 		workunit: null,
 		paneNum: 0,
@@ -91,16 +96,12 @@ define([
 
 		addResultTab: function (resultIndex) {
 			var result = this.workunit.results[resultIndex];
+			var structure = result.getStructure();
 			var paneID = this.getNextPaneID();
-			var grid = EnhancedGrid({
-				resultIndex: resultIndex,
-				store: result.getObjectStore(),
-				query: { id: "*" },
-				structure: result.getStructure(),
-				canSort: function (col) {
-					return false;
-				},
-				plugins: {
+
+			var plugins = null;
+			if (!result.hasChildDataset) {
+				plugins = {
 					//					nestedSorting: true,
 					pagination: {
 						pageSizes: [25, 50, 100, "All"],
@@ -112,7 +113,17 @@ define([
 						maxPageStep: 4,
 						position: "bottom"
 					}
-				}
+				};
+			}
+			var grid = EnhancedGrid({
+				resultIndex: resultIndex,
+				store: result.getObjectStore(),
+				query: { id: "*" },
+				structure: structure,
+				canSort: function (col) {
+					return false;
+				},
+				plugins: plugins
 			});
 			this.delayLoad[paneID] = grid;
 			this.sequenceResultStoreMap[result.Sequence] = result.store;

esp/services/ws_ecl/ws_ecl_service.cpp

@@ -2128,7 +2128,7 @@ int CWsEclBinding::submitWsEclWorkunit(IEspContext & context, WsEclWuInfo &wsinf
     Owned <IWorkUnit> workunit = factory->createWorkUnit(NULL, "wsecl", context.queryUserId());
 
     IExtendedWUInterface *ext = queryExtendedWU(workunit);
-    ext->copyWorkUnit(wsinfo.wu);
+    ext->copyWorkUnit(wsinfo.wu, false);
 
     workunit->clearExceptions();
     workunit->resetWorkflow();

esp/services/ws_workunits/ws_workunitsService.cpp

@@ -267,7 +267,7 @@ void copyWsWorkunit(IEspContext &context, IWorkUnit &wu, const char *srcWuid)
     SCMStringBuffer wuid;
     wu.getWuid(wuid);
 
-    queryExtendedWU(&wu)->copyWorkUnit(src);
+    queryExtendedWU(&wu)->copyWorkUnit(src, false);
 
     SCMStringBuffer token;
     wu.setSecurityToken(createToken(wuid.str(), context.queryUserId(), context.queryPassword(), token).str());
@@ -1103,7 +1103,7 @@ bool CWsWorkunitsEx::onWUResubmit(IEspContext &context, IEspWUResubmitRequest &r
                     Owned<IConstWorkUnit> src(factory->openWorkUnit(wuid.str(), false));
                     NewWsWorkunit wu(factory, context);
                     wu->getWuid(wuid);
-                    queryExtendedWU(wu)->copyWorkUnit(src);
+                    queryExtendedWU(wu)->copyWorkUnit(src, false);
 
                     SCMStringBuffer token;
                     wu->setSecurityToken(createToken(wuid.str(), context.queryUserId(), context.queryPassword(), token).str());
@@ -3776,7 +3776,7 @@ void CWsWorkunitsEx::deploySharedObject(IEspContext &context, StringBuffer &wuid
     {
         Owned<ILocalWorkUnit> embeddedWU = createLocalWorkUnit();
         embeddedWU->loadXML(dllXML.str());
-        queryExtendedWU(wu)->copyWorkUnit(embeddedWU);
+        queryExtendedWU(wu)->copyWorkUnit(embeddedWU, true);
     }
 
     wu.associateDll(dllpath.str(), dllname.str());

roxie/ccd/ccdactivities.cpp

@@ -353,7 +353,9 @@ protected:
         if (variableFileName) // note - in keyed join with dependent index case, kj itself won't have variableFileName but indexread might
         {
             CDateTime cacheDate(serializedCreate);
-            varFileInfo.setown(querySlaveDynamicFileCache()->lookupDynamicFile(logctx, queryDynamicFileName(), cacheDate, &packet->queryHeader(), isOpt, true));
+            unsigned checksum;
+            serializedCreate.read(checksum);
+            varFileInfo.setown(querySlaveDynamicFileCache()->lookupDynamicFile(logctx, queryDynamicFileName(), cacheDate, checksum, &packet->queryHeader(), isOpt, true));
             setVariableFileInfo();
         }
     }
@@ -924,7 +926,7 @@ public:
         {
             bool isOpt = (helper->getFlags() & TDRoptional) != 0;
             const char *fileName = helper->getFileName();
-            datafile.setown(_queryFactory.queryPackage().lookupFileName(fileName, isOpt, true));
+            datafile.setown(_queryFactory.queryPackage().lookupFileName(fileName, isOpt, true, _queryFactory.queryWorkUnit()));
             if (datafile)
             {
                 unsigned channel = queryFactory.queryChannel();
@@ -3017,7 +3019,7 @@ public:
         if (!variableFileName)
         {
             bool isOpt = (helper->getFlags() & TIRoptional) != 0;
-            datafile.setown(queryFactory.queryPackage().lookupFileName(helper->getFileName(), isOpt, true));
+            datafile.setown(queryFactory.queryPackage().lookupFileName(helper->getFileName(), isOpt, true, queryFactory.queryWorkUnit()));
             if (datafile)
                 keyArray.setown(datafile->getKeyArray(activityMeta, layoutTranslators, isOpt, queryFactory.queryChannel(), queryFactory.getEnableFieldTranslation()));
         }
@@ -4213,7 +4215,7 @@ public:
         if (!variableFileName)
         {
             bool isOpt = (fetchContext->getFetchFlags() & FFdatafileoptional) != 0;
-            datafile.setown(_queryFactory.queryPackage().lookupFileName(fetchContext->getFileName(), isOpt, true));
+            datafile.setown(_queryFactory.queryPackage().lookupFileName(fetchContext->getFileName(), isOpt, true, _queryFactory.queryWorkUnit()));
             if (datafile)
                 fileArray.setown(datafile->getIFileIOArray(isOpt, queryFactory.queryChannel()));
         }
@@ -4561,7 +4563,7 @@ public:
         if (!variableFileName)
         {
             bool isOpt = (helper->getJoinFlags() & JFindexoptional) != 0;
-            datafile.setown(_queryFactory.queryPackage().lookupFileName(helper->getIndexFileName(), isOpt, true));
+            datafile.setown(_queryFactory.queryPackage().lookupFileName(helper->getIndexFileName(), isOpt, true, _queryFactory.queryWorkUnit()));
             if (datafile)
                 keyArray.setown(datafile->getKeyArray(activityMeta, layoutTranslators, isOpt, queryFactory.queryChannel(), queryFactory.getEnableFieldTranslation()));
         }
@@ -4907,7 +4909,7 @@ public:
         {
             bool isOpt = (helper->getFetchFlags() & FFdatafileoptional) != 0;
             const char *fileName = helper->getFileName();
-            datafile.setown(_queryFactory.queryPackage().lookupFileName(fileName, isOpt, true));
+            datafile.setown(_queryFactory.queryPackage().lookupFileName(fileName, isOpt, true, _queryFactory.queryWorkUnit()));
             if (datafile)
                 fileArray.setown(datafile->getIFileIOArray(isOpt, queryFactory.queryChannel()));
         }
@@ -5256,13 +5258,13 @@ public:
             bool isOpt = _graphNode.getPropBool("att[@name='_isOpt']/@value") || pretendAllOpt;
             if (queryNodeIndexName(_graphNode))
             {
-                indexfile.setown(_queryFactory.queryPackage().lookupFileName(queryNodeIndexName(_graphNode), isOpt, true));
+                indexfile.setown(_queryFactory.queryPackage().lookupFileName(queryNodeIndexName(_graphNode), isOpt, true, _queryFactory.queryWorkUnit()));
                 if (indexfile)
                     keyArray.setown(indexfile->getKeyArray(NULL, &layoutTranslators, isOpt, queryFactory.queryChannel(), queryFactory.getEnableFieldTranslation()));
             }
             if (queryNodeFileName(_graphNode))
             {
-                datafile.setown(_queryFactory.queryPackage().lookupFileName(queryNodeFileName(_graphNode), isOpt, true));
+                datafile.setown(_queryFactory.queryPackage().lookupFileName(queryNodeFileName(_graphNode), isOpt, true, _queryFactory.queryWorkUnit()));
                 if (datafile)
                     fileArray.setown(datafile->getIFileIOArray(isOpt, queryFactory.queryChannel()));
             }

roxie/ccd/ccddali.cpp

@@ -425,7 +425,7 @@ public:
             {
                 Owned<ILocalWorkUnit> localWU = createLocalWorkUnit();
                 localWU->loadXML(wuXML);
-                queryExtendedWU(w)->copyWorkUnit(localWU);
+                queryExtendedWU(w)->copyWorkUnit(localWU, true);
             }
             else
                 throw MakeStringException(ROXIE_DALI_ERROR, "Failed to locate dll workunit info");

roxie/ccd/ccdfile.cpp

@@ -87,8 +87,6 @@ protected:
     bool remote;
     offset_t fileSize;
     CDateTime fileDate;
-    bool fileIsMemFile;
-    bool copyInForeground;
     unsigned crc;
     Owned<ILazyFileIO> patchFile;
     StringBuffer baseIndexFileName;
@@ -117,8 +115,6 @@ public:
         readCount = 0;
 #endif
         memFileRequested = _memFileRequested;
-        fileIsMemFile = false;
-        copyInForeground = false;
         lastAccess = msTick();
         copying = false;
         cached = NULL;
@@ -397,13 +393,6 @@ public:
         return patchFile;
     }
 
-    // the following calls are always made from inside of a critical block...
-    virtual void setFileIsMemFile(bool val) { fileIsMemFile = val; }
-    virtual bool getFileIsMemFile() { return fileIsMemFile; }
-    virtual void setCopyInForeground(bool val) { copyInForeground = val; }
-    virtual bool getCopyInForeground() { return copyInForeground; }
-
-
     virtual size32_t write(offset_t pos, size32_t len, const void * data) { throwUnexpected(); }
     virtual void setSize(offset_t size) { throwUnexpected(); }
     virtual offset_t appendFile(IFile *file,offset_t pos,offset_t len) { throwUnexpected(); return 0; }
@@ -1156,29 +1145,43 @@ public:
                 if ((size != -1 && size != f->getSize()) ||
                     (!modified.isNull() && !modified.equals(*f->queryDateTime(), false)))
                 {
-                    StringBuffer modifiedDt;
-                    if (!modified.isNull())
-                        modified.getString(modifiedDt);
-                    StringBuffer fileDt;
-                    f->queryDateTime()->getString(fileDt);
-                    if (fileErrorList.find(id) == 0)
+                    if (!f->IsShared())
                     {
-                        switch (fileType)
+                        // kill it
+                        files.remove(localLocation);
+                        ForEachItemInRev(idx, todo)
                         {
-                            case ROXIE_KEY:
-                                fileErrorList.setValue(id, "Key");
-                                break;
-                        
-                            case ROXIE_FILE:
-                                fileErrorList.setValue(id, "File");
-                                break;
+                            if (f == &todo.item(idx))
+                            {
+                                todo.remove(idx);
+                            }
                         }
                     }
-        
-                    throw MakeStringException(ROXIE_MISMATCH, "Different version of %s already loaded: sizes = %"I64F"d %"I64F"d  Date = %s  %s", id, size, f->getSize(), modifiedDt.str(), fileDt.str());
-                }
+                    else
+                    {
+                        StringBuffer modifiedDt;
+                        if (!modified.isNull())
+                            modified.getString(modifiedDt);
+                        StringBuffer fileDt;
+                        f->queryDateTime()->getString(fileDt);
+                        if (fileErrorList.find(id) == 0)
+                        {
+                            switch (fileType)
+                            {
+                                case ROXIE_KEY:
+                                    fileErrorList.setValue(id, "Key");
+                                    break;
 
-                return LINK(f);
+                                case ROXIE_FILE:
+                                    fileErrorList.setValue(id, "File");
+                                    break;
+                            }
+                        }
+                        throw MakeStringException(ROXIE_MISMATCH, "Different version of %s already loaded: sizes = %"I64F"d %"I64F"d  Date = %s  %s", id, size, f->getSize(), modifiedDt.str(), fileDt.str());
+                    }
+                }
+                else
+                    return LINK(f);
             }
 
             ret.setown(openFile(id, partNo, fileType, localLocation, peerRoxieCopiedLocationInfo, deployedLocationInfo, size, modified, memFile, crc, isCompressed));  // for now don't check crcs
@@ -1215,15 +1218,6 @@ public:
 //                  toCopy.signal();
                 }
             }
-            else
-            {
-                ret->setFileIsMemFile(memFile);
-                ret->setCopyInForeground(doForegroundCopy);
-                                    
-                todo.append(*ret);
-                atomic_inc(&numFilesToProcess);  // must increment counter for SNMP accuracy
-                toCopy.signal();
-            }
 
             if (!lazyOpen || fileType == ROXIE_PATCH)  // patch file MUST be open at this point - make sure we open it
                 ret->checkOpen();
@@ -1592,7 +1586,7 @@ inline void appendRemoteLocations(IPartDescriptor *pdesc, StringArray &locations
     }
 }
 
-ILazyFileIO *createDynamicFile(const char *id, IPartDescriptor *pdesc, IPartDescriptor *remotePDesc, RoxieFileType fileType, int numParts)
+ILazyFileIO *createDynamicFile(const char *id, IPartDescriptor *pdesc, IPartDescriptor *remotePDesc, RoxieFileType fileType, int numParts, bool startCopy)
 {
     IPropertyTree &partProps = pdesc->queryProperties();
     offset_t dfsSize = partProps.getPropInt64("@size");
@@ -1619,13 +1613,14 @@ ILazyFileIO *createDynamicFile(const char *id, IPartDescriptor *pdesc, IPartDesc
 
     const char *logicalname = dlfn.get();
 
-    makePhysicalPartName(logicalname, partNo, numParts, localFileName, false, DFD_OSdefault, baseDataDirectory);  // MORE - if we get the dataDirectory we can pass it in and possibly` reuse an existing file
+    makePhysicalPartName(logicalname, partNo, numParts, localFileName, false, DFD_OSdefault, baseDataDirectory);  // MORE - if we get the dataDirectory we can pass it in and possibly reuse an existing file
 
     appendRemoteLocations(pdesc, remoteLocations, true);
     if (remotePDesc)
         appendRemoteLocations(remotePDesc, remoteLocations, false);
 
-    return queryFileCache().lookupFile(id, partNo, fileType, localFileName, NULL, NULL, localLocations, remoteLocations, dfsSize, fileDate, false, true, false, false, crcResources ? crc : 0, pdesc->queryOwner().isCompressed(), NULL);
+    bool foregroundCopy = numParts==1 || (partNo==numParts && fileType==ROXIE_KEY);
+    return queryFileCache().lookupFile(id, partNo, fileType, localFileName, NULL, NULL, localLocations, remoteLocations, dfsSize, fileDate, false, true, startCopy, foregroundCopy, crcResources ? crc : 0, pdesc->queryOwner().isCompressed(), NULL);
 }
 
 //====================================================================================================
@@ -1879,7 +1874,7 @@ public:
 
 template <class X> class PerChannelCacheOf
 {
-    PointerArrayOf<X> cache;
+    PointerIArrayOf<X> cache;
     IntArray channels;
 public:
     void set(X *value, unsigned channel)
@@ -1906,10 +1901,12 @@ class CResolvedFile : public CInterface, implements IResolvedFileCreator
 protected:
     const IRoxiePackage *cached;
     StringAttr lfn;
+    StringAttr physicalName;
     Owned<IDistributedFile> dFile; // NULL on copies serialized to slaves. Note that this implies we keep a lock on dali file for the lifetime of this object.
     CDateTime fileTimeStamp;
     RoxieFileType fileType;
     offset_t fileSize;
+    unsigned fileCheckSum;
 
     StringArray subNames;
     PointerIArrayOf<IFileDescriptor> subFiles; // note - on slaves, the file descriptors may have incomplete info. On originating server is always complete
@@ -1951,10 +1948,12 @@ protected:
 
 public:
     IMPLEMENT_IINTERFACE;
-    CResolvedFile(const char *_lfn, IDistributedFile *_dFile, RoxieFileType _fileType) : lfn(_lfn), dFile(_dFile), fileType(_fileType)
+    CResolvedFile(const char *_lfn, const char *_physicalName, IDistributedFile *_dFile, RoxieFileType _fileType)
+    : lfn(_lfn), physicalName(_physicalName), dFile(_dFile), fileType(_fileType)
     {
         cached = NULL;
         fileSize = 0;
+        fileCheckSum = 0;
         if (dFile)
         {
             if (traceLevel > 5)
@@ -1972,6 +1971,7 @@ public:
             else // normal file, not superkey
                 addFile(dFile->queryLogicalName(), dFile->getFileDescriptor());
             bool tsSet = dFile->getModificationTime(fileTimeStamp);
+            bool csSet = dFile->getFileCheckSum(fileCheckSum);
             assertex(tsSet); // per Nigel, is always set
             properties.set(&dFile->queryAttributes());
         }
@@ -2014,7 +2014,6 @@ public:
         }
         return subNames.length();
     }
-    inline const char *queryLFN() const { return lfn; }
     inline bool isKey() const
     {
         return fileType==ROXIE_KEY;
@@ -2039,6 +2038,7 @@ public:
         byte type = (byte) fileType;
         mb.append(type);
         fileTimeStamp.serialize(mb);
+        mb.append(fileCheckSum);
         mb.append(fileSize);
         unsigned numSubFiles = subFiles.length();
         mb.append(numSubFiles);
@@ -2084,38 +2084,41 @@ public:
     {
         Owned<CFileIOArray> f = new CFileIOArray();
         f->addFile(NULL, 0);
-        IFileDescriptor *fdesc = subFiles.item(0);
-        Owned<IFileDescriptor> remoteFDesc = checkCloneFrom(subNames.item(0), fdesc);
-        if (fdesc)
+        if (subFiles.length())
         {
-            unsigned numParts = fdesc->numParts();
-            for (unsigned i = 1; i <= numParts; i++)
+            IFileDescriptor *fdesc = subFiles.item(0);
+            Owned<IFileDescriptor> remoteFDesc = checkCloneFrom(subNames.item(0), fdesc);
+            if (fdesc)
             {
-                if (!channel || getBondedChannel(i)==channel)
+                unsigned numParts = fdesc->numParts();
+                for (unsigned i = 1; i <= numParts; i++)
                 {
-                    try
+                    if (!channel || getBondedChannel(i)==channel)
                     {
-                        IPartDescriptor *pdesc = fdesc->queryPart(i-1);
-                        assertex(pdesc);
-                        IPartDescriptor *remotePDesc = queryMatchingRemotePart(pdesc, remoteFDesc, i-1);
-                        Owned<ILazyFileIO> file = createDynamicFile(subNames.item(0), pdesc, remotePDesc, ROXIE_FILE, numParts);
-                        IPropertyTree &partProps = pdesc->queryProperties();
-                        f->addFile(LINK(file), partProps.getPropInt64("@offset"));
+                        try
+                        {
+                            IPartDescriptor *pdesc = fdesc->queryPart(i-1);
+                            assertex(pdesc);
+                            IPartDescriptor *remotePDesc = queryMatchingRemotePart(pdesc, remoteFDesc, i-1);
+                            Owned<ILazyFileIO> file = createDynamicFile(subNames.item(0), pdesc, remotePDesc, ROXIE_FILE, numParts, cached != NULL);
+                            IPropertyTree &partProps = pdesc->queryProperties();
+                            f->addFile(file.getClear(), partProps.getPropInt64("@offset"));
+                        }
+                        catch (IException *E)
+                        {
+                            StringBuffer err;
+                            err.append("Could not load file ");
+                            fdesc->getTraceName(err);
+                            DBGLOG(E, err.str());
+                            if (!isOpt)
+                                throw;
+                            E->Release();
+                            f->addFile(NULL, 0);
+                        }
                     }
-                    catch (IException *E)
-                    {
-                        StringBuffer err;
-                        err.append("Could not load file ");
-                        fdesc->getTraceName(err);
-                        DBGLOG(E, err.str());
-                        if (!isOpt)
-                            throw;
-                        E->Release();
+                    else
                         f->addFile(NULL, 0);
-                    }
                 }
-                else
-                    f->addFile(NULL, 0);
             }
         }
         return f.getClear();
@@ -2175,7 +2178,7 @@ public:
                             IPartDescriptor *remotePDesc = queryMatchingRemotePart(pdesc, remoteFDesc, partNo-1);
                             if (pdesc)
                             {
-                                part.setown(createDynamicFile(subNames.item(idx), pdesc, remotePDesc, ROXIE_KEY, fdesc->numParts()));
+                                part.setown(createDynamicFile(subNames.item(idx), pdesc, remotePDesc, ROXIE_KEY, fdesc->numParts(), cached != NULL));
                                 pdesc->getCrc(crc);
                             }
                         }
@@ -2206,7 +2209,7 @@ public:
                     assertex(numParts > 0);
                     IPartDescriptor *pdesc = fdesc->queryPart(numParts - 1);
                     IPartDescriptor *remotePDesc = queryMatchingRemotePart(pdesc, remoteFDesc, numParts - 1);
-                    Owned<ILazyFileIO> keyFile = createDynamicFile(subNames.item(idx), pdesc, remotePDesc, ROXIE_KEY, numParts);
+                    Owned<ILazyFileIO> keyFile = createDynamicFile(subNames.item(idx), pdesc, remotePDesc, ROXIE_KEY, numParts, cached != NULL);
                     unsigned crc = 0;
                     pdesc->getCrc(crc);
                     StringBuffer pname;
@@ -2251,6 +2254,11 @@ public:
         return fileTimeStamp;
     }
 
+    virtual unsigned queryCheckSum() const
+    {
+        return fileCheckSum;
+    }
+
     virtual offset_t getFileSize() const
     {
         return fileSize;
@@ -2272,6 +2280,17 @@ public:
     {
         addFile(lfn, _sub);
     }
+    virtual void addSubFile(const char *localFileName)
+    {
+        Owned<IFile> file = createIFile(localFileName);
+        assertex(file->exists());
+        offset_t size = file->size();
+        Owned<IFileDescriptor> fdesc = createFileDescriptor();
+        Owned<IPropertyTree> pp = createPTree("Part");
+        pp->setPropInt64("@size",size);
+        fdesc->setPart(0, queryMyNode(), localFileName, pp);
+        addSubFile(fdesc.getClear());
+    }
 
     virtual void setCache(const IRoxiePackage *cache)
     {
@@ -2288,6 +2307,10 @@ public:
     {
         return lfn.get();
     }
+    virtual const char *queryPhysicalName() const
+    {
+        return physicalName.get();
+    }
     virtual const IPropertyTree *queryProperties() const
     {
         return properties;
@@ -2304,7 +2327,7 @@ public:
         {
             try
             {
-                Owned<IFile> file = createIFile(lfn.get());
+                Owned<IFile> file = createIFile(physicalName.get());
                 file->remove();
             }
             catch (IException *e)
@@ -2314,6 +2337,15 @@ public:
             }
         }
     }
+    virtual bool exists() const
+    {
+        // MORE - this is a little bizarre. We sometimes create a resolvedFile for a file that we are intending to create.
+        // This will make more sense if/when we start to lock earlier.
+        if (dFile)
+            return true; // MORE - may need some thought
+        else
+            return checkFileExists(lfn.get());
+    }
 };
 
 
@@ -2332,7 +2364,7 @@ public:
 
 public:
     CSlaveDynamicFile(const IRoxieContextLogger &logctx, const char *_lfn, RoxiePacketHeader *header, bool _isOpt, bool _isLocal) 
-        : CResolvedFile(_lfn, NULL, ROXIE_FILE), channel(header->channel), serverIdx(header->serverIdx), isOpt(_isOpt), isLocal(_isLocal)
+        : CResolvedFile(_lfn, NULL, NULL, ROXIE_FILE), channel(header->channel), serverIdx(header->serverIdx), isOpt(_isOpt), isLocal(_isLocal)
     {
         // call back to the server to get the info
         IPendingCallback *callback = ROQ->notePendingCallback(*header, lfn); // note that we register before the send to avoid a race.
@@ -2371,6 +2403,7 @@ public:
                 serverData.read(type);
                 fileType = (RoxieFileType) type;
                 fileTimeStamp.deserialize(serverData);
+                serverData.read(fileCheckSum);
                 serverData.read(fileSize);
                 unsigned numSubFiles;
                 serverData.read(numSubFiles);
@@ -2416,15 +2449,15 @@ public:
     }
 };
 
-extern IResolvedFileCreator *createResolvedFile(const char *lfn)
+extern IResolvedFileCreator *createResolvedFile(const char *lfn, const char *physical)
 {
-    return new CResolvedFile(lfn, NULL, ROXIE_FILE);
+    return new CResolvedFile(lfn, physical, NULL, ROXIE_FILE);
 }
 
-extern IResolvedFile *createResolvedFile(const char *lfn, IDistributedFile *dFile)
+extern IResolvedFile *createResolvedFile(const char *lfn, const char *physical, IDistributedFile *dFile)
 {
     const char *kind = dFile ? dFile->queryAttributes().queryProp("@kind") : NULL;
-    return new CResolvedFile(lfn, dFile, kind && stricmp(kind, "key")==0 ? ROXIE_KEY : ROXIE_FILE);
+    return new CResolvedFile(lfn, physical, dFile, kind && stricmp(kind, "key")==0 ? ROXIE_KEY : ROXIE_FILE);
 }
 
 class CSlaveDynamicFileCache : public CInterface, implements ISlaveDynamicFileCache
@@ -2437,14 +2470,14 @@ public:
     IMPLEMENT_IINTERFACE;
     CSlaveDynamicFileCache(unsigned _limit) : tableSize(_limit) {}
 
-    virtual IResolvedFile *lookupDynamicFile(const IRoxieContextLogger &logctx, const char *lfn, CDateTime &cacheDate, RoxiePacketHeader *header, bool isOpt, bool isLocal)
+    virtual IResolvedFile *lookupDynamicFile(const IRoxieContextLogger &logctx, const char *lfn, CDateTime &cacheDate, unsigned checksum, RoxiePacketHeader *header, bool isOpt, bool isLocal)
     {
         if (logctx.queryTraceLevel() > 5)
         {
             StringBuffer s;
             logctx.CTXLOG("lookupDynamicFile %s for packet %s", lfn, header->toString(s).str());
         }
-        // we use a fixed-size array with linear lookup for ease of initial coding - but unless we start making heavy use of the feaure this may be adequate.
+        // we use a fixed-size array with linear lookup for ease of initial coding - but unless we start making heavy use of the feature this may be adequate.
         CriticalBlock b(crit);
         if (!cacheDate.isNull())
         {
@@ -2452,12 +2485,12 @@ public:
             while (files.isItem(idx))
             {
                 CSlaveDynamicFile &f = files.item(idx);
-                if (f.channel==header->channel && f.serverIdx==header->serverIdx && stricmp(f.queryLFN(), lfn)==0)
+                if (f.channel==header->channel && f.serverIdx==header->serverIdx && stricmp(f.queryFileName(), lfn)==0)
                 {
-                    if (!cacheDate.equals(f.queryTimeStamp()))
+                    if (!cacheDate.equals(f.queryTimeStamp()) || checksum != f.queryCheckSum())
                     {
                         if (f.isKey())
-                            clearKeyStoreCacheEntry(f.queryLFN());
+                            clearKeyStoreCacheEntry(f.queryFileName());
                         files.remove(idx);
                         idx--;
                     }

roxie/ccd/ccdfile.hpp

@@ -41,10 +41,6 @@ interface ILazyFileIO : extends IFileIO
     virtual IFile *queryTarget() = 0;
     virtual void copyComplete() = 0;
     virtual int getLinkCount() const = 0;
-    virtual void setFileIsMemFile(bool val) = 0;
-    virtual bool getFileIsMemFile() = 0;
-    virtual void setCopyInForeground(bool val) = 0;
-    virtual bool getCopyInForeground() = 0;
     virtual bool createHardFileLink() = 0;
 
     virtual void setBaseIndexFileName(const char *val) =0;
@@ -112,23 +108,27 @@ interface IResolvedFile : extends ISimpleSuperFileEnquiry
     virtual offset_t getFileSize() const = 0;
 
     virtual const CDateTime &queryTimeStamp() const = 0;
+    virtual unsigned queryCheckSum() const = 0;
 
+    virtual const char *queryPhysicalName() const = 0; // Returns NULL unless in local file mode.
     virtual const char *queryFileName() const = 0;
     virtual void setCache(const IRoxiePackage *cache) = 0;
     virtual bool isAlive() const = 0;
     virtual const IPropertyTree *queryProperties() const = 0;
 
     virtual void remove() = 0;
+    virtual bool exists() const = 0;
 };
 
 interface IResolvedFileCreator : extends IResolvedFile
 {
+    virtual void addSubFile(const char *localFileName) = 0;
     virtual void addSubFile(const IResolvedFile *sub) = 0;
     virtual void addSubFile(IFileDescriptor *sub) = 0;
 };
 
-extern IResolvedFileCreator *createResolvedFile(const char *lfn);
-extern IResolvedFile *createResolvedFile(const char *lfn, IDistributedFile *dFile);
+extern IResolvedFileCreator *createResolvedFile(const char *lfn, const char *physical);
+extern IResolvedFile *createResolvedFile(const char *lfn, const char *physical, IDistributedFile *dFile);
 
 interface IRoxiePublishCallback
 {

roxie/ccd/ccdquery.cpp

@@ -189,7 +189,6 @@ extern void addXrefLibraryInfo(IPropertyTree &reply, const char *libraryName)
 class CQueryFactory : public CInterface, implements IQueryFactory, implements IResourceContext
 {
 protected:
-    IRoxieLibraryLookupContext *libraryContext;  // has linked to me
     const IRoxiePackage &package;
     Owned<const IQueryDll> dll;
     MapStringToActivityArray graphMap;
@@ -738,8 +737,8 @@ public:
     IMPLEMENT_IINTERFACE;
     unsigned channelNo;
 
-    CQueryFactory(const char *_id, const IQueryDll *_dll, const IRoxiePackage &_package, hash64_t _hashValue, unsigned _channelNo, IRoxieLibraryLookupContext *_libraryContext)
-        : id(_id), package(_package), dll(_dll), channelNo(_channelNo), hashValue(_hashValue), libraryContext(_libraryContext)
+    CQueryFactory(const char *_id, const IQueryDll *_dll, const IRoxiePackage &_package, hash64_t _hashValue, unsigned _channelNo)
+        : id(_id), package(_package), dll(_dll), channelNo(_channelNo), hashValue(_hashValue)
     {
         package.Link();
         isSuspended = false;
@@ -765,7 +764,7 @@ public:
 
     virtual IQueryFactory *lookupLibrary(const char *libraryName, unsigned expectedInterfaceHash, const IRoxieContextLogger &logctx) const
     {
-        return libraryContext->lookupLibrary(libraryName, expectedInterfaceHash, logctx);
+        return globalPackageSetManager->lookupLibrary(package, libraryName, expectedInterfaceHash, logctx);
     }
 
     virtual void beforeDispose()
@@ -788,12 +787,10 @@ public:
             return NULL;
     }
 
-    static hash64_t getQueryHash(const char *id, const IQueryDll *dll, const IRoxiePackage &package, const IPropertyTree *stateInfo, IRoxieLibraryLookupContext *libraryContext)
+    static hash64_t getQueryHash(const char *id, const IQueryDll *dll, const IRoxiePackage &package, const IPropertyTree *stateInfo)
     {
         hash64_t hashValue = rtlHash64VStr(dll->queryDll()->queryName(), package.queryHash());
         hashValue = rtlHash64VStr(id, hashValue);
-        if (libraryContext)  // Unit tests don't set it
-            hashValue = rtlHash64VStr(libraryContext->queryId(), hashValue);  // MORE - bit odd...
         if (stateInfo)
         {
             StringBuffer xml;
@@ -1049,6 +1046,10 @@ public:
     {
         return dll->queryDll();
     }
+    virtual IConstWorkUnit *queryWorkUnit() const
+    {
+        return dll->queryWorkUnit();
+    }
     virtual const IRoxiePackage &queryPackage() const
     {
         return package;
@@ -1190,8 +1191,8 @@ protected:
     }
 
 public:
-    CRoxieServerQueryFactory(const char *_id, const IQueryDll *_dll, const IRoxiePackage &_package, hash64_t _hashValue, IRoxieLibraryLookupContext *_libraryContext)
-        : CQueryFactory(_id, _dll, _package, _hashValue, 0, _libraryContext)
+    CRoxieServerQueryFactory(const char *_id, const IQueryDll *_dll, const IRoxiePackage &_package, hash64_t _hashValue)
+        : CQueryFactory(_id, _dll, _package, _hashValue, 0)
     {
         queryStats.setown(createQueryStatsAggregator(id.get(), statsExpiryTime));
     }
@@ -1308,28 +1309,28 @@ public:
     }
 };
 
-extern IQueryFactory *createServerQueryFactory(const char *id, const IQueryDll *dll, const IRoxiePackage &package, const IPropertyTree *stateInfo, IRoxieLibraryLookupContext *libraryContext)
+extern IQueryFactory *createServerQueryFactory(const char *id, const IQueryDll *dll, const IRoxiePackage &package, const IPropertyTree *stateInfo)
 {
     CriticalBlock b(CQueryFactory::queryCreateLock);
-    hash64_t hashValue = CQueryFactory::getQueryHash(id, dll, package, stateInfo, libraryContext);
+    hash64_t hashValue = CQueryFactory::getQueryHash(id, dll, package, stateInfo);
     IQueryFactory *cached = getQueryFactory(hashValue, 0);
     if (cached)
     {
         ::Release(dll);
         return cached;
     }
-    Owned<CRoxieServerQueryFactory> newFactory = new CRoxieServerQueryFactory(id, dll, package, hashValue, libraryContext);
+    Owned<CRoxieServerQueryFactory> newFactory = new CRoxieServerQueryFactory(id, dll, package, hashValue);
     newFactory->load(stateInfo);
     return newFactory.getClear();
 }
 
-extern IQueryFactory *createServerQueryFactoryFromWu(IConstWorkUnit *wu, IRoxieLibraryLookupContext *libraryContext)
+extern IQueryFactory *createServerQueryFactoryFromWu(IConstWorkUnit *wu)
 {
     Owned<const IQueryDll> dll = createWuQueryDll(wu);
     if (!dll)
         return NULL;
     SCMStringBuffer wuid;
-    return createServerQueryFactory(wu->getWuid(wuid).str(), dll.getClear(), queryRootPackage(), NULL, libraryContext); // MORE - if use a constant for id might cache better?
+    return createServerQueryFactory(wu->getWuid(wuid).str(), dll.getClear(), queryRootPackage(), NULL); // MORE - if use a constant for id might cache better?
 }
 
 //==============================================================================================================================================
@@ -1507,8 +1508,8 @@ class CSlaveQueryFactory : public CQueryFactory
     }
 
 public:
-    CSlaveQueryFactory(const char *_id, const IQueryDll *_dll, const IRoxiePackage &_package, hash64_t _hashValue, unsigned _channelNo, IRoxieLibraryLookupContext *_libraryContext)
-        : CQueryFactory(_id, _dll, _package, _hashValue, _channelNo, _libraryContext)
+    CSlaveQueryFactory(const char *_id, const IQueryDll *_dll, const IRoxiePackage &_package, hash64_t _hashValue, unsigned _channelNo)
+        : CQueryFactory(_id, _dll, _package, _hashValue, _channelNo)
     {
     }
 
@@ -1557,28 +1558,28 @@ public:
     }
 };
 
-IQueryFactory *createSlaveQueryFactory(const char *id, const IQueryDll *dll, const IRoxiePackage &package, unsigned channel, const IPropertyTree *stateInfo, IRoxieLibraryLookupContext *libraryContext)
+IQueryFactory *createSlaveQueryFactory(const char *id, const IQueryDll *dll, const IRoxiePackage &package, unsigned channel, const IPropertyTree *stateInfo)
 {
     CriticalBlock b(CQueryFactory::queryCreateLock);
-    hash64_t hashValue = CQueryFactory::getQueryHash(id, dll, package, stateInfo, libraryContext);
+    hash64_t hashValue = CQueryFactory::getQueryHash(id, dll, package, stateInfo);
     IQueryFactory *cached = getQueryFactory(hashValue, channel);
     if (cached)
     {
         ::Release(dll);
         return cached;
     }
-    Owned<CSlaveQueryFactory> newFactory = new CSlaveQueryFactory(id, dll, package, hashValue, channel, libraryContext);
+    Owned<CSlaveQueryFactory> newFactory = new CSlaveQueryFactory(id, dll, package, hashValue, channel);
     newFactory->load(stateInfo);
     return newFactory.getClear();
 }
 
-extern IQueryFactory *createSlaveQueryFactoryFromWu(IConstWorkUnit *wu, unsigned channelNo, IRoxieLibraryLookupContext *libraryContext)
+extern IQueryFactory *createSlaveQueryFactoryFromWu(IConstWorkUnit *wu, unsigned channelNo)
 {
     Owned<const IQueryDll> dll = createWuQueryDll(wu);
     if (!dll)
         return NULL;
     SCMStringBuffer wuid;
-    return createSlaveQueryFactory(wu->getWuid(wuid).str(), dll.getClear(), queryRootPackage(), channelNo, NULL, libraryContext);  // MORE - if use a constant for id might cache better?
+    return createSlaveQueryFactory(wu->getWuid(wuid).str(), dll.getClear(), queryRootPackage(), channelNo, NULL);  // MORE - if use a constant for id might cache better?
 }
 
 IRecordLayoutTranslator * createRecordLayoutTranslator(const char *logicalName, IDefRecordMeta const * diskMeta, IDefRecordMeta const * activityMeta)

roxie/ccd/ccdquery.hpp

@@ -91,6 +91,7 @@ interface IQueryFactory : extends IInterface
     virtual unsigned queryChannel() const = 0;
     virtual ILoadedDllEntry *queryDll() const = 0;
     virtual bool getEnableFieldTranslation() const = 0;
+    virtual IConstWorkUnit *queryWorkUnit() const = 0;
 
     virtual const IRoxiePackage &queryPackage() const = 0;
     virtual IPropertyTree &queryOnceContext() const = 0;
@@ -226,14 +227,12 @@ extern const IQueryDll *createQueryDll(const char *dllName);
 extern const IQueryDll *createExeQueryDll(const char *exeName);
 extern const IQueryDll *createWuQueryDll(IConstWorkUnit *wu);
 
-interface IRoxieLibraryLookupContext;
-
 extern IRecordLayoutTranslator *createRecordLayoutTranslator(const char *logicalName, IDefRecordMeta const * diskMeta, IDefRecordMeta const * activityMeta);
-extern IQueryFactory *createServerQueryFactory(const char *id, const IQueryDll *dll, const IRoxiePackage &package, const IPropertyTree *stateInfo, IRoxieLibraryLookupContext *libraryContext);
-extern IQueryFactory *createSlaveQueryFactory(const char *id, const IQueryDll *dll, const IRoxiePackage &package, unsigned _channelNo, const IPropertyTree *stateInfo, IRoxieLibraryLookupContext *libraryContext);
+extern IQueryFactory *createServerQueryFactory(const char *id, const IQueryDll *dll, const IRoxiePackage &package, const IPropertyTree *stateInfo);
+extern IQueryFactory *createSlaveQueryFactory(const char *id, const IQueryDll *dll, const IRoxiePackage &package, unsigned _channelNo, const IPropertyTree *stateInfo);
 extern IQueryFactory *getQueryFactory(hash64_t hashvalue, unsigned channel);
-extern IQueryFactory *createServerQueryFactoryFromWu(IConstWorkUnit *wu, IRoxieLibraryLookupContext *libraryContext);
-extern IQueryFactory *createSlaveQueryFactoryFromWu(IConstWorkUnit *wu, unsigned channelNo, IRoxieLibraryLookupContext *libraryContext);
+extern IQueryFactory *createServerQueryFactoryFromWu(IConstWorkUnit *wu);
+extern IQueryFactory *createSlaveQueryFactoryFromWu(IConstWorkUnit *wu, unsigned channelNo);
 
 inline unsigned findParentId(IPropertyTree &node)
 {

roxie/ccd/ccdqueue.cpp

@@ -595,7 +595,6 @@ void decIbytiDelay(unsigned channel, unsigned factor = 2)
 
 static SpinLock onDemandQueriesCrit;
 static MapXToMyClass<hash64_t, hash64_t, IQueryFactory> onDemandQueryCache;
-static MapXToMyClass<hash64_t, hash64_t, IRoxieLibraryLookupContext> onDemandLibraryLookupCache;
 
 void sendUnloadMessage(hash64_t hash, const char *id, const IRoxieContextLogger &logctx)
 {
@@ -623,14 +622,12 @@ void doUnload(IRoxieQueryPacket *packet, const IRoxieContextLogger &logctx)
     hash64_t hashValue = header.queryHash;
     SpinBlock b(onDemandQueriesCrit);
     onDemandQueryCache.remove(hashValue+channelNo);
-    onDemandLibraryLookupCache.remove(hashValue+channelNo);
 }
 
-void cacheOnDemandQuery(hash64_t hashValue, unsigned channelNo, IQueryFactory *query, IRoxieLibraryLookupContext *libraryContext)
+void cacheOnDemandQuery(hash64_t hashValue, unsigned channelNo, IQueryFactory *query)
 {
     SpinBlock b(onDemandQueriesCrit);
     onDemandQueryCache.setValue(hashValue+channelNo, query);
-    onDemandLibraryLookupCache.setValue(hashValue+channelNo, libraryContext);
 }
 
 //=================================================================================
@@ -997,18 +994,15 @@ public:
         hash64_t queryHash = packet->queryHeader().queryHash;
         unsigned activityId = packet->queryHeader().activityId & ~ROXIE_PRIORITY_MASK;
         Owned<IQueryFactory> queryFactory = getQueryFactory(queryHash, channel);
-        Owned<IRoxieLibraryLookupContext> libraryContext;
         if (!queryFactory && logctx.queryWuid())
         {
-            // Ensure that any library lookup is done in the correct QuerySet...
             Owned <IRoxieDaliHelper> daliHelper = connectToDali();
             Owned<IConstWorkUnit> wu = daliHelper->attachWorkunit(logctx.queryWuid(), NULL);
             SCMStringBuffer target;
             wu->getClusterName(target);
-            libraryContext.setown(globalPackageSetManager->getLibraryLookupContext(target.str()));
-            queryFactory.setown(createSlaveQueryFactoryFromWu(wu, channel, libraryContext));
+            queryFactory.setown(createSlaveQueryFactoryFromWu(wu, channel));
             if (queryFactory)
-                cacheOnDemandQuery(queryHash, channel, queryFactory, libraryContext);
+                cacheOnDemandQuery(queryHash, channel, queryFactory);
         }
         if (!queryFactory)
         {

roxie/ccd/ccdserver.cpp

@@ -902,23 +902,6 @@ public:
         createPending = true;
     }
     
-    CRoxieServerActivity(const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager, IHThorArg &_helper)
-      : factory(_factory), basehelper(_helper)
-    {
-        input = NULL;
-        ctx = NULL;
-        meta.set(basehelper.queryOutputMeta());
-        processed = 0;
-        totalCycles = 0;
-        if (factory)
-            factory->createChildQueries(childGraphs, this, _probeManager, *this);
-        state=STATEreset;
-        rowAllocator = NULL;
-        debugging = _probeManager != NULL; // Don't want to collect timing stats from debug sessions
-        colocalParent = NULL;
-        createPending = true;
-    }
-
     CRoxieServerActivity(IHThorArg & _helper) : factory(NULL), basehelper(_helper)
     {
         activityId = 0;
@@ -1430,6 +1413,7 @@ public:
     CRoxieServerLateStartActivity(const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager)
         : CRoxieServerActivity(_factory, _probeManager)
     {
+        input = NULL;
         prefiltered = false;
         eof = false;
     }
@@ -3421,7 +3405,10 @@ private:
                 activity.serializeCreateStartContext(cachedContext.clear());
                 activity.serializeExtra(cachedContext);
                 if (activity.queryVarFileInfo())
+                {
                     activity.queryVarFileInfo()->queryTimeStamp().serialize(cachedContext);
+                    cachedContext.append(activity.queryVarFileInfo()->queryCheckSum());
+                }
                 contextCached = true;
             }
 
@@ -4082,7 +4069,7 @@ public:
                             RecordLengthType *rowlen = (RecordLengthType *) len.get();
                             OwnedConstRoxieRow row = callbackData->getNext(*rowlen);
                             char *rowdata = (char *) row.get();
-                            //if (ctxTraceLevel > 5)
+                            if (ctxTraceLevel > 5)
                             {
                                 StringBuffer s;
                                 activity.queryLogCtx().CTXLOG("Callback on query %s for debug", header.toString(s).str());
@@ -4121,7 +4108,7 @@ public:
                             bool isOpt = * (bool *) rowdata;
                             bool isLocal = * (bool *) (rowdata+1);
                             const char *lfn = rowdata+2;
-                            //if (ctxTraceLevel > 5)
+                            if (ctxTraceLevel > 5)
                             {
                                 StringBuffer s;
                                 activity.queryLogCtx().CTXLOG("Callback on query %s file %s", header.toString(s).str(),(const char *) lfn);
@@ -10633,7 +10620,9 @@ protected:
     CachedOutputMetaData diskmeta;
     Owned<IRoxieWriteHandler> writer;
 
+    bool tallycrc;
     unsigned __int64 uncompressedBytesWritten;
+    CRC32 crc;
 
     void updateWorkUnitResult(unsigned __int64 reccount)
     {
@@ -10715,6 +10704,7 @@ public:
         diskmeta.set(helper.queryDiskRecordSize());
         blockcompressed = (((helper.getFlags() & TDWnewcompress) != 0) || (((helper.getFlags() & TDXcompress) != 0) && (diskmeta.getFixedSize() >= MIN_ROWCOMPRESS_RECSIZE))); //always use new compression
         encrypted = false; // set later
+        tallycrc = true;
         uncompressedBytesWritten = 0;
     }
 
@@ -10745,17 +10735,17 @@ public:
             encrypted = true;
             blockcompressed = true;
         }
-        if(blockcompressed)
+        if (blockcompressed)
             io.setown(createCompressedFileWriter(writer->queryFile(), (diskmeta.isFixedSize() ? diskmeta.getFixedSize() : 0), extend, true, ecomp));
         else
             io.setown(writer->queryFile()->open(extend ? IFOwrite : IFOcreate));
-        if(!io)
+        if (!io)
             throw MakeStringException(errno, "Failed to create%s file %s for writing", (encrypted ? " encrypted" : (blockcompressed ? " compressed" : "")), writer->queryFile()->queryFilename());
         diskout.setown(createBufferedIOStream(io));
-        if(extend)
+        if (extend)
             diskout->seek(0, IFSend);
         rowSerializer.setown(input->queryOutputMeta()->createRowSerializer(ctx->queryCodeContext(), activityId)); 
-        bool tallycrc = !factory->queryQueryFactory().getDebugValueBool("skipFileFormatCrcCheck", false) && !(helper.getFlags() & TDRnocrccheck); 
+        tallycrc = !factory->queryQueryFactory().getDebugValueBool("skipFileFormatCrcCheck", false) && !(helper.getFlags() & TDRnocrccheck) && !blockcompressed;
         outSeq.setown(createRowWriter(diskout, rowSerializer, rowAllocator, grouped, tallycrc, true )); 
     }
 
@@ -10768,7 +10758,7 @@ public:
         }
         else
         {
-            outSeq->flush();
+            outSeq->flush(&crc);
             updateWorkUnitResult(processed);
             uncompressedBytesWritten = outSeq->getPosition();
             writer->finish(true, this);
@@ -10784,6 +10774,7 @@ public:
         outSeq.clear();
         writer.clear();
         uncompressedBytesWritten = 0;
+        crc.reset();
     }
 
     virtual void onExecute()
@@ -10814,6 +10805,8 @@ public:
             fileProps.setPropInt64("@size", uncompressedBytesWritten);
             partProps.setPropInt64("@size", uncompressedBytesWritten);
         }
+        else if (tallycrc)
+            partProps.setPropInt64("@fileCrc", crc.get());
 
         if (encrypted)
             fileProps.setPropBool("@encrypted", true);
@@ -19581,26 +19574,33 @@ protected:
     bool isKeyed;
     bool variableFileName;
     bool isOpt;
+    bool sorted;
     bool maySkip;
     bool isLocal;
     CachedOutputMetaData diskSize;
     Owned<const IResolvedFile> varFileInfo;
     Owned<IFileIOArray> varFiles;
 
+    inline bool useRemote()
+    {
+        return remote != NULL && numParts > 1;
+    }
+
 public:
     IMPLEMENT_IINTERFACE;
 
-    CRoxieServerDiskReadBaseActivity(const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager, const RemoteActivityId &_remoteId, unsigned _numParts, bool _isLocal, bool sorted, bool _maySkip, IInMemoryIndexManager *_manager)
+    CRoxieServerDiskReadBaseActivity(const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager, const RemoteActivityId &_remoteId, unsigned _numParts, bool _isLocal, bool _sorted, bool _maySkip, IInMemoryIndexManager *_manager)
         : CRoxieServerActivity(_factory, _probeManager), 
           helper((IHThorDiskReadBaseArg &)basehelper),
           numParts(_numParts),
           remoteId(_remoteId),
           manager(_manager),
           isLocal(_isLocal),
+          sorted(_sorted),
           maySkip(_maySkip),
           deserializeSource(NULL)
     {
-        if (numParts != 1 && !isLocal)
+        if (numParts != 1 && !isLocal)  // NOTE : when numParts == 0 (variable case) we create, even though we may not use
             remote.setown(new CSkippableRemoteResultAdaptor(remoteId, meta.queryOriginal(), helper, *this, sorted, false, _maySkip));
         compoundHelper = NULL;
         eof = false;
@@ -19634,30 +19634,37 @@ public:
             rowLimit = compoundHelper->getRowLimit();
             stopAfter = compoundHelper->getChooseNLimit();
         }
-        if (remote)
+        if (!helper.canMatchAny())
+            eof = true;
+        else
         {
-            remote->onStart(parentExtractSize, parentExtract);
-            remote->setLimits(rowLimit, (unsigned __int64) -1, stopAfter);
-            if (helper.canMatchAny())
+            if (variableFileName)
             {
-                if (variableFileName)
-                    varFileInfo.setown(resolveLFN(helper.getFileName(), isOpt));
+                varFileInfo.setown(resolveLFN(helper.getFileName(), isOpt));
+                Owned<IFilePartMap> map = varFileInfo->getFileMap();
+                if (map)
+                    numParts = map->getNumParts();
+                else
+                {
+                    numParts = 0;
+                    eof = true;
+                    return;
+                }
+            }
+            if (useRemote())
+            {
+                remote->onStart(parentExtractSize, parentExtract);
+                remote->setLimits(rowLimit, (unsigned __int64) -1, stopAfter);
                 unsigned fileNo = 0;        // MORE - superfiles require us to do this per file part... maybe (needs thought)
                 // Translation into a message per channel done elsewhere....
                 remote->getMem(0, fileNo, 0);
+                remote->flush();
+                remote->senddone();
             }
-            remote->flush();
-            remote->senddone();
-        }
-        else
-        {
-            if (!helper.canMatchAny())
-                eof = true;
             else
             {
                 if (variableFileName)
                 {
-                    varFileInfo.setown(resolveLFN(helper.getFileName(), isOpt));
                     unsigned channel = isLocal ? factory->queryQueryFactory().queryChannel() : 0;
                     varFiles.setown(varFileInfo->getIFileIOArray(isOpt, channel));
                     manager.setown(varFileInfo->getIndexManager(isOpt, channel, varFiles, diskSize, false, 0));
@@ -19707,14 +19714,14 @@ public:
 
     virtual void stop(bool aborting)
     {
-        if (remote)
+        if (useRemote())
             remote->onStop(aborting);
         CRoxieServerActivity::stop(aborting);
     }
 
     virtual void reset()
     {
-        if (remote)
+        if (useRemote())
         {
             processed = remote->processed;
             remote->processed = 0;
@@ -19733,21 +19740,6 @@ public:
         throw MakeStringException(ROXIE_SET_INPUT, "Internal error: setInput() called for source activity");
     }
 
-    virtual IRoxieInput *queryOutput(unsigned idx)
-    {
-        if (idx==(unsigned)-1)
-            idx = 0;
-        if (idx == 0)
-        {
-            if (remote)
-                return remote;
-            else
-                return this;
-        }
-        else
-            return NULL;
-    }
-
     virtual void onLimitExceeded(bool isKeyed)
     {
         if (traceLevel > 4)
@@ -19830,10 +19822,11 @@ public:
 
     virtual const void *nextInGroup()
     {
-        // Note - in remote case this never gets called as input chain is routed to remoteResultAdaptor
         if (eof)
             return NULL;
-        if (maySkip)
+        else if (useRemote())
+            return remote->nextInGroup();
+        else if (maySkip)
         {
             if (!readAheadDone)
             {
@@ -19973,7 +19966,7 @@ public:
         rowLimit = readHelper->getRowLimit();
         stopAfter = readHelper->getChooseNLimit();
         CRoxieServerDiskReadBaseActivity::start(parentExtractSize, parentExtract, paused);
-        if (!remote)
+        if (!useRemote())
         {
             rowTransformer.set(readHelper->queryTransformer());
             assertex(reader != NULL);
@@ -20077,7 +20070,7 @@ public:
         rowLimit = readHelper->getRowLimit();
         stopAfter = readHelper->getChooseNLimit();
         CRoxieServerDiskReadBaseActivity::start(parentExtractSize, parentExtract, paused);
-        if (!remote)
+        if (!useRemote())
         {
             headerLines = csvInfo->queryHeaderLen(); 
             if (headerLines && isLocal && reader->queryFilePart() != 1)
@@ -20357,7 +20350,7 @@ public:
         unsigned __int64 totalCount = 0;
         if (helper.canMatchAny())
         {
-            if (remote)
+            if (useRemote())
             {
                 loop
                 {
@@ -20458,7 +20451,7 @@ public:
     {
         RtlDynamicRowBuilder rowBuilder(rowAllocator, false);
         size32_t finalSize = 0;
-        if (remote)
+        if (useRemote())
         {
             const void * firstRow = remote->nextInGroup();
             if (!firstRow)
@@ -20555,7 +20548,7 @@ public:
 
     void gatherMerged()
     {
-        if (remote)
+        if (useRemote())
         {
             loop
             {
@@ -20628,7 +20621,7 @@ public:
         {
             bool isOpt = (helper->getFlags() & TDRoptional) != 0;
             const char *fileName = helper->getFileName();
-            datafile.setown(_queryFactory.queryPackage().lookupFileName(fileName, isOpt, true));
+            datafile.setown(_queryFactory.queryPackage().lookupFileName(fileName, isOpt, true, _queryFactory.queryWorkUnit()));
             if (datafile)
                 map.setown(datafile->getFileMap());
             bool isSimple = (map && map->getNumParts()==1);
@@ -21728,7 +21721,7 @@ public:
         if (!variableFileName)
         {
             bool isOpt = (flags & TIRoptional) != 0;
-            indexfile.setown(queryFactory.queryPackage().lookupFileName(indexHelper->getFileName(), isOpt, true));
+            indexfile.setown(queryFactory.queryPackage().lookupFileName(indexHelper->getFileName(), isOpt, true, queryFactory.queryWorkUnit()));
             if (indexfile)
                 keySet.setown(indexfile->getKeyArray(activityMeta, translatorArray, isOpt, isLocal ? queryFactory.queryChannel() : 0, enableFieldTranslation));
         }
@@ -22645,7 +22638,7 @@ public:
             assertex(recsize);
             const char *fileName = helper->getFileName();
             bool isOpt = (helper->getFlags() & TDRoptional) != 0;
-            datafile.setown(queryFactory.queryPackage().lookupFileName(fileName, isOpt, true));
+            datafile.setown(queryFactory.queryPackage().lookupFileName(fileName, isOpt, true, queryFactory.queryWorkUnit()));
             offset_t filesize = datafile ? datafile->getFileSize() : 0;
             if (filesize % recsize != 0)
                 throw MakeStringException(ROXIE_MISMATCH, "Record size mismatch for file %s - %"I64F"d is not a multiple of fixed record size %d", fileName, filesize, recsize);
@@ -22859,7 +22852,10 @@ public:
         variableFileName = (fetchContext->getFetchFlags() & (FFvarfilename|FFdynamicfilename)) != 0;
         if (!variableFileName)
         {
-            datafile.setown(_queryFactory.queryPackage().lookupFileName(fetchContext->getFileName(), (fetchContext->getFetchFlags() & FFdatafileoptional) != 0, true));
+            datafile.setown(_queryFactory.queryPackage().lookupFileName(fetchContext->getFileName(),
+                                                                        (fetchContext->getFetchFlags() & FFdatafileoptional) != 0,
+                                                                        true,
+                                                                        _queryFactory.queryWorkUnit()));
             if (datafile)
                 map.setown(datafile->getFileMap());
         }
@@ -22905,13 +22901,13 @@ public:
             bool isOpt = _graphNode.getPropBool("att[@name='_isOpt']/@value") || pretendAllOpt;
             if (queryNodeIndexName(_graphNode))
             {
-                indexfile.setown(queryFactory.queryPackage().lookupFileName(queryNodeIndexName(_graphNode), isOpt, true));
+                indexfile.setown(queryFactory.queryPackage().lookupFileName(queryNodeIndexName(_graphNode), isOpt, true, queryFactory.queryWorkUnit()));
                 if (indexfile)
                     keySet.setown(indexfile->getKeyArray(NULL, &layoutTranslators, isOpt, isLocal ? queryFactory.queryChannel() : 0, false));
             }
             if (queryNodeFileName(_graphNode))
             {
-                datafile.setown(_queryFactory.queryPackage().lookupFileName(queryNodeFileName(_graphNode), isOpt, true));
+                datafile.setown(_queryFactory.queryPackage().lookupFileName(queryNodeFileName(_graphNode), isOpt, true, queryFactory.queryWorkUnit()));
                 if (datafile)
                 {
                     if (isLocal)
@@ -23389,7 +23385,10 @@ public:
             MemoryBuffer tmp;
             rootIndex->queryActivity()->serializeCreateStartContext(tmp);
             if (rootIndex->queryActivity()->queryVarFileInfo())
+            {
                 rootIndex->queryActivity()->queryVarFileInfo()->queryTimeStamp().serialize(tmp);
+                tmp.append(rootIndex->queryActivity()->queryVarFileInfo()->queryCheckSum());
+            }
             unsigned ctxlen = tmp.length();
             out.append(ctxlen).append(tmp);
         }
@@ -24173,7 +24172,10 @@ public:
             MemoryBuffer tmp;
             rootIndex->queryActivity()->serializeCreateStartContext(tmp);
             if (rootIndex->queryActivity()->queryVarFileInfo())
+            {
                 rootIndex->queryActivity()->queryVarFileInfo()->queryTimeStamp().serialize(tmp);
+                tmp.append(rootIndex->queryActivity()->queryVarFileInfo()->queryCheckSum());
+            }
             unsigned ctxlen = tmp.length();
             out.append(ctxlen).append(tmp);
         }
@@ -24421,7 +24423,7 @@ public:
         if (!variableIndexFileName)
         {
             bool isOpt = (joinFlags & JFindexoptional) != 0;
-            indexfile.setown(queryFactory.queryPackage().lookupFileName(helper->getIndexFileName(), isOpt, true));
+            indexfile.setown(queryFactory.queryPackage().lookupFileName(helper->getIndexFileName(), isOpt, true, queryFactory.queryWorkUnit()));
             if (indexfile)
                 keySet.setown(indexfile->getKeyArray(activityMeta, translatorArray, isOpt, isLocal ? queryFactory.queryChannel() : 0, enableFieldTranslation));
         }
@@ -24440,7 +24442,7 @@ public:
         if (!isHalfKeyed && !variableFetchFileName)
         {
             bool isFetchOpt = (helper->getFetchFlags() & FFdatafileoptional) != 0;
-            datafile.setown(_queryFactory.queryPackage().lookupFileName(queryNodeFileName(_graphNode), isFetchOpt, true));
+            datafile.setown(_queryFactory.queryPackage().lookupFileName(queryNodeFileName(_graphNode), isFetchOpt, true, _queryFactory.queryWorkUnit()));
             if (datafile)
             {
                 if (isLocal)
@@ -28063,7 +28065,7 @@ public:
     virtual const IResolvedFile *resolveLFN(const char *filename, bool isOpt)
     {
         CDateTime cacheDate; // Note - this is empty meaning we don't know...
-        return querySlaveDynamicFileCache()->lookupDynamicFile(*this, filename, cacheDate, header, isOpt, false);
+        return querySlaveDynamicFileCache()->lookupDynamicFile(*this, filename, cacheDate, 0, header, isOpt, false);
     }
 
     virtual IRoxieWriteHandler *createLFN(const char *filename, bool overwrite, bool extend, const StringArray &clusters)
@@ -30002,7 +30004,7 @@ public:
         {
             dynamicPackage.setown(createPackage(NULL));
         }
-        return dynamicPackage->lookupFileName(filename, isOpt, true);
+        return dynamicPackage->lookupFileName(filename, isOpt, true, workUnit);
     }
 
     virtual IRoxieWriteHandler *createLFN(const char *filename, bool overwrite, bool extend, const StringArray &clusters)
@@ -30012,7 +30014,7 @@ public:
         {
             dynamicPackage.setown(createPackage(NULL));
         }
-        return dynamicPackage->createFileName(filename, overwrite, extend, clusters);
+        return dynamicPackage->createFileName(filename, overwrite, extend, clusters, workUnit);
     }
 
     virtual void onFileCallback(const RoxiePacketHeader &header, const char *lfn, bool isOpt, bool isLocal)
@@ -31367,14 +31369,9 @@ public:
         daliHelper->noteWorkunitRunning(wuid.get(), true);
         if (!wu)
             throw MakeStringException(ROXIE_DALI_ERROR, "Failed to open workunit %s", wuid.get());
-        // Ensure that any library lookup is done in the correct QuerySet...
-        // MORE - Not 100% sure if this is right
-        // - there's no package file resolution in play for WUs read from a queue (should there be?),
-        // but as this stands we will resolve libraries using those packages defined as loading for this QuerySet.
         SCMStringBuffer target;
         wu->getClusterName(target);
-        Owned<IRoxieLibraryLookupContext> libraryContext = globalPackageSetManager->getLibraryLookupContext(target.str());
-        Owned<IQueryFactory> queryFactory = createServerQueryFactoryFromWu(wu, libraryContext);
+        Owned<IQueryFactory> queryFactory = createServerQueryFactoryFromWu(wu);
         Owned<StringContextLogger> logctx = new StringContextLogger(wuid.get());
         doMain(wu, queryFactory, *logctx);
         sendUnloadMessage(queryFactory->queryHash(), wuid.get(), *logctx);
@@ -32419,7 +32416,7 @@ protected:
         package.setown(createPackage(NULL));
         ctx.setown(createSlaveContext(NULL, logctx, 0, 50*1024*1024, NULL));
         queryDll.setown(createExeQueryDll("roxie"));
-        queryFactory.setown(createServerQueryFactory("test", queryDll.getLink(), *package, NULL, NULL));
+        queryFactory.setown(createServerQueryFactory("test", queryDll.getLink(), *package, NULL));
         timer->reset();
     }
 

roxie/ccd/ccdstate.cpp

@@ -309,7 +309,7 @@ protected:
             IPropertyTree *fileInfo = node->queryPropTree(xpath.appendf("File[@id='%s']", fileName).str());
             if (fileInfo)
             {
-                Owned <IResolvedFileCreator> result = createResolvedFile(fileName);
+                Owned <IResolvedFileCreator> result = createResolvedFile(fileName, NULL);
                 result->addSubFile(createFileDescriptorFromRoxieXML(fileInfo));
                 return result.getClear();
             }
@@ -317,15 +317,16 @@ protected:
         return NULL;
     }
     // Use dali to resolve subfile into physical file info
-    IResolvedFile *resolveLFNusingDali(const char *fileName, bool cacheIt, bool writeAccess) const
+    IResolvedFile *resolveLFNusingDali(const char *fileName, bool cacheIt, bool writeAccess, bool alwaysCreate) const
     {
+        // MORE - look at alwaysCreate... This may be useful to implement earlier locking semantics.
         if (daliHelper)
         {
             if (daliHelper->connected())
             {
                 Owned<IDistributedFile> dFile = daliHelper->resolveLFN(fileName, cacheIt, writeAccess);
                 if (dFile)
-                    return createResolvedFile(fileName, dFile.getClear());
+                    return createResolvedFile(fileName, NULL, dFile.getClear());
             }
             else if (!writeAccess)  // If we need write access and expect a dali, but don't have one, we should probably fail
             {
@@ -333,7 +334,7 @@ protected:
                 Owned<IFileDescriptor> fd = daliHelper->resolveCachedLFN(fileName);
                 if (fd)
                 {
-                    Owned <IResolvedFileCreator> result = createResolvedFile(fileName);
+                    Owned <IResolvedFileCreator> result = createResolvedFile(fileName, NULL);
                     result->addSubFile(fd.getClear());
                     return result.getClear();
                 }
@@ -342,17 +343,32 @@ protected:
         return NULL;
     }
     // Use local package file's localFile info to resolve subfile into physical file info
-    IResolvedFile *resolveLFNusingLocal(const char *fileName) const
+    IResolvedFile *resolveLFNusingLocal(const char *fileName, bool writeAccess, bool alwaysCreate) const
     {
         if (node && node->getPropBool("@localFiles"))
         {
-            Owned <IResolvedFileCreator> result = createResolvedFile(fileName);
-            return result.getClear();
+            StringBuffer useName;
+            if (strstr(fileName,"::"))
+            {
+                bool wasDFS;
+                // MORE - really we don't want the 1 of 1 bit of this...
+                makeSinglePhysicalPartName(fileName, useName, true, wasDFS, baseDataDirectory.str());
+            }
+            else
+                useName.append(fileName);
+            bool exists = checkFileExists(useName);
+            if (exists || alwaysCreate)
+            {
+                Owned <IResolvedFileCreator> result = createResolvedFile(fileName, useName);
+                if (exists)
+                    result->addSubFile(useName);
+                return result.getClear();
+            }
         }
         return NULL;
     }
     // Use local package and its bases to resolve existing file into physical file info via all supported resolvers
-    IResolvedFile *lookupFile(const char *fileName, bool cache, bool writeAccess) const
+    IResolvedFile *lookupFile(const char *fileName, bool cache, bool writeAccess, bool alwaysCreate) const
     {
         // Order of resolution: 
         // 1. Files named in package
@@ -373,7 +389,7 @@ protected:
                 // Optimize the common case of a single subfile
                 StringBuffer subFileName;
                 subFileInfo->getSubFileName(0, subFileName);
-                return lookupFile(subFileName, cache, writeAccess);
+                return lookupFile(subFileName, cache, writeAccess, alwaysCreate);
             }
             else
             {
@@ -383,11 +399,11 @@ protected:
                 {
                     StringBuffer subFileName;
                     subFileInfo->getSubFileName(idx, subFileName);
-                    Owned<const IResolvedFile> subFileInfo = lookupFile(subFileName, cache, writeAccess);
+                    Owned<const IResolvedFile> subFileInfo = lookupFile(subFileName, cache, writeAccess, alwaysCreate);
                     if (subFileInfo)
                     {
                         if (!super) 
-                            super.setown(createResolvedFile(fileName));
+                            super.setown(createResolvedFile(fileName, NULL));
                         super->addSubFile(subFileInfo);
                     }
                 }
@@ -398,9 +414,9 @@ protected:
         }
         result = resolveLFNusingPackage(fileName);
         if (!result)
-            result = resolveLFNusingDali(fileName, cache, writeAccess);
+            result = resolveLFNusingDali(fileName, cache, writeAccess, alwaysCreate);
         if (!result)
-            result = resolveLFNusingLocal(fileName);
+            result = resolveLFNusingLocal(fileName, writeAccess, alwaysCreate);
         if (result)
         {
             if (cache)
@@ -410,7 +426,7 @@ protected:
         ForEachItemIn(idx, bases)
         {
             const CRoxiePackage &basePackage = bases.item(idx);
-            IResolvedFile *result = basePackage.lookupFile(fileName, cache, writeAccess);
+            IResolvedFile *result = basePackage.lookupFile(fileName, cache, writeAccess, alwaysCreate);
             if (result)
                 return result;
         }
@@ -517,12 +533,12 @@ public:
         return lookupElements(xpath.str(), "MemIndex");
     }
 
-    virtual const IResolvedFile *lookupFileName(const char *_fileName, bool opt, bool cache) const
+    virtual const IResolvedFile *lookupFileName(const char *_fileName, bool opt, bool cache, IConstWorkUnit *wu) const
     {
         StringBuffer fileName;
-        expandLogicalFilename(fileName, _fileName, NULL, false);   // MORE - if we have a wu, and we have not yet got rid of the concept of scope, we should use it here
+        expandLogicalFilename(fileName, _fileName, wu, false);
 
-        const IResolvedFile *result = lookupFile(fileName, cache, false);
+        const IResolvedFile *result = lookupFile(fileName, cache, false, false);
         if (!result)
         {
             if (!opt)
@@ -533,33 +549,31 @@ public:
         return result;
     }
 
-    virtual IRoxieWriteHandler *createFileName(const char *_fileName, bool overwrite, bool extend, const StringArray &clusters) const
+    virtual IRoxieWriteHandler *createFileName(const char *_fileName, bool overwrite, bool extend, const StringArray &clusters, IConstWorkUnit *wu) const
     {
         StringBuffer fileName;
-        expandLogicalFilename(fileName, _fileName, NULL, false);   // MORE - if we have a wu, and we have not yet got rid of the concept of scope, we should use it here
-        // Dali filenames used locally
-        bool disconnected = !daliHelper->connected();
-        if (disconnected && strstr(fileName,"::"))
-        {
-            StringBuffer name;
-            bool wasDFS;
-            makeSinglePhysicalPartName(fileName, name, true, wasDFS, baseDataDirectory.str());
-            fileName.clear().append(name);
-        }
-        Owned<IResolvedFile> resolved = lookupFile(fileName, false, true);
+        expandLogicalFilename(fileName, _fileName, wu, false);
+        Owned<IResolvedFile> resolved = lookupFile(fileName, false, true, true);
         if (resolved)
         {
-            if (!overwrite)
-                throw MakeStringException(99, "Cannot write %s, file already exists (missing OVERWRITE attribute?)", resolved->queryFileName());
-            if (extend)
-                UNIMPLEMENTED; // How does extend fit in with the clusterwritemanager stuff? They can't specify cluster and extend together...
-            removeCache(resolved);
-            resolved->remove();
+            if (resolved->exists())
+            {
+                if (!overwrite)
+                    throw MakeStringException(99, "Cannot write %s, file already exists (missing OVERWRITE attribute?)", resolved->queryFileName());
+                if (extend)
+                    UNIMPLEMENTED; // How does extend fit in with the clusterwritemanager stuff? They can't specify cluster and extend together...
+                removeCache(resolved);
+                resolved->remove();
+            }
+            if (resolved->queryPhysicalName())
+                fileName.clear().append(resolved->queryPhysicalName());
             resolved.clear();
         }
-        Owned<ILocalOrDistributedFile> ldFile = createLocalOrDistributedFile(fileName, NULL, disconnected, false, true); // MORE - is onlyDFS right?
+        bool disconnected = !daliHelper->connected();
+        // MORE - not sure this is really the right test. If there SHOULD be a dali but is's unavailable, we should fail.
+        Owned<ILocalOrDistributedFile> ldFile = createLocalOrDistributedFile(fileName, NULL, disconnected, !disconnected, true);
         if (!ldFile)
-            throw MakeStringException(ROXIE_FILE_ERROR, "Cannot write %s, %s file not found", fileName.str(), (disconnected?"local":"DFS"));
+            throw MakeStringException(ROXIE_FILE_ERROR, "Cannot write %s", fileName.str());
 
         return createRoxieWriteHandler(daliHelper, ldFile.getClear(), clusters);
     }
@@ -751,7 +765,7 @@ protected:
             throw MakeStringException(ROXIE_INTERNAL_ERROR, "Invalid parameters to addAlias");
     }
 
-    virtual IQueryFactory *loadQueryFromDll(const char *id, const IQueryDll *dll, const IRoxiePackage &package, const IPropertyTree *stateInfo, IRoxieLibraryLookupContext *libraryContext) = 0;
+    virtual IQueryFactory *loadQueryFromDll(const char *id, const IQueryDll *dll, const IRoxiePackage &package, const IPropertyTree *stateInfo) = 0;
 
 public:
     IMPLEMENT_IINTERFACE;
@@ -771,7 +785,7 @@ public:
         return active;
     }
 
-    virtual void load(const IPropertyTree *querySet, const IPackageMap &packages, hash64_t &hash, IRoxieLibraryLookupContext *libraryContext)
+    virtual void load(const IPropertyTree *querySet, const IPackageMap &packages, hash64_t &hash)
     {
         Owned<IPropertyTreeIterator> queryNames = querySet->getElements("Query");
         ForEach (*queryNames)
@@ -799,7 +813,7 @@ public:
                     if (!package) package = &queryRootPackage();
                 }
                 assertex(package);
-                addQuery(id, loadQueryFromDll(id, queryDll.getClear(), *package, &query, libraryContext), hash);
+                addQuery(id, loadQueryFromDll(id, queryDll.getClear(), *package, &query), hash);
             }
             catch (IException *E)
             {
@@ -894,29 +908,6 @@ public:
         }
     }
 
-    virtual IQueryFactory * lookupLibrary(const char * libraryName, unsigned expectedInterfaceHash, const IRoxieContextLogger &logctx) const
-    {
-#ifdef _DEBUG
-        DBGLOG("Lookup library %s (hash %d)", libraryName, expectedInterfaceHash);
-#endif
-        Owned<IQueryFactory> query = getQuery(libraryName, logctx);
-        if (query)
-        {
-            if (query->isQueryLibrary())
-            {
-                unsigned foundInterfaceHash = query->getQueryLibraryInterfaceHash();
-                if (!foundInterfaceHash  || (foundInterfaceHash == expectedInterfaceHash))
-                    return query.getClear();
-                else
-                    throw MakeStringException(ROXIE_LIBRARY_ERROR, "The library interface found in %s is not compatible (found %d, expected %d)", libraryName, foundInterfaceHash, expectedInterfaceHash);
-            }
-            else
-                throw MakeStringException(ROXIE_LIBRARY_ERROR, "The query resolved by %s is not a library", libraryName);
-        }
-        else
-            throw MakeStringException(ROXIE_LIBRARY_ERROR, "No compatible library available for %s", libraryName);
-    }
-
     virtual IQueryFactory *getQuery(const char *id, const IRoxieContextLogger &logctx) const
     {
         IQueryFactory *ret;
@@ -941,9 +932,9 @@ public:
     {
     }
 
-    virtual IQueryFactory * loadQueryFromDll(const char *id, const IQueryDll *dll, const IRoxiePackage &package, const IPropertyTree *stateInfo, IRoxieLibraryLookupContext *libraryContext)
+    virtual IQueryFactory * loadQueryFromDll(const char *id, const IQueryDll *dll, const IRoxiePackage &package, const IPropertyTree *stateInfo)
     {
-        return createServerQueryFactory(id, dll, package, stateInfo, libraryContext);
+        return createServerQueryFactory(id, dll, package, stateInfo);
     }
 
 };
@@ -965,9 +956,9 @@ public:
         channelNo = _channelNo;
     }
 
-    virtual IQueryFactory *loadQueryFromDll(const char *id, const IQueryDll *dll, const IRoxiePackage &package, const IPropertyTree *stateInfo, IRoxieLibraryLookupContext *libraryContext)
+    virtual IQueryFactory *loadQueryFromDll(const char *id, const IQueryDll *dll, const IRoxiePackage &package, const IPropertyTree *stateInfo)
     {
-        return createSlaveQueryFactory(id, dll, package, channelNo, stateInfo, libraryContext);
+        return createSlaveQueryFactory(id, dll, package, channelNo, stateInfo);
     }
 
 };
@@ -1006,11 +997,11 @@ public:
         return managers[idx];
     }
 
-    virtual void load(const IPropertyTree *querySets, const IPackageMap &packages, hash64_t &hash, IRoxieLibraryLookupContext *libraryContext)
+    virtual void load(const IPropertyTree *querySets, const IPackageMap &packages, hash64_t &hash)
     {
         for (unsigned channel = 0; channel < numChannels; channel++)
             if (managers[channel])
-                managers[channel]->load(querySets, packages, hash, libraryContext); // MORE - this means the hash depends on the number of channels. Is that desirable?
+                managers[channel]->load(querySets, packages, hash); // MORE - this means the hash depends on the number of channels. Is that desirable?
     }
 
 private:
@@ -1268,8 +1259,8 @@ public:
         Owned<IPropertyTree> newQuerySet = daliHelper->getQuerySet(querySet);
         Owned<CRoxieSlaveQuerySetManagerSet> newSlaveManagers = new CRoxieSlaveQuerySetManagerSet(numChannels, querySet);
         Owned<IRoxieQuerySetManager> newServerManager = createServerManager(querySet);
-        newServerManager->load(newQuerySet, *packages, newHash, newServerManager);
-        newSlaveManagers->load(newQuerySet, *packages, newHash, newServerManager);
+        newServerManager->load(newQuerySet, *packages, newHash);
+        newSlaveManagers->load(newQuerySet, *packages, newHash);
         reloadQueryManagers(newSlaveManagers.getClear(), newServerManager.getClear(), newHash);
         clearKeyStoreCache(false);   // Allows us to fully release files we no longer need because of unloaded queries
     }
@@ -1301,8 +1292,8 @@ public:
         newQuerySet->addPropTree("Query", standaloneDll.getLink());
         Owned<CRoxieSlaveQuerySetManagerSet> newSlaveManagers = new CRoxieSlaveQuerySetManagerSet(numChannels, querySet);
         Owned<IRoxieQuerySetManager> newServerManager = createServerManager(querySet);
-        newServerManager->load(newQuerySet, *packages, newHash, newServerManager);
-        newSlaveManagers->load(newQuerySet, *packages, newHash, newServerManager);
+        newServerManager->load(newQuerySet, *packages, newHash);
+        newSlaveManagers->load(newQuerySet, *packages, newHash);
         reloadQueryManagers(newSlaveManagers.getClear(), newServerManager.getClear(), newHash);
     }
 };
@@ -1321,7 +1312,7 @@ class CRoxiePackageSetManager : public CInterface, implements IRoxieQueryPackage
 public:
     IMPLEMENT_IINTERFACE;
     CRoxiePackageSetManager(const IQueryDll *_standAloneDll) :
-        standAloneDll(_standAloneDll)
+        standAloneDll(_standAloneDll), stateHash(0)
     {
         daliHelper.setown(connectToDali(ROXIE_DALI_CONNECT_TIMEOUT));
     }
@@ -1370,16 +1361,31 @@ public:
         controlSem.signal();
     }
 
-    virtual IRoxieLibraryLookupContext *getLibraryLookupContext(const char *querySet) const
+    virtual IQueryFactory *lookupLibrary(const IRoxiePackage &package, const char *libraryName, unsigned expectedInterfaceHash, const IRoxieContextLogger &logctx) const
     {
         ReadLockBlock b(packageCrit);
         ForEachItemIn(idx, allQueryPackages)
         {
             Owned<IRoxieQuerySetManager> sm = allQueryPackages.item(idx).getRoxieServerManager();
-            if (sm->isActive() && strcmp(sm->queryId(), querySet)==0)
-                return sm.getClear();
+            if (sm->isActive())
+            {
+                Owned<IQueryFactory> library = sm->getQuery(libraryName, logctx);
+                if (library && (&library->queryPackage() == &package))  // MORE - is this check too restrictive?
+                {
+                    if (library->isQueryLibrary())
+                    {
+                        unsigned foundInterfaceHash = library->getQueryLibraryInterfaceHash();
+                        if (!foundInterfaceHash || (foundInterfaceHash == expectedInterfaceHash))
+                            return library.getClear();
+                        else
+                            throw MakeStringException(ROXIE_LIBRARY_ERROR, "The library interface found in %s is not compatible (found %d, expected %d)", libraryName, foundInterfaceHash, expectedInterfaceHash);
+                    }
+                    else
+                        throw MakeStringException(ROXIE_LIBRARY_ERROR, "The query resolved by %s is not a library", libraryName);
+                }
+            }
         }
-        return NULL;
+        throw MakeStringException(ROXIE_LIBRARY_ERROR, "No compatible library available for %s", libraryName);
     }
 
     virtual IQueryFactory *getQuery(const char *id, const IRoxieContextLogger &logctx) const

roxie/ccd/ccdstate.hpp

@@ -66,9 +66,9 @@ interface IRoxiePackage : extends IInterface
     // Return entire XML tree for package
     virtual const IPropertyTree *queryTree() const = 0;
     // Lookup information in package to resolve existing logical file name
-    virtual const IResolvedFile *lookupFileName(const char *fileName, bool opt, bool cacheDaliResults) const = 0;
+    virtual const IResolvedFile *lookupFileName(const char *fileName, bool opt, bool cacheDaliResults, IConstWorkUnit *wu) const = 0;
     // Lookup information in package to create new logical file name
-    virtual IRoxieWriteHandler *createFileName(const char *fileName, bool overwrite, bool extend, const StringArray &clusters) const = 0;
+    virtual IRoxieWriteHandler *createFileName(const char *fileName, bool overwrite, bool extend, const StringArray &clusters, IConstWorkUnit *wu) const = 0;
     // Lookup information in package about what in-memory indexes should be built for file
     virtual IPropertyTreeIterator *getInMemoryIndexInfo(const IPropertyTree &graphNode) const = 0;
     // Retrieve hash for the package
@@ -83,7 +83,7 @@ extern IRoxiePackage *createPackage(IPropertyTree *p);
 
 interface ISlaveDynamicFileCache : extends IInterface
 {
-    virtual IResolvedFile *lookupDynamicFile(const IRoxieContextLogger &logctx, const char *lfn, CDateTime &cacheDate, RoxiePacketHeader *header, bool isOpt, bool isLocal) = 0; 
+    virtual IResolvedFile *lookupDynamicFile(const IRoxieContextLogger &logctx, const char *lfn, CDateTime &cacheDate, unsigned checksum, RoxiePacketHeader *header, bool isOpt, bool isLocal) = 0;
 };
 extern ISlaveDynamicFileCache *querySlaveDynamicFileCache();
 extern void releaseSlaveDynamicFileCache();
@@ -99,17 +99,11 @@ interface IFileIOArray : extends IInterface
     virtual StringBuffer &getId(StringBuffer &) const = 0;
 };
 
-interface IRoxieLibraryLookupContext : extends IInterface
-{
-    virtual IQueryFactory *lookupLibrary(const char * libraryName, unsigned expectedInterfaceHash, const IRoxieContextLogger &logctx) const = 0;
-    virtual const char *queryId() const = 0;
-};
-
-interface IRoxieQuerySetManager : extends IRoxieLibraryLookupContext
+interface IRoxieQuerySetManager : extends IInterface
 {
     virtual bool isActive() const = 0;
     virtual IQueryFactory *getQuery(const char *id, const IRoxieContextLogger &ctx) const = 0;
-    virtual void load(const IPropertyTree *querySet, const IPackageMap &packages, hash64_t &hash, IRoxieLibraryLookupContext *libraryContext) = 0;
+    virtual void load(const IPropertyTree *querySet, const IPackageMap &packages, hash64_t &hash) = 0;
     virtual void getStats(const char *queryName, const char *graphName, StringBuffer &reply, const IRoxieContextLogger &logctx) const = 0;
     virtual void resetQueryTimings(const char *queryName, const IRoxieContextLogger &logctx) = 0;
     virtual void resetAllQueryTimings() = 0;
@@ -126,7 +120,7 @@ interface IRoxieDebugSessionManager : extends IInterface
 
 interface IRoxieQuerySetManagerSet : extends IInterface
 {
-    virtual void load(const IPropertyTree *querySets, const IPackageMap &packages, hash64_t &hash, IRoxieLibraryLookupContext *libraryContext) = 0;
+    virtual void load(const IPropertyTree *querySets, const IPackageMap &packages, hash64_t &hash) = 0;
 };
 
 interface IRoxieQueryPackageManagerSet : extends IInterface
@@ -134,7 +128,7 @@ interface IRoxieQueryPackageManagerSet : extends IInterface
     virtual void load() = 0;
     virtual void doControlMessage(IPropertyTree *xml, StringBuffer &reply, const IRoxieContextLogger &ctx) = 0;
     virtual IQueryFactory *getQuery(const char *id, const IRoxieContextLogger &logctx) const = 0;
-    virtual IRoxieLibraryLookupContext *getLibraryLookupContext(const char *querySet) const = 0;
+    virtual IQueryFactory *lookupLibrary(const IRoxiePackage &package, const char *libraryName, unsigned expectedInterfaceHash, const IRoxieContextLogger &logctx) const = 0;
 };
 
 extern IRoxieDebugSessionManager &queryRoxieDebugSessionManager();

rtl/eclrtl/CMakeLists.txt

@@ -67,6 +67,7 @@ target_link_libraries ( eclrtl
     )
 
 FOREACH( iFILES
+    ${CMAKE_CURRENT_SOURCE_DIR}/eclinclude.hpp
     ${CMAKE_CURRENT_SOURCE_DIR}/eclrtl.hpp
     ${CMAKE_CURRENT_SOURCE_DIR}/eclrtl_imp.hpp
     ${CMAKE_CURRENT_SOURCE_DIR}/rtldistr.hpp

rtl/eclrtl/eclinclude.hpp

@@ -0,0 +1,78 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#ifndef eclinclude_incl
+#define eclinclude_incl
+
+// Gather all the includes used by generated code into a single file so that we can
+// take advantage of precompiled headers more easily
+
+#ifdef _WIN32
+#define ECL_API __declspec(dllexport)
+#define LOCAL_API
+#define SERVICE_API __declspec(dllimport)
+#define RTL_API __declspec(dllimport)
+#define BCD_API __declspec(dllimport)
+#else
+  #ifdef USE_VISIBILITY
+    #define ECL_API __attribute__ ((visibility("default")))
+    #define LOCAL_API __attribute__ ((visibility("hidden")))
+  #else
+    #define ECL_API
+    #define LOCAL_API
+  #endif
+  #define SERVICE_API
+  #define RTL_API
+  #define BCD_API
+#endif
+
+#include <string.h>
+#include <malloc.h>
+#include <stdlib.h>
+#include <ctype.h>
+#include <math.h>
+
+#define CHEAP_UCHAR_DEF
+#if __WIN32
+typedef wchar_t UChar;
+#else //__WIN32
+typedef unsigned short UChar;
+#endif //__WIN32
+
+#ifdef _WIN32
+typedef unsigned int size32_t; // avoid pulling in platform.h (which pulls in windows.h etc) just for this...
+#else
+#include "platform.h"
+#endif
+
+#include "eclrtl.hpp"
+#include "eclhelper.hpp"
+#include "rtlkey.hpp"
+#include "eclrtl_imp.hpp"
+#include "rtlfield_imp.hpp"
+#include "rtlds_imp.hpp"
+#include "eclhelper_base.hpp"
+
+extern __declspec(dllimport) void _fastcall DecLock();
+extern __declspec(dllimport) void _fastcall DecUnlock();
+struct BcdCriticalBlock
+{
+    BcdCriticalBlock()      { DecLock(); }
+    ~BcdCriticalBlock()     { DecUnlock(); }
+};
+
+#endif

rtl/ecltpl/childtpl.cpp

@@ -16,60 +16,7 @@ $?doNotIncludeInGeneratedCode$
 ##############################################################################*/
 
 $?$/* Template for generating a child module for query */
-#ifdef _WIN32
-#define ECL_API __declspec(dllexport)
-#define LOCAL_API
-#define SERVICE_API __declspec(dllimport)
-#define RTL_API __declspec(dllimport)
-#define BCD_API __declspec(dllimport)
-#else
-  #ifdef USE_VISIBILITY
-    #define ECL_API __attribute__ ((visibility("default")))
-    #define LOCAL_API __attribute__ ((visibility("hidden")))
-  #else
-    #define ECL_API
-    #define LOCAL_API
-  #endif
-  #define SERVICE_API
-  #define RTL_API
-  #define BCD_API
-#endif
-
-#include <string.h>
-#include <malloc.h>
-#include <stdlib.h>
-#include <ctype.h>
-#include <math.h>
-
-#define CHEAP_UCHAR_DEF
-#if __WIN32
-typedef wchar_t UChar;
-#else //__WIN32
-typedef unsigned short UChar;
-#endif //__WIN32
-
-#ifdef _WIN32
-typedef unsigned int size32_t; // avoid pulling in platform.h (which pulls in windows.h etc) just for this...
-#else
-#include "platform.h"
-#endif
-
-#include "eclrtl.hpp"
-#include "eclhelper.hpp"
-#include "rtlkey.hpp"
-#include "eclrtl_imp.hpp"
-#include "rtlfield_imp.hpp"
-#include "rtlds_imp.hpp"
-#include "eclhelper_base.hpp"
-
-extern __declspec(dllimport) void _fastcall DecLock();
-extern __declspec(dllimport) void _fastcall DecUnlock();
-struct BcdCriticalBlock
-{
-    BcdCriticalBlock()      { DecLock(); }
-    ~BcdCriticalBlock()     { DecUnlock(); }
-};
-
+#include "eclinclude.hpp"
 @include@
 @prototype@
 

rtl/ecltpl/thortpl.cpp

@@ -16,60 +16,7 @@ $?doNotIncludeInGeneratedCode$
 ############################################################################## */
 
 $?$/* Template for generating thor/hthor/roxie output */
-#ifdef _WIN32
-#define ECL_API __declspec(dllexport)
-#define LOCAL_API
-#define SERVICE_API __declspec(dllimport)
-#define RTL_API __declspec(dllimport)
-#define BCD_API __declspec(dllimport)
-#else
-  #ifdef USE_VISIBILITY
-    #define ECL_API __attribute__ ((visibility("default")))
-    #define LOCAL_API __attribute__ ((visibility("hidden")))
-  #else
-    #define ECL_API
-    #define LOCAL_API
-  #endif
-  #define SERVICE_API
-  #define RTL_API
-  #define BCD_API
-#endif
-
-#include <string.h>
-#include <malloc.h>
-#include <stdlib.h>
-#include <ctype.h>
-#include <math.h>
-
-#define CHEAP_UCHAR_DEF
-#if __WIN32
-typedef wchar_t UChar;
-#else //__WIN32
-typedef unsigned short UChar;
-#endif //__WIN32
-
-#ifdef _WIN32
-typedef unsigned int size32_t; // avoid pulling in platform.h (which pulls in windows.h etc) just for this...
-#else
-#include "platform.h"
-#endif
-
-#include "eclrtl.hpp"
-#include "eclhelper.hpp"
-#include "rtlkey.hpp"
-#include "eclrtl_imp.hpp"
-#include "rtlfield_imp.hpp"
-#include "rtlds_imp.hpp"
-#include "eclhelper_base.hpp"
-
-extern __declspec(dllimport) void _fastcall DecLock();
-extern __declspec(dllimport) void _fastcall DecUnlock();
-struct BcdCriticalBlock
-{
-    BcdCriticalBlock()      { DecLock(); }
-    ~BcdCriticalBlock()     { DecUnlock(); }
-};
-
+#include "eclinclude.hpp"
 @include@
 @prototype@
 $?multiFile$#include "$headerName$"

system/include/platform.h

@@ -126,6 +126,7 @@ typedef memsize_t rowsize_t;
 #define TEXT_TRANS "t"
 #define LLC(NUM) NUM
 #define ENVSEPCHAR ';'
+#define ENVSEPSTR ";"
 
 #define SEPARATE_LIB_DLL_FILES
 #define SharedObjectPrefix         ""
@@ -282,6 +283,7 @@ typedef int socklen_t;
 #define __TIMESTAMP__ "<__TIMESTAMP__ unsupported>"
 #endif
 #define ENVSEPCHAR ':'
+#define ENVSEPSTR ":"
 #define PATHSEPCHAR '/'
 #define PATHSEPSTR "/"
 #define TEXT_TRANS

system/jlib/jcomp.cpp

@@ -72,6 +72,7 @@ static const char * USE_INCLUDE_TAIL[] = { "\"", "\"" };
 static const char * INCLUDEPATH[] = { "\"#\\include\"", "\"#/include\"" };
 static const char * LINK_SEPARATOR[] = { " /link ", " " };
 static const char * OBJECT_FILE_EXT[] = { "obj", "o" };
+static const char * PCH_FILE_EXT[] = { "", "gch" };
 
 static const char * LIBFLAG_DEBUG[] = { "/MDd", "" };
 static const char * LIBFLAG_RELEASE[] = { "/MD", "" };
@@ -86,6 +87,8 @@ static const char * EXE_LINK_OPTION_DEBUG[] = { "/BASE:" BASE_ADDRESS " /NOLOGO
 
 static const char * CC_OPTION_RELEASE[] = { "/Zm500 /EHsc /GR /Oi /Ob1 /GF /nologo /bigobj", "-fPIC -pipe -O0" };
 
+static const char * CC_OPTION_PRECOMPILEHEADER[] = { "", " -x c++-header" };
+
 static const char * DLL_LINK_OPTION_RELEASE[] = { "/BASE:" BASE_ADDRESS " /NOLOGO /LARGEADDRESSAWARE /INCREMENTAL:NO", "-shared -L. -fPIC -pipe -O0" };
 static const char * EXE_LINK_OPTION_RELEASE[] = { "/BASE:" BASE_ADDRESS " /NOLOGO /LARGEADDRESSAWARE /INCREMENTAL:NO", "-L. -Wl,-E -fPIC -pipe -O0" };
 
@@ -254,6 +257,7 @@ CppCompiler::CppCompiler(const char * _coreName, const char * _sourceDir, const
     verbose = _verbose;
     saveTemps = false;
     abortChecker = NULL;
+    precompileHeader = false;
 }
 
 void CppCompiler::addCompileOption(const char * option)
@@ -261,6 +265,13 @@ void CppCompiler::addCompileOption(const char * option)
     compilerOptions.append(' ').append(option);
 }
 
+void CppCompiler::setPrecompileHeader(bool _pch)
+{
+    if (targetCompiler!=GccCppCompiler)
+        throw MakeStringException(0, "precompiled header generation only supported for g++ and compatible compilers");
+    precompileHeader = _pch;
+}
+
 void CppCompiler::addDefine(const char * symbolName, const char * value)
 {
     compilerOptions.append(" ").append(USE_DEFINE_FLAG[targetCompiler]).append(symbolName);
@@ -398,7 +409,7 @@ bool CppCompiler::compile()
 
     if (atomic_read(&numFailed) > 0)
         ret = false;
-    else if (!onlyCompile)
+    else if (!onlyCompile && !precompileHeader)
         ret = doLink();
 
     if (!saveTemps && !onlyCompile)
@@ -436,13 +447,19 @@ bool CppCompiler::compileFile(IThreadPool * pool, const char * filename, Semapho
         return false;
 
     StringBuffer cmdline;
-    cmdline.append(CC_NAME[targetCompiler]).append(" \"");
+    cmdline.append(CC_NAME[targetCompiler]);
+    if (precompileHeader)
+        cmdline.append(CC_OPTION_PRECOMPILEHEADER[targetCompiler]);
+    cmdline.append(" \"");
     if (sourceDir.length())
     {
         cmdline.append(sourceDir);
         addPathSepChar(cmdline);
     }
-    cmdline.append(filename).append(".cpp\" ");
+    cmdline.append(filename);
+    if (!precompileHeader)
+        cmdline.append(".cpp");
+    cmdline.append("\" ");
     expandCompileOptions(cmdline);
 
     if (useDebugLibrary)
@@ -460,7 +477,12 @@ bool CppCompiler::compileFile(IThreadPool * pool, const char * filename, Semapho
     }
     else
     {
-        cmdline.append(" -o ").append("\"").append(targetDir).append(filename).append('.').append(OBJECT_FILE_EXT[targetCompiler]).append("\"");
+        cmdline.append(" -o ").append("\"").append(targetDir).append(filename).append('.');
+        if (precompileHeader)
+            cmdline.append(PCH_FILE_EXT[targetCompiler]);
+        else
+            cmdline.append(OBJECT_FILE_EXT[targetCompiler]);
+        cmdline.append("\"");
     }
     
     StringBuffer expanded;

system/jlib/jcomp.hpp

@@ -61,6 +61,7 @@ public:
     virtual void setMaxCompileThreads(const unsigned max) = 0;
     virtual void setCCLogPath(const char* path) = 0;
     virtual void setSaveTemps(bool _save) = 0;
+    virtual void setPrecompileHeader(bool _pch) = 0;
     virtual void setAbortChecker(IAbortRequestCallback * abortChecker) = 0;
 };
 

system/jlib/jcomp.ipp

@@ -46,6 +46,7 @@ public:
     virtual IPooledThread *createNew();
     virtual void setCCLogPath(const char* path);
     virtual void setSaveTemps(bool _save) { saveTemps = _save; }
+    virtual void setPrecompileHeader(bool _pch);
     virtual void setAbortChecker(IAbortRequestCallback * _abortChecker) {abortChecker = _abortChecker;}
 
 protected:
@@ -78,6 +79,7 @@ protected:
     bool            verbose;
     void _addInclude(StringBuffer &s, const char *paths);
     bool            saveTemps;
+    bool            precompileHeader;
     IAbortRequestCallback * abortChecker;
 };
 

system/jlib/jcrc.hpp

@@ -37,7 +37,7 @@ class jlib_decl CRC32
 public:
     CRC32(unsigned crc = ~0U) { reset(crc); }
     inline void reset(unsigned _crc = ~0U) { crc = _crc; }
-    inline unsigned get() { return ~crc; }
+    inline unsigned get() const { return ~crc; }
     void skip(offset_t length);
     void tally(unsigned len, const void * buf);
 

system/jlib/jdebug.cpp

@@ -519,9 +519,8 @@ public:
         CriticalBlock b(c);
         if (numSections())
         {
-            str.append("Timings:\n");
             for (unsigned i = 0; i < numSections(); i++)
-                getSection(i, str.append("  ")).append(" total=")
+                getSection(i, str.append("Timing: ")).append(" total=")
                                          .append(getTime(i)/1000000)
                                          .append("ms max=")
                                          .append(getMaxTime(i)/1000)

testing/ecl/key/rewrite.xml

@@ -0,0 +1,15 @@
+<Dataset name='Result 1'>
+</Dataset>
+<Dataset name='Result 2'>
+ <Row><a>1</a></Row>
+</Dataset>
+<Dataset name='Result 3'>
+</Dataset>
+<Dataset name='Result 4'>
+ <Row><a>2</a></Row>
+</Dataset>
+<Dataset name='Result 5'>
+</Dataset>
+<Dataset name='Result 6'>
+ <Row><a>3</a></Row>
+</Dataset>

testing/ecl/rewrite.ecl

@@ -0,0 +1,17 @@
+#option ('allowVariableRoxieFilenames', 1);
+string prefix := '' : stored('prefix');
+
+
+d1 := dataset([{1}], { INTEGER a });
+d2 := dataset([{2}], { INTEGER a });
+d3 := dataset([{3}], { INTEGER a });
+i := dataset('regress::outfile'+prefix, { INTEGER a }, FLAT);
+
+sequential(
+  output(d1,,'regress::outfile'+prefix, OVERWRITE),
+  output(i);
+  output(d2,,'regress::outfile'+prefix, OVERWRITE),
+  output(i);
+  output(d3,,'regress::outfile'+prefix, OVERWRITE),
+  output(i);
+);

testing/ecl/roxie/key/rewrite.xml

@@ -0,0 +1,15 @@
+<Dataset name='Result 1'>
+</Dataset>
+<Dataset name='Result 2'>
+ <Row><a>1</a></Row>
+</Dataset>
+<Dataset name='Result 3'>
+</Dataset>
+<Dataset name='Result 4'>
+ <Row><a>2</a></Row>
+</Dataset>
+<Dataset name='Result 5'>
+</Dataset>
+<Dataset name='Result 6'>
+ <Row><a>3</a></Row>
+</Dataset>

thorlcr/activities/loop/thloop.cpp

@@ -239,8 +239,11 @@ public:
                     initLoopResults(loopCounter);
                 boundGraph->execute(*this, (flags & IHThorLoopArg::LFcounter)?loopCounter:0, ownedResults, (IRowWriterMultiReader *)NULL, 0, extractBuilder.size(), extractBuilder.getbytes());
                 ++loopCounter;
-                if (!barrier->wait(false))
-                    break;
+                if (flags & IHThorLoopArg::LFnewloopagain)
+                {
+                    if (!barrier->wait(false))
+                        break;
+                }
             }
         }
         else