Merge branch 'candidate-5.4.0'

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

common/workunit/workunit.cpp

@@ -1676,6 +1676,7 @@ public:
     virtual void        setQueryMainDefinition(const char * str);
     virtual void        addAssociatedFile(WUFileType type, const char * name, const char * ip, const char * desc, unsigned crc);
     virtual void        removeAssociatedFiles();
+    virtual void        removeAssociatedFile(WUFileType type, const char * name, const char * desc);
 };
 
 class CLocalWUWebServicesInfo : public CInterface, implements IWUWebServicesInfo
@@ -7023,6 +7024,23 @@ void CLocalWUQuery::addAssociatedFile(WUFileType type, const char * name, const
     associated.append(*q);
 }
 
+void CLocalWUQuery::removeAssociatedFile(WUFileType type, const char * name, const char * desc)
+{
+    CriticalBlock block(crit);
+    associatedCached = false;
+    associated.kill();
+    StringBuffer xpath;
+    xpath.append("Associated/File");
+    if (type)
+        xpath.append("[@type=\"").append(getEnumText(type, queryFileTypes)).append("\"]");
+    if (name)
+        xpath.append("[@filename=\"").append(name).append("\"]");
+    if (desc)
+        xpath.append("[@desc=\"").append(desc).append("\"]");
+
+    p->removeProp(xpath.str());
+}
+
 void CLocalWUQuery::removeAssociatedFiles()
 {
     associatedCached = false;

common/workunit/workunit.hpp

@@ -407,6 +407,7 @@ interface IWUQuery : extends IConstWUQuery
     virtual void addAssociatedFile(WUFileType type, const char * name, const char * ip, const char * desc, unsigned crc) = 0;
     virtual void removeAssociatedFiles() = 0;
     virtual void setQueryMainDefinition(const char * str) = 0;
+    virtual void removeAssociatedFile(WUFileType type, const char * name, const char * desc) = 0;
 };
 
 

dali/base/dasds.cpp

@@ -4469,6 +4469,7 @@ void CSDSTransactionServer::processMessage(CMessageBuffer &mb)
                 if (queryTransactionLogging())
                     transactionLog.log("xpath='%s'", xpath.get());
                 mb.clear();
+                CHECKEDDALIREADLOCKBLOCK(manager.dataRWLock, readWriteTimeout);
                 Owned<IPropertyTree> matchTree = SDSManager->getXPaths(serverId, xpath, DAMP_SDSCMD_GETXPATHSPLUSIDS==action);
                 if (matchTree)
                 {
@@ -4499,6 +4500,7 @@ void CSDSTransactionServer::processMessage(CMessageBuffer &mb)
                         ascending?"true":"false", from, limit);
                 }
                 mb.clear();
+                CHECKEDDALIREADLOCKBLOCK(manager.dataRWLock, readWriteTimeout);
                 Owned<IPropertyTree> matchTree = SDSManager->getXPathsSortLimitMatchTree(xpath, matchXPath, sortBy, caseinsensitive, ascending, from, limit);
                 if (matchTree)
                 {

dali/dfu/dfurun.cpp

@@ -1220,7 +1220,6 @@ public:
                             opttree->setPropBool("@recordStructurePresent", true);
 
                         opttree->setPropBool("@quotedTerminator", options->getQuotedTerminator());
-
                         Owned<IFileDescriptor> fdesc = destination->getFileDescriptor(iskey,options->getSuppressNonKeyRepeats()&&!iskey);
                         if (fdesc) {
                             if (options->getSubfileCopy()) {// need to set destination compressed or not

dali/dfu/dfuwu.cpp

@@ -2057,6 +2057,15 @@ public:
         queryRoot()->setPropBool("@quotedTerminator",val);
     }
 
+    bool getPreserveCompression() const
+    {
+        return queryRoot()->getPropBool("@preserveCompression");
+    }
+
+    void setPreserveCompression(bool val)
+    {
+        queryRoot()->setPropBool("@preserveCompression",val);
+    }
 };
 
 class CExceptionIterator: public CInterface, implements IExceptionIterator

dali/dfu/dfuwu.hpp

@@ -165,13 +165,11 @@ interface IConstDFUoptions : extends IInterface
     virtual bool getSuppressNonKeyRepeats() const = 0;
     virtual bool getSubfileCopy() const = 0;                                // i.e. called by supercopy
     virtual bool getEncDec(StringAttr &enc,StringAttr &dec) = 0;
-
     virtual IPropertyTree *queryTree() const = 0;                   // used by DFU server
     virtual bool getFailIfNoSourceFile() const = 0;
-
     virtual bool getRecordStructurePresent() const = 0;
-
     virtual bool getQuotedTerminator() const = 0;
+    virtual bool getPreserveCompression() const = 0;
 };
 
 interface IDFUoptions : extends IConstDFUoptions
@@ -209,6 +207,7 @@ interface IDFUoptions : extends IConstDFUoptions
     virtual void setFailIfNoSourceFile(bool val=false) = 0;
     virtual void setRecordStructurePresent(bool val=false) = 0;
     virtual void setQuotedTerminator(bool val=true) = 0;
+    virtual void setPreserveCompression(bool val=true) = 0;
 };
 
 interface IConstDFUfileSpec: extends IInterface

dali/dfuplus/dfuplus.cpp

@@ -766,6 +766,8 @@ int CDfuPlusHelper::copy()
     }
     if(globals->hasProp("compress"))
         req->setCompress(globals->getPropBool("compress", false));
+    if(globals->hasProp("preserveCompression"))
+        req->setPreserveCompression(globals->getPropBool("preserveCompression", true));
     if(globals->hasProp("encrypt"))
         req->setEncrypt(globals->queryProp("encrypt"));
     if(globals->hasProp("decrypt"))

dali/dfuplus/main.cpp

@@ -110,6 +110,7 @@ void handleSyntax()
     out.append("        diffkeysrc=<old-key-name>   -- use keydiff/keypatch (src old name)\n");
     out.append("        diffkeydst=<old-key-name>   -- use keydiff/keypatch (dst old name)\n");
     out.append("        multicopy=0|1   -- each destination part gets whole file\n");
+    out.append("        preservecompression=1|0 -- optional, default is 1 (preserve compression)\n");
     out.append("    remove options:\n");
     out.append("        name=<logical-name>\n");
     out.append("        names=<multiple-logical-names-separated-by-comma>\n");

dali/ft/daft.cpp

@@ -57,6 +57,14 @@ void CDistributedFileSystem::copy(IDistributedFile * from, IDistributedFile * to
     sprayer->setPartFilter(filter);
     sprayer->setSource(from);
     sprayer->setTarget(to);
+
+    bool compressInput = from->isCompressed();
+    bool compressOutput = options->getPropBool("@compress");
+    bool preserveCompression = options->getPropBool("@preserveCompression");
+    LOG(MCdebugInfo, unknownJob, "DFS: compressInput:%d, compressOutput:%d, preserveCompression:%d ", compressInput, compressOutput, preserveCompression);
+    if (preserveCompression & compressInput)
+        sprayer->setTargetCompression(compressInput);
+
     sprayer->spray();
 }
 

dali/ft/filecopy.cpp

@@ -2500,6 +2500,11 @@ void FileSprayer::setSourceTarget(IFileDescriptor * fd, DaftReplicateMode mode)
         setCopyCompressedRaw();
 }
 
+void FileSprayer::setTargetCompression(bool compress)
+{
+    compressOutput = compress;
+}
+
 void FileSprayer::setTarget(IDistributedFile * target)
 {
     distributedTarget.set(target);
@@ -2733,6 +2738,8 @@ void FileSprayer::spray()
     if ((sourceSize == 0) && failIfNoSourceFile)
         throwError(DFTERR_NoFilesMatchWildcard);
 
+    LOG(MCdebugInfo, job, "compressedInput:%d, compressOutput:%d", compressedInput, compressOutput);
+
     LocalAbortHandler localHandler(daftAbortHandler);
 
     if (allowRecovery && progressTree->getPropBool(ANcomplete))
@@ -2973,7 +2980,10 @@ void FileSprayer::updateTargetProperties()
                      (stricmp(aname,"@eclCRC")==0)||
                      (stricmp(aname,"@formatCrc")==0)||
                      (stricmp(aname,"@owner")==0)||
-                     ((stricmp(aname,FArecordCount)==0)&&!gotrc))
+                     ((stricmp(aname,FArecordCount)==0)&&!gotrc) ||
+                     ((stricmp(aname,"@blockCompressed")==0)&&copyCompressed) ||
+                     ((stricmp(aname,"@rowCompressed")==0)&&copyCompressed)
+                     )
                     )
                     curProps.setProp(aname,aiter->queryValue());
             }

dali/ft/filecopy.hpp

@@ -122,6 +122,7 @@ public:
     virtual void setSource(IDistributedFilePart * part) = 0;
     virtual void setSourceTarget(IFileDescriptor * fd, DaftReplicateMode mode) = 0;
     virtual void setTarget(IDistributedFile * target) = 0;
+    virtual void setTargetCompression(bool compress) = 0;
     virtual void setTarget(IFileDescriptor * target, unsigned copy=0) = 0;
     virtual void setTarget(IGroup * target) = 0;
     virtual void setTarget(INode * target) = 0;

dali/ft/filecopy.ipp

@@ -192,6 +192,7 @@ public:
     virtual void setSource(IDistributedFilePart * part);
     virtual void setSourceTarget(IFileDescriptor * fd, DaftReplicateMode mode);
     virtual void setTarget(IDistributedFile * target);
+    virtual void setTargetCompression(bool compress);
     virtual void setTarget(IFileDescriptor * target, unsigned copy);
     virtual void setTarget(IGroup * target);
     virtual void setTarget(INode * target);
@@ -315,6 +316,7 @@ protected:
     size32_t                transferBufferSize;
     StringAttr              encryptKey;
     StringAttr              decryptKey;
+    bool                    preserveCompression;
 };
 
 

docs/HPCCSystemAdmin/HPCCSystemAdministratorsGuide.xml

@@ -525,7 +525,7 @@
       clusters. The number of Roxie nodes should never exceed the number of
       Thor nodes. In addition, the number of Thor nodes should be evenly
       divisible by the number of Roxie nodes. This ensures an efficient
-      distribution of file parts from Thor to Roxie. </para>
+      distribution of file parts from Thor to Roxie.</para>
     </sect1>
 
     <sect1>
@@ -1238,9 +1238,9 @@ lock=/var/lock/HPCCSystems</programlisting>
 
         <para>If you are adding or removing Thor cluster nodes but
         <emphasis>all previous nodes remain part of the environment and
-        accessible</emphasis>, you can <emphasis role="bold">rename</emphasis>
-        the group that is associated with the Thor cluster (or the Cluster
-        name if there is no group name).</para>
+        accessible</emphasis>, you must <emphasis
+        role="bold">rename</emphasis> the group that is associated with the
+        Thor cluster (or the Cluster name if there is no group name).</para>
 
         <para>This will ensure all previously existing files, continue to use
         the old group structure, while new files use the new group

ecl/eclcc/eclcc.hpp

@@ -114,6 +114,7 @@ const char * const helpText[] = {
     "?!  -fmaxCompileThreads     Number of compiler instances to compile the c++",
     "?!  -fnoteRecordSizeInGraph Add estimates of record sizes to the graph",
     "?!  -fpickBestEngine        Allow simple thor queries to be passed to thor",
+    "?!  -fobfuscateOutput       Remove details of the original ECL from output",
     "?!  -freportCppWarnings     Report warnings from c++ compilation",
     "?!  -fsaveCpp -fsaveCppTempFiles  Retain the generated c++ files",
     "?!  -fshowActivitySizeInGraph  Show estimates of generated c++ size in the graph",

ecl/eclccserver/eclccserver.cpp

@@ -398,8 +398,11 @@ class EclccCompileThread : public CInterface, implements IPooledThread, implemen
                     const char *jobname = embeddedWU->queryJobName();
                     if (jobname && *jobname) //let ECL win naming job during initial compile
                         workunit->setJobName(jobname);
-                    Owned<IWUQuery> query = workunit->updateQuery();
-                    query->setQueryText(eclQuery.s.str());
+                    if (!workunit->getDebugValueBool("obfuscateOutput", false))
+                    {
+                        Owned<IWUQuery> query = workunit->updateQuery();
+                        query->setQueryText(eclQuery.s.str());
+                    }
                 }
 
                 createUNCFilename(realdllfilename.str(), dllurl);

ecl/hql/hqlatoms.cpp

@@ -189,6 +189,7 @@ IAtom * gctxmethodAtom;
 IAtom * getAtom;
 IAtom * globalAtom;
 IAtom * graphAtom;
+IAtom * _graphLocal_Atom;
 IAtom * groupAtom;
 IAtom * groupedAtom;
 IAtom * hashAtom;
@@ -226,6 +227,7 @@ IAtom * keyedAtom;
 IAtom * labeledAtom;
 IAtom * languageAtom;
 IAtom * lastAtom;
+IAtom * _lazy_Atom;
 IAtom * leftAtom;
 IAtom * leftonlyAtom;
 IAtom * leftouterAtom;
@@ -622,6 +624,7 @@ MODULE_INIT(INIT_PRIORITY_HQLATOM)
     MAKEATOM(global);
     MAKEATOM(globalContext);
     MAKEATOM(graph);
+    MAKESYSATOM(graphLocal);
     MAKEATOM(group);
     MAKEATOM(grouped);
     MAKEATOM(hash);
@@ -660,6 +663,7 @@ MODULE_INIT(INIT_PRIORITY_HQLATOM)
     MAKEATOM(labeled);
     MAKEATOM(language);
     MAKEATOM(last);
+    MAKESYSATOM(lazy);
     MAKEATOM(left);
     leftonlyAtom = createLowerCaseAtom("left only");
     leftouterAtom = createLowerCaseAtom("left outer");

ecl/hql/hqlatoms.hpp

@@ -229,6 +229,7 @@ extern HQL_API IAtom * keyedAtom;
 extern HQL_API IAtom * labeledAtom;
 extern HQL_API IAtom * languageAtom;
 extern HQL_API IAtom * lastAtom;
+extern HQL_API IAtom * _lazy_Atom;
 extern HQL_API IAtom * leftAtom;
 extern HQL_API IAtom * leftonlyAtom;
 extern HQL_API IAtom * leftouterAtom;

ecl/hql/hqlexpr.cpp

@@ -15896,7 +15896,6 @@ extern HQL_API IPropertyTree * gatherAttributeDependencies(IEclRepository * data
     {
         HqlScopeArray scopes;   
         getRootScopes(scopes, dataServer, ctx);
-        scopes.sort(compareScopesByName);
         ForEachItemIn(i, scopes)
         {
             IHqlScope & cur = scopes.item(i);

ecl/hql/hqlir.cpp

@@ -2302,11 +2302,11 @@ static const char * const expectedIR1 [] = {
 "%c8 = constant 10 : %t1;",
 "%as9 = assign(%e7,%c8);",
 "%e10 = %as9 {location '',6,3};",
-"%e11 = %e3 {symbol r};",
+"%e11 = %e3 {symbol r@1};",
 "%t12 = type %t4 {original(%e11)};",
 "%t13 = type transform(%t12);",
 "%e14 = transform(%e10) : %t13;",
-"%e15 = %e14 {symbol t};",
+"%e15 = %e14 {symbol t@5};",
 "%t16 = type int8;",
 "%c17 = constant 1 : %t16;",
 "%c18 = constant 10 : %t16;",
@@ -2315,17 +2315,17 @@ static const char * const expectedIR1 [] = {
 "%as21 = assign(%e7,%e20);",
 "%e22 = %as21 {location '',6,3};",
 "%e23 = transform(%e22) : %t13;",
-"%e24 = %e23 {symbol t};",
+"%e24 = %e23 {symbol t@5};",
 "%t25 = type null;",
 "%e26 = transformlist(%e15,%e24) : %t25;",
 "%t27 = type table(%t5);",
 "%e28 = inlinetable(%e26,%e3) : %t27;",
 "%e29 = %e28 {location '',9,7};",
-"%e30 = %e29 {symbol ds};",
+"%e30 = %e29 {symbol ds@9};",
 "%e31 = null : <null>;",
 "%e32 = selectfields(%e30,%e31) : %t27;",
-"%t33 = type uint4;",
-"%c34 = constant 3219609901 : %t33;",
+"%t33 = type int4;",
+"%c34 = constant 706706620 : %t33;",
 "%e35 = attr#always;",
 "%e36 = attr#update(%c34,%e35);",
 "%e37 = output(%e32,%e36);",

ecl/hql/hqlopt.cpp

@@ -934,10 +934,22 @@ bool CTreeOptimizer::expandFilterCondition(HqlExprArray & expanded, HqlExprArray
         {
             ExpandComplexityMonitor expandMonitor(*this);
             OwnedHqlExpr expandedFilter;
-            if (moveOver)
-                expandedFilter.setown(expandFields(mapper, cur, child, grandchild, &expandMonitor));
-            else
-                expandedFilter.setown(mapper->expandFields(cur, child, grandchild, grandchild, &expandMonitor));
+            try
+            {
+                if (moveOver)
+                    expandedFilter.setown(expandFields(mapper, cur, child, grandchild, &expandMonitor));
+                else
+                    expandedFilter.setown(mapper->expandFields(cur, child, grandchild, grandchild, &expandMonitor));
+            }
+            catch (IException * e)
+            {
+                //Highly unusual, but assertwild doesn't force the fields into the output, so the output of the
+                //project project may not include the fields that are wildcarded.
+                if (cur->getOperator() != no_assertwild)
+                    throw;
+                e->Release();
+                expandedFilter.set(cur);
+            }
 
             if (expandedFilter->isConstant())
             {

ecl/hql/hqlrepository.cpp

@@ -23,6 +23,7 @@
 #include "eclrtl.hpp"
 #include "hqlexpr.ipp"
 #include "hqlerror.hpp"
+#include "hqlutil.hpp"
 
 //-------------------------------------------------------------------------------------------------------------------
 
@@ -30,6 +31,7 @@ static void getRootScopes(HqlScopeArray & rootScopes, IHqlScope * scope)
 {
     HqlExprArray rootSymbols;
     scope->getSymbols(rootSymbols);
+    rootSymbols.sort(compareSymbolsByName);
     ForEachItemIn(i, rootSymbols)
     {
         IHqlExpression & cur = rootSymbols.item(i);

ecl/hql/hqlutil.cpp

@@ -619,6 +619,16 @@ static IHqlExpression * findCommonExpression(IHqlExpression * lower, IHqlExpress
 
 //---------------------------------------------------------------------------------------------------------------------
 
+bool isFileOutput(IHqlExpression * expr)
+{
+    return (expr->getOperator() == no_output) && (queryRealChild(expr, 1) != NULL);
+}
+
+bool isWorkunitOutput(IHqlExpression * expr)
+{
+    return (expr->getOperator() == no_output) && (queryRealChild(expr, 1) == NULL);
+}
+
 bool isCommonSubstringRange(IHqlExpression * expr)
 {
     if (expr->getOperator() != no_substring)

ecl/hql/hqlutil.hpp

@@ -681,6 +681,8 @@ extern HQL_API IHqlExpression * queryTransformAssign(IHqlExpression * transform,
 extern HQL_API IHqlExpression * queryTransformAssignValue(IHqlExpression * transform, IHqlExpression * searchField);
 
 extern HQL_API bool isCommonSubstringRange(IHqlExpression * expr);
+extern HQL_API bool isFileOutput(IHqlExpression * expr);
+extern HQL_API bool isWorkunitOutput(IHqlExpression * expr);
 
 class HQL_API AtmostLimit
 {

ecl/hqlcpp/hqlcatom.cpp

@@ -93,7 +93,6 @@ IAtom * _spill_Atom;
 IAtom * _spillReason_Atom;
 IAtom * _steppedMeta_Atom;
 IAtom * subgraphAtom;
-IAtom * _tempCount_Atom;
 IAtom * _translated_Atom;
 IAtom * utf8Atom;
 IAtom * wrapperAtom;
@@ -1488,7 +1487,6 @@ MODULE_INIT(INIT_PRIORITY_HQLATOM-1)
     MAKESYSATOM(spill);
     MAKESYSATOM(spillReason);
     MAKESYSATOM(steppedMeta);
-    MAKESYSATOM(tempCount);
     MAKESYSATOM(translated);
     return true;
 }

ecl/hqlcpp/hqlcatom.hpp

@@ -93,7 +93,6 @@ extern IAtom * _spill_Atom;
 extern IAtom * _spillReason_Atom;
 extern IAtom * _steppedMeta_Atom;
 extern IAtom * subgraphAtom;
-extern IAtom * _tempCount_Atom;
 extern IAtom * _translated_Atom;
 extern IAtom * utf8Atom;
 extern IAtom * wrapperAtom;

ecl/hqlcpp/hqlcpp.cpp

@@ -1758,8 +1758,11 @@ void HqlCppTranslator::cacheOptions()
         DebugOption(options.newBalancedSpotter,"newBalancedSpotter",true),
         DebugOption(options.keyedJoinPreservesOrder,"keyedJoinPreservesOrder",true),
         DebugOption(options.expandSelectCreateRow,"expandSelectCreateRow",false),
+        DebugOption(options.obfuscateOutput,"obfuscateOutput",false),
+        DebugOption(options.showEclInGraph,"showEclInGraph",true),
         DebugOption(options.optimizeSortAllFields,"optimizeSortAllFields",true),
         DebugOption(options.optimizeSortAllFieldsStrict,"optimizeSortAllFieldsStrict",false),
+        DebugOption(options.alwaysReuseGlobalSpills,"alwaysReuseGlobalSpills",true),
     };
 
     //get options values from workunit
@@ -4365,7 +4368,7 @@ void HqlCppTranslator::createTempFor(BuildCtx & ctx, ITypeInfo * _exprType, CHql
 
 void HqlCppTranslator::buildTempExpr(BuildCtx & ctx, BuildCtx & declareCtx, CHqlBoundTarget & tempTarget, IHqlExpression * expr, ExpressionFormat format, bool ignoreSetAll)
 {
-    if (options.addLocationToCpp)
+    if (options.addLocationToCpp && !options.obfuscateOutput)
     {
         IHqlExpression * location = queryLocation(expr);
         if (location)

ecl/hqlcpp/hqlcpp.ipp

@@ -745,8 +745,11 @@ struct HqlCppOptions
     bool                newBalancedSpotter;
     bool                keyedJoinPreservesOrder;
     bool                expandSelectCreateRow;
+    bool                obfuscateOutput;
+    bool                showEclInGraph;
     bool                optimizeSortAllFields;
     bool                optimizeSortAllFieldsStrict;
+    bool                alwaysReuseGlobalSpills;
 };
 
 //Any information gathered while processing the query should be moved into here, rather than cluttering up the translator class

ecl/hqlcpp/hqlhtcpp.cpp

@@ -1834,7 +1834,7 @@ void ActivityInstance::addAttribute(const char * name, IHqlExpression * expr)
 
 void ActivityInstance::addLocationAttribute(IHqlExpression * location)
 {
-    if (!translator.queryOptions().reportLocations)
+    if (!translator.queryOptions().reportLocations || translator.queryOptions().obfuscateOutput)
         return;
 
     unsigned line = location->getStartLine();
@@ -1857,6 +1857,9 @@ void ActivityInstance::addLocationAttribute(IHqlExpression * location)
 
 void ActivityInstance::addNameAttribute(IHqlExpression * symbol)
 {
+    if (translator.queryOptions().obfuscateOutput)
+        return;
+
     //Not so sure about adding a location for a named symbol if there are other locations already present....
     //We should probably perform some deduping instead.
     addLocationAttribute(symbol);
@@ -1933,9 +1936,12 @@ void ActivityInstance::processHint(IHqlExpression * attr)
 
 void ActivityInstance::processSection(IHqlExpression * section)
 {
-    StringBuffer sectionName;
-    getStringValue(sectionName, section->queryChild(0));
-    addAttribute("section", sectionName);
+    if (!translator.queryOptions().obfuscateOutput)
+    {
+        StringBuffer sectionName;
+        getStringValue(sectionName, section->queryChild(0));
+        addAttribute("section", sectionName);
+    }
 }
 
 void ActivityInstance::processHints(IHqlExpression * hintAttr)
@@ -1973,18 +1979,24 @@ void ActivityInstance::createGraphNode(IPropertyTree * defaultSubGraph, bool alw
     IPropertyTree * parentGraphNode = subgraph ? subgraph->tree.get() : defaultSubGraph;
     if (!parentGraphNode)
         return;
+
+    HqlCppOptions const & options = translator.queryOptions();
     assertex(kind < TAKlast);
     graphNode.set(parentGraphNode->addPropTree("node", createPTree()));
 
     graphNode->setPropInt64("@id", activityId);
-    StringBuffer label;
-    if (isGrouped)
-        label.append("Grouped ");
-    else if (isLocal)
-        label.append("Local ");
-    label.append(getActivityText(kind));
 
-    graphNode->setProp("@label", graphLabel ? graphLabel.get() : label.str());
+    if (!options.obfuscateOutput)
+    {
+        StringBuffer label;
+        if (isGrouped)
+            label.append("Grouped ");
+        else if (isLocal)
+            label.append("Local ");
+        label.append(getActivityText(kind));
+
+        graphNode->setProp("@label", graphLabel ? graphLabel.get() : label.str());
+    }
 
     IHqlExpression * cur = dataset;
     loop
@@ -2026,82 +2038,88 @@ void ActivityInstance::createGraphNode(IPropertyTree * defaultSubGraph, bool alw
     if (isNoAccess)
         addAttributeBool("noAccess", true);
 
-    if (graphEclText.length() == 0)
-        toECL(dataset->queryBody(), graphEclText, false, true);
-
-    elideString(graphEclText, MAX_GRAPH_ECL_LENGTH);
-    if (strcmp(graphEclText.str(), "<>") != 0)
-        addAttribute("ecl", graphEclText.str());
-
-    if (translator.queryOptions().showSeqInGraph)
+    if (!options.obfuscateOutput)
     {
-        IHqlExpression * selSeq = querySelSeq(dataset);
-        if (selSeq)
-            addAttributeInt("selSeq", selSeq->querySequenceExtra());
-    }
+        if (graphEclText.length() == 0)
+            toECL(dataset->queryBody(), graphEclText, false, true);
 
+        elideString(graphEclText, MAX_GRAPH_ECL_LENGTH);
+        if (options.showEclInGraph)
+        {
+            if (strcmp(graphEclText.str(), "<>") != 0)
+                addAttribute("ecl", graphEclText.str());
+        }
 
-    if (translator.queryOptions().showMetaInGraph)
-    {
-        StringBuffer s;
-        if (translator.targetThor())
+        if (options.showSeqInGraph)
         {
-            IHqlExpression * distribution = queryDistribution(dataset);
-            if (distribution && distribution->queryName() != localAtom)
-                addAttribute("metaDistribution", getExprECL(distribution, s.clear(), true).str());
+            IHqlExpression * selSeq = querySelSeq(dataset);
+            if (selSeq)
+                addAttributeInt("selSeq", selSeq->querySequenceExtra());
         }
 
-        IHqlExpression * grouping = queryGrouping(dataset);
-        if (grouping)
-            addAttribute("metaGrouping", getExprECL(grouping, s.clear(), true).str());
 
-        if (translator.targetThor())
+        if (options.showMetaInGraph)
         {
-            IHqlExpression * globalSortOrder = queryGlobalSortOrder(dataset);
-            if (globalSortOrder)
-                addAttribute("metaGlobalSortOrder", getExprECL(globalSortOrder, s.clear(), true).str());
-        }
+            StringBuffer s;
+            if (translator.targetThor())
+            {
+                IHqlExpression * distribution = queryDistribution(dataset);
+                if (distribution && distribution->queryName() != localAtom)
+                    addAttribute("metaDistribution", getExprECL(distribution, s.clear(), true).str());
+            }
 
-        IHqlExpression * localSortOrder = queryLocalUngroupedSortOrder(dataset);
-        if (localSortOrder)
-            addAttribute("metaLocalSortOrder", getExprECL(localSortOrder, s.clear(), true).str());
+            IHqlExpression * grouping = queryGrouping(dataset);
+            if (grouping)
+                addAttribute("metaGrouping", getExprECL(grouping, s.clear(), true).str());
 
-        IHqlExpression * groupSortOrder = queryGroupSortOrder(dataset);
-        if (groupSortOrder)
-            addAttribute("metaGroupSortOrder", getExprECL(groupSortOrder, s.clear(), true).str());
-    }
+            if (translator.targetThor())
+            {
+                IHqlExpression * globalSortOrder = queryGlobalSortOrder(dataset);
+                if (globalSortOrder)
+                    addAttribute("metaGlobalSortOrder", getExprECL(globalSortOrder, s.clear(), true).str());
+            }
 
-    if (translator.queryOptions().noteRecordSizeInGraph)
-    {
-        IHqlExpression * record = dataset->queryRecord();
-        if (!record && (getNumChildTables(dataset) == 1))
-            record = dataset->queryChild(0)->queryRecord();
-        if (record)
+            IHqlExpression * localSortOrder = queryLocalUngroupedSortOrder(dataset);
+            if (localSortOrder)
+                addAttribute("metaLocalSortOrder", getExprECL(localSortOrder, s.clear(), true).str());
+
+            IHqlExpression * groupSortOrder = queryGroupSortOrder(dataset);
+            if (groupSortOrder)
+                addAttribute("metaGroupSortOrder", getExprECL(groupSortOrder, s.clear(), true).str());
+        }
+
+        if (options.noteRecordSizeInGraph)
         {
-            size32_t maxSize = getMaxRecordSize(record, translator.getDefaultMaxRecordSize());
-            if (isVariableSizeRecord(record))
+            IHqlExpression * record = dataset->queryRecord();
+            if (!record && (getNumChildTables(dataset) == 1))
+                record = dataset->queryChild(0)->queryRecord();
+            if (record)
             {
-                size32_t minSize = getMinRecordSize(record);
-                size32_t expectedSize = getExpectedRecordSize(record);
-                StringBuffer temp;
-                temp.append(minSize).append("..");
-                if (maxRecordSizeUsesDefault(record))
-                    temp.append("?");
+                size32_t maxSize = getMaxRecordSize(record, translator.getDefaultMaxRecordSize());
+                if (isVariableSizeRecord(record))
+                {
+                    size32_t minSize = getMinRecordSize(record);
+                    size32_t expectedSize = getExpectedRecordSize(record);
+                    StringBuffer temp;
+                    temp.append(minSize).append("..");
+                    if (maxRecordSizeUsesDefault(record))
+                        temp.append("?");
+                    else
+                        temp.append(maxSize);
+                    temp.append("(").append(expectedSize).append(")");
+                    addAttribute("recordSize", temp.str());
+                }
                 else
-                    temp.append(maxSize);
-                temp.append("(").append(expectedSize).append(")");
-                addAttribute("recordSize", temp.str());
+                    addAttributeInt("recordSize", maxSize);
             }
-            else
-                addAttributeInt("recordSize", maxSize);
         }
-    }
 
-    if (translator.queryOptions().showRecordCountInGraph && !dataset->isAction())
-    {
-        StringBuffer text;
-        getRecordCountText(text, dataset);
-        addAttribute("predictedCount", text);
+        if (options.showRecordCountInGraph && !dataset->isAction())
+        {
+            StringBuffer text;
+            getRecordCountText(text, dataset);
+            addAttribute("predictedCount", text);
+        }
     }
 
     processAnnotations(dataset);
@@ -2247,7 +2265,7 @@ void ActivityInstance::buildSuffix()
             if ((options.complexClassesActivityFilter == 0) || (kind == options.complexClassesActivityFilter))
                 translator.WARNING2(CategoryEfficiency, HQLWRN_ComplexHelperClass, activityId, approxSize);
         }
-        if (options.showActivitySizeInGraph)
+        if (!options.obfuscateOutput && options.showActivitySizeInGraph)
             addAttributeInt("approxClassSize", approxSize);
     }
 
@@ -5784,6 +5802,12 @@ bool HqlCppTranslator::buildCpp(IHqlCppInstance & _code, HqlQueryContext & query
         wu()->setCodeVersion(ACTIVITY_INTERFACE_VERSION,BUILD_TAG,LANGUAGE_VERSION);
         cacheOptions();
 
+        if (options.obfuscateOutput)
+        {
+            Owned<IWUQuery> query = wu()->updateQuery();
+            query->setQueryText(NULL);
+        }
+
         useLibrary(ECLRTL_LIB);
         useInclude("eclrtl.hpp");
 
@@ -7685,7 +7709,7 @@ static bool isFilePersist(IHqlExpression * expr)
             expr = expr->queryChild(1);
             break;
         case no_output:
-            return (queryRealChild(expr, 1) != NULL);
+            return isFileOutput(expr);
         case no_actionlist:
         case no_orderedactionlist:
             expr = expr->queryChild(expr->numChildren()-1);

ecl/hqlcpp/hqlnlp.cpp

@@ -736,7 +736,7 @@ ABoundActivity * HqlCppTranslator::doBuildActivityParse(BuildCtx & ctx, IHqlExpr
     wu()->setDebugValue("maxMemory", text.str(), true);
 #endif
 
-    if (options.debugNlp != 0)
+    if ((options.debugNlp != 0) && !options.obfuscateOutput)
     {
         BuildCtx subctx(instance->classctx);
         subctx.addQuotedLiteral("#if 0\nHuman readable form of the grammar");

ecl/hqlcpp/hqlresource.ipp

@@ -38,44 +38,21 @@ enum ResourceType {
 class CResourceOptions
 {
 public:
-    CResourceOptions(UniqueSequenceCounter & _spillSequence)
-    : spillSequence(_spillSequence)
-    {
-        filteredSpillThreshold = 0;
-        minimizeSpillSize = 0;
-        allowThroughSpill = false;
-        allowThroughResult = false;
-        cloneFilteredIndex = false;
-        spillSharedConditionals = false;
-        shareDontExpand = false;
-        useGraphResults = false;
-        noConditionalLinks = false;
-        minimiseSpills = false;
-        hoistResourced = false;
-        isChildQuery = false;
-        groupedChildIterators = false;
-        allowSplitBetweenSubGraphs = false;
-        preventKeyedSplit = false;
-        preventSteppedSplit = false;
-        minimizeSkewBeforeSpill = false;
-        expandSingleConstRow = false;
-        createSpillAsDataset = false;
-        optimizeSharedInputs = false;
-        combineSiblings = false;
-        actionLinkInNewGraph = false;
-        convertCompoundToExecuteWhen = false;
-        useResultsForChildSpills = false;
-        alwaysUseGraphResults = false;
-        newBalancedSpotter = false;
-        graphIdExpr = NULL;
-        nextResult = 0;
-        clusterSize = 0;
-        targetClusterType = ThorLCRCluster;
-        state.updateSequence = 0;
-    }
+    CResourceOptions(ClusterType _targetClusterType, unsigned _clusterSize, const HqlCppOptions & _translatorOptions, UniqueSequenceCounter & _spillSequence);
+
+    IHqlExpression * createResultSpillName();
+    IHqlExpression * createDiskSpillName();
+    IHqlExpression * createGlobalSpillName();
 
-    IHqlExpression * createSpillName(bool isGraphResult);
     void noteGraphsChanged() { state.updateSequence++; }
+    void setChildQuery(bool value);
+    void setNewChildQuery(IHqlExpression * graphIdExpr, unsigned numResults);
+    void setUseGraphResults(bool _useGraphResults)
+    {
+        useGraphResults = _useGraphResults;
+    }
+    bool useGraphResult(bool linkedFromChild);
+    bool useGlobalResult(bool linkedFromChild);
 
 public:
     UniqueSequenceCounter & spillSequence;
@@ -105,6 +82,9 @@ public:
     bool     useResultsForChildSpills;
     bool     alwaysUseGraphResults;
     bool     newBalancedSpotter;
+    bool     spillMultiCondition;
+    bool     spotThroughAggregate;
+    bool     alwaysReuseGlobalSpills;
 
     IHqlExpression * graphIdExpr;
     unsigned nextResult;
@@ -160,7 +140,8 @@ public:
     virtual void changeSourceGraph(ResourceGraphInfo * newGraph);
     virtual void changeSinkGraph(ResourceGraphInfo * newGraph);
     virtual bool isRedundantLink();
-    virtual bool isDependency() { return false; }
+    virtual bool isDependency() const { return false; }
+    virtual IHqlExpression * queryDependency() const { return NULL; }
 
 protected:
     void trace(const char * name);
@@ -176,12 +157,16 @@ public:
 class ResourceGraphDependencyLink : public ResourceGraphLink
 {
 public:
-    ResourceGraphDependencyLink(ResourceGraphInfo * _sourceGraph, IHqlExpression * _sourceNode, ResourceGraphInfo * _sinkGraph, IHqlExpression * _sinkNode);
+    ResourceGraphDependencyLink(ResourceGraphInfo * _sourceGraph, IHqlExpression * _sourceNode, ResourceGraphInfo * _sinkGraph, IHqlExpression * _sinkNode, IHqlExpression * _dependency);
 
     virtual void changeSourceGraph(ResourceGraphInfo * newGraph);
     virtual void changeSinkGraph(ResourceGraphInfo * newGraph);
     virtual bool isRedundantLink()          { return false; }
-    virtual bool isDependency() { return true; }
+    virtual bool isDependency() const { return true; }
+    virtual IHqlExpression * queryDependency() const { return dependency; }
+
+protected:
+    LinkedHqlExpr dependency;
 };
 
 typedef CIArrayOf<ResourceGraphInfo> ResourceGraphArray;
@@ -204,7 +189,7 @@ public:
     bool isDependentOn(ResourceGraphInfo & other, bool allowDirect);
     bool isVeryCheap();
     bool mergeInSibling(ResourceGraphInfo & other, const CResources & limit);
-    bool mergeInSource(ResourceGraphInfo & other, const CResources & limit);
+    bool mergeInSource(ResourceGraphInfo & other, const CResources & limit, bool ignoreConditional);
     void removeResources(const CResources & value);
 
     bool isSharedInput(IHqlExpression * expr);
@@ -219,7 +204,7 @@ protected:
 public:
     OwnedHqlExpr createdGraph;
     CResourceOptions * options;
-    GraphLinkArray dependsOn;           // NB: These do no link....
+    GraphLinkArray dependsOn;           // NB: These do not link....
     GraphLinkArray sources;
     GraphLinkArray sinks;
     HqlExprArray conditions;
@@ -327,14 +312,40 @@ public:
     bool ignoreExternalDependencies;
 };
 
-class ResourcerInfo : public CInterfaceOf<IInterface>
+class SpillerInfo : public NewTransformInfo
+{
+public:
+    SpillerInfo(IHqlExpression * _original, CResourceOptions * _options);
+
+    IHqlExpression * createSpilledRead(IHqlExpression * spillReason);
+    IHqlExpression * createSpilledWrite(IHqlExpression * transformed, bool lazy);
+    bool isUsedFromChild() const { return linkedFromChild; }
+    IHqlExpression * queryOutputSpillFile() const { return outputToUseForSpill; }
+    void setPotentialSpillFile(IHqlExpression * expr);
+
+
+protected:
+    void addSpillFlags(HqlExprArray & args, bool isRead);
+    IHqlExpression * createSpillName();
+    bool useGraphResult();
+    bool useGlobalResult();
+    IHqlExpression * wrapRowOwn(IHqlExpression * expr);
+
+protected:
+    CResourceOptions * options;
+    HqlExprAttr spillName;
+    HqlExprAttr spilledDataset;
+    IHqlExpression * outputToUseForSpill;
+    bool linkedFromChild; // could reuse a spare byte in the parent class
+};
+
+class ResourcerInfo : public SpillerInfo
 {
 public:
     enum { PathUnknown, PathConditional, PathUnconditional };
 
     ResourcerInfo(IHqlExpression * _original, CResourceOptions * _options);
 
-    IHqlExpression * createSpilledRead(IHqlExpression * spillReason);
     IHqlExpression * createTransformedExpr(IHqlExpression * expr);
 
     bool addCondition(IHqlExpression * condition);
@@ -356,8 +367,12 @@ public:
     void resetBalanced();
     void setConditionSource(IHqlExpression * condition, bool isFirst);
 
-    //hthor - don't merge anything to a global result because we don't allow splitters.
-    inline bool preventMerge()          { return !options->canSplit() && useGlobalResult(); }
+    inline bool preventMerge()
+    {
+        //If we need to create a spill global result, but engine can't split then don't merge
+        //Only required because hthor doesn't support splitters (or through-workunit results).
+        return !options->canSplit() && options->useGlobalResult(linkedFromChild);
+    }
     inline bool isUnconditional()       { return (pathToExpr == ResourcerInfo::PathUnconditional); }
     inline bool isConditionExpr()
     {
@@ -393,34 +408,20 @@ public:
 
 protected:
     bool spillSharesSplitter();
-    bool useGraphResult();
-    bool useGlobalResult();
     IHqlExpression * createAggregation(IHqlExpression * expr);
-    IHqlExpression * createSpilledWrite(IHqlExpression * transformed);
     IHqlExpression * createSpiller(IHqlExpression * transformed, bool reuseSplitter);
     IHqlExpression * createSplitter(IHqlExpression * transformed);
 
-protected:
-    void addSpillFlags(HqlExprArray & args, bool isRead);
-    IHqlExpression * createSpillName();
-    IHqlExpression * wrapRowOwn(IHqlExpression * expr);
-
 public:
-    HqlExprAttr original;
     Owned<ResourceGraphInfo> graph;
-    HqlExprAttr spillName;
-    IHqlExpression * transformed;
-    IHqlExpression * outputToUseForSpill;
-    CResourceOptions * options;
     HqlExprAttr pathToSplitter;
     HqlExprArray aggregates;
     HqlExprArray conditions;
-    ChildDependentArray childDependents;
-    HqlExprAttr spilledDataset;
     HqlExprAttr splitterOutput;
     HqlExprArray projected;
     HqlExprAttr projectedExpr;
     CIArrayOf<CSplitterLink> balancedLinks;
+    GraphLinkArray dependsOn;           // NB: These do not link....
 
     unsigned numUses;
     unsigned numExternalUses;
@@ -440,7 +441,6 @@ public:
     bool isSpillPoint:1;
     bool balanced:1;
     bool isAlreadyInScope:1;
-    bool linkedFromChild:1;
     bool forceHoist:1;
     bool neverSplit:1;
     bool isConditionalFilter:1;
@@ -449,32 +449,19 @@ public:
     bool balancedVisiting:1;
 };
 
-struct DependencySourceInfo
-{
-    HqlExprArray                    search;
-    CIArrayOf<ResourceGraphInfo>    graphs;
-    HqlExprArray                    exprs;
-};
-
-
+class EclResourceDependencyGatherer;
 class EclResourcer
 {
     friend class SelectHoistTransformer;
     friend class CSplitterInfo;
 public:
-    EclResourcer(IErrorReceiver & _errors, IConstWorkUnit * _wu, ClusterType _targetClusterType, unsigned _clusterSize, const HqlCppOptions & _translatorOptions, UniqueSequenceCounter & _spillSequence);
+    EclResourcer(IErrorReceiver & _errors, IConstWorkUnit * _wu, const HqlCppOptions & _translatorOptions, CResourceOptions & _options);
     ~EclResourcer();
 
     void resourceGraph(IHqlExpression * expr, HqlExprArray & transformed);
     void resourceRemoteGraph(IHqlExpression * expr, HqlExprArray & transformed);
-    void setChildQuery(bool value);
-    void setNewChildQuery(IHqlExpression * graphIdExpr, unsigned numResults);
     void setSequential(bool _sequential) { sequential = _sequential; }
-    void setUseGraphResults(bool _useGraphResults) 
-    { 
-        options.useGraphResults = _useGraphResults; 
-    }
-    void tagActiveCursors(HqlExprCopyArray & activeRows);
+    void tagActiveCursors(HqlExprCopyArray * activeRows);
     inline unsigned numGraphResults() { return options.nextResult; }
 
 protected:
@@ -487,15 +474,11 @@ protected:
     void replaceGraphReferences(ResourceGraphInfo * oldGraph, ResourceGraphInfo * newGraph);
 
 //Pass 1
-    void gatherChildSplitPoints(IHqlExpression * expr, ResourcerInfo * info, unsigned first, unsigned last);
     bool findSplitPoints(IHqlExpression * expr, bool isProjected);
     void findSplitPoints(HqlExprArray & exprs);
     void noteConditionalChildren(BoolArray & alwaysHoistChild);
     void deriveUsageCounts(IHqlExpression * expr);
     void deriveUsageCounts(const HqlExprArray & exprs);
-    void extendSplitPoints();
-    void projectChildDependents();
-    IHqlExpression * projectChildDependent(IHqlExpression * expr);
 
 //Pass 2
     void createInitialGraph(IHqlExpression * expr, IHqlExpression * owner, ResourceGraphInfo * ownerGraph, LinkKind linkKind, bool forceNewGraph);
@@ -520,13 +503,7 @@ protected:
     void resourceSubGraphs(HqlExprArray & exprs);
 
 //Pass 5
-    void addDependencySource(IHqlExpression * search, ResourceGraphInfo * curGraph, IHqlExpression * expr);
-    void addDependencyUse(IHqlExpression * search, ResourceGraphInfo * curGraph, IHqlExpression * expr);
-    bool addExprDependency(IHqlExpression * expr, ResourceGraphInfo * curGraph, IHqlExpression * activityExpr);
-    void addRefExprDependency(IHqlExpression * expr, ResourceGraphInfo * curGraph, IHqlExpression * activityExpr);
-    void doAddChildDependencies(IHqlExpression * expr, ResourceGraphInfo * graph, IHqlExpression * activityExpr);
-    void addChildDependencies(IHqlExpression * expr, ResourceGraphInfo * graph, IHqlExpression * activityExpr);
-    void addDependencies(IHqlExpression * expr, ResourceGraphInfo * graph, IHqlExpression * activityExpr);
+    void addDependencies(EclResourceDependencyGatherer & gatherer, IHqlExpression * expr, ResourceGraphInfo * graph, IHqlExpression * activityExpr);
     void addDependencies(HqlExprArray & exprs);
 
 //Pass 6
@@ -560,7 +537,6 @@ protected:
     void moveExternalSpillPoints();
 
 //Pass 9
-    IHqlExpression * replaceResourcedReferences(ResourcerInfo * info, IHqlExpression * expr);
     IHqlExpression * doCreateResourced(IHqlExpression * expr, ResourceGraphInfo * graph, bool expandInParent, bool defineSideEffect);
     IHqlExpression * createResourced(IHqlExpression * expr, ResourceGraphInfo * graph, bool expandInParent, bool defineSideEffect);
     void createResourced(HqlExprArray & transformed);
@@ -582,21 +558,16 @@ protected:
     CIArrayOf<ResourceGraphInfo> graphs;
     CIArrayOf<ResourceGraphLink> links;
     ClusterType targetClusterType;
-    DependencySourceInfo dependencySource;                  
-    unsigned clusterSize;
     CResources * resourceLimit;
     IErrorReceiver * errors;
     unsigned thisPass;
     bool spilled;
-    bool spillMultiCondition;
-    bool spotThroughAggregate;
     bool insideNeverSplit;
     bool insideSteppedNeverSplit;
     bool sequential;
-    CResourceOptions options;
+    CResourceOptions & options;
     HqlExprArray rootConditions;
     HqlExprCopyArray activeSelectors;
-    ChildDependentArray allChildDependents;
 };
 
 #endif

ecl/hqlcpp/hqlsource.cpp

@@ -1991,7 +1991,7 @@ ABoundActivity * SourceBuilder::buildActivity(BuildCtx & ctx, IHqlExpression * e
 
     IHqlExpression * spillReason = tableExpr ? queryAttributeChild(tableExpr, _spillReason_Atom, 0) : NULL;
 
-    if (spillReason)
+    if (spillReason && !translator.queryOptions().obfuscateOutput)
     {
         StringBuffer text;
         getStringValue(text, spillReason);

ecl/hqlcpp/hqlttcpp.cpp

@@ -4536,27 +4536,27 @@ IHqlExpression * CompoundActivityTransformer::createTransformed(IHqlExpression *
         {
             if (transformed->hasAttribute(onFailAtom))
                 break;
+
             LinkedHqlExpr dataset = transformed->queryChild(0);
             if (dataset->hasAttribute(limitAtom) || transformed->hasAttribute(skipAtom))
                 break;
+
+            // A limited KEYED-JOIN should never read more than limit rows from the index for each left row
+            // so add a LIMIT attribute onto the keyed join to ensure it doesn't return too many records.
             switch (dataset->getOperator())
             {
             case no_join:
-            case no_denormalize:
-            case no_denormalizegroup:
                 if (isKeyedJoin(dataset))
                     break;
                 return transformed.getClear();
             default:
                 return transformed.getClear();
             }
-            if (!isThorCluster(targetClusterType))
-                return mergeLimitIntoDataset(dataset, transformed);
+
             HqlExprArray args;
             unwindChildren(args, transformed);
             args.replace(*mergeLimitIntoDataset(dataset, transformed), 0);
             return transformed->clone(args);
-
         }
     }
 

ecllibrary/std/File.ecl

@@ -554,8 +554,8 @@ EXPORT Despray(varstring logicalName, varstring destinationIP, varstring destina
  * @return              The DFU workunit id for the job.
  */
 
-EXPORT varstring fCopy(varstring sourceLogicalName, varstring destinationGroup, varstring destinationLogicalName, varstring sourceDali='', integer4 timeOut=-1, varstring espServerIpPort=GETENV('ws_fs_server'), integer4 maxConnections=-1, boolean allowOverwrite=FALSE, boolean replicate=FALSE, boolean asSuperfile=FALSE, boolean compress=FALSE, boolean forcePush=FALSE, integer4 transferBufferSize=0) :=
-    lib_fileservices.FileServices.fCopy(sourceLogicalName, destinationGroup, destinationLogicalName, sourceDali, timeOut, espServerIpPort, maxConnections, allowOverwrite, replicate, asSuperfile, compress, forcePush, transferBufferSize);
+EXPORT varstring fCopy(varstring sourceLogicalName, varstring destinationGroup, varstring destinationLogicalName, varstring sourceDali='', integer4 timeOut=-1, varstring espServerIpPort=GETENV('ws_fs_server'), integer4 maxConnections=-1, boolean allowOverwrite=FALSE, boolean replicate=FALSE, boolean asSuperfile=FALSE, boolean compress=FALSE, boolean forcePush=FALSE, integer4 transferBufferSize=0, boolean preserveCompression=TRUE) :=
+    lib_fileservices.FileServices.fCopy(sourceLogicalName, destinationGroup, destinationLogicalName, sourceDali, timeOut, espServerIpPort, maxConnections, allowOverwrite, replicate, asSuperfile, compress, forcePush, transferBufferSize, preserveCompression);
 
 /**
  * Same as fCopy, but does not return the DFU Workunit ID.
@@ -563,8 +563,8 @@ EXPORT varstring fCopy(varstring sourceLogicalName, varstring destinationGroup,
  * @see fCopy
  */
 
-EXPORT Copy(varstring sourceLogicalName, varstring destinationGroup, varstring destinationLogicalName, varstring sourceDali='', integer4 timeOut=-1, varstring espServerIpPort=GETENV('ws_fs_server'), integer4 maxConnections=-1, boolean allowOverwrite=FALSE, boolean replicate=FALSE, boolean asSuperfile=FALSE, boolean compress=FALSE, boolean forcePush=FALSE, integer4 transferBufferSize=0) :=
-    lib_fileservices.FileServices.Copy(sourceLogicalName, destinationGroup, destinationLogicalName, sourceDali, timeOut, espServerIpPort, maxConnections, allowOverwrite, replicate, asSuperfile, compress, forcePush, transferBufferSize);
+EXPORT Copy(varstring sourceLogicalName, varstring destinationGroup, varstring destinationLogicalName, varstring sourceDali='', integer4 timeOut=-1, varstring espServerIpPort=GETENV('ws_fs_server'), integer4 maxConnections=-1, boolean allowOverwrite=FALSE, boolean replicate=FALSE, boolean asSuperfile=FALSE, boolean compress=FALSE, boolean forcePush=FALSE, integer4 transferBufferSize=0, boolean preserveCompression=TRUE) :=
+    lib_fileservices.FileServices.Copy(sourceLogicalName, destinationGroup, destinationLogicalName, sourceDali, timeOut, espServerIpPort, maxConnections, allowOverwrite, replicate, asSuperfile, compress, forcePush, transferBufferSize, preserveCompression);
 
 /**
  * Ensures the specified file is replicated to its mirror copies.

esp/eclwatch/ws_XSLT/wuidcommon.xslt

@@ -44,14 +44,8 @@
                 <xsl:otherwise>
                   <xsl:value-of select="$wuid"/>
                   &nbsp;
-                  <xsl:choose>
-                    <xsl:when test="WUXMLSize &lt; 5000000">
-                      <a href="/esp/iframe?esp_iframe_title=ECL Workunit XML - {$wuid}&amp;inner=/WsWorkunits/WUFile%3fWuid%3d{$wuid}%26Type%3dXML%26Option%3d0" >XML</a><xsl:value-of select="WUXMLSize"/>
-                    </xsl:when>
-                    <xsl:otherwise>
-                      <a href="/esp/iframe?esp_iframe_title=Download ECL Workunit XML - {$wuid}&amp;inner=/WsWorkunits/WUFile%3fWuid%3d{$wuid}%26Type%3dXML%26Option%3d2" >Download XML</a>
-                    </xsl:otherwise>
-                  </xsl:choose>
+                  <a href="/esp/iframe?esp_iframe_title=ECL Workunit XML - {$wuid}&amp;inner=/WsWorkunits/WUFile%3fWuid%3d{$wuid}%26Type%3dXML%26Option%3d0%26SizeLimit%3d5000000" >XML</a>
+                  <a href="/esp/iframe?esp_iframe_title=Download ECL Workunit XML - {$wuid}&amp;inner=/WsWorkunits/WUFile%3fWuid%3d{$wuid}%26Type%3dXML%26Option%3d2" >Download XML</a>
                   &nbsp;
                   <a href="/esp/iframe?esp_iframe_title=ECL Playground - {$wuid}&amp;inner=/esp/files/stub.htm%3fWidget%3dECLPlaygroundWidget%26Wuid%3d{$wuid}%26Target%3d{Cluster}" >ECL Playground</a>
                 </xsl:otherwise>

esp/files/gen_form_wsecl.js

@@ -0,0 +1,499 @@
+/*##############################################################################
+#    HPCC SYSTEMS software Copyright (C) 2015 HPCC Systems.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+############################################################################## */
+
+// for test purpose
+function getAttributes(ctrl)
+{
+    var results = "";
+    var attrs = ctrl.attributes;
+    for (var i = 0; i < attrs.length; i++) {
+       var attr = attrs[i];
+       results += attr.nodeName + '=' + attr.nodeValue + ' (' + attr.specified + ')<BR>';
+    }
+    return results;
+}
+
+//==================================================================
+// Restore dynamically generate content, and user input values(non-IE browsers only)
+
+function restoreDataFromCache()
+{
+    var vals = document.getElementById("esp_vals_").value;
+    if (vals && vals!="")
+    {
+        // alert("esp_vals_ = "+vals);
+        var end = vals.indexOf('|');
+        while (end>0)
+        {
+            var name = vals.substring(0,end);
+            vals = vals.substring(end+1);
+            end = vals.indexOf("|");
+            if (end<0) break;
+            var ctrl = document.getElementsByName(name)[0];
+            if (ctrl)
+            {
+                if (ctrl.type == 'checkbox')
+                {
+                    ctrl.checked = vals.substring(0,end)=='1';
+                    //  alert("Name = "+name+", string = "+ vals.substring(0,end) + ", ctrl.checked = "+ ctrl.checked);
+                }
+                else if (ctrl.type == 'text' || ctrl.type == 'textarea')
+                    ctrl.value = decodeURI(vals.substring(0,end));
+                else if(ctrl.type == 'radio')
+                {
+                    //alert("Name = "+name+", string = "+ vals.substring(0,end) + ", ctrl.checked = "+ ctrl.checked);
+                    if(vals.substring(0,end) == '0')
+                    {
+                         ctrl = document.getElementsByName(name)[1];
+                    }
+
+                    if(ctrl)
+                        ctrl.checked = true;
+                }
+                else if(ctrl.type == 'select-one')
+            {
+                    //alert("Select: name="+ctrl.name+"; value="+vals.substring(0,end));
+                    ctrl.options[vals.substring(0,end)].selected = true;
+                }
+                //TODO: more types
+            }
+            vals = vals.substring(end+1);
+            end = vals.indexOf("|");
+        }
+    }
+}
+
+function disableAllInputs(self)
+{
+    var toEnable = self.checked ? 1 : 0;
+    var form = document.forms['esp_form'];
+    var ctrls = form.elements;
+    for (var idx=0; idx<ctrls.length; idx++)
+    {
+        var c = ctrls[idx];
+        if ( (c.id!='') && (c.id.substr(0,3) == '$V.'))
+        {
+            if ( (c.checked && !toEnable) || (!c.checked && toEnable))
+                c.click();
+        }
+    }
+}
+
+function disableInputControls(form)
+{
+    var ctrls = form.elements;
+    for (var idx=0; idx<ctrls.length; idx++)
+    {
+        var c = ctrls[idx];
+        if ( (c.id!='') && (c.id.substr(0,3) == '$V.') && !c.checked)
+        {
+             var id = c.id.substring(3);
+             var ctrl = document.getElementById(id);
+             if (ctrl) // struct name has no id
+                disableInputControl(ctrl,true);
+             var label = document.getElementById('$L.'+id)
+             disableInputLabel(label,true);
+        }
+    }
+}
+
+
+function onPageLoad()
+{
+   var ctrl = document.getElementById('esp_html_');
+   //alert("onPageLoad(): ctrl="+ctrl+"; length="+ctrl.value.length+";value='"+ctrl.value+"'");
+   if (ctrl && ctrl.value != undefined && ctrl.value!="")
+   {
+      //alert("Restore ctrol value: " + ctrl.value);
+        document.forms['esp_form'].innerHTML = ctrl.value;
+   }
+
+   var form = document.forms['esp_form'];
+   initFormValues(form, getUrlFormValues(top.location.href));
+
+   disableInputControls(form);
+
+   // IE seems need this now too
+   //if (isIE) return true;
+   // FF 1.5 history cache works, but seems to stop working afterwards
+   restoreDataFromCache();
+
+   return true;
+}
+
+function getUrlFormValues(url)
+{
+    var idx = url.indexOf('?');
+    if (idx>0)
+        url = url.substring(idx+1);
+    var a = url.split('&');
+
+    var ps = new Hashtable();
+    for (var i=0; i<a.length; i++)
+    {
+         idx = a[i].indexOf('=');
+         if (a[i].charAt(0) == '.' && idx>0)
+         {
+            var key = a[i].substring(0,idx);
+            var val =  a[i].substring(idx+1);
+            if (val != '')
+                ps.put(key, val);
+        }
+    }
+
+    return ps;
+}
+
+function getUrlEspFlags(url)
+{
+    var idx = url.indexOf('?');
+    if (idx>0)
+        url = url.substring(idx+1);
+    var a = url.split('&');
+
+    var ps = new Hashtable();
+    for (var i=0; i<a.length; i++)
+    {
+        if (a[i].charAt(0) != '.')
+        {
+            idx = a[i].indexOf('=');
+            if (idx>0)
+            {
+                var key = a[i].substring(0,idx);
+                var val =   a[i].substring(idx+1);
+                ps.put(key,val);
+            } else
+                ps.put(a[i],"");
+        }
+    }
+
+    return ps;
+}
+
+function createArray(ps)
+{
+    var remains = new Hashtable();
+    ps.moveFirst();
+    while (ps.next())
+    {
+         var name = ps.getKey();
+         var val = ps.getValue();
+         // alert(name + ": " + val);
+         if (val > 0 && name.substring(name.length-11)==".itemcount!")
+         {
+             var id = name.substring(1, name.length-10) + '_AddBtn';
+             var ctrl = document.getElementById(id);
+             if (ctrl)
+             {
+                //   alert("name: " + ctrl.tagName + ", type " + ctrl.type);
+                for (var i=0; i<val; i++)
+                    ctrl.click();
+             }
+             else {
+                //alert("Can not find control: " + id);
+                remains.put(name,val);
+             }
+         }
+    }
+
+    if (remains.size()>0)
+      return remains;
+    else
+      return null;
+}
+
+function initFormValues(form, ps)
+{
+    // create array controls
+    // Implementation NOTE: The Add order is important: if array A contains array B, item in A must be created first before B can be created.
+
+    var working = ps;
+    do
+    {
+        working  = createArray(working);
+        //alert("Left: " + working);
+    } while (working!=null);
+
+    // init values
+    ps.moveFirst();
+    while (ps.next())
+    {
+        var name = ps.getKey();
+        if (name.charAt(0) != '.')
+        {
+            //alert("Skip " + name);
+            continue;
+        }
+        name = name.substring(1);
+        var val = ps.getValue();
+        ctrl = document.getElementsByName(name)[0];
+
+        // alert("Set value for " + name + ": " + val + ". Ctrl type: " + ctrl.type);
+        if (ctrl)
+        {
+            if (ctrl.type == 'checkbox') {
+                ctrl.checked = val =='1';
+            }
+            else if (ctrl.type == 'text') {
+                ctrl.value =  decodeURIComponent(val); //    decodeURI(vals.substring(0,end)); //TODO: do we need encoding
+            }
+            else if (ctrl.type == 'textarea') {
+                ctrl.value =  decodeURIComponent(val);
+            }
+            else if(ctrl.type == 'radio')  {
+                if(val == '0')
+                    ctrl = document.getElementsByName(name)[1];
+                if(ctrl)
+                    ctrl.checked = true;
+            }
+            else if (ctrl.type=='select-one')  {
+                //alert("Set select value: " + val);
+                ctrl.options[val].selected=true;
+            }
+        }
+        else
+            alert("failed to find contrl: " + name);
+    }
+}
+
+function doBookmark(form)
+{
+    var ps = getUrlEspFlags(form.action);
+
+    var ctrls = form.elements;
+    for (var idx=0; idx<ctrls.length; idx++)
+    {
+        var c = ctrls[idx];
+        if ( (c.name!='') && (c.value != '') )
+        {
+            if (c.tagName == 'TEXTAREA') {
+                ps.put('.'+c.name,encodeURIComponent(c.value));
+            } else if (c.tagName == "SELECT") {
+                ps.put('.'+c.name, c.selectedIndex); // use the index
+            } else if (c.tagName == 'INPUT')  {
+                if ( c.type == 'text' || c.type=='password')  {
+                    ps.put('.'+c.name,encodeURIComponent(c.value)); // existing one is overwrotten
+                } else if (c.type == 'radio' && c.checked) {
+                    if (c.id.substring(c.id.length-5) == '.true')
+                        ps.put('.'+c.name,"1");
+                    else if (c.id.substring(c.id.length-6) == '.false')
+                        ps.put('.'+c.name,"0");
+                } else if ( c.type=='hidden') {
+                    // alert("hidden:"+c.name+", value " + c.value + ", sub = " +  c.name.substring(c.name.length-10));
+                    if (c.value!='0' && c.name.substring(c.name.length-11)=='.itemcount!')
+                        ps.put('.'+c.name,c.value);
+                }
+            }
+        }
+    }
+
+    var idx = form.action.indexOf('?');
+    var action = (idx>0) ? form.action.substring(0,idx) : form.action;
+
+    action += "?form";
+
+    var parm = "";
+    ps.moveFirst();
+    while (ps.next())
+       parm += '&' + ps.getKey() + '=' + ps.getValue();
+    //alert("parm="+parm);
+
+    /*
+    // TODO: make inner frame work
+    var url = "/?inner=.." + path + "%3Fform";
+    top.location.href = url + parm;
+    */
+    top.location.href = action + parm;
+}
+
+//==================================================================
+// Save dynamically generate content, and user input values(non-IE browsers only)
+function onSubmit(reqType)  // reqType: 0: regular form, 1: soap, 2: form param passing
+{
+    var form = document.forms['esp_form'];
+    if (!form)  return false;
+
+    // remove "soap_builder_" (somehow FF (not IE) remembers this changed form.action )
+    if (reqType != 1)
+    {
+        var action = form.action;
+        var idx = action.indexOf('soap_builder_');
+        if (idx>0)
+        {
+            if (action.length <= idx + 13) // no more char after 'soap_builder_'
+            {
+                var ch = action.charAt(idx-1);
+                if (ch == '&' || ch == '?')
+                    action = action.substring(0,idx-1);
+            } else {
+                var ch = action.charAt(idx+13) // the char after 'soap_builder_';
+                if (ch == '&')
+                   action = action.substring(0,idx) + action.substring(idx+13);
+            }
+
+            // alert("Old action: " + form.action + "\nNew action: " + action);
+            form.action = action;
+        }
+    }
+
+    // --  change action if user wants to
+    var dest = document.getElementById('esp_dest');
+    if (dest && dest.checked)
+    {
+         form.action = document.getElementById('dest_url').value;
+    }
+    if (reqType==1)
+    {
+         if (form.action.indexOf('soap_builder_')<0) // add only if does not exist already
+         {
+                var c =  (form.action.indexOf('?')>0) ? '&' : '?';
+                form.action += c + "soap_builder_";
+         }
+    }
+    else if (reqType==2)
+    {
+         doBookmark(form);
+    }
+    if (reqType==3)
+    {
+         if (form.action.indexOf('roxie_builder_')<0) // add only if does not exist already
+         {
+                var c =  (form.action.indexOf('?')>0) ? '&' : '?';
+                form.action += c + "roxie_builder_";
+         }
+    }
+    if (reqType==4)
+    {
+         if (form.action.indexOf('json_builder_')<0) // add only if does not exist already
+         {
+                var c =  (form.action.indexOf('?')>0) ? '&' : '?';
+                form.action += c + "json_builder_";
+         }
+    }
+    // alert("Form action = " + form.action);
+
+    // firefox now save input values (version 1.5)
+    saveInputValues(form);
+
+    return true;
+}
+
+//==================================================================
+// Save dynamically generate content, and user input values(non-IE browsers only)
+function onWsEcl2Submit(path)  // reqType: 0: regular form, 1: soap, 2: form param passing, 3: roxiexml
+{
+    var form = document.forms['esp_form'];
+    if (!form)  return false;
+
+    var dest = document.getElementById('esp_dest');
+    if (dest && dest.checked)
+    {
+         form.action = document.getElementById('dest_url').value;
+    }
+    else if (path=="bookmark")
+    {
+         doBookmark(form);
+    }
+    else
+    {
+        form.action = path;
+    }
+
+    alert("Form action = " + form.action);
+
+    // firefox now save input values (version 1.5)
+    saveInputValues(form);
+
+    return true;
+}
+
+
+function saveInputValues(form)
+{
+    // -- save values in input for browser
+    var ctrl = document.getElementById('esp_html_');
+    // IE seems to need this too
+    //if (isIE || !ctrl)  return true;
+
+    ctrl.value=form.innerHTML;
+
+    // save all user input
+    var ctrls = form.elements;
+    var items = ctrls.length;
+    var inputValues = "";
+
+    for (var idx=0; idx<ctrls.length; idx++)
+    {
+        var item = ctrls[idx];
+        var name = item.name;
+
+        if (!name) continue;
+
+        //NOTE: we can not omit empty value since it can be different from the default
+        if (item.type == 'checkbox')
+            inputValues += name+"|"+(item.checked ? "1" : "0")+"|";
+        else if (item.type =='text' || item.type=='textarea')
+        {
+           inputValues += name+"|" + encodeURI(item.value) +"|";
+           //alert("value added: "+inputValues[inputValues.length-1] +", input items: " + inputValues.length+", values="+inputValues.toString());
+        }
+        else if (item.type == 'radio')
+        {
+            //if(item.checked) // the unchecked value can be different from the default
+            inputValues += name+"|"+item.value+"|";
+        }
+        else if (item.type == 'select-one')
+        {
+        inputValues += name+"|"+item.selectedIndex+"|";
+        //alert("inputValues=" + inputValues + "; index="+item.selectedIndex);
+        }
+        // TODO: other control types
+    }
+
+    document.getElementById("esp_vals_").value = inputValues;
+}
+
+//==================================================================
+// Reset all values to orginal (all dynamically generated arrays are removed)
+function onClearAll()
+{
+    // reset dynamic generated content
+    var reqCtrl = document.getElementById('esp_dyn');
+    reqCtrl.innerHTML = getRequestFormHtml();;
+
+    // clear cache
+    var ctrl = document.getElementById('esp_html_');
+    if (ctrl)
+        ctrl.value = "";
+    ctrl = document.getElementById("esp_vals_");
+    if (ctrl)
+        ctrl.value = "";
+}
+
+//  Exclusive selectable
+function  onClickSort(chked)
+{
+    if (chked) {
+         document.getElementById("esp_validate").checked = false;
+    }
+}
+
+function  onClickValidate(chked)
+{
+    if (chked) {
+        document.getElementById("esp_sort_result").checked = false;
+    }
+}

esp/scm/ws_fs.ecm

@@ -82,10 +82,9 @@ ESPStruct [nil_remove] DFUWorkunit
     string decrypt;
 
     [min_ver("1.08")] bool failIfNoSourceFile(false);
-
     [min_ver("1.09")] bool recordStructurePresent(false);
-
     [min_ver("1.10")] bool quotedTerminator(true);
+    [min_ver("1.12")] bool preserveCompression(true);
 };
 
 ESPStruct [nil_remove] GroupNode
@@ -455,6 +454,7 @@ ESPrequest [nil_remove] Copy
     string encrypt;
     string decrypt;
 
+    [min_ver("1.12")] bool preserveCompression(true);
 };
 
 ESPresponse [exceptions_inline] 
@@ -629,7 +629,7 @@ ESPresponse [exceptions_inline, nil_remove] GetSprayTargetsResponse
 };
 
 ESPservice [
-    version("1.11"),
+    version("1.12"),
     exceptions_inline("./smc_xslt/exceptions.xslt")] FileSpray
 {
     ESPuses ESPstruct DFUWorkunit;

esp/scm/ws_workunits.ecm

@@ -236,7 +236,6 @@ ESPStruct [nil_remove] ECLWorkunit
     [min_ver("1.29")] string ApplicationValuesDesc;
     [min_ver("1.29")] string WorkflowsDesc;
     [min_ver("1.31")] bool HasArchiveQuery(false);
-    [min_ver("1.49")] int64 WUXMLSize;
     [min_ver("1.38")] ESParray<ESPstruct ThorLogInfo> ThorLogList;
     [min_ver("1.47")] ESParray<string, URL> ResourceURLs;
     [min_ver("1.50")] int ResultViewCount;
@@ -656,6 +655,7 @@ ESPrequest WULogFileRequest
     [min_ver("1.38")] string ClusterGroup;
     [min_ver("1.38")] string LogDate;
     [min_ver("1.38")] int SlaveNumber(1);
+    [min_ver("1.55")] int64 SizeLimit(0);
     string PlainText;
 };
 
@@ -1610,7 +1610,7 @@ ESPresponse [exceptions_inline] WUGetStatsResponse
 };
 
 ESPservice [
-    version("1.55"),
+    version("1.55"), default_client_version("1.55"),
     noforms,exceptions_inline("./smc_xslt/exceptions.xslt"),use_method_name] WsWorkunits
 {
     ESPmethod [resp_xsl_default("/esp/xslt/workunits.xslt")]     WUQuery(WUQueryRequest, WUQueryResponse);

esp/services/common/jsonhelpers.hpp

@@ -111,7 +111,9 @@ namespace HttpParamHelpers
         Owned<IPropertyIterator> props = parameters->getIterator();
         ForEach(*props)
         {
-            const char *key = props->getPropKey();
+            StringBuffer key = props->getPropKey();
+            if (!key.length() || key.charAt(key.length()-1)=='!')
+                continue;
             const char *value = parameters->queryProp(key);
             if (value && *value)
                 ensureParameter(pt, key, value);

esp/services/ws_fs/ws_fsService.cpp

@@ -2438,6 +2438,7 @@ bool CFileSprayEx::onCopy(IEspContext &context, IEspCopy &req, IEspCopyResponse
         wuFSpecDest->setLogicalName(dstname);
         wuFSpecDest->setFileMask(fileMask.str());
         wuOptions->setOverwrite(req.getOverwrite());
+        wuOptions->setPreserveCompression(req.getPreserveCompression());
 
         if (bRoxie)
         {

esp/services/ws_workunits/ws_workunitsHelpers.cpp

@@ -1023,12 +1023,6 @@ void WsWuInfo::getInfo(IEspECLWorkunit &info, unsigned flags)
     info.setDescription(cw->getDebugValue("description", s).str());
     if (version > 1.21)
         info.setXmlParams(cw->getXmlParams(s).str());
-    if (version >= 1.49)
-    {
-        SCMStringBuffer xml;
-        exportWorkUnitToXML(cw, xml, true, false);
-        info.setWUXMLSize(xml.length());
-    }
 
     info.setResultLimit(cw->getResultLimit());
     info.setArchived(false);

esp/services/ws_workunits/ws_workunitsService.cpp

@@ -2383,15 +2383,25 @@ void getWsWuResult(IEspContext &context, const char* wuid, const char *name, con
     }
 }
 
-void openSaveFile(IEspContext &context, int opt, const char* filename, const char* origMimeType, MemoryBuffer& buf, IEspWULogFileResponse &resp)
+void checkFileSizeLimit(unsigned long xmlSize, unsigned long sizeLimit)
+{
+    if ((sizeLimit > 0) && (xmlSize > sizeLimit))
+        throw MakeStringException(ECLWATCH_CANNOT_OPEN_FILE,
+            "The file size (%ld bytes) exceeds the size limit (%ld bytes). You may set 'Option > 1' or use 'Download_XML' link to get compressed file.",
+            xmlSize, sizeLimit);
+}
+
+void openSaveFile(IEspContext &context, int opt, __int64 sizeLimit, const char* filename, const char* origMimeType, MemoryBuffer& buf, IEspWULogFileResponse &resp)
 {
     if (opt < 1)
     {
+        checkFileSizeLimit(buf.length(), sizeLimit);
         resp.setThefile(buf);
         resp.setThefile_mimetype(origMimeType);
     }
     else if (opt < 2)
     {
+        checkFileSizeLimit(buf.length(), sizeLimit);
         StringBuffer headerStr("attachment;");
         if (filename && *filename)
         {
@@ -2507,12 +2517,12 @@ bool CWsWorkunitsEx::onWUFile(IEspContext &context,IEspWULogFileRequest &req, IE
             if (strieq(File_ArchiveQuery, req.getType()))
             {
                 winfo.getWorkunitArchiveQuery(mb);
-                openSaveFile(context, opt, "ArchiveQuery.xml", HTTP_TYPE_APPLICATION_XML, mb, resp);
+                openSaveFile(context, opt, req.getSizeLimit(), "ArchiveQuery.xml", HTTP_TYPE_APPLICATION_XML, mb, resp);
             }
             else if (strieq(File_Cpp,req.getType()) && notEmpty(req.getName()))
             {
                 winfo.getWorkunitCpp(req.getName(), req.getDescription(), req.getIPAddress(),mb, opt > 0);
-                openSaveFile(context, opt, req.getName(), HTTP_TYPE_TEXT_PLAIN, mb, resp);
+                openSaveFile(context, opt, req.getSizeLimit(), req.getName(), HTTP_TYPE_TEXT_PLAIN, mb, resp);
             }
             else if (strieq(File_DLL,req.getType()))
             {
@@ -2520,17 +2530,17 @@ bool CWsWorkunitsEx::onWUFile(IEspContext &context,IEspWULogFileRequest &req, IE
                 winfo.getWorkunitDll(name, mb);
                 resp.setFileName(name.str());
                 resp.setDaliServer(daliServers.get());
-                openSaveFile(context, opt, req.getName(), HTTP_TYPE_OCTET_STREAM, mb, resp);
+                openSaveFile(context, opt, req.getSizeLimit(), req.getName(), HTTP_TYPE_OCTET_STREAM, mb, resp);
             }
             else if (strieq(File_Res,req.getType()))
             {
                 winfo.getWorkunitResTxt(mb);
-                openSaveFile(context, opt, "res.txt", HTTP_TYPE_TEXT_PLAIN, mb, resp);
+                openSaveFile(context, opt, req.getSizeLimit(), "res.txt", HTTP_TYPE_TEXT_PLAIN, mb, resp);
             }
             else if (strncmp(req.getType(), File_ThorLog, 7) == 0)
             {
                 winfo.getWorkunitThorLog(req.getName(), mb);
-                openSaveFile(context, opt, "thormaster.log", HTTP_TYPE_TEXT_PLAIN, mb, resp);
+                openSaveFile(context, opt, req.getSizeLimit(), "thormaster.log", HTTP_TYPE_TEXT_PLAIN, mb, resp);
             }
             else if (strieq(File_ThorSlaveLog,req.getType()))
             {
@@ -2538,12 +2548,12 @@ bool CWsWorkunitsEx::onWUFile(IEspContext &context,IEspWULogFileRequest &req, IE
                 getConfigurationDirectory(directories, "log", "thor", req.getProcess(), logDir);
 
                 winfo.getWorkunitThorSlaveLog(req.getClusterGroup(), req.getIPAddress(), req.getLogDate(), logDir.str(), req.getSlaveNumber(), mb, false);
-                openSaveFile(context, opt, "ThorSlave.log", HTTP_TYPE_TEXT_PLAIN, mb, resp);
+                openSaveFile(context, opt, req.getSizeLimit(), "ThorSlave.log", HTTP_TYPE_TEXT_PLAIN, mb, resp);
             }
             else if (strieq(File_EclAgentLog,req.getType()))
             {
                 winfo.getWorkunitEclAgentLog(req.getName(), req.getProcess(), mb);
-                openSaveFile(context, opt, "eclagent.log", HTTP_TYPE_TEXT_PLAIN, mb, resp);
+                openSaveFile(context, opt, req.getSizeLimit(), "eclagent.log", HTTP_TYPE_TEXT_PLAIN, mb, resp);
             }
             else if (strieq(File_XML,req.getType()) && notEmpty(req.getName()))
             {
@@ -2555,7 +2565,7 @@ bool CWsWorkunitsEx::onWUFile(IEspContext &context,IEspWULogFileRequest &req, IE
                     ptr = name;
 
                 winfo.getWorkunitAssociatedXml(name, req.getIPAddress(), req.getPlainText(), req.getDescription(), opt > 0, mb);
-                openSaveFile(context, opt, ptr, HTTP_TYPE_APPLICATION_XML, mb, resp);
+                openSaveFile(context, opt, req.getSizeLimit(), ptr, HTTP_TYPE_APPLICATION_XML, mb, resp);
             }
             else if (strieq(File_XML,req.getType()) || strieq(File_WUECL,req.getType()))
             {
@@ -2583,7 +2593,7 @@ bool CWsWorkunitsEx::onWUFile(IEspContext &context,IEspWULogFileRequest &req, IE
                         mimeType.set(HTTP_TYPE_APPLICATION_XML);
                     }
                 }
-                openSaveFile(context, opt, fileName.str(), mimeType.str(), mb, resp);
+                openSaveFile(context, opt, req.getSizeLimit(), fileName.str(), mimeType.str(), mb, resp);
             }
         }
     }
@@ -3370,6 +3380,54 @@ bool isRunning(IConstWorkUnit &cw)
     }
 }
 
+void CWsWorkunitsEx::readGraph(IEspContext& context, const char* subGraphId, WUGraphIDType& id, bool running,
+    IConstWUGraph* graph, IArrayOf<IEspECLGraphEx>& graphs)
+{
+    SCMStringBuffer name, label, type;
+    graph->getName(name);
+    graph->getLabel(label);
+    graph->getTypeName(type);
+
+    Owned<IEspECLGraphEx> g = createECLGraphEx("","");
+    g->setName(name.str());
+    g->setLabel(label.str());
+    g->setType(type.str());
+
+    WUGraphState graphState = graph->getState();
+    if (running && (WUGraphRunning == graphState))
+    {
+        g->setRunning(true);
+        g->setRunningId(id);
+    }
+    else if (context.getClientVersion() > 1.20)
+    {
+        if (WUGraphComplete == graphState)
+            g->setComplete(true);
+        else if (WUGraphFailed == graphState)
+            g->setFailed(true);
+    }
+
+    Owned<IPropertyTree> xgmml = graph->getXGMMLTree(true);
+
+    // New functionality, if a subgraph id is specified and we only want to load the xgmml for that subgraph
+    // then we need to conditionally pull a propertytree from the xgmml graph one and use that for the xgmml.
+
+    //JCSMORE this should be part of the API and therefore allow *only* the subtree to be pulled from the backend.
+
+    StringBuffer xml;
+    if (notEmpty(subGraphId))
+    {
+        VStringBuffer xpath("//node[@id='%s']", subGraphId);
+        toXML(xgmml->queryPropTree(xpath.str()), xml);
+    }
+    else
+        toXML(xgmml, xml);
+
+    g->setGraph(xml.str());
+
+    graphs.append(*g.getClear());
+}
+
 bool CWsWorkunitsEx::onWUGetGraph(IEspContext& context, IEspWUGetGraphRequest& req, IEspWUGetGraphResponse& resp)
 {
     try
@@ -3388,65 +3446,16 @@ bool CWsWorkunitsEx::onWUGetGraph(IEspContext& context, IEspWUGetGraphRequest& r
         bool running = (isRunning(*cw) && cw->getRunningGraph(runningGraph,id));
 
         IArrayOf<IEspECLGraphEx> graphs;
-
-        Owned<IConstWUGraphIterator> it;
-        IConstWUGraph *graph = NULL;
         if (isEmpty(req.getGraphName())) // JCS->GS - is this really required??
         {
-            it.setown(&cw->getGraphs(GraphTypeAny));
-            if (it->first())
-                graph = &it->query();
+            Owned<IConstWUGraphIterator> it = &cw->getGraphs(GraphTypeAny);
+            ForEach(*it)
+                readGraph(context, req.getSubGraphId(), id, running, &it->query(), graphs);
         }
         else
-            graph = cw->getGraph(req.getGraphName());
-        while (graph)
         {
-            SCMStringBuffer name, label, type;
-            graph->getName(name);
-            graph->getLabel(label);
-            graph->getTypeName(type);
-
-            Owned<IEspECLGraphEx> g = createECLGraphEx("","");
-            g->setName(name.str());
-            g->setLabel(label.str());
-            g->setType(type.str());
-            WUGraphState graphState = graph->getState();
-
-            if (running && (WUGraphRunning == graphState))
-            {
-                g->setRunning(true);
-                g->setRunningId(id);
-            }
-            else if (context.getClientVersion() > 1.20)
-            {
-                if (WUGraphComplete == graphState)
-                    g->setComplete(true);
-                else if (WUGraphFailed == graphState)
-                    g->setFailed(true);
-            }
-
-            Owned<IPropertyTree> xgmml = graph->getXGMMLTree(true);
-
-            // New functionality, if a subgraph id is specified and we only want to load the xgmml for that subgraph
-            // then we need to conditionally pull a propertytree from the xgmml graph one and use that for the xgmml.
-
-            //JCSMORE this should be part of the API and therefore allow *only* the subtree to be pulled from the backend.
-
-            StringBuffer xml;
-            if (notEmpty(req.getSubGraphId()))
-            {
-                VStringBuffer xpath("//node[@id='%s']", req.getSubGraphId());
-                toXML(xgmml->queryPropTree(xpath.str()), xml);
-            }
-            else
-                toXML(xgmml, xml);
-
-            g->setGraph(xml.str());
-
-            graphs.append(*g.getClear());
-            if (!it || !it->next())
-                break;
-            graph = &it->query();
+            Owned<IConstWUGraph> graph = cw->getGraph(req.getGraphName());
+            readGraph(context, req.getSubGraphId(), id, running, graph, graphs);
         }
         resp.setGraphs(graphs);
     }

esp/services/ws_workunits/ws_workunitsService.hpp

@@ -265,6 +265,8 @@ private:
     void cleanZAPFolder(IFile* zipDir, bool removeFolder);
     IPropertyTree* sendControlQuery(IEspContext &context, const char* target, const char* query, unsigned timeout);
     bool resetQueryStats(IEspContext &context, const char* target, IProperties* queryIds, IEspWUQuerySetQueryActionResponse& resp);
+    void readGraph(IEspContext& context, const char* subGraphId, WUGraphIDType& id, bool running,
+        IConstWUGraph* graph, IArrayOf<IEspECLGraphEx>& graphs);
 
     unsigned awusCacheMinutes;
     StringBuffer queryDirectory;

esp/src/eclwatch/ESPWorkunit.js

@@ -122,7 +122,12 @@ define([
         _SourceFilesSetter: function (SourceFiles) {
             var sourceFiles = [];
             for (var i = 0; i < SourceFiles.ECLSourceFile.length; ++i) {
-                sourceFiles.push(ESPResult.Get(lang.mixin({ wu: this.wu, Wuid: this.Wuid }, SourceFiles.ECLSourceFile[i])));
+                sourceFiles.push(ESPResult.Get(lang.mixin({ wu: this.wu, Wuid: this.Wuid, __hpcc_parentName: "" }, SourceFiles.ECLSourceFile[i])));
+                if (lang.exists("ECLSourceFiles.ECLSourceFile", SourceFiles.ECLSourceFile[i])) {
+                    for (var j = 0; j < SourceFiles.ECLSourceFile[i].ECLSourceFiles.ECLSourceFile.length; ++j) {
+                        sourceFiles.push(ESPResult.Get(lang.mixin({ wu: this.wu, Wuid: this.Wuid, __hpcc_parentName: SourceFiles.ECLSourceFile[i].Name }, SourceFiles.ECLSourceFile[i].ECLSourceFiles.ECLSourceFile[j])));
+                    }
+                }
             }
             this.set("sourceFiles", sourceFiles);
         },

esp/src/eclwatch/EventScheduleWorkunitWidget.js

@@ -147,7 +147,7 @@ define([
                     JobName: { label: this.i18n.JobName, sortable: true },
                     EventName: { label: this.i18n.EventName, width: 180, sortable: true },
                     EventText: { label: this.i18n.EventText, width: 180, sortable: true },
-                    Owner: { label: this.i18n.Owner, width: 180, sortable: false },
+                    Owner: { label: this.i18n.Owner, width: 180, sortable: true },
                     State: { label: this.i18n.State, width: 180, sortable: false }
                 }
             }, this.id + "EventGrid");

esp/src/eclwatch/SFDetailsWidget.js

@@ -75,6 +75,7 @@ define([
         postCreate: function (args) {
             this.inherited(arguments);
             this.summaryWidget = registry.byId(this.id + "_Summary");
+            this.deleteBtn = registry.byId(this.id + "Delete");
         },
 
         startup: function (args) {
@@ -229,6 +230,15 @@ define([
                 var tab = context.ensureLFPane(item.Name, item);
                 context.selectChild(tab, true);
             });
+            this.subfilesGrid.on("dgrid-select", function (evt) {
+                context.deleteBtn.set("disabled", true);
+            });
+            this.subfilesGrid.on("dgrid-deselect", function (evt) {
+                var selections = context.subfilesGrid.getSelected();
+                if (selections.length === 0) {
+                    context.deleteBtn.set("disabled", false);
+                }
+            });
             this.subfilesGrid.on(".dgrid-row:dblclick", function (evt) {
                 var item = context.subfilesGrid.row(evt).data;
                 var tab = context.ensureLFPane(item.Name, item);

esp/src/eclwatch/SourceFilesWidget.js

@@ -21,6 +21,7 @@ define([
     "dojo/_base/array",
     "dojo/on",
 
+    "dgrid/tree",
     "dgrid/selector",
 
     "hpcc/GridDetailsWidget",
@@ -29,7 +30,7 @@ define([
     "hpcc/ESPUtil"
 
 ], function (declare, lang, i18n, nlsHPCC, arrayUtil, on,
-                selector,
+                tree, selector,
                 GridDetailsWidget, ESPWorkunit, DelayLoadWidget, ESPUtil) {
     return declare("SourceFilesWidget", [GridDetailsWidget], {
         i18n: nlsHPCC,
@@ -57,6 +58,12 @@ define([
         },
 
         createGrid: function (domID) {
+            this.store.mayHaveChildren = function (item) {
+                return item.IsSuperFile;
+            };
+            this.store.getChildren = function (parent, options) {
+                return context.store.query({__hpcc_parentName: parent.Name});
+            };
             var retVal = new declare([ESPUtil.Grid(false, true)])({
                 store: this.store,
                 columns: {
@@ -64,18 +71,18 @@ define([
                         width: 27,
                         selectorType: 'checkbox'
                     }),
-                    Name: {
+                    Name: tree({
                         label: "Name", sortable: true,
                         formatter: function (Name, row) {
                             return dojoConfig.getImageHTML(row.IsSuperFile ? "folder_table.png" : "file.png") + "&nbsp;<a href='#' class='dgrid-row-url'>" + Name + "</a>";
                         }
-                    },
+                    }),
                     Count: { label: "Usage", width: 72, sortable: true }
                 }
             }, domID);
 
             var context = this;
-            retVal.on("." + this.id + "dgrid-row-url:click", function (evt) {
+            retVal.on(".dgrid-row-url:click", function (evt) {
                 if (context._onRowDblClick) {
                     var row = context.grid.row(evt).data;
                     context._onRowDblClick(row);
@@ -127,7 +134,7 @@ define([
                         row.sequence = idx;
                     });
                     context.store.setData(sourceFiles);
-                    context.grid.refresh();
+                    context.grid.set("query", { __hpcc_parentName: "" });
                 }
             });
         }

esp/src/eclwatch/WUQueryWidget.js

@@ -208,7 +208,6 @@ define([
         //  Implementation  ---
         getFilter: function () {
             var retVal = this.filter.toObject();
-            retval.Wuid =  retVal.Wuid.toUpperCase().trim();
             if (retVal.StartDate && retVal.FromTime) {
                 lang.mixin(retVal, {
                     StartDate: this.getISOString("FromDate", "FromTime")

esp/src/eclwatch/templates/WUQueryWidget.html

@@ -20,7 +20,7 @@
                     <div id="${id}Filter" data-dojo-type="FilterDropDownWidget">
                         <p id="${id}ArchivedWarning" style="display:none">${i18n.ArchivedWarning}</p>
                         <input id="${id}Type" title="${i18n.ArchivedOnly}" name="Type" colspan="2" data-dojo-attach-event="onClick:_onFilterType" data-dojo-props="value:'archived workunits'" data-dojo-type="dijit.form.CheckBox" />
-                        <input id="${id}Wuid" title="${i18n.WUID}:" name="Wuid" colspan="2" style="width:100%" data-dojo-props="trim: true, placeHolder:'W20130222-171723'" data-dojo-type="dijit.form.TextBox" />
+                        <input id="${id}Wuid" title="${i18n.WUID}:" name="Wuid" colspan="2" style="width:100%" data-dojo-props="trim: true, uppercase: true, placeHolder:'W20130222-171723'" data-dojo-type="dijit.form.TextBox" />
                         <input id="${id}Owner" title="${i18n.Owner}:" name="Owner" colspan="2" data-dojo-props="trim: true, placeHolder:'${i18n.jsmi}'" data-dojo-type="dijit.form.TextBox" />
                         <input id="${id}Jobname" title="${i18n.JobName}:" name="Jobname" colspan="2" data-dojo-props="trim: true, placeHolder:'${i18n.log_analysis_1}'" data-dojo-type="dijit.form.TextBox" />
                         <input id="${id}ClusterTargetSelect" title="${i18n.Cluster}:" name="Cluster" colspan="2" data-dojo-props="trim: true, placeHolder:'${i18n.Owner}'" data-dojo-type="TargetSelectWidget" />

esp/xslt/wsecl3_form.xsl

@@ -92,7 +92,7 @@
                 <link rel="stylesheet" type="text/css" href="/esp/files/gen_form.css"/>
                 <script type="text/javascript" src="/esp/files/req_array.js"/>
                 <script type="text/javascript" src="/esp/files/hashtable.js"/>
-                <script type="text/javascript" src="/esp/files/gen_form.js"/>
+                <script type="text/javascript" src="/esp/files/gen_form_wsecl.js"/>
                 <script type="text/javascript"><xsl:text disable-output-escaping="yes">
                 <![CDATA[
   var isIE = (navigator.appName == "Microsoft Internet Explorer");
@@ -114,7 +114,7 @@
                         <xsl:text disable-output-escaping="yes"><![CDATA[ + "<table id='"+newId+"'> </table></hr>"]]></xsl:text>
                     </xsl:if>
                     <xsl:text disable-output-escaping="yes"><![CDATA[
-       + "<input type='hidden' id='"+newId+"_ItemCt' name='"+newId+".itemcount' value='0' />"
+       + "<input type='hidden' id='"+newId+"_ItemCt' name='"+newId+".itemcount!' value='0' />"
           + "&nbsp;<input type='button' id='"+newId+"_AddBtn' onclick='appendRow(\""+newId+"\",\""+itemName+"\",get_"+typeName+"_Item)' value='Add' /> "
           + "<input type='button' id='"+newId+"_RvBtn' onclick='removeRow(\""+newId+"\",-1)' value='Delete' disabled='true' />" ]]></xsl:text>
                     <xsl:if test="not($useTableBorder)">

initfiles/bash/etc/init.d/pid.sh

@@ -14,112 +14,89 @@
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
 ################################################################################
+
+# Checks if Pid Directory exists and creates if it doesn't exist
 checkPidDir () {
-    
-    # Checks if Pid Directory exists and creates if it doesn't exist
     PIDFILEPATH=$1
-    #echo -n "Pid Path exists"
-    if [ -e ${PIDFILEPATH} ]; then
+    if [[ -e ${PIDFILEPATH} ]]; then
         log_success_msg ""
     else
         log_failure_msg "" 
         echo "Creating a Pid directory"
         /bin/mkdir -P ${PIDFILEPATH} 
-        if [ !-e ${PIDFILEPATH} ]; then
+        if [[ ! -e ${PIDFILEPATH} ]]; then
             echo "Can not create a Pid directory $PIDFILEPATH"
         else
             log_success_msg
         fi
     fi
-
 }
 
+# Creats a Pid file
 createPid () {
-
-    # Creats a Pid file
     PIDFILEPATH=$1
     PIDNO=$2
-    #echo -n "Creating Pid file $PIDFILEPATH"
     checkPid ${PIDFILEPATH}
-    if [ $__flagPid -eq 1 ]; then
-        if [ ${DEBUG} != "NO_DEBUG" ]; then
-            log_failure_msg "Pid file already exists"
-        fi
+    if [[ $__flagPid -eq 1 ]]; then
+        [[ ${DEBUG} != "NO_DEBUG" ]] && log_failure_msg "Pid file already exists"
         __pidCreated=0
     else
-        echo $PIDNO > ${PIDFILEPATH} 
+        echo $PIDNO > ${PIDFILEPATH}
         checkPid ${PIDFILEPATH}
-        if [ $__flagPid -eq 1 ]; then
-            if [ ${DEBUG} != "NO_DEBUG" ]; then
-                log_success_msg 
-            fi
+        if [[ $__flagPid -eq 1 ]]; then
+            [[ ${DEBUG} != "NO_DEBUG" ]] && log_success_msg 
             __pidCreated=1
         else
-            if [ ${DEBUG} != "NO_DEBUG" ]; then
-                log_failure_msg "Failed to create Pid"
-            fi
+            [[ ${DEBUG} != "NO_DEBUG" ]] && log_failure_msg "Failed to create Pid"
             __pidCreated=0
         fi
     fi
-
 }
 
+# Checks if Pid file exists
 checkPid () {
-
-    # Checks if Pid file exists
     PIDFILEPATH=$1
-    if [ -e ${PIDFILEPATH} ]; then
+    if [[ -e ${PIDFILEPATH} ]]; then
         __flagPid=1
     else
         __flagPid=0
     fi
-
 }
 
+# Reads the Pid file if Pidfile exists
 getPid () {
-    
-    # Reads the Pid file if Pidfile exists
     PIDFILEPATH=$1
     checkPid ${PIDFILEPATH}
-    if [ $__flagPid -eq 1 ]; then
-        __pidValue=`/bin/cat $PIDFILEPATH`
+    if [[ $__flagPid -eq 1 ]]; then
+        __pidValue=$(/bin/cat $PIDFILEPATH)
     else
         __pidValue=0
     fi
-
 }
 
+# Removes a Pid file 
 removePid () {
-
-    # Removes a Pid file 
     PIDFILEPATH=$1
-    #echo -n "Removing Pid file $PIDFILEPATH"
     checkPid ${PIDFILEPATH}
-    if [ $__flagPid -eq 0 ]; then
-        if [ ${DEBUG} != "NO_DEBUG" ]; then
-            log_failure_msg "Pidfile doesn't exist"
-        fi
+    if [[ $__flagPid -eq 0 ]]; then
+        [[ ${DEBUG} != "NO_DEBUG" ]] && log_failure_msg "Pidfile doesn't exist"
         __pidRemoved=0
     else
-        /bin/rm -rf ${PIDFILEPATH}
-        if [ ! -e ${PIDFILEPATH} ]; then
+        rm -rf ${PIDFILEPATH} > /dev/null 2>&1
+        if [[ ! -e ${PIDFILEPATH} ]]; then
             __pidRemoved=1
         else
-            if [ ${DEBUG} != "NO_DEBUG" ]; then
-                log_failure_msg "Failed to remove pid"
-            fi
+            [[ ${DEBUG} != "NO_DEBUG" ]] && log_failure_msg "Failed to remove pid"
             __pidRemoved=0
         fi
     fi
-
- 
 }
     
 checkPidExist() {
-        PIDFILEPATH=$1
+    PIDFILEPATH=$1
     getPid ${PIDFILEPATH}
-    if [ $__pidValue -ne 0 ]; then
-        ! /bin/kill -0 $__pidValue > /dev/null 2>&1 
+    if [[ $__pidValue -ne 0 ]]; then
+        ! kill -0 $__pidValue > /dev/null 2>&1 
         __pidExists=$?
     else
         __pidExists=0
@@ -145,10 +122,6 @@ check_status() {
     COMPPIDFILEPATH=$3
     SENTINELFILECHK=$4
 
-    checkPid $PIDFILEPATH
-    local pidfilepathExists=$__flagPid
-    checkPid $COMPPIDFILEPATH
-    local comppidfilepathExists=$__flagPid
     locked $LOCKFILEPATH
     local componentLocked=$flagLocked
     checkPidExist $PIDFILEPATH
@@ -159,64 +132,41 @@ check_status() {
     local sentinelFlag=$?
 
     # check if running and healthy
-    if [ $pidfilepathExists -eq 1 ] && [ $comppidfilepathExists -eq 1 ] && [ $componentLocked -eq 1 ] && [ $initRunning -eq 1 ] && [ $compRunning -eq 1 ]; then
-      if [ ${DEBUG} != "NO_DEBUG" ]; then
-        echo "everything is up except sentinel"
-      fi
-      if [ ${SENTINELFILECHK} -eq 1 ]; then
-        if [ ${sentinelFlag} -eq 0 ]; then
-          if [ ${DEBUG} != "NO_DEBUG" ]; then
-            echo "Sentinel is now up"
-          fi
-          return 0
-        else
-          if [ ${DEBUG} != "NO_DEBUG" ]; then
-            echo "Sentinel not yet located, process currently unhealthy"
-          fi
-          return 2
+    if [[ $componentLocked -eq 1 ]] && [[ $initRunning -eq 1 ]] && [[ $compRunning -eq 1 ]]; then
+        [[ ${DEBUG} != "NO_DEBUG" ]] && echo "everything is up except sentinel"
+        if [[ ${SENTINELFILECHK} -eq 1 ]]; then
+            if [[ ${sentinelFlag} -eq 0 ]]; then
+                [[ ${DEBUG} != "NO_DEBUG" ]] && echo "Sentinel not yet located, process currently unhealthy"
+                return 2 
+            fi
+            [[ ${DEBUG} != "NO_DEBUG" ]] && echo "Sentinel is now up"
         fi
-      else
         return 0
-      fi
     # check if shutdown and healthy
-    elif [ $pidfilepathExists -eq 0 ] && [ $comppidfilepathExists -eq 0 ] && [ $componentLocked -eq 0 ] && [ $initRunning -eq 0 ] && [ $compRunning -eq 0 ]; then
-      if [ ${SENTINELFILECHK} -eq 1 ]; then
-        if [ ${sentinelFlag} -eq 0 ]; then
-          if [ ${DEBUG} != "NO_DEBUG" ]; then
-            echo "Sentinel is up but orphaned"
-          fi
-          return 3
-        else
-          if [ ${DEBUG} != "NO_DEBUG" ]; then
-            echo "Sentinel is now down"
-          fi
-          return 1
+    elif [[ $componentLocked -eq 0 ]] && [[ $initRunning -eq 0 ]] && [[ $compRunning -eq 0 ]]; then
+        if [[ ${SENTINELFILECHK} -eq 1 ]]; then
+            if [[ ${sentinelFlag} -eq 1 ]]; then
+                [[ ${DEBUG} != "NO_DEBUG" ]] && echo "Sentinel is up but orphaned"
+                return 3
+            fi
+            [[ ${DEBUG} != "NO_DEBUG" ]] && echo "Sentinel is now down"
         fi
-      else
         return 1
-      fi
     else
-      if [ "${DEBUG}" != "NO_DEBUG" ]; then
-        [ $pidfilepathExists -eq 0 ]     && log_failure_msg "pid file path does not exist: $1"
-        [ $comppidfilepathExists -eq 0 ] && log_failure_msg "comp pid file path does not exist: $3"
-        [ $componentLocked -eq 0 ]       && log_failure_msg "component is not locked: $2"
-        [ $initRunning -eq 0 ]           && log_failure_msg "process for ${compName}_init.pid is not running"
-        [ $compRunning -eq 0 ]           && log_failure_msg "process for ${compName}.pid is not running"
-      fi
-      return 4
+        if [[ "${DEBUG}" != "NO_DEBUG" ]]; then
+            [[ $componentLocked -eq 0 ]] && log_failure_msg "component is not locked: $2"
+            [[ $initRunning -eq 0 ]]     && log_failure_msg "process for ${compName}_init.pid is not running"
+            [[ $compRunning -eq 0 ]]     && log_failure_msg "process for ${compName}.pid is not running"
+        fi
+        return 4
     fi
 }
 
 checkSentinelFile() {
     FILEPATH="${runtime}/${compName}"
-    if [ -d ${FILEPATH} ];then
-       fileCheckOP=`find ${FILEPATH} -name "*senti*"`
-       if [ ! -z "${fileCheckOP}" ]; then
-         return 0
-       else
-         return 3
-       fi
-    else
-       return 3
+    if [[ -d ${FILEPATH} ]]; then
+       fileCheckOP=$(find ${FILEPATH} -name "*senti*")
+       [[ ! -z "${fileCheckOP}" ]] && return 1
     fi
+    return 0
 }

initfiles/bin/init_thor

@@ -28,7 +28,7 @@ rm -f ${SENTINEL}
 
 killed() {
         echo "Stopping"
-        $deploydir/stop_thor $deploydir
+        kill_process ${SENTINEL} ${PID_NAME} 3
         exit 255
 }
 

initfiles/componentfiles/thor/run_thor

@@ -120,6 +120,7 @@ while [[ 1 ]]; do
         esac
     else
         echo failed to start thormaster$LCR, pausing for 30 seconds
+        $deploydir/stop_thor $deploydir
         sleep 30
     fi
     if [[ ! -e $SENTINEL ]]; then

initfiles/componentfiles/thor/start_backupnode.in

@@ -71,7 +71,7 @@ if [ ! -z ${THORPRIMARY} ]; then
 else
     groupName=${THORNAME}
 fi
-daliadmin server=$DALISERVER dfsgroup ${groupName} > $INSTANCE_DIR/slaves
+daliadmin server=$DALISERVER dfsgroup ${groupName} $INSTANCE_DIR/backupnode.slaves
 errcode=$?
 if [ 0 != ${errcode} ]; then
     echo 'failed to lookup dali group for $groupName'
@@ -90,7 +90,7 @@ rm -f $BACKUPNODE_DATA/*.ERR
 rm -f $BACKUPNODE_DATA/*.DAT
 
 echo Using backupnode directory $BACKUPNODE_DATA
-echo Reading slaves file $INSTANCE_DIR/slaves
+echo Reading slaves file $INSTANCE_DIR/backupnode.slaves
 echo Scanning files from dali ...
 
 NODEGROUP=$THORPRIMARY
@@ -112,16 +112,16 @@ fi
 # maximum number of threads frunssh will be permitted to use (capped by # slaves)
 MAXTHREADS=1000
 
-frunssh $INSTANCE_DIR/slaves "killall backupnode" -i:$SSHidentityfile -u:$SSHusername -pe:$SSHpassword -t:$SSHtimeout -a:$SSHretries -n:$MAXTHREADS -b >> $LOGFILE 2>&1
-echo frunssh $INSTANCE_DIR/slaves "/bin/sh -c 'mkdir -p `dirname $LOGPATH/${LOGDATE}_node%n.log`; mkdir -p $INSTANCE_DIR; $DEPLOY_DIR/backupnode -T -X $BACKUPNODE_REMOTEDATA %n %c %a %x $2 > $LOGPATH/${LOGDATE}_node%n.log 2>&1'" -i:$SSHidentityfile -u:$SSHusername -pe:$SSHpassword -t:$SSHtimeout -a:$SSHretries -n:$MAXTHREADS -b >> $LOGFILE 2>&1
-frunssh $INSTANCE_DIR/slaves "/bin/sh -c 'mkdir -p `dirname $LOGPATH/${LOGDATE}_node%n.log`; mkdir -p $INSTANCE_DIR; $DEPLOY_DIR/backupnode -T -X $BACKUPNODE_REMOTEDATA %n %c %a %x $2 > $LOGPATH/${LOGDATE}_node%n.log 2>&1'" -i:$SSHidentityfile -u:$SSHusername -pe:$SSHpassword -t:$SSHtimeout -a:$SSHretries -n:$MAXTHREADS -b >> $LOGFILE 2>&1
+frunssh $INSTANCE_DIR/backupnode.slaves "killall backupnode" -i:$SSHidentityfile -u:$SSHusername -pe:$SSHpassword -t:$SSHtimeout -a:$SSHretries -n:$MAXTHREADS -b >> $LOGFILE 2>&1
+echo frunssh $INSTANCE_DIR/backupnode.slaves "/bin/sh -c 'mkdir -p `dirname $LOGPATH/${LOGDATE}_node%n.log`; mkdir -p $INSTANCE_DIR; $DEPLOY_DIR/backupnode -T -X $BACKUPNODE_REMOTEDATA %n %c %a %x $2 > $LOGPATH/${LOGDATE}_node%n.log 2>&1'" -i:$SSHidentityfile -u:$SSHusername -pe:$SSHpassword -t:$SSHtimeout -a:$SSHretries -n:$MAXTHREADS -b >> $LOGFILE 2>&1
+frunssh $INSTANCE_DIR/backupnode.slaves "/bin/sh -c 'mkdir -p `dirname $LOGPATH/${LOGDATE}_node%n.log`; mkdir -p $INSTANCE_DIR; $DEPLOY_DIR/backupnode -T -X $BACKUPNODE_REMOTEDATA %n %c %a %x $2 > $LOGPATH/${LOGDATE}_node%n.log 2>&1'" -i:$SSHidentityfile -u:$SSHusername -pe:$SSHpassword -t:$SSHtimeout -a:$SSHretries -n:$MAXTHREADS -b >> $LOGFILE 2>&1
 
 echo ------------------------------
 sleep 5
 echo ------------------------------
 echo Waiting for backup to complete
 
-nohup backupnode -W $INSTANCE_DIR/slaves $BACKUPNODE_DATA >> $LOGFILE 2>&1 &
+nohup backupnode -W $INSTANCE_DIR/backupnode.slaves $BACKUPNODE_DATA >> $LOGFILE 2>&1 &
 pid=`${PIDOF} backupnode`
 trap "echo start_backupnode exiting, backupnode process $pid still continuing; exit 0" 2
 if [ -n "$pid" ]; then

plugins/fileservices/fileservices.hpp

@@ -58,6 +58,7 @@ FILESERVICES_API void FILESERVICES_CALL fsSprayVariable_v5(ICodeContext *ctx, co
 FILESERVICES_API void FILESERVICES_CALL fsSprayXml(ICodeContext *ctx, const char * sourceIP, const char * sourcePath, int sourceMaxRecordSize, const char *sourceRowTag, const char *sourceEncoding, const char * destinationGroup, const char * destinationLogicalName, int timeOut, const char * espServerIpPort, int maxConnections, bool overwrite, bool replicate, bool compress=false, bool failIfNoSourceFile=false);
 FILESERVICES_API void FILESERVICES_CALL fsDespray(ICodeContext *ctx, const char * sourceLogicalName, const char * destinationIP, const char * destinationPath, int timeOut, const char * espServerIpPort, int maxConnections, bool overwrite=false);
 FILESERVICES_API void FILESERVICES_CALL fsCopy(ICodeContext *ctx, const char * sourceLogicalName, const char *destinationGroup, const char * destinationLogicalName, const char * sourceDali, int timeOut, const char * espServerIpPort, int maxConnections, bool overwrite, bool replicate, bool asSuperfile, bool compress, bool forcePush, int transferBufferSize);
+FILESERVICES_API void FILESERVICES_CALL fsCopy_v2(ICodeContext *ctx, const char * sourceLogicalName, const char *destinationGroup, const char * destinationLogicalName, const char * sourceDali, int timeOut, const char * espServerIpPort, int maxConnections, bool overwrite, bool replicate, bool asSuperfile, bool compress, bool forcePush, int transferBufferSize, bool preserveCompression);
 FILESERVICES_API void FILESERVICES_CALL fsDkc(ICodeContext *ctx, const char * sourceLogicalName, const char * destinationIP, const char * destinationPath, int timeOut, const char * espServerIpPort, int maxConnections, bool overwrite);
 FILESERVICES_API void FILESERVICES_CALL fsReplicate(ICodeContext *ctx, const char * sourceLogicalName, int timeOut, const char * espServerIpPort);
 FILESERVICES_API void FILESERVICES_CALL fsCreateSuperFile(ICodeContext *ctx, const char *lsuperfn, bool sequentialparts, bool ifdoesnotexist);
@@ -89,6 +90,7 @@ FILESERVICES_API char * FILESERVICES_CALL fsfSprayVariable_v5(ICodeContext *ctx,
 FILESERVICES_API char * FILESERVICES_CALL fsfSprayXml(ICodeContext *ctx, const char * sourceIP, const char * sourcePath, int sourceMaxRecordSize, const char *sourceRowTag, const char *sourceEncoding, const char * destinationGroup, const char * destinationLogicalName, int timeOut, const char * espServerIpPort, int maxConnections, bool overwrite, bool replicate, bool compress=false, bool failIfNoSourceFile=false);
 FILESERVICES_API char * FILESERVICES_CALL fsfDespray(ICodeContext *ctx, const char * sourceLogicalName, const char * destinationIP, const char * destinationPath, int timeOut, const char * espServerIpPort, int maxConnections, bool overwrite=false);
 FILESERVICES_API char * FILESERVICES_CALL fsfCopy(ICodeContext *ctx, const char * sourceLogicalName, const char *destinationGroup, const char * destinationLogicalName, const char * sourceDali, int timeOut, const char * espServerIpPort, int maxConnections, bool overwrite, bool replicate, bool asSuperfile, bool compress, bool forcePush, int transferBufferSize);
+FILESERVICES_API char * FILESERVICES_CALL fsfCopy_v2(ICodeContext *ctx, const char * sourceLogicalName, const char *destinationGroup, const char * destinationLogicalName, const char * sourceDali, int timeOut, const char * espServerIpPort, int maxConnections, bool overwrite, bool replicate, bool asSuperfile, bool compress, bool forcePush, int transferBufferSize, bool preserveCompression);
 FILESERVICES_API char * FILESERVICES_CALL fsfDkc(ICodeContext *ctx, const char * sourceLogicalName, const char * destinationIP, const char * destinationPath, int timeOut, const char * espServerIpPort, int maxConnections, bool overwrite);
 FILESERVICES_API char * FILESERVICES_CALL fsfReplicate(ICodeContext *ctx, const char * sourceLogicalName, int timeOut, const char * espServerIpPort);
 FILESERVICES_API char *  FILESERVICES_CALL fsfMonitorLogicalFileName(ICodeContext *ctx, const char *eventname, const char *_lfn,int shotcount, const char * espServerIpPort);

system/jlib/jdebug.cpp

@@ -50,6 +50,9 @@
 #ifdef __APPLE__
  #include <sys/param.h>
  #include <sys/mount.h>
+ #include <sys/sysctl.h>
+ #include <mach/task.h>
+ #include <mach/mach_init.h>
 #endif
 
 //===========================================================================
@@ -1116,8 +1119,21 @@ void getMemStats(StringBuffer &out, unsigned &memused, unsigned &memtot)
 #endif
     memused = mu+su;
     memtot = mt+st;
-#endif
-#if defined (__FreeBSD__) || defined (__APPLE__)
+#elif defined (__APPLE__)
+    __uint64 bytes;
+    size_t len = sizeof(bytes);
+    sysctlbyname("hw.memsize", &bytes, &len, NULL, 0);
+    // See http://miknight.blogspot.com/2005/11/resident-set-size-in-mac-os-x.html
+    struct task_basic_info t_info;
+    mach_msg_type_number_t t_info_count = TASK_BASIC_INFO_COUNT;
+    task_info(current_task(), TASK_BASIC_INFO, (task_info_t)&t_info, &t_info_count);
+    out.appendf("RES=%" I64F "uMiB VIRT=%" I64F "uMiB TOT=%" I64F "uMiB",
+            (__uint64) t_info.resident_size/(1024*1024),
+            (__uint64) t_info.virtual_size/(1024*1024),
+            bytes/(1024*1024));
+    memused = t_info.resident_size;
+    memtot = t_info.virtual_size;
+#elif defined (__FreeBSD___)
     UNIMPLEMENTED;
 #endif
 }
@@ -1799,9 +1815,6 @@ class CExtendedStats  // Disk network and cpu stats
             kbuf = NULL;
         }
 #endif
-#if defined (__FreeBSD__) || defined (__APPLE__)
-        UNIMPLEMENTED;
-#endif
         data = NULL;
         return 0;
     }

system/security/LdapSecurity/ldapconnection.cpp

@@ -1036,6 +1036,61 @@ public:
     }
 };
 
+class CLDAPMessage
+{
+public:
+    LDAPMessage *msg;
+    CLDAPMessage()       { msg = NULL; }
+    ~CLDAPMessage()      { ldapMsgFree(); }
+    inline void ldapMsgFree()  { if (msg) { ldap_msgfree(msg); msg = NULL;} }
+    inline operator LDAPMessage *() const { return msg; }
+};
+
+static CriticalSection  mpaCrit;
+static __int64 getMaxPwdAge(Owned<ILdapConnectionPool> _conns, const char * _baseDN)
+{
+    static time_t   lastPwdAgeCheck = 0;
+    static __int64  maxPwdAge = PWD_NEVER_EXPIRES;
+    #define HOURLY  ((time_t)(60*60*1000))
+
+    CriticalBlock block(mpaCrit);
+    if (lastPwdAgeCheck != 0 && (((msTick() - lastPwdAgeCheck) < HOURLY)))//in case it was retrieved whilst this thread blocked
+        return maxPwdAge;
+
+    DBGLOG("Retrieving LDAP 'maxPwdAge'");
+    char* attrs[] = {"maxPwdAge", NULL};
+    CLDAPMessage searchResult;
+    TIMEVAL timeOut = {LDAPTIMEOUT,0};
+    Owned<ILdapConnection> lconn = _conns->getConnection();
+    LDAP* sys_ld = ((CLdapConnection*)lconn.get())->getLd();
+    int result = ldap_search_ext_s(sys_ld, (char*)_baseDN, LDAP_SCOPE_BASE, NULL,
+        attrs, 0, NULL, NULL, &timeOut, LDAP_NO_LIMIT, &searchResult.msg);
+    if(result != LDAP_SUCCESS)
+    {
+        DBGLOG("ldap_search_ext_s error: %s, when searching maxPwdAge", ldap_err2string( result ));
+        return 0;
+    }
+    unsigned entries = ldap_count_entries(sys_ld, searchResult);
+    if(entries == 0)
+    {
+        DBGLOG("ldap_search_ext_s error: Could not find maxPwdAge");
+        return 0;
+    }
+    maxPwdAge = 0;
+    CLDAPGetValuesWrapper vals(sys_ld, searchResult.msg, "maxPwdAge");
+    if (vals.hasValues())
+    {
+        char *val = vals.queryValues()[0];
+        if (*val == '-')
+            ++val;
+        for (int x=0; val[x]; x++)
+            maxPwdAge = maxPwdAge * 10 + ( (int)val[x] - '0');
+    }
+    else
+        maxPwdAge = PWD_NEVER_EXPIRES;
+    lastPwdAgeCheck = msTick();
+    return maxPwdAge;
+}
 
 class CLdapClient : public CInterface, implements ILdapClient
 {
@@ -1047,18 +1102,6 @@ private:
     Owned<CLdapConfig>   m_ldapconfig;
     StringBuffer         m_pwscheme;
     bool                 m_domainPwdsNeverExpire;//no domain policy for password expiration
-    __int64              m_maxPwdAge;
-    time_t               m_lastPwdAgeCheck;
-
-    class CLDAPMessage
-    {
-    public:
-        LDAPMessage *msg;
-        CLDAPMessage()       { msg = NULL; }
-        ~CLDAPMessage()      { ldapMsgFree(); }
-        inline void ldapMsgFree()  { if (msg) { ldap_msgfree(msg); msg = NULL;} }
-        inline operator LDAPMessage *() const { return msg; }
-    };
 
 public:
     IMPLEMENT_IINTERFACE
@@ -1071,7 +1114,6 @@ public:
         else
             m_connections.setown(new CLdapConnectionPool(m_ldapconfig.get()));  
         m_pp = NULL;
-        m_lastPwdAgeCheck = 0;
         //m_defaultFileScopePermission = -2;
         //m_defaultWorkunitScopePermission = -2;
     }
@@ -1121,50 +1163,12 @@ public:
         createLdapBasedn(NULL, m_ldapconfig->getResourceBasedn(rtype), PT_DEFAULT);
     }
 
-    virtual __int64 getMaxPwdAge()
-    {
-        if ((msTick() - m_lastPwdAgeCheck) < (60*1000))
-            return m_maxPwdAge;
-        char* attrs[] = {"maxPwdAge", NULL};
-        CLDAPMessage searchResult;
-        TIMEVAL timeOut = {LDAPTIMEOUT,0};
-        Owned<ILdapConnection> lconn = m_connections->getConnection();
-        LDAP* sys_ld = ((CLdapConnection*)lconn.get())->getLd();
-        int result = ldap_search_ext_s(sys_ld, (char*)m_ldapconfig->getBasedn(), LDAP_SCOPE_BASE, NULL,
-                                        attrs, 0, NULL, NULL, &timeOut, LDAP_NO_LIMIT, &searchResult.msg);
-        if(result != LDAP_SUCCESS)
-        {
-            DBGLOG("ldap_search_ext_s error: %s, when searching maxPwdAge", ldap_err2string( result ));
-            return 0;
-        }
-        unsigned entries = ldap_count_entries(sys_ld, searchResult);
-        if(entries == 0)
-        {
-            DBGLOG("ldap_search_ext_s error: Could not find maxPwdAge");
-            return 0;
-        }
-        m_maxPwdAge = 0;
-        CLDAPGetValuesWrapper vals(sys_ld, searchResult.msg, "maxPwdAge");
-        if (vals.hasValues())
-        {
-            char *val = vals.queryValues()[0];
-            if (*val == '-')
-                ++val;
-            for (int x=0; val[x]; x++)
-                m_maxPwdAge = m_maxPwdAge * 10 + ( (int)val[x] - '0');
-        }
-        else
-            m_maxPwdAge = PWD_NEVER_EXPIRES;
-        m_lastPwdAgeCheck = msTick();
-        return m_maxPwdAge;
-    }
-
     void calcPWExpiry(CDateTime &dt, unsigned len, char * val)
     {
         __int64 time = 0;
         for (unsigned x=0; x < len; x++)
             time = time * 10 + ( (int)val[x] - '0');
-        time += m_maxPwdAge;
+        time += getMaxPwdAge(m_connections,(char*)m_ldapconfig->getBasedn() );
         dt.setFromFILETIME(time);
         dt.adjustTime(dt.queryUtcToLocalDelta());
     }
@@ -1180,8 +1184,7 @@ public:
             if(!username || !*username || !password || !*password)
                 return false;
 
-            getMaxPwdAge();//sets m_maxPwdAge
-            if (m_maxPwdAge != PWD_NEVER_EXPIRES)
+            if (getMaxPwdAge(m_connections,(char*)m_ldapconfig->getBasedn()) != PWD_NEVER_EXPIRES)
                 m_domainPwdsNeverExpire = false;
             else
                 m_domainPwdsNeverExpire = true;

system/security/shared/authmap.ipp

@@ -46,6 +46,15 @@ public:
     IMPLEMENT_IINTERFACE;
 
     CAuthMap(ISecManager* secmgr) {m_secmgr = secmgr;};
+    virtual ~CAuthMap()
+    {
+        ForEachItemIn(x, m_resourcelists)
+        {
+            ISecResourceList* rlist = m_resourcelists.item(x).list();
+            if (rlist)
+                rlist->Release();
+        }
+    }
     int add(const char* path, ISecResourceList* resourceList);
     bool shouldAuth(const char* path);
     ISecResourceList* queryResourceList(const char* path);

testing/regress/ecl/aggidx1.ecl

@@ -24,6 +24,8 @@ useSequential := #IFDEFINED(root.useSequential, false);
 
 //--- end of version configuration ---
 
+#option ('obfuscateOutput', true);
+
 import $.setup;
 sq := setup.sq(multiPart);
 

testing/regress/ecl/childds1.ecl

@@ -0,0 +1,58 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2015 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//This is needed to prevent cntBad being hoisted before the resourcing, and becoming unconditional
+//The tests are part of the work aiming to remove this code.
+#option ('workunitTemporaries', false);
+
+idRec := { unsigned id; };
+mainRec := { unsigned seq, dataset(idRec) ids };
+
+idRec createId(unsigned id) := TRANSFORM
+    SELF.id := id;
+END; 
+
+mainRec createMain(unsigned c, unsigned num) := TRANSFORM
+    SELF.seq := c;
+    SELF.ids := DATASET(num, createId(c + (COUNTER-1)));
+END;
+
+ds := NOFOLD(DATASET(4, createMain(COUNTER, 3)));
+
+boolean assertTrue(boolean x, const varstring msg = 'Condition should have been true') := BEGINC++
+    #option pure
+    if (!x)
+        rtlFail(0, msg);
+    return x;
+ENDC++;
+
+trueValue := true : stored('trueValue');
+falseValue := false : stored('falseValue');
+
+//Case 1 - a child query where the filter is always correct
+cnt := COUNT(ds(assertTrue(seq < 10, 'seq < 10'))) + NOFOLD(100000);
+output(ds(seq != cnt));
+
+
+//Case 2 - a condition that is always false is used from a branch that should never be executed
+cntBad := COUNT(ds(assertTrue(seq > 10, 'seq > 10'))) + NOFOLD(100000);
+
+//NOFOLD is required to stop code generator converting this to a filter (trueValue && ...) which means that cntBad is evaluated always
+//See childds1err.ecl
+cond := IF(trueValue, ds, NOFOLD(ds)(seq != cntBad));
+output(cond);
+

testing/regress/ecl/childds1err.ecl

@@ -0,0 +1,62 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2015 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//fail
+//NOTE: This example shouldn't really fail, it illustates a problem converting
+//IF(c, ds, ds(f1)) to ds(c OR f1)
+//which causes dependencies of f1 to be evaluated when they wouldn't be otherwise.
+
+//This is needed to prevent cntBad being hoisted before the resourcing, and becoming unconditional
+//The tests are part of the work aiming to remove this code.
+#option ('workunitTemporaries', false);
+
+idRec := { unsigned id; };
+mainRec := { unsigned seq, dataset(idRec) ids };
+
+idRec createId(unsigned id) := TRANSFORM
+    SELF.id := id;
+END; 
+
+mainRec createMain(unsigned c, unsigned num) := TRANSFORM
+    SELF.seq := c;
+    SELF.ids := DATASET(num, createId(c + (COUNTER-1)));
+END;
+
+ds := NOFOLD(DATASET(4, createMain(COUNTER, 3)));
+
+boolean assertTrue(boolean x, const varstring msg = 'Condition should have been true') := BEGINC++
+    #option pure
+    if (!x)
+        rtlFail(0, msg);
+    return x;
+ENDC++;
+
+trueValue := true : stored('trueValue');
+falseValue := false : stored('falseValue');
+
+//Case 1 - a child query where the filter is always correct
+cnt := COUNT(ds(assertTrue(seq < 10, 'seq < 10'))) + NOFOLD(100000);
+output(ds(seq != cnt));
+
+
+//Case 2 - a condition that is always false is used from a branch that should never be executed
+cntBad := COUNT(ds(assertTrue(seq > 10, 'seq > 10'))) + NOFOLD(100000);
+
+//Problem1: Converting this to a filtered disk read means that cntBad is evaluated always
+cond := IF(trueValue, ds, ds(seq != cntBad));
+output(cond);
+

testing/regress/ecl/childds2.ecl

@@ -0,0 +1,53 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2015 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//This is needed to prevent cntBad being hoisted before the resourcing, and becoming unconditional
+//The tests are part of the work aiming to remove this code.
+#option ('workunitTemporaries', false);
+
+idRec := { unsigned id; };
+mainRec := { unsigned seq, dataset(idRec) ids };
+
+idRec createId(unsigned id) := TRANSFORM
+    SELF.id := id;
+END; 
+
+mainRec createMain(unsigned c, unsigned num) := TRANSFORM
+    SELF.seq := c;
+    SELF.ids := DATASET(num, createId(c + (COUNTER-1)));
+END;
+
+ds := NOFOLD(DATASET(4, createMain(COUNTER, 3)));
+
+boolean assertTrue(boolean x, const varstring msg = 'Condition should have been true') := BEGINC++
+    #option pure
+    if (!x)
+        rtlFail(0, msg);
+    return x;
+ENDC++;
+
+trueValue := true : stored('trueValue');
+falseValue := false : stored('falseValue');
+
+//Case 2 - a condition that is always false is used from a multiple branches that should never be executed
+cntBad := COUNT(ds(assertTrue(seq > 10, 'seq > 10'))) + NOFOLD(100000);
+
+//Problem1: Converting this to a filtered disk read means that cntBad is evaluated always
+cond1 := IF(trueValue, ds, NOFOLD(ds)(seq != cntBad));
+cond2 := IF(falseValue, NOFOLD(ds)(seq != cntBad), ds);
+output(cond1+cond2);
+

testing/regress/ecl/childds3.ecl

@@ -0,0 +1,55 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2015 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+idRec := { unsigned id; };
+mainRec := { unsigned seq, dataset(idRec) ids };
+
+idRec createId(unsigned id) := TRANSFORM
+    SELF.id := id;
+END; 
+
+mainRec createMain(unsigned c, unsigned num) := TRANSFORM
+    SELF.seq := c;
+    SELF.ids := DATASET(num, createId(c + (COUNTER-1)));
+END;
+
+ds := NOFOLD(DATASET(4, createMain(COUNTER, 3)));
+
+boolean assertTrue(boolean x, const varstring msg = 'Condition should have been true') := BEGINC++
+    #option pure
+    if (!x)
+        rtlFail(0, msg);
+    return x;
+ENDC++;
+
+trueValue := true : stored('trueValue');
+falseValue := false : stored('falseValue');
+
+evalFilter(mainRec l) := FUNCTION
+
+    sortedIds := nofold(sort(l.ids, id));
+    cntGood := COUNT(sortedIds(assertTrue(id < 10, 'seq < 10')));
+    RETURN count(sortedIds(id != cntGood)) = 3;
+END;
+
+mainRec t(mainRec l) := TRANSFORM,SKIP(NOT evalFilter(l))
+    SELF := l;
+END;
+
+output(PROJECT(ds, t(LEFT)));
+
+output(ds(evalFilter(ds)));

testing/regress/ecl/childds4.ecl

@@ -0,0 +1,57 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2015 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+idRec := { unsigned id; };
+mainRec := { unsigned seq, dataset(idRec) ids };
+
+idRec createId(unsigned id) := TRANSFORM
+    SELF.id := id;
+END; 
+
+mainRec createMain(unsigned c, unsigned num) := TRANSFORM
+    SELF.seq := c;
+    SELF.ids := DATASET(num, createId(c + (COUNTER-1)));
+END;
+
+ds := NOFOLD(DATASET(4, createMain(COUNTER, 3)));
+
+boolean assertTrue(boolean x, const varstring msg = 'Condition should have been true') := BEGINC++
+    #option pure
+    if (!x)
+        rtlFail(0, msg);
+    return x;
+ENDC++;
+
+trueValue := true : stored('trueValue');
+falseValue := false : stored('falseValue');
+
+evalFilter(mainRec l) := FUNCTION
+
+    sortedIds := nofold(sort(l.ids, id));
+    cntBad := COUNT(sortedIds(assertTrue(id > 10, 'seq > 10')));
+    
+    f := IF(trueValue, sortedIds, NOFOLD(sortedIds(id != cntBad)));
+    RETURN COUNT(NOFOLD(f)) != 0;
+END;
+
+mainRec t(mainRec l) := TRANSFORM,SKIP(NOT evalFilter(l))
+    SELF := l;
+END;
+
+output(PROJECT(ds, t(LEFT)));
+
+output(ds(evalFilter(ds)));

testing/regress/ecl/childds5.ecl

@@ -0,0 +1,58 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2015 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+idRec := { unsigned id; };
+mainRec := { unsigned seq, dataset(idRec) ids };
+
+idRec createId(unsigned id) := TRANSFORM
+    SELF.id := id;
+END; 
+
+mainRec createMain(unsigned c, unsigned num) := TRANSFORM
+    SELF.seq := c;
+    SELF.ids := DATASET(num, createId(c + (COUNTER-1)));
+END;
+
+ds := NOFOLD(DATASET(4, createMain(COUNTER, 3)));
+
+boolean assertTrue(boolean x, const varstring msg = 'Condition should have been true') := BEGINC++
+    #option pure
+    if (!x)
+        rtlFail(0, msg);
+    return x;
+ENDC++;
+
+trueValue := true : stored('trueValue');
+falseValue := false : stored('falseValue');
+
+evalFilter(mainRec l) := FUNCTION
+
+    sortedIds := nofold(sort(l.ids, id));
+    cntBad1 := COUNT(sortedIds(assertTrue(id > 10, 'seq > 10')));
+    cntBad2 := COUNT(sortedIds(assertTrue(id > 11, 'seq > 11')));
+    
+    f := IF(trueValue, sortedIds, NOFOLD(sortedIds(id != cntBad1 * cntBad2)));
+    RETURN COUNT(NOFOLD(f)) != 0;
+END;
+
+mainRec t(mainRec l) := TRANSFORM,SKIP(NOT evalFilter(l))
+    SELF := l;
+END;
+
+output(PROJECT(ds, t(LEFT)));
+
+output(ds(evalFilter(ds)));

testing/regress/ecl/childds6.ecl

@@ -0,0 +1,59 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2015 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+idRec := { unsigned id; };
+mainRec := { unsigned seq, dataset(idRec) ids };
+
+idRec createId(unsigned id) := TRANSFORM
+    SELF.id := id;
+END; 
+
+mainRec createMain(unsigned c, unsigned num) := TRANSFORM
+    SELF.seq := c;
+    SELF.ids := DATASET(num, createId(c + (COUNTER-1)));
+END;
+
+ds := NOFOLD(DATASET(4, createMain(COUNTER, 3)));
+
+boolean assertTrue(boolean x, const varstring msg = 'Condition should have been true') := BEGINC++
+    #option pure
+    if (!x)
+        rtlFail(0, msg);
+    return x;
+ENDC++;
+
+trueValue := true : stored('trueValue');
+falseValue := false : stored('falseValue');
+
+evalFilter(mainRec l) := FUNCTION
+
+    sortedIds := nofold(sort(l.ids, id));
+    bad := sortedIds(assertTrue(id > 10, 'seq > 10'));
+    cntBad1 := COUNT(bad);
+    cntBad2 := SUM(bad, id);
+    
+    f := IF(trueValue, sortedIds, NOFOLD(sortedIds(id != cntBad1 * cntBad2)));
+    RETURN COUNT(NOFOLD(f)) != 0;
+END;
+
+mainRec t(mainRec l) := TRANSFORM,SKIP(NOT evalFilter(l))
+    SELF := l;
+END;
+
+output(PROJECT(ds, t(LEFT)));
+
+output(ds(evalFilter(ds)));

testing/regress/ecl/childds7.ecl

@@ -0,0 +1,58 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2015 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+idRec := { unsigned id; };
+mainRec := { unsigned seq, dataset(idRec) ids };
+
+idRec createId(unsigned id) := TRANSFORM
+    SELF.id := id;
+END; 
+
+mainRec createMain(unsigned c, unsigned num) := TRANSFORM
+    SELF.seq := c;
+    SELF.ids := DATASET(num, createId(c + (COUNTER-1)));
+END;
+
+ds := NOFOLD(DATASET(4, createMain(COUNTER, 3)));
+
+boolean assertTrue(boolean x, const varstring msg = 'Condition should have been true') := BEGINC++
+    #option pure
+    if (!x)
+        rtlFail(0, msg);
+    return x;
+ENDC++;
+
+trueValue := true : stored('trueValue');
+falseValue := false : stored('falseValue');
+
+evalFilter(mainRec l) := FUNCTION
+
+    dedupIds := DEDUP(l.ids, id);
+    sortedIds := nofold(sort(l.ids, id));
+    bad := sortedIds(assertTrue(id > 10, 'seq > 10'));
+    cntGood := COUNT(sortedIds(id != 10000));
+    cntBad1 := COUNT(bad);
+    
+    f := IF(trueValue, dedupIds(id != cntGood), NOFOLD(dedupIds(id != cntBad1)));
+    RETURN COUNT(NOFOLD(f)) != 0;
+END;
+
+mainRec t(mainRec l) := TRANSFORM,SKIP(NOT evalFilter(l))
+    SELF := l;
+END;
+
+output(PROJECT(ds, t(LEFT)));

testing/regress/ecl/childds7b.ecl

@@ -0,0 +1,58 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2015 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+idRec := { unsigned id; };
+mainRec := { unsigned seq, dataset(idRec) ids };
+
+idRec createId(unsigned id) := TRANSFORM
+    SELF.id := id;
+END; 
+
+mainRec createMain(unsigned c, unsigned num) := TRANSFORM
+    SELF.seq := c;
+    SELF.ids := DATASET(num, createId(c + (COUNTER-1)));
+END;
+
+ds := NOFOLD(DATASET(4, createMain(COUNTER, 3)));
+
+boolean assertTrue(boolean x, const varstring msg = 'Condition should have been true') := BEGINC++
+    #option pure
+    if (!x)
+        rtlFail(0, msg);
+    return x;
+ENDC++;
+
+trueValue := true : stored('trueValue');
+falseValue := false : stored('falseValue');
+
+evalFilter(mainRec l) := FUNCTION
+
+    dedupIds := l.ids;
+    sortedIds := nofold(sort(l.ids, id));
+    bad := sortedIds(assertTrue(id > 10, 'seq > 10'));
+    cntGood := COUNT(sortedIds(id != 10000));
+    cntBad1 := COUNT(bad);
+    
+    f := IF(trueValue, dedupIds(id != cntGood), NOFOLD(dedupIds(id != cntBad1)));
+    RETURN COUNT(NOFOLD(f)) != 0;
+END;
+
+mainRec t(mainRec l) := TRANSFORM,SKIP(NOT evalFilter(l))
+    SELF := l;
+END;
+
+output(PROJECT(ds, t(LEFT)));

testing/regress/ecl/filecompcopy.ecl

@@ -65,6 +65,8 @@ outdata := NORMALIZE(one_per_node, numrecs, fillRow(LEFT, counter));
 
 copiedcmp1 := DATASET('nhtest::testfile_exp_copy_cmp', rec, flat, __compressed__);
 copiedcmp2 := DATASET('nhtest::testfile_cmp_copy_cmp', rec, flat, __compressed__);
+copiedcmp3 := DATASET('nhtest::testfile_cmp_copy_cmp2', rec, flat, __compressed__);
+copiedcmp4 := DATASET('nhtest::testfile_cmp_copy_cmp3', rec, flat, __compressed__);
 copiedexp := DATASET('nhtest::testfile_cmp_copy_exp', rec, flat);
 
 unsigned compareDatasets(dataset(rec) ds1,dataset(rec) ds2) := FUNCTION
@@ -106,6 +108,34 @@ sequential (
             true                        // compress
            ),  
 
+// test copy compressed to compressed with preservecompression flag
+
+FileServices.Copy('nhtest::testfile_cmp',   // sourceLogicalName
+            '',                             // destinationGroup
+            'nhtest::testfile_cmp_copy_cmp2', // destinationLogicalName
+            ,                               // sourceDali
+            5*60*1000,                      // timeOut
+            ,                               // espServerIpPort
+            ,                               // maxConnections
+            true,                           // allowoverwrite
+            false,                          // replicate
+            ,                               // asSuperfile,
+            ,
+            PRESERVECOMPRESSION:=true
+           ),
+
+// test copy compressed to compressed without preservecompression flag
+
+FileServices.Copy('nhtest::testfile_cmp',   // sourceLogicalName
+            '',                             // destinationGroup
+            'nhtest::testfile_cmp_copy_cmp3', // destinationLogicalName
+            ,                               // sourceDali
+            5*60*1000,                      // timeOut
+            ,                               // espServerIpPort
+            ,                               // maxConnections
+            true,                           // allowoverwrite
+            false                           // replicate
+           ),
 
 // test copy compressed to expanded  
 
@@ -117,13 +147,15 @@ sequential (
             ,                               // espServerIpPort
             ,                               // maxConnections, 
             true,                           // allowoverwrite
-            true,                       // replicate=false, 
+            false,                       // replicate=false,
             ,                           // asSuperfile, 
             false                       // compress
            ),
            
    OUTPUT(compareDatasets(outdata,copiedcmp1)),
    OUTPUT(compareDatasets(outdata,copiedcmp2)),
+   OUTPUT(compareDatasets(outdata,copiedcmp3)),
+   OUTPUT(compareDatasets(outdata,copiedcmp4)),
    OUTPUT(compareDatasets(outdata,copiedexp))
            
  );

testing/regress/ecl/key/childds1.xml

@@ -0,0 +1,12 @@
+<Dataset name='Result 1'>
+ <Row><seq>1</seq><ids><Row><id>1</id></Row><Row><id>2</id></Row><Row><id>3</id></Row></ids></Row>
+ <Row><seq>2</seq><ids><Row><id>2</id></Row><Row><id>3</id></Row><Row><id>4</id></Row></ids></Row>
+ <Row><seq>3</seq><ids><Row><id>3</id></Row><Row><id>4</id></Row><Row><id>5</id></Row></ids></Row>
+ <Row><seq>4</seq><ids><Row><id>4</id></Row><Row><id>5</id></Row><Row><id>6</id></Row></ids></Row>
+</Dataset>
+<Dataset name='Result 2'>
+ <Row><seq>1</seq><ids><Row><id>1</id></Row><Row><id>2</id></Row><Row><id>3</id></Row></ids></Row>
+ <Row><seq>2</seq><ids><Row><id>2</id></Row><Row><id>3</id></Row><Row><id>4</id></Row></ids></Row>
+ <Row><seq>3</seq><ids><Row><id>3</id></Row><Row><id>4</id></Row><Row><id>5</id></Row></ids></Row>
+ <Row><seq>4</seq><ids><Row><id>4</id></Row><Row><id>5</id></Row><Row><id>6</id></Row></ids></Row>
+</Dataset>

testing/regress/ecl/key/childds1err.xml

@@ -0,0 +1 @@
+<Exception><Source>eclagent</Source><Message>System error: 0: Graph[4], filter[7]: SLAVE #1 [192.168.0.200:20100]: seq &gt; 10, </Message></Exception>

testing/regress/ecl/key/childds2.xml

@@ -0,0 +1,10 @@
+<Dataset name='Result 1'>
+ <Row><seq>1</seq><ids><Row><id>1</id></Row><Row><id>2</id></Row><Row><id>3</id></Row></ids></Row>
+ <Row><seq>2</seq><ids><Row><id>2</id></Row><Row><id>3</id></Row><Row><id>4</id></Row></ids></Row>
+ <Row><seq>3</seq><ids><Row><id>3</id></Row><Row><id>4</id></Row><Row><id>5</id></Row></ids></Row>
+ <Row><seq>4</seq><ids><Row><id>4</id></Row><Row><id>5</id></Row><Row><id>6</id></Row></ids></Row>
+ <Row><seq>1</seq><ids><Row><id>1</id></Row><Row><id>2</id></Row><Row><id>3</id></Row></ids></Row>
+ <Row><seq>2</seq><ids><Row><id>2</id></Row><Row><id>3</id></Row><Row><id>4</id></Row></ids></Row>
+ <Row><seq>3</seq><ids><Row><id>3</id></Row><Row><id>4</id></Row><Row><id>5</id></Row></ids></Row>
+ <Row><seq>4</seq><ids><Row><id>4</id></Row><Row><id>5</id></Row><Row><id>6</id></Row></ids></Row>
+</Dataset>

testing/regress/ecl/key/childds3.xml

@@ -0,0 +1,6 @@
+<Dataset name='Result 1'>
+ <Row><seq>4</seq><ids><Row><id>4</id></Row><Row><id>5</id></Row><Row><id>6</id></Row></ids></Row>
+</Dataset>
+<Dataset name='Result 2'>
+ <Row><seq>4</seq><ids><Row><id>4</id></Row><Row><id>5</id></Row><Row><id>6</id></Row></ids></Row>
+</Dataset>

testing/regress/ecl/key/childds4.xml

@@ -0,0 +1,12 @@
+<Dataset name='Result 1'>
+ <Row><seq>1</seq><ids><Row><id>1</id></Row><Row><id>2</id></Row><Row><id>3</id></Row></ids></Row>
+ <Row><seq>2</seq><ids><Row><id>2</id></Row><Row><id>3</id></Row><Row><id>4</id></Row></ids></Row>
+ <Row><seq>3</seq><ids><Row><id>3</id></Row><Row><id>4</id></Row><Row><id>5</id></Row></ids></Row>
+ <Row><seq>4</seq><ids><Row><id>4</id></Row><Row><id>5</id></Row><Row><id>6</id></Row></ids></Row>
+</Dataset>
+<Dataset name='Result 2'>
+ <Row><seq>1</seq><ids><Row><id>1</id></Row><Row><id>2</id></Row><Row><id>3</id></Row></ids></Row>
+ <Row><seq>2</seq><ids><Row><id>2</id></Row><Row><id>3</id></Row><Row><id>4</id></Row></ids></Row>
+ <Row><seq>3</seq><ids><Row><id>3</id></Row><Row><id>4</id></Row><Row><id>5</id></Row></ids></Row>
+ <Row><seq>4</seq><ids><Row><id>4</id></Row><Row><id>5</id></Row><Row><id>6</id></Row></ids></Row>
+</Dataset>

testing/regress/ecl/key/childds5.xml

@@ -0,0 +1,12 @@
+<Dataset name='Result 1'>
+ <Row><seq>1</seq><ids><Row><id>1</id></Row><Row><id>2</id></Row><Row><id>3</id></Row></ids></Row>
+ <Row><seq>2</seq><ids><Row><id>2</id></Row><Row><id>3</id></Row><Row><id>4</id></Row></ids></Row>
+ <Row><seq>3</seq><ids><Row><id>3</id></Row><Row><id>4</id></Row><Row><id>5</id></Row></ids></Row>
+ <Row><seq>4</seq><ids><Row><id>4</id></Row><Row><id>5</id></Row><Row><id>6</id></Row></ids></Row>
+</Dataset>
+<Dataset name='Result 2'>
+ <Row><seq>1</seq><ids><Row><id>1</id></Row><Row><id>2</id></Row><Row><id>3</id></Row></ids></Row>
+ <Row><seq>2</seq><ids><Row><id>2</id></Row><Row><id>3</id></Row><Row><id>4</id></Row></ids></Row>
+ <Row><seq>3</seq><ids><Row><id>3</id></Row><Row><id>4</id></Row><Row><id>5</id></Row></ids></Row>
+ <Row><seq>4</seq><ids><Row><id>4</id></Row><Row><id>5</id></Row><Row><id>6</id></Row></ids></Row>
+</Dataset>

testing/regress/ecl/key/childds6.xml

@@ -0,0 +1,12 @@
+<Dataset name='Result 1'>
+ <Row><seq>1</seq><ids><Row><id>1</id></Row><Row><id>2</id></Row><Row><id>3</id></Row></ids></Row>
+ <Row><seq>2</seq><ids><Row><id>2</id></Row><Row><id>3</id></Row><Row><id>4</id></Row></ids></Row>
+ <Row><seq>3</seq><ids><Row><id>3</id></Row><Row><id>4</id></Row><Row><id>5</id></Row></ids></Row>
+ <Row><seq>4</seq><ids><Row><id>4</id></Row><Row><id>5</id></Row><Row><id>6</id></Row></ids></Row>
+</Dataset>
+<Dataset name='Result 2'>
+ <Row><seq>1</seq><ids><Row><id>1</id></Row><Row><id>2</id></Row><Row><id>3</id></Row></ids></Row>
+ <Row><seq>2</seq><ids><Row><id>2</id></Row><Row><id>3</id></Row><Row><id>4</id></Row></ids></Row>
+ <Row><seq>3</seq><ids><Row><id>3</id></Row><Row><id>4</id></Row><Row><id>5</id></Row></ids></Row>
+ <Row><seq>4</seq><ids><Row><id>4</id></Row><Row><id>5</id></Row><Row><id>6</id></Row></ids></Row>
+</Dataset>

testing/regress/ecl/key/childds7.xml

@@ -0,0 +1,6 @@
+<Dataset name='Result 1'>
+ <Row><seq>1</seq><ids><Row><id>1</id></Row><Row><id>2</id></Row><Row><id>3</id></Row></ids></Row>
+ <Row><seq>2</seq><ids><Row><id>2</id></Row><Row><id>3</id></Row><Row><id>4</id></Row></ids></Row>
+ <Row><seq>3</seq><ids><Row><id>3</id></Row><Row><id>4</id></Row><Row><id>5</id></Row></ids></Row>
+ <Row><seq>4</seq><ids><Row><id>4</id></Row><Row><id>5</id></Row><Row><id>6</id></Row></ids></Row>
+</Dataset>

testing/regress/ecl/key/childds7b.xml

@@ -0,0 +1,6 @@
+<Dataset name='Result 1'>
+ <Row><seq>1</seq><ids><Row><id>1</id></Row><Row><id>2</id></Row><Row><id>3</id></Row></ids></Row>
+ <Row><seq>2</seq><ids><Row><id>2</id></Row><Row><id>3</id></Row><Row><id>4</id></Row></ids></Row>
+ <Row><seq>3</seq><ids><Row><id>3</id></Row><Row><id>4</id></Row><Row><id>5</id></Row></ids></Row>
+ <Row><seq>4</seq><ids><Row><id>4</id></Row><Row><id>5</id></Row><Row><id>6</id></Row></ids></Row>
+</Dataset>

testing/regress/ecl/key/filecompcopy.xml

@@ -11,3 +11,9 @@
 <Dataset name='Result 5'>
  <Row><Result_5>0</Result_5></Row>
 </Dataset>
+<Dataset name='Result 6'>
+ <Row><Result_6>0</Result_6></Row>
+</Dataset>
+<Dataset name='Result 7'>
+ <Row><Result_7>0</Result_7></Row>
+</Dataset>

testing/regress/hpcc/util/ecl/command.py

@@ -139,8 +139,12 @@ class ECLcmd(Shell):
                     logging.debug("%3d. Ignore result (ecl:'%s')", eclfile.getTaskId(),  eclfile.getBaseEclRealName())
                     test = True
                 elif eclfile.testFail():
-                    logging.debug("%3d. Fail is the expected result (ecl:'%s')", eclfile.getTaskId(),  eclfile.getBaseEclRealName())
-                    test = True
+                   if res['state'] == 'completed':
+                        logging.debug("%3d. Completed but Fail is the expected result (ecl:'%s')", eclfile.getTaskId(),  eclfile.getBaseEclRealName())
+                        test = False
+                   else:
+                        logging.debug("%3d. Fail is the expected result (ecl:'%s')", eclfile.getTaskId(),  eclfile.getBaseEclRealName())
+                        test = True
                 elif eclfile.testNoKey():
                     # keyfile comparaison disabled with //nokey tag
                     if eclfile.testNoOutput():

thorlcr/activities/funnel/thfunnelslave.cpp

@@ -76,10 +76,6 @@ class CParallelFunnel : public CSimpleInterface, implements IRowStream
             bool started = false;
             try
             {
-                { 
-                    CriticalBlock b(stopCrit);
-                    if (stopping) return;
-                }
                 if (funnel.startInputs)
                 {
                     IThorDataLink *_input = QUERYINTERFACE(input.get(), IThorDataLink);

thorlcr/activities/lookupjoin/thlookupjoinslave.cpp

@@ -97,6 +97,7 @@ class CBroadcaster : public CSimpleInterface
     InterruptableSemaphore allDoneSem;
     CriticalSection allDoneLock, bcastOtherCrit;
     bool allDone, allDoneWaiting, allRequestStop, stopping, stopRecv;
+    broadcast_flags stopFlag;
     Owned<IBitSet> slavesDone, slavesStopping;
 
     class CRecv : implements IThreaded
@@ -367,6 +368,7 @@ public:
         slavesStopping.setown(createThreadSafeBitSet());
         mpTag = TAG_NULL;
         recvInterface = NULL;
+        stopFlag = bcastflag_null;
     }
     void start(IBCastReceive *_recvInterface, mptag_t _mpTag, bool _stopping)
     {
@@ -382,6 +384,7 @@ public:
     void reset()
     {
         allDone = allDoneWaiting = allRequestStop = stopping = false;
+        stopFlag = bcastflag_null;
         slavesDone->reset();
         slavesStopping->reset();
     }
@@ -435,10 +438,22 @@ public:
     {
         return slavesStopping->test(myNode-1);
     }
+    broadcast_flags queryStopFlag() { return stopFlag; }
+    bool stopRequested()
+    {
+        if (bcastflag_null != queryStopFlag()) // if this node has requested to stop immediately
+            return true;
+        return allRequestStop; // if not, if all have request to stop
+    }
     void setStopping()
     {
         slavesStopping->set(myNode-1, true);
     }
+    void stop(broadcast_flags flag)
+    {
+        setStopping();
+        stopFlag = flag;
+    }
 };
 
 /* CMarker processes a sorted set of rows, comparing every adjacent row.
@@ -923,17 +938,17 @@ protected:
                         throw MakeActivityException(this, 0, "Out of memory: Unable to add any more rows to RHS");
 
                     rightSerializer->serialize(mbser, (const byte *)row.get());
-                    if (mb.length() >= MAX_SEND_SIZE || broadcaster.isStopping())
+                    if (mb.length() >= MAX_SEND_SIZE || broadcaster.stopRequested())
                         break;
                 }
                 if (0 == mb.length())
                     break;
-                if (broadcaster.isStopping())
-                    sendItem->setFlag(bcastflag_spilt);
+                if (broadcaster.stopRequested())
+                    sendItem->setFlag(broadcaster.queryStopFlag());
                 ThorCompress(mb, sendItem->queryMsg());
                 if (!broadcaster.send(sendItem))
                     break;
-                if (broadcaster.isStopping())
+                if (broadcaster.stopRequested())
                     break;
                 mb.clear();
                 broadcaster.resetSendItem(sendItem);
@@ -946,8 +961,8 @@ protected:
         }
 
         sendItem.setown(broadcaster.newSendItem(bcast_stop));
-        if (broadcaster.isStopping())
-            sendItem->setFlag(bcastflag_spilt);
+        if (broadcaster.stopRequested())
+            sendItem->setFlag(broadcaster.queryStopFlag());
         ActPrintLog("Sending final RHS broadcast packet");
         broadcaster.send(sendItem); // signals stop to others
     }
@@ -1445,7 +1460,7 @@ protected:
         setLocalLookup(true);
         ActPrintLog("Clearing non-local rows - cause: %s", msg);
 
-        broadcaster.setStopping(); // signals to broadcast to start stopping
+        broadcaster.stop(bcastflag_spilt); // signals to broadcast to start stopping immediately and to signal spilt to others
 
         rowidx_t clearedRows = 0;
         if (rhsCollated)