Merge branch 'candidate-6.0.0'

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

common/environment/environment.cpp

@@ -896,6 +896,7 @@ CLocalEnvironment::CLocalEnvironment(const char* environmentFile)
    }
 
    machineCacheBuilt = false;
+   dropZoneCacheBuilt = false;
 }
 
 CLocalEnvironment::CLocalEnvironment(IRemoteConnection *_conn, IPropertyTree* root/*=NULL*/, 
@@ -910,6 +911,7 @@ CLocalEnvironment::CLocalEnvironment(IRemoteConnection *_conn, IPropertyTree* ro
       p.setown(conn->getRoot());
 
     machineCacheBuilt = false;
+    dropZoneCacheBuilt = false;
 }
 
 CLocalEnvironment::~CLocalEnvironment()
@@ -1329,6 +1331,7 @@ void CLocalEnvironment::clearCache()
     }
     cache.kill();
     machineCacheBuilt = false;
+    dropZoneCacheBuilt = false;
     resetPasswordsFromSDS();
 }
 

common/fileview2/fvresultset.cpp

@@ -1111,14 +1111,12 @@ xdouble CResultSetCursor::getDouble(int columnIndex)
             return (xdouble)rtlGetPackedUnsigned(cur);
     case type_decimal:
         {
-            DecLock();
+            BcdCriticalBlock bcdBlock;
             if (type.isSigned())
                 DecPushDecimal(cur, type.getSize(), type.getPrecision());
             else
                 DecPushUDecimal(cur, type.getSize(), type.getPrecision());
-            xdouble ret = DecPopReal();
-            DecUnlock();
-            return ret;
+            return DecPopReal();
         }
     case type_real:
         if (size == 4)
@@ -1188,14 +1186,12 @@ __int64 CResultSetCursor::getInt(int columnIndex)
             return rtlGetPackedUnsigned(cur);
     case type_decimal:
         {
-            DecLock();
+            BcdCriticalBlock bcdBlock;
             if (type.isSigned())
                 DecPushDecimal(cur, type.getSize(), type.getPrecision());
             else
                 DecPushUDecimal(cur, type.getSize(), type.getPrecision());
-            __int64 ret = DecPopInt64();
-            DecUnlock();
-            return ret;
+            return DecPopInt64();
         }
     case type_real:
         if (size == 4)
@@ -1346,13 +1342,12 @@ IStringVal & CResultSetCursor::getString(IStringVal & ret, int columnIndex)
         }
     case type_decimal:
         {
-            DecLock();
+            BcdCriticalBlock bcdBlock;
             if (type.isSigned())
                 DecPushDecimal(cur, type.getSize(), type.getPrecision());
             else
                 DecPushUDecimal(cur, type.getSize(), type.getPrecision());
             DecPopStringX(resultLen, resultStr);
-            DecUnlock();
             ret.setLen(resultStr, resultLen);
             return ret;
         }
@@ -1466,13 +1461,12 @@ IStringVal & CResultSetCursor::getDisplayText(IStringVal &ret, int columnIndex)
         }
     case type_decimal:
         {
-            DecLock();
+            BcdCriticalBlock bcdBlock;
             if (type.isSigned())
                 DecPushDecimal(cur, type.getSize(), type.getPrecision());
             else
                 DecPushUDecimal(cur, type.getSize(), type.getPrecision());
             DecPopStringX(resultLen, resultStr);
-            DecUnlock();
             ret.setLen(resultStr, resultLen);
             return ret;
         }
@@ -2473,13 +2467,13 @@ void CColumnFilter::addValue(unsigned sizeText, const char * text)
     case type_decimal:
         {
             void * target = next->allocate(size);
-            DecLock();
+
+            BcdCriticalBlock bcdBlock;
             rtlDecPushUtf8(lenText, text);
             if (type->isSigned())
                 DecPopDecimal(target, size, type->getPrecision());
             else
                 DecPopUDecimal(target, size, type->getPrecision());
-            DecUnlock();
             break;
         }
     case type_real:

common/thorhelper/thorxmlwrite.cpp

@@ -752,7 +752,8 @@ inline void outputEncodedXmlDecimal(const void *field, unsigned size, unsigned p
     char dec[50];
     if (fieldname)
         out.append('<').append(fieldname).append(" xsi:type=\"xsd:decimal\">");
-    DecLock();
+
+    BcdCriticalBlock bcdBlock;
     if (DecValid(true, size*2-1, field))
     {
         DecPushDecimal(field, size, precision);
@@ -763,7 +764,7 @@ inline void outputEncodedXmlDecimal(const void *field, unsigned size, unsigned p
     }
     else
         out.append("####");
-    DecUnlock();
+
     if (fieldname)
         out.append("</").append(fieldname).append('>');
 }
@@ -773,7 +774,8 @@ inline void outputEncodedXmlUDecimal(const void *field, unsigned size, unsigned
     char dec[50];
     if (fieldname)
         out.append('<').append(fieldname).append(" xsi:type=\"xsd:decimal\">");
-    DecLock();
+
+    BcdCriticalBlock bcdBlock;
     if (DecValid(false, size*2, field))
     {
         DecPushUDecimal(field, size, precision);
@@ -784,7 +786,7 @@ inline void outputEncodedXmlUDecimal(const void *field, unsigned size, unsigned
     }
     else
         out.append("####");
-    DecUnlock();
+
     if (fieldname)
         out.append("</").append(fieldname).append('>');
 }

docs/ECLLanguageReference/ECLR_mods/BltInFunc-LOOP.xml

@@ -31,13 +31,13 @@
   role="bold"></emphasis></para>
 
   <para><emphasis role="bold">LOOP(</emphasis><emphasis>
-  dataset,</emphasis><emphasis role="bold">
-  </emphasis><emphasis>loopcondition, loopbody </emphasis><emphasis
-  role="bold">)</emphasis><emphasis role="bold"></emphasis></para>
+  dataset, rowfilter,
+  loopcondition, loopbody </emphasis><emphasis role="bold">)</emphasis></para>
 
   <para><emphasis role="bold">LOOP(</emphasis><emphasis>
-  dataset,</emphasis><emphasis role="bold"> </emphasis><emphasis>rowfilter,
-  loopcondition, loopbody </emphasis><emphasis role="bold">)</emphasis></para>
+  dataset,</emphasis><emphasis role="bold"> </emphasis><emphasis>loopfilter,
+  loopcondition, loopbody </emphasis><emphasis
+  role="bold">)</emphasis></para>
 
   <informaltable colsep="1" frame="all" rowsep="1">
     <tgroup cols="2">

docs/ECLLanguageReference/ECLR_mods/ExtrSvcs-ExternalServicesImpl.xml

@@ -462,7 +462,7 @@
   <sect2 id="DLL_code_module">
     <title>.SO code module:</title>
 
-    <programlisting>  //******************************************************
+    <para><programlisting>//******************************************************
   // hqlplugins.hpp : Defines standard values included
               in
   // the plugin header file.
@@ -490,9 +490,9 @@
   
   typedef bool (*EclPluginDefinition) (ECLPluginDefinitionBlock *);
   
-  #endif //__HQLPLUGIN_INCL
-  
-  //******************************************************
+  #endif //__HQLPLUGIN_INCL</programlisting></para>
+
+    <programlisting>//******************************************************
   // examplelib.hpp : Defines standard values included in
   // the plugin code file.
   //******************************************************
@@ -515,65 +515,75 @@
   
   extern "C" {
   EXAMPLELIB_API bool getECLPluginDefinition(ECLPluginDefinitionBlock *pb);
+  EXAMPLELIB_API void setPluginContext(IPluginContext * _ctx);
   EXAMPLELIB_API unsigned EXAMPLELIB_CALL elStringFind(unsigned srcLen,
        const char * src, unsigned hitLen, const char * hit,
        unsigned instance);
   }
   
   #endif //EXAMPLELIB_INCL
-  
-  //******************************************************
-  // examplelib.cpp : Defines the plugin code.
-  //******************************************************
-  #include &lt;memory.h&gt;
-  #include "examplelib.hpp"
-  
-  static char buildVersion[] = "$Name$ $Id$";
-  
-  #define EXAMPLELIB_VERSION "EXAMPLELIB 1.0.00"
-  
-  static const char * const HoleDefinition =
-    "SYSTEM\n"
-    "MODULE (SYSTEM)\n"
-    " FUNCTION StringFind(string src, string search,
-        unsigned4 instance),unsigned4,c,name('elStringFind')\n"
-    "END\n";
-  
-  static const char * const EclDefinition =
-    "export ExampleLib := SERVICE\n"
-    " unsigned integer4 StringFind(const string src,
-        const string tofind, unsigned4 instance )
-            : c, pure,entrypoint='elStringFind'; \n"
-    "END;";
-  
-  EXAMPLELIB_API bool getECLPluginDefinition(ECLPluginDefinitionBlock *pb)
-  {
+  </programlisting>
+
+    <para></para>
+
+    <programlisting>//******************************************************
+// examplelib.cpp : Defines the plugin code.
+//******************************************************
+#include &lt;time.h&gt;
+#include &lt;stdlib.h&gt;
+#include &lt;string.h&gt;
+#include &lt;ctype.h&gt;
+#include "examplelib.hpp"
+
+#define EXAMPLELIB_VERSION "EXAMPLELIB 1.0.00"
+
+static const char * HoleDefinition = NULL;
+
+static const char * EclDefinition =
+"export ExampleLib := SERVICE\n"
+"  string EchoString(const string src) : c, pure,fold,entrypoint='elEchoString'; \n"
+"END;";
+
+EXAMPLELIB_API bool getECLPluginDefinition(ECLPluginDefinitionBlock *pb) 
+{
+    //  Warning:    This function may be called without the plugin being loaded fully.  
+    //              It should not make any library calls or assume that dependent modules
+    //              have been loaded or that it has been initialised.
+    //
+    //              Specifically:  "The system does not call DllMain for process and thread 
+    //              initialization and termination.  Also, the system does not load 
+    //              additional executable modules that are referenced by the specified module."
+
     if (pb-&gt;size != sizeof(ECLPluginDefinitionBlock))
-       return false;
+        return false;
+
     pb-&gt;magicVersion = PLUGIN_VERSION;
-    pb-&gt;version = EXAMPLELIB_VERSION " $Name$ $Id$";
+    pb-&gt;version = EXAMPLELIB_VERSION " $Revision: 62376 $";
     pb-&gt;moduleName = "lib_examplelib";
     pb-&gt;ECL = EclDefinition;
     pb-&gt;Hole = HoleDefinition;
     pb-&gt;flags = PLUGIN_IMPLICIT_MODULE;
     pb-&gt;description = "ExampleLib example services library";
     return true;
-  }
-  
-  //----------------------------------------------------------------
-  EXAMPLELIB_API unsigned EXAMPLELIB_CALL elStringFind(unsigned srcLen,
-    const char * src, unsigned hitLen, const char * hit,
-    unsigned instance)
-  {
-    if ( srcLen &lt; hitLen )
-       return 0;
-    unsigned steps = srcLen-hitLen+1;
-    for ( unsigned i = 0; i &lt; steps; i++ )
-       if ( !memcmp((char *)src+i,hit,hitLen) )
-            if ( !--instance )
-                 return i+1;
-    return 0;
-  }
+}
+
+namespace nsExamplelib {
+    IPluginContext * parentCtx = NULL;
+}
+using namespace nsExamplelib;
+
+EXAMPLELIB_API void setPluginContext(IPluginContext * _ctx) { parentCtx = _ctx; }
+
+//-------------------------------------------------------------------------------------------------------------------------------------------
+
+EXAMPLELIB_API unsigned EXAMPLELIB_CALL elStringFind(unsigned srcLen,
+ const char * src, unsigned hitLen, const char * hit,
+ unsigned instance)
+{
+    tgt = (char *)CTXMALLOC(parentCtx, srcLen);
+    memcpy(tgt,src,srcLen);
+    tgtLen = srcLen;
+}
 </programlisting>
   </sect2>
 </sect1>

docs/RoxieReference/RoxieRefMods/directAccessToRoxie.xml

@@ -341,5 +341,37 @@ Regs.Reg.1.Codes.Code=MCO</programlisting></para>
 to
 /WsEcl/<emphasis role="bold">submit</emphasis>/query/RoxieTargetName/QueryName/expanded</programlisting></para>
     </sect2>
+
+    <sect2 id="RESTful_access" role="brk">
+      <title>Direct RESTful access to Roxie</title>
+
+      <para>You can access your Roxie queries directly using a RESTful
+      interface in the following manner:</para>
+
+      <para><programlisting>http://&lt;ip&gt;:9876/&lt;target&gt;/&lt;queryid&gt;?&lt;stored1&gt;=&lt;value&gt;
+       &amp;&lt;storeddataset&gt;.Row.0.name=abc&amp;storeddataset.Row.0.id=123</programlisting>where,
+      </para>
+
+      <para>ip is the IP address or hostname of your Roxie server or a VIP to
+      a range of IPs for a farm of Roxie servers</para>
+
+      <para><emphasis>target</emphasis> is the name of the target
+      cluster</para>
+
+      <para><emphasis>queryid</emphasis> is the published Query's Query
+      Id.</para>
+
+      <para><emphasis>stored1</emphasis> is an input variable (using STORED in
+      ECL) and value is the <emphasis>value</emphasis> to submit</para>
+
+      <para><emphasis>storeddataset</emphasis> is a dataset to be passed in to
+      the query</para>
+
+      <para>For example:</para>
+
+      <para><programlisting>http://127.0.0.1:9876/roxie/echotest.1?echoValue=Ziggy%20played%20guitar</programlisting></para>
+
+      <para></para>
+    </sect2>
   </sect1>
 </chapter>

ecl/hql/hqlerror.cpp

@@ -230,7 +230,7 @@ class HQL_API ThrowingErrorReceiver : public ErrorReceiverSink
 
 void ThrowingErrorReceiver::report(IError* error)
 {
-    throw error;
+    throw LINK(error);
 }
 
 IErrorReceiver * createThrowingErrorReceiver()

ecl/hqlcpp/hqlcppds.cpp

@@ -2132,9 +2132,7 @@ void HqlCppTranslator::doBuildDataset(BuildCtx & ctx, IHqlExpression * expr, CHq
             Owned<BoundRow> rowBuilder = createRowBuilder(ctx, tempRow);
             Owned<IReferenceSelector> createdRef = createReferenceSelector(rowBuilder);
 
-            BuildCtx subctx(ctx);
-            subctx.addGroup();
-            doBuildRowAssignAggregate(subctx, createdRef, expr);
+            doBuildRowAssignAggregate(ctx, createdRef, expr);
             finalizeTempRow(ctx, tempRow, rowBuilder);
 
             convertBoundRowToDataset(ctx, tgt, tempRow, format);
@@ -3332,13 +3330,11 @@ void HqlCppTranslator::buildDatasetAssignJoin(BuildCtx & ctx, IHqlCppDatasetBuil
 
 void HqlCppTranslator::buildDatasetAssignAggregate(BuildCtx & ctx, IHqlCppDatasetBuilder * target, IHqlExpression * expr)
 {
-    BuildCtx subctx(ctx);
-    subctx.addGroup();
-    BoundRow * targetRow = target->buildCreateRow(subctx);
+    BoundRow * targetRow = target->buildCreateRow(ctx);
 
-    Owned<IReferenceSelector> targetRef = buildActiveRow(subctx, targetRow->querySelector());
-    doBuildRowAssignAggregate(subctx, targetRef, expr);
-    target->finishRow(subctx, targetRow);
+    Owned<IReferenceSelector> targetRef = buildActiveRow(ctx, targetRow->querySelector());
+    doBuildRowAssignAggregate(ctx, targetRef, expr);
+    target->finishRow(ctx, targetRow);
 }
 
 void HqlCppTranslator::buildDatasetAssignChoose(BuildCtx & ctx, IHqlCppDatasetBuilder * target, IHqlExpression * expr)
@@ -3577,7 +3573,6 @@ void HqlCppTranslator::buildDatasetAssign(BuildCtx & ctx, IHqlCppDatasetBuilder
     if (isRowAssign)
     {
         bool done = false;
-        subctx.addGroup();
 
         //Some code primarily here to improve the generated code for productions inside parse statements for text parsing.
         //see if we can replace a memcpy of the child record with a link...
@@ -4390,8 +4385,6 @@ void HqlCppTranslator::doBuildRowAssignSerializeRow(BuildCtx & ctx, IReferenceSe
         //MORE: This doesn't associated the returned size with the target if assigned to a child field.
         //very unusual code, so not too concerned.
     }
-
-    subctx.removeAssociation(selfCursor);
 }
         
 void HqlCppTranslator::doBuildRowAssignUserTable(BuildCtx & ctx, IReferenceSelector * target, IHqlExpression * expr)
@@ -4797,7 +4790,6 @@ IHqlExpression * HqlCppTranslator::ensureIteratedRowIsLive(BuildCtx & initctx, B
     }
 
     BuildCtx childctx(iterctx);
-    childctx.addGroup();
 
     Owned<BoundRow> tempRow = declareTempRow(childctx, childctx, rowExpr);
     Owned<BoundRow> rowBuilder = createRowBuilder(childctx, tempRow);

ecl/hqlcpp/hqlcppsys.ecl

@@ -356,8 +356,6 @@ const char * cppSystemText[]  = {
     "   DecSwap() : eclrtl,library='eclrtl',entrypoint='DecSwap';",
     "   DecUint4Power(unsigned4 pow) :  eclrtl,library='eclrtl',entrypoint='DecUint4Power';",
     "   string DecPopStringX() :    eclrtl,library='eclrtl',entrypoint='DecPopStringX';",
-    "   DecLock() : eclrtl,library='eclrtl',entrypoint='DecLock';",
-    "   DecUnlock() :   eclrtl,library='eclrtl',entrypoint='DecUnlock';",
     "   boolean DecValid(boolean isSigned, const data src) : eclrtl,pure,library='eclrtl',entrypoint='DecValid';",
     "   boolean DecValidTos() : eclrtl,pure,library='eclrtl',entrypoint='DecValidTos';",
 

ecl/hqlcpp/hqlcset.cpp

@@ -1684,7 +1684,6 @@ void DatasetBuilderBase::doFinishRow(BuildCtx & ctx, BoundRow * selfCursor, IHql
     s.append(instanceName).append(".finalizeRow(");
     translator.generateExprCpp(s, boundSize.expr).append(");");
     ctx.addQuoted(s);
-    ctx.removeAssociation(selfCursor);
 }
 
 //---------------------------------------------------------------------------

ecl/hqlcpp/hqlhtcpp.cpp

@@ -7107,7 +7107,6 @@ void HqlCppTranslator::finishSelf(BuildCtx & ctx, BoundRow * self, BoundRow * ta
         OwnedHqlExpr sizeofTarget = createSizeof(target->querySelector());
         ctx.associateExpr(sizeofTarget, bound);
     }
-    ctx.removeAssociation(self);
 }
 
 
@@ -8035,7 +8034,7 @@ void HqlCppTranslator::doBuildExprSizeof(BuildCtx & ctx, IHqlExpression * expr,
 
 }
 
-void HqlCppTranslator::doBuildExprRowDiff(BuildCtx & ctx, const CHqlBoundTarget & target, IHqlExpression * expr, IHqlExpression * rightRecord, IHqlExpression * leftSelector, IHqlExpression * rightSelector, StringBuffer & selectorText, bool isCount)
+void HqlCppTranslator::doBuildExprRowDiff(BuildCtx & ctx, const CHqlBoundTarget & target, IHqlExpression * expr, IHqlExpression * leftSelector, IHqlExpression * rightRecord, IHqlExpression * rightSelector, StringBuffer & selectorText, bool isCount)
 {
     switch (expr->getOperator())
     {
@@ -12615,48 +12614,30 @@ void HqlCppTranslator::buildProcessTransformFunction(BuildCtx & ctx, IHqlExpress
 
     LinkedHqlExpr skipReturnValue = queryZero();
     associateSkipReturnMarker(funcctx, skipReturnValue, NULL);
-    if (!recordTypesMatch(dataset, right))
-    {
-        //self won't clash, so can generate efficient code.
-        //Perform cse on both transforms
-        OwnedHqlExpr comma = createComma(LINK(transformRow), LINK(transformRight));
-        comma.setown(spotScalarCSE(comma, NULL, queryOptions().spotCseInIfDatasetConditions));
-        if (comma->getOperator() == no_alias_scope)
-            comma.set(comma->queryChild(0));
 
-        HqlExprArray unwound;
-        comma->unwindList(unwound, no_comma);
-        unsigned max = unwound.ordinality();
+    //Perform cse on both transforms
+    OwnedHqlExpr comma = createComma(LINK(transformRow), LINK(transformRight));
+    comma.setown(spotScalarCSE(comma, NULL, queryOptions().spotCseInIfDatasetConditions));
+    if (comma->getOperator() == no_alias_scope)
+        comma.set(comma->queryChild(0));
 
-        BoundRow * selfCursor = bindSelf(funcctx, dataset, "crSelf");
-        BoundRow * selfRowCursor = bindSelf(funcctx, right, "crSelfRight");
+    HqlExprArray unwound;
+    comma->unwindList(unwound, no_comma);
+    unsigned max = unwound.ordinality();
 
-        for (unsigned i=0; i<max-2; i++)
-            buildStmt(funcctx, &unwound.item(i));
+    BoundRow * selfCursor = bindSelf(funcctx, dataset, "crSelf");
+    BoundRow * selfRowCursor = bindSelf(funcctx, right, "crSelfRight");
 
-        IHqlExpression * newTransformRow = queryExpandAliasScope(funcctx, &unwound.item(max-2));
-        IHqlExpression * newTransformRight = queryExpandAliasScope(funcctx, &unwound.item(max-1));
-        assertex(newTransformRow->getOperator() == no_transform && newTransformRight->getOperator() == no_transform);
+    for (unsigned i=0; i<max-2; i++)
+        buildStmt(funcctx, &unwound.item(i));
 
-        doTransform(funcctx, newTransformRow, selfCursor);
-        doTransform(funcctx, newTransformRight, selfRowCursor);
-        buildReturnRecordSize(funcctx, selfCursor);
-    }
-    else
-    {
-        BuildCtx ctx1(funcctx);
+    IHqlExpression * newTransformRow = queryExpandAliasScope(funcctx, &unwound.item(max-2));
+    IHqlExpression * newTransformRight = queryExpandAliasScope(funcctx, &unwound.item(max-1));
+    assertex(newTransformRow->getOperator() == no_transform && newTransformRight->getOperator() == no_transform);
 
-        ctx1.addGroup();
-        BoundRow * selfRowCursor = bindSelf(ctx1, right, "crSelfRight");
-        doTransform(ctx1, transformRight, selfRowCursor);
-
-        BuildCtx ctx2(funcctx);
-        ctx2.addGroup();
-        BoundRow * selfCursor = bindSelf(ctx2, dataset, "crSelf");
-        doTransform(ctx2, transformRow, selfCursor);
-
-        buildReturnRecordSize(ctx2, selfCursor);
-    }
+    doTransform(funcctx, newTransformRow, selfCursor);
+    doTransform(funcctx, newTransformRight, selfRowCursor);
+    buildReturnRecordSize(funcctx, selfCursor);
 }
 
 

ecl/hqlcpp/hqlsource.cpp

@@ -1231,7 +1231,7 @@ void SourceBuilder::buildTargetCursor(Shared<BoundRow> & tempRow, Shared<BoundRo
 void SourceBuilder::associateTargetCursor(BuildCtx & subctx, BuildCtx & ctx, BoundRow * tempRow, BoundRow * rowBuilder, IHqlExpression * expr)
 {
     //First remove the old active dataset
-    //NOT sure this is needed
+    //This is not strictly necessary, but it avoids the redundant row being serialized to any child queries
     BoundRow * oldCursor = translator.resolveSelectorDataset(ctx, expr->queryChild(0));
     ctx.removeAssociation(oldCursor);
 

ecl/regress/rowdiff.ecl

@@ -26,16 +26,18 @@ string20            middle;
                 end;
             end;
 
-
-in1rec :=   record
+idRec := record
 unsigned    id;
+END;
+
+in1rec :=   record(idRec)
 complexName name;
 unsigned    age;
 string      title;
         end;
 
 in2rec := record
-unsigned    id;
+idRec;
 complexName name;
 real8       age;
 boolean     dead;

esp/scm/ws_fs.ecm

@@ -522,6 +522,8 @@ ESPStruct PhysicalFileStruct
     bool isDir;
     int64 filesize;
     string modifiedtime;
+    [min_ver("1.13")] string Path;
+    [min_ver("1.13")] ESParray<ESPstruct PhysicalFileStruct> Files;
 };
 
 ESPresponse [exceptions_inline] FileListResponse
@@ -535,6 +537,20 @@ ESPresponse [exceptions_inline] FileListResponse
     ESParray<ESPStruct PhysicalFileStruct> files;   
 };
 
+ESPrequest [nil_remove] DropZoneFileSearchRequest
+{
+    string DropZoneName;
+    string FileName;
+    string Path;
+    bool DirectoryOnly(false);
+    bool FileOnly(false);
+};
+
+ESPresponse [exceptions_inline] DropZoneFileSearchResponse
+{
+    ESParray<ESPStruct PhysicalFileStruct> Files;
+};
+
 ESPrequest OpenSaveRequest
 {
     string Location;
@@ -638,7 +654,7 @@ ESPresponse [exceptions_inline, nil_remove] GetSprayTargetsResponse
 };
 
 ESPservice [
-    version("1.12"),
+    version("1.13"),
     exceptions_inline("./smc_xslt/exceptions.xslt")] FileSpray
 {
     ESPuses ESPstruct DFUWorkunit;
@@ -671,6 +687,7 @@ ESPservice [
     ESPmethod [resp_xsl_default("/esp/xslt/opensave.xslt")] OpenSave(OpenSaveRequest, OpenSaveResponse);
     ESPmethod [resp_xsl_default("/esp/xslt/dropzonefile.xslt")] DropZoneFiles(DropZoneFilesRequest, DropZoneFilesResponse);
     ESPmethod [resp_xsl_default("/esp/xslt/dfuwuaction_results.xslt")] DeleteDropZoneFiles(DeleteDropZoneFilesRequest, DFUWorkunitsActionResponse);
+    ESPmethod [min_ver("1.13")] DropZoneFileSearch(DropZoneFileSearchRequest, DropZoneFileSearchResponse);
     ESPmethod GetSprayTargets(GetSprayTargetsRequest, GetSprayTargetsResponse);
 };
 

esp/services/ws_fs/ws_fsService.cpp

@@ -2711,6 +2711,8 @@ bool CFileSprayEx::onFileList(IEspContext &context, IEspFileListRequest &req, IE
 
         double version = context.getClientVersion();
         const char* netaddr = req.getNetaddr();
+        if (!netaddr || !*netaddr)
+            throw MakeStringException(ECLWATCH_INVALID_INPUT, "Network address not specified.");
         const char* mask = req.getMask();
         bool directoryOnly = req.getDirectoryOnly();
 
@@ -2803,6 +2805,153 @@ bool CFileSprayEx::onFileList(IEspContext &context, IEspFileListRequest &req, IE
     return true;
 }
 
+void CFileSprayEx::queryDropZoneInfo(const char* dropZone, const char* pathReq, StringBuffer& path,
+    EnvMachineOS& os, IpAddress& ip)
+{
+    Owned<IEnvironmentFactory> factory = getEnvironmentFactory();
+    factory->validateCache();
+
+    Owned<IConstEnvironment> env = factory->openEnvironment();
+    if (!env)
+        throw MakeStringException(ECLWATCH_CANNOT_GET_ENV_INFO,"Cannot get environment information.");
+
+    Owned<IPropertyTree> root = &env->getPTree();
+    VStringBuffer xpath("Software/DropZone[@name='%s']", dropZone);
+    IPropertyTree *pt = root->queryPropTree(xpath.str());
+    if (!pt)
+        throw MakeStringException(ECLWATCH_INVALID_INPUT, "DropZone %s not found.", dropZone);
+
+    const char* computer = pt->queryProp("@computer");
+    IConstMachineInfo* machine = env->getMachine(computer);
+    if (!machine)
+        throw MakeStringException(ECLWATCH_INVALID_INPUT, "DropZone %s: machine %s not found.", dropZone, computer);
+
+    SCMStringBuffer netAddr;
+    machine->getNetAddress(netAddr);
+    ip.ipset(netAddr.str());
+
+    os = machine->getOS();
+    pt->getProp("@directory", path);
+    if (!pathReq || !*pathReq)
+        return;//Return the path since the "pathReq" is not set from user request.
+
+    //If the "pathReq" is set from user request, we need to verify it and return it.
+    StringBuffer s = pathReq;
+    const char pathSep = (os == MachineOsW2K) ? '\\' : '/';
+    s.replace(pathSep=='\\'?'/':'\\', pathSep);
+    if (strncmp(s, path.str(), path.length()))
+    {
+        throw MakeStringException(ECLWATCH_INVALID_INPUT, "DropZone %s: path %s not in dropzone path %s.",
+            dropZone, pathReq, path.str());
+    }
+    path.set(s);
+}
+
+void CFileSprayEx::addDropZoneFile(IEspContext& context, IDirectoryIterator* di, const char* name, const char* path,
+    IArrayOf<IEspPhysicalFileStruct>& filesInFolder, IArrayOf<IEspPhysicalFileStruct>& files)
+{
+    Owned<IEspPhysicalFileStruct> aFile = createPhysicalFileStruct();
+    aFile->setName(name);
+    aFile->setPath(path);
+    aFile->setIsDir(di->isDir());
+    CDateTime modtime;
+    StringBuffer timestr;
+    di->getModifiedTime(modtime);
+    unsigned y,m,d,h,min,sec,nsec;
+    modtime.getDate(y,m,d,true);
+    modtime.getTime(h,min,sec,nsec,true);
+    timestr.appendf("%04d-%02d-%02d %02d:%02d:%02d", y,m,d,h,min,sec);
+    aFile->setModifiedtime(timestr.str());
+    aFile->setFilesize(di->getFileSize());
+    if (di->isDir() && filesInFolder.ordinality())
+        aFile->setFiles(filesInFolder);
+    files.append(*aFile.getLink());
+}
+
+bool CFileSprayEx::searchDropZoneFileInFolder(IEspContext& context, IFile* f, CDropZoneFileFilter& filter,
+    bool returnAll, StringBuffer& folder, EnvMachineOS& os, IArrayOf<IEspPhysicalFileStruct>& files)
+{
+    bool foundMatch = false;
+    const char pathSep = (os == MachineOsW2K) ? '\\' : '/';
+    Owned<IDirectoryIterator> di = f->directoryFiles(NULL, false, true);
+    ForEach(*di)
+    {
+        StringBuffer fname;
+        di->getName(fname);
+        if (!fname.length())
+            continue;
+
+        StringBuffer newPath = folder;
+        newPath.append(pathSep).append(fname.str());
+
+        IArrayOf<IEspPhysicalFileStruct> filesInFolder;
+        if (returnAll) //Every files in this folder have to be returned
+        {
+            if (di->isDir())
+                searchDropZoneFileInFolder(context, &di->get(), filter, returnAll, newPath, os, filesInFolder);
+            addDropZoneFile(context, di, fname.str(), folder.str(), filesInFolder, files);
+            continue;
+        }
+
+        bool foundMatchNew = false;
+        if (((!filter.fileOnly && di->isDir()) || (!filter.dirOnly && !di->isDir()))
+            && (!filter.name.length() || WildMatch(fname.str(), filter.name.str(), true)))
+            foundMatchNew = true;
+        if (di->isDir() && searchDropZoneFileInFolder(context, &di->get(), filter, foundMatchNew, newPath, os, filesInFolder))
+            foundMatchNew = true;
+        if (foundMatchNew)
+        {
+            addDropZoneFile(context, di, fname.str(), folder.str(), filesInFolder, files);
+            foundMatch = true;
+        }
+    }
+    return foundMatch;
+}
+
+void CFileSprayEx::searchDropZoneFileInFolder(IEspContext& context, CDropZoneFileFilter& filter,
+    IpAddress& ip, EnvMachineOS& os, const char* path, IArrayOf<IEspPhysicalFileStruct>& files)
+{
+    RemoteFilename rfn;
+    SocketEndpoint ep;
+    ep.ipset(ip);
+    rfn.setPath(ep, path);
+    Owned<IFile> f = createIFile(rfn);
+    if(!f->isDirectory())
+        throw MakeStringException(ECLWATCH_INVALID_DIRECTORY, "%s is not a directory.", path);
+
+    StringBuffer folder = path;
+    searchDropZoneFileInFolder(context, f, filter, false, folder, os, files);
+}
+
+bool CFileSprayEx::onDropZoneFileSearch(IEspContext &context, IEspDropZoneFileSearchRequest &req, IEspDropZoneFileSearchResponse &resp)
+{
+    try
+    {
+        if (!context.validateFeatureAccess(FILE_SPRAY_URL, SecAccess_Access, false))
+            throw MakeStringException(ECLWATCH_FILE_SPRAY_ACCESS_DENIED, "Failed to do FileList. Permission denied.");
+
+        const char* dropZone = req.getDropZoneName();
+        if (!dropZone || !*dropZone)
+            throw MakeStringException(ECLWATCH_INVALID_INPUT, "DropZone not specified.");
+
+        IpAddress ip;
+        EnvMachineOS os;
+        StringBuffer path;
+        queryDropZoneInfo(dropZone, req.getPath(), path, os, ip);
+
+        IArrayOf<IEspPhysicalFileStruct> files;
+        CDropZoneFileFilter filter(req.getFileName(), req.getDirectoryOnly(), req.getFileOnly());
+        searchDropZoneFileInFolder(context, filter, ip, os, path.str(), files);
+        resp.setFiles(files);
+    }
+    catch(IException* e)
+    {
+        FORWARDEXCEPTION(context, e,  ECLWATCH_INTERNAL_ERROR);
+    }
+
+    return true;
+}
+
 bool CFileSprayEx::onDfuMonitor(IEspContext &context, IEspDfuMonitorRequest &req, IEspDfuMonitorResponse &resp)
 {
     try

esp/services/ws_fs/ws_fsService.hpp

@@ -22,6 +22,16 @@
 #include "msgbuilder.hpp"
 #include "jthread.hpp"
 #include "dfuwu.hpp"
+#include "environment.hpp"
+
+struct CDropZoneFileFilter
+{
+    StringAttr name;
+    bool dirOnly;
+    bool fileOnly;
+    CDropZoneFileFilter(const char* _name, bool _dirOnly, bool _fileOnly)
+        : name(_name), dirOnly(_dirOnly), fileOnly(_fileOnly) {};
+};
 
 class Schedule : public Thread
 {
@@ -92,6 +102,7 @@ public:
     virtual bool onDfuMonitor(IEspContext &context, IEspDfuMonitorRequest &req, IEspDfuMonitorResponse &resp);
     virtual bool onGetDFUProgress(IEspContext &context, IEspProgressRequest &req, IEspProgressResponse &resp);
     virtual bool onOpenSave(IEspContext &context, IEspOpenSaveRequest &req, IEspOpenSaveResponse &resp);
+    virtual bool onDropZoneFileSearch(IEspContext &context, IEspDropZoneFileSearchRequest &req, IEspDropZoneFileSearchResponse &resp);
     virtual bool onDropZoneFiles(IEspContext &context, IEspDropZoneFilesRequest &req, IEspDropZoneFilesResponse &resp);
     virtual bool onDeleteDropZoneFiles(IEspContext &context, IEspDeleteDropZoneFilesRequest &req, IEspDFUWorkunitsActionResponse &resp);
     virtual bool onGetSprayTargets(IEspContext &context, IEspGetSprayTargetsRequest &req, IEspGetSprayTargetsResponse &resp);
@@ -115,6 +126,13 @@ protected:
     void appendGroupNode(IArrayOf<IEspGroupNode>& groupNodes, const char* nodeName, const char* clusterType, bool replicateOutputs);
     bool getOneDFUWorkunit(IEspContext& context, const char* wuid, IEspGetDFUWorkunitsResponse& resp);
     const char* getDropZoneDirByIP(const char* destIP, StringBuffer& dir);
+    void queryDropZoneInfo(const char* dropZone, const char* pathReq, StringBuffer& path, EnvMachineOS& os, IpAddress& ip);
+    void addDropZoneFile(IEspContext& context, IDirectoryIterator* di, const char* name, const char* path,
+        IArrayOf<IEspPhysicalFileStruct>& filesInFolder, IArrayOf<IEspPhysicalFileStruct>&files);
+    bool searchDropZoneFileInFolder(IEspContext& context, IFile* f, CDropZoneFileFilter& filter, bool skipMask,
+        StringBuffer& relPath, EnvMachineOS& os, IArrayOf<IEspPhysicalFileStruct>& files);
+    void searchDropZoneFileInFolder(IEspContext& context, CDropZoneFileFilter& filter, IpAddress& ip,
+        EnvMachineOS& os, const char* path, IArrayOf<IEspPhysicalFileStruct>& files);
 };
 
 #endif //_ESPWIZ_FileSpray_HPP__

esp/src/eclwatch/ESPWorkunit.js

@@ -24,13 +24,14 @@ define([
     "dojo/store/Observable",
     "dojo/topic",
 
+    "hpcc/Utility",
     "hpcc/WsWorkunits",
     "hpcc/WsTopology",
     "hpcc/ESPUtil",
     "hpcc/ESPRequest",
     "hpcc/ESPResult"
 ], function (declare, arrayUtil, lang, i18n, nlsHPCC, Deferred, all, Observable, topic,
-    WsWorkunits, WsTopology, ESPUtil, ESPRequest, ESPResult) {
+    Utility, WsWorkunits, WsTopology, ESPUtil, ESPRequest, ESPResult) {
 
     var _workunits = {};
 
@@ -135,53 +136,10 @@ define([
             }
             this.set("sourceFiles", sourceFiles);
         },
-        ExtractTime: function (Timer) {
-            //  GH:  <n>ns or <m>ms or <s>s or [<d> days ][<h>:][<m>:]<s>[.<ms>]
-            var nsIndex = Timer.indexOf("ns");
-            if (nsIndex !== -1) {
-                return parseFloat(Timer.substr(0, nsIndex)) / 1000000000;
-            }
-            var msIndex = Timer.indexOf("ms");
-            if (msIndex !== -1) {
-                return parseFloat(Timer.substr(0, msIndex)) / 1000;
-            }
-            var sIndex = Timer.indexOf("s");
-            if (sIndex !== -1 && Timer.indexOf("days") === -1) {
-                return parseFloat(Timer.substr(0, sIndex));
-            }
-
-            var dayTimeParts = Timer.split(" days ");
-            var days = parseFloat(dayTimeParts.length > 1 ? dayTimeParts[0] : 0.0);
-            var time = dayTimeParts.length > 1 ? dayTimeParts[1] : dayTimeParts[0];
-            var secs = 0.0;
-            var timeParts = time.split(":").reverse();
-            for (var j = 0; j < timeParts.length; ++j) {
-                secs += parseFloat(timeParts[j]) * Math.pow(60, j);
-            }
-            return (days * 24 * 60 * 60) + secs;
-        },
-        ExtractTimeTests: function () {
-            var tests = [
-                { str: "1.1s", expected: 1.1 },
-                { str: "2.2ms", expected: 0.0022 },
-                { str: "3.3ns", expected: 0.0000000033 },
-                { str: "4.4", expected: 4.4 },
-                { str: "5:55.5", expected: 355.5 },
-                { str: "6:06:06.6", expected: 21966.6 },
-                { str: "6:06:6.6", expected: 21966.6 },
-                { str: "6:6:6.6", expected: 21966.6 },
-                { str: "7 days 7:07:7.7", expected: 630427.7 }
-            ];
-            tests.forEach(function (test, idx) {
-                if (this.ExtractTime(test.str) !== test.expected) {
-                    console.log("ExtractTimeTests failed with " + this.ExtractTime(test.str) + " !== " +  test.expected);
-                }
-            }, this);
-        },
         _TimersSetter: function (Timers) {
             var timers = [];
             for (var i = 0; i < Timers.ECLTimer.length; ++i) {
-                var secs = this.ExtractTime(Timers.ECLTimer[i].Value);
+                var secs = Utility.espTime2Seconds(Timers.ECLTimer[i].Value);
                 timers.push(lang.mixin(Timers.ECLTimer[i], {
                     __hpcc_id: i + 1,
                     Seconds: Math.round(secs * 1000) / 1000,

esp/src/eclwatch/FilterDropDownWidget.js

@@ -22,6 +22,7 @@ define([
     "dojo/dom",
     "dojo/dom-form",
     "dojo/on",
+    "dojo/dom-style",
 
     "dijit/registry",
     "dijit/form/Select",
@@ -37,7 +38,7 @@ define([
 
     "hpcc/TableContainer"
 
-], function (declare, lang, i18n, nlsHPCC, arrayUtil, dom, domForm, on,
+], function (declare, lang, i18n, nlsHPCC, arrayUtil, dom, domForm, on, domStyle,
                 registry, Select,
                 _Widget,
                 template) {
@@ -50,11 +51,13 @@ define([
         iconFilter: null,
         filterDropDown: null,
         filterForm: null,
+        filterLabel: null,
 
         postCreate: function (args) {
             this.inherited(arguments);
             this.filterDropDown = registry.byId(this.id + "FilterDropDown");
             this.filterForm = registry.byId(this.id + "FilterForm");
+            this.filterLabel = registry.byId(this.id + "FilterLabel");
         },
 
         startup: function (args) {
@@ -118,13 +121,25 @@ define([
         close: function (event) {
             this.filterDropDown.closeDropDown();
         },
-        
+
         disable: function(disable) {
             this.filterDropDown.set("disabled", disable);
         },
 
         refreshState: function () {
-            this.iconFilter.src = this.exists() ? dojoConfig.getImageURL("filter.png") : dojoConfig.getImageURL("noFilter.png");
+            if (this.exists()) {
+                this.iconFilter.src = dojoConfig.getImageURL("filter1.png");
+                dom.byId(this.id + "FilterDropDown_label").innerHTML = this.i18n.FilterSet;
+                domStyle.set(this.id + "FilterDropDown_label", {
+                    "font-weight": "bold"
+                });
+            } else {
+                this.iconFilter.src = dojoConfig.getImageURL("noFilter1.png");
+                dom.byId(this.id + "FilterDropDown_label").innerHTML = this.i18n.Filter;
+                domStyle.set(this.id + "FilterDropDown_label", {
+                    "font-weight": "normal"
+                });
+            }
         }
     });
 });

esp/src/eclwatch/GraphTreeWidget.js

@@ -98,6 +98,7 @@ define([
             this.inherited(arguments);
             this._initGraphControls();
             this._initTimings();
+            this._initActivitiesMap();
             this._initDialogs();
         },
 
@@ -159,9 +160,12 @@ define([
             this.widget.TimingsTreeMap.onClick = function (value) {
                 context.syncSelectionFrom(context.widget.TimingsTreeMap);
             }
-            this.widget.TimingsTreeMap.onDblClick = function (value) {
-                var mainItem = context.main.getItem(value.SubGraphId);
-                context.main.centerOnItem(mainItem, true);
+        },
+
+        _initActivitiesMap: function () {
+            var context = this;
+            this.widget.ActivitiesTreeMap.onClick = function (value) {
+                context.syncSelectionFrom(context.widget.ActivitiesTreeMap);
             }
         },
 
@@ -442,6 +446,15 @@ define([
                 },
                 hideHelp: true
             }, params));
+
+            this.widget.ActivitiesTreeMap.init(lang.mixin({
+                query: {
+                    activitiesOnly: true,
+                    graphName: this.graphName,
+                    subGraphId: "*"
+                },
+                hideHelp: true
+            }, params));
         },
 
         refreshData: function () {
@@ -628,6 +641,7 @@ define([
             this.verticesStore.appendColumns(columns, ["name"], ["ecl", "definition"]);
             this.verticesGrid.set("columns", columns);
             this.verticesGrid.refresh();
+            this.widget.ActivitiesTreeMap.setActivities(vertices);
         },
 
         loadEdges: function () {
@@ -660,6 +674,13 @@ define([
                         selectedGlobalIDs.push(items[i].SubGraphId);
                     }
                 }
+            } else if (sourceControl == this.widget.ActivitiesTreeMap) {
+                    var items = sourceControl.getSelected();
+                    for (var i = 0; i < items.length; ++i) {
+                        if (items[i].ActivityID) {
+                            selectedGlobalIDs.push(items[i].ActivityID);
+                        }
+                    }
             } else if (sourceControl == this.verticesGrid || sourceControl == this.edgesGrid || sourceControl == this.subgraphsGrid || sourceControl == this.treeGrid) {
                 var items = sourceControl.getSelected();
                 for (var i = 0; i < items.length; ++i) {
@@ -680,6 +701,9 @@ define([
             if (sourceControl != this.widget.TimingsTreeMap) {
                 this.widget.TimingsTreeMap.setSelectedAsGlobalID(selectedGlobalIDs);
             }
+            if (sourceControl != this.widget.ActivitiesTreeMap) {
+                this.widget.ActivitiesTreeMap.setSelectedAsGlobalID(selectedGlobalIDs);
+            }
             if (sourceControl != this.subgraphsGrid && this.subgraphsGrid.store) {
                 this.subgraphsGrid.setSelection(selectedGlobalIDs);
             }

esp/src/eclwatch/SourceFilesWidget.js

@@ -77,7 +77,8 @@ define([
                             return dojoConfig.getImageHTML(row.IsSuperFile ? "folder_table.png" : "file.png") + "&nbsp;<a href='#' class='dgrid-row-url'>" + Name + "</a>";
                         }
                     }),
-                    Count: { label: "Usage", width: 72, sortable: true }
+                    FileCluster: { label: this.i18n.FileCluster, width: 300, sortable: false },
+                    Count: { label: this.i18n.Usage, width: 72, sortable: true }
                 }
             }, domID);
 

esp/src/eclwatch/TimingTreeMapWidget.js

@@ -22,20 +22,22 @@ define([
     "dojo/store/Memory",
     "dojo/dom",
     "dojo/dom-class",
+    "dojo/dom-style",
 
     "dijit/registry",
 
     "dojox/treemap/TreeMap",
 
     "hpcc/_Widget",
+    "hpcc/Utility",
     "hpcc/ESPWorkunit",
 
     "dojo/text!../templates/TimingTreeMapWidget.html"
 ],
-    function (declare, lang, i18n, nlsHPCC, arrayUtil, Memory, dom, domClass,
+    function (declare, lang, i18n, nlsHPCC, arrayUtil, Memory, dom, domClass, domStyle,
             registry, 
             TreeMap,
-            _Widget, ESPWorkunit,
+            _Widget, Utility, ESPWorkunit,
             template) {
         return declare("TimingTreeMapWidget", [_Widget], {
             templateString: template,
@@ -66,8 +68,18 @@ define([
                 this.inherited(arguments);
             },
 
+            calcHeight: function (elmID) {
+                var elmHeight, elmMargin, elm = document.getElementById(elmID);
+                var computedStyle = domStyle.getComputedStyle(elm);
+                elmHeight = parseFloat(computedStyle.getPropertyValue("height"));
+                elmMargin = parseFloat(computedStyle.getPropertyValue('margin-top')) + parseInt(computedStyle.getPropertyValue('margin-bottom'));
+                return elmHeight + elmMargin;
+            },
+
             resize: function (args) {
                 this.inherited(arguments);
+                var helpHeight = this.params.hideHelp ? 0 : this.calcHeight(this.id + "Help");
+                args.h -= helpHeight + 2;
                 this.treeMap._dataChanged = true;
                 this.treeMap.resize(args);
             },
@@ -155,6 +167,21 @@ define([
                 }
             },
 
+            setActivities: function (activities) {
+                var context = this;
+                setTimeout(function () {
+                    context.loadTimers(activities.map(function (activity) {
+                        return {
+                            __hpcc_prefix: "Activites",
+                            __hpcc_id: activity._globalID,
+                            ActivityID: activity._globalID,
+                            Name: activity.label,
+                            Seconds: Utility.espTime2Seconds(activity.TimeMaxLocalExecute)
+                        };
+                    }));
+                }, 20);
+            },
+
             refreshTreeMap: function () {
                 var context = this;
                 this.wu.fetchTimers(function (timers) {
@@ -163,41 +190,42 @@ define([
                 });
             },
 
-            loadTimers: function (timers) {
-                this.largestValue = 0;
+            timerFilter: function (timer) {
+                if (lang.exists("params.query.graphsOnly", this) && this.params.query.graphsOnly) {
+                    return (timer.SubGraphId && (this.params.query.graphName === "*" || this.params.query.graphName === timer.GraphName) && (this.params.query.subGraphId === "*" || this.params.query.subGraphId === timer.SubGraphId));
+                }
+                return (timer.Name != "Process" &&
+                        timer.Name != "compile" &&
+                        timer.Name != "Total thor time" &&
+                        timer.Name != "Total cluster time" &&
+                        timer.Name.indexOf(":TimeElapsed") < 0);
+            },
+
+            loadTimers: function (_timers) {
+                var context = this;
+                var timers = _timers.filter(function (d) { return context.timerFilter(d); });
                 var timerData = [];
                 if (timers) {
+                    this.avg = timers.reduce(function (sum, timer) { return sum + timer.Seconds; }, 0) / timers.length;
+                    var sqrDiffs = timers.map(function (timer) { return Math.pow(timer.Seconds - context.avg, 2); });
+                    var variance = sqrDiffs.reduce(function (sum, value) { return sum + value; }, 0) / sqrDiffs.length;
+                    this.stdDev = Math.sqrt(variance);
                     for (var i = 0; i < timers.length; ++i) {
-                        if (this.params.query.graphsOnly) {
-                            if (timers[i].SubGraphId && (this.params.query.graphName === "*" || this.params.query.graphName === timers[i].GraphName) && (this.params.query.subGraphId === "*" || this.params.query.subGraphId === timers[i].SubGraphId)) {
-                                timerData.push(lang.mixin({
-                                    __hpcc_prefix: timers[i].GraphName
-                                }, timers[i]));
-                                if (this.largestValue < timers[i].Seconds * 1000) {
-                                    this.largestValue = timers[i].Seconds * 1000;
-                                }
+                        var prefix = "other";
+                        if (timers[i].Name.indexOf("Graph graph") == 0) {
+                            if (!timers[i].SubGraphId) {
+                                continue;
                             }
-                        } else if ( timers[i].Name != "Process" &&
-                                    timers[i].Name != "Total thor time") {
-                            var prefix = "other";
-                            if (timers[i].Name.indexOf("Graph graph") == 0) {
-                                if (!timers[i].SubGraphId) {
-                                    continue;
-                                }
-                                prefix = timers[i].GraphName;
-                            } else {
-                                var nameParts = timers[i].Name.split(":");
-                                if (nameParts.length > 1) {
-                                    prefix = nameParts[0];
-                                }
-                            }
-                            timerData.push(lang.mixin({
-                                __hpcc_prefix: prefix
-                            }, timers[i]));
-                            if (this.largestValue < timers[i].Seconds * 1000) {
-                                this.largestValue = timers[i].Seconds * 1000;
+                            prefix = timers[i].GraphName;
+                        } else {
+                            var nameParts = timers[i].Name.split(":");
+                            if (nameParts.length > 1) {
+                                prefix = nameParts[0];
                             }
                         }
+                        timerData.push(lang.mixin({
+                            __hpcc_prefix: prefix
+                        }, timers[i]));
                     }
                 }
                 this.store = new Memory({
@@ -209,11 +237,18 @@ define([
                 this.treeMap.set("store", this.store);
                 this.treeMap.set("areaAttr", "Seconds");
                 this.treeMap.set("colorFunc", function (item) {
-                    var redness = Math.floor(255 * (item.Seconds * 1000 / context.largestValue));
+                    var deviation = (item.Seconds - context.avg) / context.stdDev;
+                    var redness = 0;
+                    var greeness = 0;
+                    if (deviation > 0) {
+                        redness = Math.min(255, Math.floor(255 * deviation / 3));
+                    } else {
+                        greeness = -Math.min(255, Math.floor(255 * deviation / 3));
+                    }
                     return {
-                        r: 255,
+                        r: 255 - greeness,
                         g: 255 - redness,
-                        b: 255 - redness
+                        b: 255 - redness - greeness
                     };
                 });
                 this.treeMap.set("groupAttrs", ["__hpcc_prefix"]);

esp/src/eclwatch/Utility.js

@@ -0,0 +1,50 @@
+define([], function () {
+    function espTime2Seconds(duration) {
+        //  GH:  <n>ns or <m>ms or <s>s or [<d> days ][<h>:][<m>:]<s>[.<ms>]
+        var nsIndex = duration.indexOf("ns");
+        if (nsIndex !== -1) {
+            return parseFloat(duration.substr(0, nsIndex)) / 1000000000;
+        }
+        var msIndex = duration.indexOf("ms");
+        if (msIndex !== -1) {
+            return parseFloat(duration.substr(0, msIndex)) / 1000;
+        }
+        var sIndex = duration.indexOf("s");
+        if (sIndex !== -1 && duration.indexOf("days") === -1) {
+            return parseFloat(duration.substr(0, sIndex));
+        }
+
+        var dayTimeParts = duration.split(" days ");
+        var days = parseFloat(dayTimeParts.length > 1 ? dayTimeParts[0] : 0.0);
+        var time = dayTimeParts.length > 1 ? dayTimeParts[1] : dayTimeParts[0];
+        var secs = 0.0;
+        var timeParts = time.split(":").reverse();
+        for (var j = 0; j < timeParts.length; ++j) {
+            secs += parseFloat(timeParts[j]) * Math.pow(60, j);
+        }
+        return (days * 24 * 60 * 60) + secs;
+    }
+
+    function espTime2SecondsTests() {
+        var tests = [
+            { str: "1.1s", expected: 1.1 },
+            { str: "2.2ms", expected: 0.0022 },
+            { str: "3.3ns", expected: 0.0000000033 },
+            { str: "4.4", expected: 4.4 },
+            { str: "5:55.5", expected: 355.5 },
+            { str: "6:06:06.6", expected: 21966.6 },
+            { str: "6:06:6.6", expected: 21966.6 },
+            { str: "6:6:6.6", expected: 21966.6 },
+            { str: "7 days 7:07:7.7", expected: 630427.7 }
+        ];
+        tests.forEach(function (test, idx) {
+            if (espTime2Seconds(test.str) !== test.expected) {
+                console.log("espTime2SecondsTests failed with " + espTime2Seconds(test.str) + " !== " + test.expected);
+            }
+        }, this);
+    }
+
+    return {
+        espTime2Seconds: espTime2Seconds
+    }
+});

esp/src/eclwatch/nls/hpcc.js

@@ -12,6 +12,7 @@ define({root:
     ActiveWorkunit: "Active Workunit",
     Activities: "Activities",
     Activity: "Activity",
+    ActivityMap: "Activity Map",
     ActualSize: "Actual Size",
     Add: "Add",
     AddFile: "Add File",
@@ -155,6 +156,7 @@ define({root:
     ExpandAll: "Expand All",
     Export: "Export",
     File: "File",
+    FileCluster: "File Cluster",
     FileCounts: "File Counts",
     FileName: "File Name",
     FileParts: "File Parts",
@@ -168,6 +170,7 @@ define({root:
     FileUploader: "File Uploader",
     FileUploadStillInProgress: "File upload still in progress",
     Filter: "Filter",
+    FilterSet: "Filter Set",
     Find: "Find",
     Finished: "Finished",
     FindNext: "Find Next",

esp/src/eclwatch/templates/FilterDropDownWidget.html

@@ -1,7 +1,7 @@
 <span class="${baseClass}">
-    <img id="${id}IconFilter" src="${dojoConfig.urlInfo.resourcePath}/img/noFilter.png" class="iconNoFilter" />
+    <img id="${id}IconFilter" src="${dojoConfig.urlInfo.resourcePath}/img/noFilter1.png" class="iconNoFilter" />
     <div id="${id}FilterDropDown" data-dojo-type="dijit.form.DropDownButton">
-        <span>${i18n.Filter}</span>
+        <span id="${id}FilterLabel">${i18n.Filter}</span>
         <div class="toolTip" data-dojo-type="dijit.TooltipDialog">
             <div id="${id}FilterForm" style="width:${_width}" onsubmit="return false;" data-dojo-type="dijit.form.Form">
                 <div id="${id}TableContainer" class="dijitDialogPaneContentArea" data-dojo-props="cols:2" data-dojo-type="hpcc.TableContainer">

esp/src/eclwatch/templates/GraphTreeWidget.html

@@ -55,6 +55,8 @@
                 </div>
                 <div id="${id}TimingsTreeMap" title="${i18n.TimingsMap}" data-dojo-props="iconClass:'iconTreeMap', showTitle: false" data-dojo-type="TimingTreeMapWidget">
                 </div>
+                <div id="${id}ActivitiesTreeMap" title="${i18n.ActivityMap}" data-dojo-props="iconClass:'iconTreeMap', showTitle: false" data-dojo-type="TimingTreeMapWidget">
+                </div>
             </div>
             <div id="${id}LocalTabContainer" style="height: 33%" data-dojo-props="region: 'bottom', splitter:true, minSize: 120, tabPosition: 'bottom'" data-dojo-type="dijit.layout.TabContainer">
                 <div id="${id}Properties" title="${i18n.Properties}" data-dojo-type="dijit.layout.ContentPane">

initfiles/bash/etc/init.d/dafilesrv.in

@@ -72,7 +72,6 @@ component=""
 runSetupOnly=0
 source ${configgen_path}/hpcc_setenv
 
-is_root
 which_service
 get_commondirs
 

initfiles/bash/etc/init.d/hpcc-init.in

@@ -90,10 +90,6 @@ source  ${INSTALL_DIR}/etc/init.d/hpcc_common
 source  ${INSTALL_DIR}/etc/init.d/init-functions
 source  ${INSTALL_DIR}/etc/init.d/export-path
 
-# Only root user can write following logfile
-is_root
-
-[ ! -e ${LOG_DIR} ] && mkdir -p ${LOG_DIR}
 export logfile=${LOG_DIR}/hpcc-init.log
 
 [ ! -e ${logfile}  ] && touch $logfile
@@ -277,6 +273,9 @@ if [ -z $arg ] || [ $# -ne 1 ]; then
 fi
 
 log "Debug log written to $LOG_DIR/hpcc-init.debug"
+[ -e $LOG_DIR/hpcc-init.debug ] && rm -rf ${LOG_DIR}/hpcc-init.debug
+touch ${LOG_DIR}/hpcc-init.debug
+chown ${user}:${user} ${LOG_DIR}/hpcc-init.debug
 exec 2>$LOG_DIR/hpcc-init.debug
 set -x
 

initfiles/bash/etc/init.d/hpcc_common.in

@@ -302,7 +302,11 @@ configGenCmd() {
     validate_configuration
     configcmd="${configgen_path}/configgen -env ${envfile} -od ${runtime} -id ${componentFile} -c ${compName}"
     log "$configcmd"
-    su ${user} -c "$configcmd" 2>/dev/null
+    if [ "${USER}" != "${user}" ]; then
+        su ${user} -c "$configcmd" 2>/dev/null
+    else
+        ${configcmd} 2>/dev/null
+    fi
     rc=$?
     if [[ $rc -ne 0 ]]; then
         log  "configGenCmd(): failure in configgen call"
@@ -335,6 +339,7 @@ createRuntime() {
     chown -c $user:$group ${lock} 1> /dev/null 2>/dev/null
     chown -c $user:$group ${log}  1> /dev/null 2>/dev/null
 
+    [ -z "$compName" ] && return
 
     # Creating Component Specific directories
     # Creating pidfile specific directory and changing its owner permissions
@@ -549,6 +554,8 @@ startCmd() {
 stop_component() {
     printf "Stopping %-21s" "${compName}... "
 
+    cd ${compPath}
+
     ####
     ## This is handling for when daemon is running as an orphan daemon. That is process is
     ## not running but associated pidfile and/or lockfiles do exist.

initfiles/bash/etc/init.d/install-init.in

@@ -297,6 +297,7 @@ if [ "${DISTRIB_NAME}" = "ubuntu" ] &&
    fi
 fi
 
-if [ -d ${CONFIG_DIR} ]; then
-    date > ${CONFIG_DIR}/installed
-fi
+createRuntime
+[ -e  ${LOG_DIR}/hpcc-init.debug ] && chown ${user}:${user} ${LOG_DIR}/hpcc-init.debug
+
+exit 0

plugins/cassandra/cassandraembed.cpp

@@ -1114,7 +1114,12 @@ public:
     {
         if (isAll)
             UNSUPPORTED("SET(ALL)");
-        collection.setown(new CassandraCollection(cass_collection_new(CASS_COLLECTION_TYPE_SET, numElements)));
+        // We don't know whether the corresponding field in Cassandra is a list or a set. Try binding a dummy list to tell which.
+        CassandraCollection temp(cass_collection_new(CASS_COLLECTION_TYPE_LIST, 0));
+        if (cass_statement_bind_collection(stmtInfo->queryStatement(), thisParam, temp) == CASS_OK)
+            collection.setown(new CassandraCollection(cass_collection_new(CASS_COLLECTION_TYPE_LIST, numElements)));
+        else
+            collection.setown(new CassandraCollection(cass_collection_new(CASS_COLLECTION_TYPE_SET, numElements)));
         return true;
     }
     virtual bool processBeginDataset(const RtlFieldInfo * field, unsigned numRows)

plugins/cassandra/cpp-driver

@@ -1 +1 @@
-Subproject commit d7bbad34db39a51f209c6fadd07c9ec3a0bb86b7
+Subproject commit b4bb435129bab533612fa2caf194555fa943f925

plugins/pyembed/pyembed.cpp

@@ -345,6 +345,9 @@ public:
         }
 
         const RtlFieldInfo * const *fields = type->queryFields();
+        if (!fields && type->queryChildType())
+            fields = type->queryChildType()->queryFields();
+        assertex(fields);
         StringBuffer names;
         while (*fields)
         {
@@ -842,7 +845,7 @@ public:
     {
         // Expect to see a tuple here, or possibly (if the ECL record has a single field), an arbitrary scalar object
         // If it's a tuple, we push it onto our stack as the active object
-        nextField(NULL);  // MORE - should it be passing field?
+        nextField(field);
         if (!PyTuple_Check(elem))
         {
             if (countFields(field->type->queryFields())==1)
@@ -1043,11 +1046,13 @@ protected:
     void push()
     {
         stack.append(args.getClear());
+        args.setown(PyList_New(0));
     }
     void pop()
     {
-        addArg(args.getClear());
+        OwnedPyObject arg = args.getClear();
         args.setown((PyObject *) stack.popGet());
+        addArg(arg.getClear());
     }
     void addArg(PyObject *arg)
     {

rtl/eclrtl/rtlbcd.cpp

@@ -22,20 +22,21 @@
 #include "jmutex.hpp"
 #include "jexcept.hpp"
 
-static CriticalSection bcdCriticalSection;
-static Decimal stack[32];
-static unsigned curStack;
+static thread_local Decimal stack[32];
+static thread_local unsigned curStack;
 
 //---------------------------------------------------------------------------------------------------------------------
 
+//These functions are retained to that old work units will load, and then report a version mismatch, rather than a
+//confusing unresolved symbol error.
 void DecLock()
 {
-    bcdCriticalSection.enter();
+    throwUnexpected();
 }
 
 void DecUnlock()
 {
-    bcdCriticalSection.leave();
+    throwUnexpected();
 }
 
 unsigned DecMarkStack()

rtl/eclrtl/rtlbcd.hpp

@@ -66,8 +66,6 @@ ECLRTL_API void  DecTruncate( void );       // truncate value on top of decimal
 ECLRTL_API void  DecTruncateAt(unsigned places);       // truncate value on top of decimal stack
 ECLRTL_API void  DecUlongPower(unsigned long pow); // calculates top of stack to the power of unsigned long and replaces with result
 
-ECLRTL_API void  DecLock();
-ECLRTL_API void  DecUnlock();
 ECLRTL_API bool  DecValid(bool isSigned, unsigned digits, const void * data);
 ECLRTL_API bool  DecValidTos();
 ECLRTL_API bool  Dec2Bool(size32_t bytes, const void * data);
@@ -87,11 +85,12 @@ ECLRTL_API void  DecUnlock();
 ECLRTL_API unsigned DecMarkStack();
 ECLRTL_API void DecReleaseStack(unsigned mark);
 
+//No longer a critical section (since stack is thread_local), but prevents problems with exceptions.
 class ECLRTL_API BcdCriticalBlock
 {
 public:
-    inline BcdCriticalBlock()       { DecLock(); mark = DecMarkStack(); }
-    inline ~BcdCriticalBlock()      { DecReleaseStack(mark); DecUnlock(); }
+    inline BcdCriticalBlock()       { mark = DecMarkStack(); }
+    inline ~BcdCriticalBlock()      { DecReleaseStack(mark); }
 
 protected:
     unsigned mark;

rtl/eclrtl/rtlfield.cpp

@@ -219,8 +219,8 @@ size32_t RtlSwapIntTypeInfo::build(ARowBuilder &builder, size32_t offset, const
 {
     builder.ensureCapacity(length+offset, str(field->name));
     __int64 val = isUnsigned() ? (__int64) source.getUnsignedResult(field) : source.getSignedResult(field);
-    // NOTE - we assume that the value returned from the source is already a swapped int
-    rtlWriteInt(builder.getSelf() + offset, val, length);
+    // NOTE - we assume that the value returned from the source is NOT already a swapped int - source doesn;t know that we are going to store it swapped
+    rtlWriteSwapInt(builder.getSelf() + offset, val, length);
     offset += length;
     return offset;
 }
@@ -298,14 +298,12 @@ size32_t RtlStringTypeInfo::build(ARowBuilder &builder, size32_t offset, const R
         builder.ensureCapacity(offset+size+sizeof(size32_t), str(field->name));
         byte *dest = builder.getSelf()+offset;
         rtlWriteInt4(dest, size);
-#if 0
-        // NOTE - you might argue that we should convert the incoming data to EBCDIC. But it seems more useful to
-        // define the semantics as being that the IFieldSource should return EBCDIC if you have declared the matching field as EBCDIC
-        // (otherwise, why did you bother?)
+        // NOTE - it has been the subject of debate whether we should convert the incoming data to EBCDIC, or expect the IFieldSource to have already returned ebcdic
+        // In order to be symmetrical with the passing of ecl data to a IFieldProcessor the former interpretation is preferred.
+        // Expecting source.getStringResult to somehow "know" that EBCDIC was expected seems odd.
         if (isEbcdic())
             rtlStrToEStr(size, (char *) dest+sizeof(size32_t), size, (char *)value);
         else
-#endif
             memcpy(dest+sizeof(size32_t), value, size);
         offset += size+sizeof(size32_t);
     }
@@ -313,12 +311,9 @@ size32_t RtlStringTypeInfo::build(ARowBuilder &builder, size32_t offset, const R
     {
         builder.ensureCapacity(offset+length, str(field->name));
         byte *dest = builder.getSelf()+offset;
-#if 0
-        // See above...
         if (isEbcdic())
             rtlStrToEStr(length, (char *) dest, size, (char *) value);
         else
-#endif
             rtlStrToStr(length, dest, size, value);
         offset += length;
     }

rtl/eclrtl/rtlxml.cpp

@@ -98,7 +98,8 @@ void outputXmlDecimal(const void *field, unsigned size, unsigned precision, cons
     char dec[50];
     if (fieldname && *fieldname)
         out.append('<').append(fieldname).append('>');
-    DecLock();
+
+    BcdCriticalBlock bcdBlock;
     if (DecValid(true, size*2-1, field))
     {
         DecPushDecimal(field, size, precision);
@@ -109,7 +110,7 @@ void outputXmlDecimal(const void *field, unsigned size, unsigned precision, cons
     }
     else
         out.append("####");
-    DecUnlock();
+
     if (fieldname && *fieldname)
         out.append("</").append(fieldname).append('>');
 }
@@ -119,7 +120,8 @@ void outputXmlUDecimal(const void *field, unsigned size, unsigned precision, con
     char dec[50];
     if (fieldname && *fieldname)
         out.append('<').append(fieldname).append('>');
-    DecLock();
+
+    BcdCriticalBlock bcdBlock;
     if (DecValid(false, size*2, field))
     {
         DecPushUDecimal(field, size, precision);
@@ -130,7 +132,7 @@ void outputXmlUDecimal(const void *field, unsigned size, unsigned precision, con
     }
     else
         out.append("####");
-    DecUnlock();
+
     if (fieldname && *fieldname)
         out.append("</").append(fieldname).append('>');
 }
@@ -253,7 +255,8 @@ void outputJsonDecimal(const void *field, unsigned size, unsigned precision, con
 {
     char dec[50];
     appendJSONNameOrDelimit(out, fieldname);
-    DecLock();
+
+    BcdCriticalBlock bcdBlock;
     if (DecValid(true, size*2-1, field))
     {
         DecPushDecimal(field, size, precision);
@@ -262,14 +265,14 @@ void outputJsonDecimal(const void *field, unsigned size, unsigned precision, con
         while(isspace(*finger)) finger++;
         out.append(finger);
     }
-    DecUnlock();
 }
 
 void outputJsonUDecimal(const void *field, unsigned size, unsigned precision, const char *fieldname, StringBuffer &out)
 {
     char dec[50];
     appendJSONNameOrDelimit(out, fieldname);
-    DecLock();
+
+    BcdCriticalBlock bcdBlock;
     if (DecValid(false, size*2, field))
     {
         DecPushUDecimal(field, size, precision);
@@ -278,5 +281,4 @@ void outputJsonUDecimal(const void *field, unsigned size, unsigned precision, co
         while(isspace(*finger)) finger++;
         out.append(finger);
     }
-    DecUnlock();
 }

system/jhtree/jhtree.cpp

@@ -609,6 +609,11 @@ public:
                 throw e;
             }
         }
+        else
+        {
+            keySize = 0;
+            keyedSize = 0;
+        }
     }
 
     virtual void reset(bool crappyHack)
@@ -2545,7 +2550,11 @@ public:
             numkeys = _keyset->numParts();
         }
         else
+        {
+            keySize = 0;
+            keyedSize = 0;
             numkeys = 0;
+        }
         killBuffers();
     }
 
@@ -2694,7 +2703,8 @@ public:
         if (!started)
         {
             started = true;
-            segs.checkSize(keyedSize, "[merger]"); //PG: not sure what keyname to use here
+            if (keyedSize)
+                segs.checkSize(keyedSize, "[merger]"); //PG: not sure what keyname to use here
         }
         if (!crappyHack)
         {

system/jlib/jsocket.cpp

@@ -24,7 +24,6 @@
     look at loopback
 */
 
-
 #include "platform.h"
 #ifdef _VER_C5
 #include <clwclib.h>
@@ -137,7 +136,7 @@ static bool IP6preferred=false;         // e.g. for DNS and socket create
 IpSubNet PreferredSubnet(NULL,NULL);    // set this if you prefer a particular subnet for debugging etc
                                         // e.g. PreferredSubnet("192.168.16.0", "255.255.255.0")
 
-static atomic_t pre_conn_unreach_cnt = ATOMIC_INIT(0);    // global count of pre_connect() ENETUNREACH error
+static atomic_t pre_conn_unreach_cnt = ATOMIC_INIT(0);    // global count of pre_connect() JSE_NETUNREACH error
 
 #define IPV6_SERIALIZE_PREFIX (0x00ff00ff)
 
@@ -315,13 +314,8 @@ struct MCASTREQ
 #define T_FD_SET fd_set
 #define XFD_SETSIZE FD_SETSIZE
 //Following are defined in more modern headers
-#ifndef ETIMEDOUT
-#define ETIMEDOUT WSAETIMEDOUT
-#define ECONNREFUSED WSAECONNREFUSED
-#endif
 #define XFD_ZERO(s) FD_ZERO(s)
 #define SEND_FLAGS 0
-#define BADSOCKERR(err) ((err==WSAEBADF)||(err==WSAENOTSOCK))
 #define CHECKSOCKRANGE(s)
 #elif defined(__FreeBSD__) || defined(__APPLE__)
 #define XFD_SETSIZE FD_SETSIZE
@@ -329,7 +323,6 @@ struct MCASTREQ
 #define XFD_ZERO(s) FD_ZERO(s)
 #define T_SOCKET int
 #define SEND_FLAGS (MSG_NOSIGNAL)
-#define BADSOCKERR(err) ((err==EBADF)||(err==ENOTSOCK))
 #define CHECKSOCKRANGE(s)
 #else
 #define XFD_SETSIZE 32768
@@ -353,13 +346,14 @@ struct xfd_set { __fd_mask fds_bits[XFD_SETSIZE / __NFDBITS]; }; // define our o
 #define XFD_ZERO(s) memset(s,0,sizeof(xfd_set))
 #define T_SOCKET int
 #define SEND_FLAGS (MSG_NOSIGNAL)
-#define BADSOCKERR(err) ((err==EBADF)||(err==ENOTSOCK))
 #endif
 #ifdef CENTRAL_NODE_RANDOM_DELAY
 static SocketEndpointArray CentralNodeArray;
 #endif
 enum SOCKETMODE { sm_tcp_server, sm_tcp, sm_udp_server, sm_udp, sm_multicast_server, sm_multicast};
 
+#define BADSOCKERR(err) ((err==JSE_BADF)||(err==JSE_NOTSOCK))
+
 class CSocket: public CInterface, public ISocket
 {
 public:
@@ -521,17 +515,20 @@ bool win_socket_library::initdone = false;
 static win_socket_library ws32_lib;
 
 #define ERRNO() WSAGetLastError()
-#ifndef EADDRINUSE
-#define EADDRINUSE WSAEADDRINUSE
-#define ECONNRESET WSAECONNRESET
-#define ECONNABORTED WSAECONNABORTED
-#define ENOTCONN WSAENOTCONN
-#define EWOULDBLOCK WSAEWOULDBLOCK
-#define EINPROGRESS WSAEINPROGRESS
-#define ENETUNREACH WSAENETUNREACH
-#define ENOTSOCK WSAENOTSOCK
-#endif
-#define EINTRCALL WSAEINTR
+
+#define JSE_ADDRINUSE WSAEADDRINUSE
+#define JSE_CONNRESET WSAECONNRESET
+#define JSE_CONNABORTED WSAECONNABORTED
+#define JSE_NOTCONN WSAENOTCONN
+#define JSE_WOULDBLOCK WSAEWOULDBLOCK
+#define JSE_INPROGRESS WSAEINPROGRESS
+#define JSE_NETUNREACH WSAENETUNREACH
+#define JSE_NOTSOCK WSAENOTSOCK
+#define JSE_TIMEDOUT WSAETIMEDOUT
+#define JSE_CONNREFUSED WSAECONNREFUSED
+#define JSE_BADF WSAEBADF
+
+#define JSE_INTR WSAEINTR
 
 struct j_sockaddr_in6 {
     short   sin6_family;        /* AF_INET6 */
@@ -637,6 +634,20 @@ int inet_aton (const char *name, struct in_addr *addr)
 
 
 #else
+
+#define JSE_ADDRINUSE EADDRINUSE
+#define JSE_CONNRESET ECONNRESET
+#define JSE_CONNABORTED ECONNABORTED
+#define JSE_NOTCONN ENOTCONN
+#define JSE_WOULDBLOCK EWOULDBLOCK
+#define JSE_INPROGRESS EINPROGRESS
+#define JSE_NETUNREACH ENETUNREACH
+#define JSE_NOTSOCK ENOTSOCK
+#define JSE_TIMEDOUT ETIMEDOUT
+#define JSE_CONNREFUSED ECONNREFUSED
+#define JSE_BADF EBADF
+
+
 #define _inet_ntop inet_ntop
 #define _inet_pton inet_pton
 
@@ -651,7 +662,7 @@ typedef union {
 #define DEFINE_SOCKADDR(name) J_SOCKADDR name; memset(&name,0,sizeof(J_SOCKADDR))
 
 
-#define EINTRCALL EINTR
+#define JSE_INTR EINTR
 #define ERRNO() (errno)
 #ifndef INADDR_NONE
 #define INADDR_NONE (-1)
@@ -825,8 +836,8 @@ int CSocket::pre_connect (bool block)
     int rc = ::connect(sock, &u.sa, ul);
     if (rc==SOCKET_ERROR) {
         err = ERRNO();
-        if ((err != EINPROGRESS)&&(err != EWOULDBLOCK)&&(err != ETIMEDOUT)&&(err!=ECONNREFUSED)) {   // handled by caller
-            if (err != ENETUNREACH) {
+        if ((err != JSE_INPROGRESS)&&(err != JSE_WOULDBLOCK)&&(err != JSE_TIMEDOUT)&&(err!=JSE_CONNREFUSED)) {   // handled by caller
+            if (err != JSE_NETUNREACH) {
                 atomic_set(&pre_conn_unreach_cnt, 0);
                 LOGERR2(err,1,"pre_connect");
             } else {
@@ -859,7 +870,7 @@ int CSocket::post_connect ()
         set_nagle(false);
         state = ss_open;
     }
-    else if ((err!=ETIMEDOUT)&&(err!=ECONNREFUSED)) // handled by caller
+    else if ((err!=JSE_TIMEDOUT)&&(err!=JSE_CONNREFUSED)) // handled by caller
         LOGERR2(err,1,"post_connect");
     return err;
 }
@@ -911,7 +922,7 @@ void CSocket::open(int listen_queue_size,bool reuseports)
     int saverr;
     if (::bind(sock, &u.sa, ul) != 0) {
         saverr = ERRNO();
-        if (saverr==EADDRINUSE) {   // don't log as error (some usages probe ports)
+        if (saverr==JSE_ADDRINUSE) {   // don't log as error (some usages probe ports)
 ErrPortInUse:
             closesock();
             char msg[1024]; 
@@ -927,7 +938,7 @@ ErrPortInUse:
     if (!connectionless()) {
         if (::listen(sock, listen_queue_size) != 0) {
             saverr = ERRNO();
-            if (saverr==EADDRINUSE)
+            if (saverr==JSE_ADDRINUSE)
                 goto ErrPortInUse;
             closesock();
             THROWJSOCKEXCEPTION(saverr);
@@ -990,7 +1001,7 @@ ISocket* CSocket::accept(bool allowcancel)
                 return NULL;
             THROWJSOCKEXCEPTION(JSOCKERR_cancel_accept);
         }
-        if (saverr != EINTRCALL) {
+        if (saverr != JSE_INTR) {
             accept_cancel_state = accept_not_cancelled;
             THROWJSOCKEXCEPTION(saverr);
         }
@@ -1203,7 +1214,7 @@ bool CSocket::connect_timeout( unsigned timeout, bool noexception)
     int err;
     while (!tm.timedout(&remaining)) {
         err = pre_connect(false);
-        if ((err == EINPROGRESS)||(err == EWOULDBLOCK)) {
+        if ((err == JSE_INPROGRESS)||(err == JSE_WOULDBLOCK)) {
             T_FD_SET fds;
             struct timeval tv;
             CHECKSOCKRANGE(sock);
@@ -1311,7 +1322,7 @@ void CSocket::connect_wait(unsigned timems)
     #ifndef BLOCK_POLLED_SINGLE_CONNECTS
             unsigned polltime = 1;
     #endif
-            while (!blockselect && ((err == EINPROGRESS)||(err == EWOULDBLOCK))) {
+            while (!blockselect && ((err == JSE_INPROGRESS)||(err == JSE_WOULDBLOCK))) {
                 T_FD_SET fds;
                 struct timeval tv;
                 CHECKSOCKRANGE(sock);
@@ -1463,7 +1474,7 @@ int CSocket::wait_read(unsigned timeout)
         }
         if (ret==SOCKET_ERROR) {
             int err = ERRNO();
-            if (err!=EINTRCALL) {   // else retry (should adjust time but for our usage don't think it matters that much)
+            if (err!=JSE_INTR) {   // else retry (should adjust time but for our usage don't think it matters that much)
                 LOGERR2(err,1,"wait_read");
                 break;
             }
@@ -1493,7 +1504,7 @@ int CSocket::wait_write(unsigned timeout)
         }
         if (ret==SOCKET_ERROR) {
             int err = ERRNO();
-            if (err!=EINTRCALL) {   // else retry (should adjust time but for our usage don't think it matters that much)
+            if (err!=JSE_INTR) {   // else retry (should adjust time but for our usage don't think it matters that much)
                 LOGERR2(err,1,"wait_write");
                 break;
             }
@@ -1554,14 +1565,14 @@ EintrRetry:
                 LOGERR2(err,1,"Socket closed during read");
                 rc = 0;
             }
-            else if ((err==EINTRCALL)&&(retrycount--!=0)) {
+            else if ((err==JSE_INTR)&&(retrycount--!=0)) {
                 LOGERR2(err,1,"EINTR retrying");
                 goto EintrRetry;
             }
             else {
                 VStringBuffer errMsg("readtms(timeoutms=%d)", timeoutms);
                 LOGERR2(err,1,errMsg.str());
-                if ((err==ECONNRESET)||(err==EINTRCALL)||(err==ECONNABORTED)) {
+                if ((err==JSE_CONNRESET)||(err==JSE_INTR)||(err==JSE_CONNABORTED)) {
                     errclose();
                     err = JSOCKERR_broken_pipe;
                 }
@@ -1631,7 +1642,7 @@ EintrRetry:
                 LOGERR2(err,3,"Socket closed during read");
                 rc = 0;
             }
-            else if ((err==EINTRCALL)&&(retrycount--!=0)) {
+            else if ((err==JSE_INTR)&&(retrycount--!=0)) {
                 if (sock==INVALID_SOCKET)
                     rc = 0;         // convert an EINTR after closed to a graceful close
                 else {
@@ -1641,7 +1652,7 @@ EintrRetry:
             }
             else {
                 LOGERR2(err,3,"read");
-                if ((err==ECONNRESET)||(err==EINTRCALL)||(err==ECONNABORTED)) {
+                if ((err==JSE_CONNRESET)||(err==JSE_INTR)||(err==JSE_CONNABORTED)) {
                     errclose();
                     err = JSOCKERR_broken_pipe;
                 }
@@ -1691,13 +1702,13 @@ EintrRetry:
                 LOGERR2(err,5,"Socket closed during read");
                 rc = 0;
             }
-            else if ((err==EINTRCALL)&&(retrycount--!=0)) {
+            else if ((err==JSE_INTR)&&(retrycount--!=0)) {
                 LOGERR2(err,5,"EINTR retrying");
                 goto EintrRetry;
             }
             else {
                 LOGERR2(err,5,"read");
-                if ((err==ECONNRESET)||(err==EINTRCALL)||(err==ECONNABORTED)) {
+                if ((err==JSE_CONNRESET)||(err==JSE_INTR)||(err==JSE_CONNABORTED)) {
                     errclose();
                     err = JSOCKERR_broken_pipe;
                 }
@@ -1747,23 +1758,23 @@ EintrRetry:
                 LOGERR2(err,7,"Socket closed during write");
                 rc = 0;
             }
-            else if ((err==EINTRCALL)&&(retrycount--!=0)) {
+            else if ((err==JSE_INTR)&&(retrycount--!=0)) {
                 LOGERR2(err,7,"EINTR retrying");
                 goto EintrRetry;
             }
             else {
-                if (((sockmode==sm_multicast)||(sockmode==sm_udp))&&(err==ECONNREFUSED))
+                if (((sockmode==sm_multicast)||(sockmode==sm_udp))&&(err==JSE_CONNREFUSED))
                     break; // ignore
                 LOGERR2(err,7,"write");
-                if ((err==ECONNRESET)||(err==EINTRCALL)||(err==ECONNABORTED)
+                if ((err==JSE_CONNRESET)||(err==JSE_INTR)||(err==JSE_CONNABORTED)
 #ifndef _WIN32
-                    ||(err==EPIPE)||(err==ETIMEDOUT)  // linux can raise these on broken pipe
+                    ||(err==EPIPE)||(err==JSE_TIMEDOUT)  // linux can raise these on broken pipe
 #endif
                     ) {
                     errclose();
                     err = JSOCKERR_broken_pipe;
                 }
-                if ((err == EWOULDBLOCK) && nonblocking)
+                if ((err == JSE_WOULDBLOCK) && nonblocking)
                     break;
                 THROWJSOCKEXCEPTION(err);
             }
@@ -1854,7 +1865,7 @@ EintrRetry:
     }
     if (rc < 0) {
         int err=ERRNO();
-        if ((err==EINTRCALL)&&(retrycount--!=0)) {
+        if ((err==JSE_INTR)&&(retrycount--!=0)) {
             LOGERR2(err,7,"EINTR retrying");
             goto EintrRetry;
         }
@@ -1879,9 +1890,9 @@ size32_t CSocket::udp_write_to(const SocketEndpoint &ep, void const* buf, size32
         int rc = sendto(sock, (char*)buf, size, 0, &u.sa, ul);
         if (rc < 0) {
             int err=ERRNO();
-            if (((sockmode==sm_multicast)||(sockmode==sm_udp))&&(err==ECONNREFUSED))
+            if (((sockmode==sm_multicast)||(sockmode==sm_udp))&&(err==JSE_CONNREFUSED))
                 break; // ignore
-            if (err!=EINTRCALL) {
+            if (err!=JSE_INTR) {
                 THROWJSOCKEXCEPTION(err);
             }
         }
@@ -1929,13 +1940,13 @@ EintrRetry:
             LOGERR2(err,8,"Socket closed during write");
             sent = 0;
         }
-        else if ((err==EINTRCALL)&&(retrycount--!=0)) {
+        else if ((err==JSE_INTR)&&(retrycount--!=0)) {
             LOGERR2(err,8,"EINTR retrying");
             goto EintrRetry;
         }
         else {
             LOGERR2(err,8,"write_multiple");
-            if ((err==ECONNRESET)||(err==EINTRCALL)||(err==ECONNABORTED)||(err==ETIMEDOUT)) {
+            if ((err==JSE_CONNRESET)||(err==JSE_INTR)||(err==JSE_CONNABORTED)||(err==JSE_TIMEDOUT)) {
                 errclose();
                 err = JSOCKERR_broken_pipe;
             }
@@ -2251,7 +2262,7 @@ void CSocket::shutdown(unsigned mode)
         int rc = ::shutdown(sock, mode);
         if (rc != 0) {
             int err=ERRNO();
-            if (err==ENOTCONN) {
+            if (err==JSE_NOTCONN) {
                 LOGERR2(err,9,"shutdown");
                 err = JSOCKERR_broken_pipe;
             }
@@ -4023,8 +4034,8 @@ public:
         if (offset>=ni)
 #endif
             offset = 0;
-        unsigned j=offset;
-        ForEachItemIn(i,items) {
+        unsigned j = offset;
+        ForEachItemIn(i, items) {
             SelectItem &si = items.element(j);
             j++;
             if (j==ni)
@@ -4105,7 +4116,7 @@ public:
                 if (n < 0) {
                     CriticalBlock block(sect);
                     int err = ERRNO();
-                    if (err != EINTRCALL) {
+                    if (err != JSE_INTR) {
                         if (dummysockopen) {
                             LOGERR(err,12,"CSocketSelectThread select error"); // should cache error ?
                             validateselecterror = err;
@@ -4633,7 +4644,7 @@ public:
                 if (n < 0) {
                     CriticalBlock block(sect);
                     int err = ERRNO();
-                    if (err != EINTRCALL) {
+                    if (err != JSE_INTR) {
                         if (dummysockopen) {
                             LOGERR(err,12,"CSocketEpollThread epoll error"); // should cache error ?
                             validateselecterror = err;
@@ -5377,7 +5388,7 @@ static CSocket *prepareSocket(unsigned idx,const SocketEndpoint &ep, ISocketConn
 {
     Owned<CSocket> sock = new CSocket(ep,sm_tcp,NULL);
     int err = sock->pre_connect(false);
-    if ((err == EINPROGRESS)||(err == EWOULDBLOCK)) 
+    if ((err == JSE_INPROGRESS)||(err == JSE_WOULDBLOCK)) 
         return sock.getClear();
     if (err==0) {
         int err = sock->post_connect();
@@ -5429,7 +5440,7 @@ void multiConnect(const SocketEndpointArray &eps,ISocketConnectNotify &inotify,u
                 CriticalUnblock unblock(*sect); // up to caller to cope with multithread
                 if (err==0) 
                     inotify->connected(idx,ep,sock);
-                else if ((err==ETIMEDOUT)||(err==ECONNREFUSED))  { 
+                else if ((err==JSE_TIMEDOUT)||(err==JSE_CONNREFUSED))  { 
                          // don't give up so easily (maybe listener not yet started (i.e. racing))
                     newsock = prepareSocket(idx,ep,*inotify);
                     Sleep(100); // not very nice but without this would just loop 
@@ -6038,7 +6049,7 @@ public:
                 isopen = true;
                 err = initerr?initerr:sock->pre_connect(false);
                 initerr = 0;
-                if ((err == EINPROGRESS)||(err == EWOULDBLOCK))
+                if ((err == JSE_INPROGRESS)||(err == JSE_WOULDBLOCK))
                     err = 0; // continue
                 else {
                     if (err==0)
@@ -6176,7 +6187,7 @@ int wait_multiple(bool isRead,               //IN   true if wait read, false it
     else
     {
         int err = ERRNO();
-        if (err != EINTRCALL)
+        if (err != JSE_INTR)
         {
             throw MakeStringException(-1,"wait_multiple::select error %d", err);
         }

testing/regress/ecl/bcd2.ecl

@@ -30,6 +30,10 @@ rec := { decimal17_4 sumx, unsigned countx };
 ds := dataset([{20,3},{10,2},{10.0001,2}], rec);
 output(nofold(ds), { sumx, countx, decimal20_10 average := sumx/countx, sumx between 10 and 10.00009, sumx between 10D and 10.00009D });
 
+ds1 := dataset(10000, TRANSFORM(rec, SELF.sumx := COUNTER * COUNTER; SELF.countx := COUNTER));
+ds2 := dataset(10000, TRANSFORM(rec, SELF.sumx := COUNTER * 2 * COUNTER; SELF.countx := COUNTER/3));
+ds3 := ds1 + ds2; // unoredered;
+output(SUM(nofold(ds3), sumx));
 
 decimal17_4 value1 := 1.6667;
 decimal17_4 value2 := 1.6667 : stored('value2');

testing/regress/ecl/key/bcd2.xml

@@ -10,7 +10,7 @@
  <Row><sumx>10.0001</sumx><countx>2</countx><average>5.00005</average><_unnamed_4>false</_unnamed_4><_unnamed_5>false</_unnamed_5></Row>
 </Dataset>
 <Dataset name='Result 4'>
- <Row><Result_4>2</Result_4></Row>
+ <Row><Result_4>1000150005000</Result_4></Row>
 </Dataset>
 <Dataset name='Result 5'>
  <Row><Result_5>2</Result_5></Row>
@@ -21,3 +21,6 @@
 <Dataset name='Result 7'>
  <Row><Result_7>2</Result_7></Row>
 </Dataset>
+<Dataset name='Result 8'>
+ <Row><Result_8>2</Result_8></Row>
+</Dataset>

testing/regress/ecl/key/rowdiff.xml

@@ -0,0 +1,10 @@
+<Dataset name='Result 1'>
+ <Row><id>1</id><diff1>                                   </diff1><diff2>                                   </diff2></Row>
+ <Row><id>2</id><diff1>name.surname,name.middle           </diff1><diff2>surname,middle                     </diff2></Row>
+ <Row><id>3</id><diff1>age                                </diff1><diff2>                                   </diff2></Row>
+</Dataset>
+<Dataset name='Result 2'>
+ <Row><id>1</id><diff1>0,0,0,0,0                          </diff1><diff2>0,0,0                              </diff2></Row>
+ <Row><id>2</id><diff1>0,0,1,1,0                          </diff1><diff2>0,1,1                              </diff2></Row>
+ <Row><id>3</id><diff1>0,0,0,0,1                          </diff1><diff2>0,0,0                              </diff2></Row>
+</Dataset>

testing/regress/ecl/key/streame.xml

@@ -23,3 +23,6 @@
 <Dataset name='Result 5'>
  <Row><Result_5>Yo</Result_5></Row>
 </Dataset>
+<Dataset name='Result 6'>
+ <Row><name1>Gavin</name1><name2>Halliday  </name2><childnames><Row><name>a</name><value>1</value></Row><Row><name>b</name><value>2</value></Row><Row><name>c</name><value>3</value></Row></childnames><childdict><Row><name>aa</name><value>11</value></Row></childdict><r><name>aaa</name><value>111</value></r><val1>250</val1><val2>-1</val2><u1>là</u1><u2>là</u2><u3>là      </u3><val3>1</val3><d>4141</d><b>false</b><ss1><Item>1</Item><Item>2</Item></ss1></Row>
+</Dataset>

testing/regress/ecl/rowdiff.ecl

@@ -0,0 +1,83 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+
+
+complexName :=
+            record
+string30        forename;
+string20        surname;
+                ifblock(self.surname <> 'Windsor')
+string20            middle;
+                end;
+            end;
+
+idRec := record
+unsigned    id;
+END;
+
+in1rec :=   record(idRec)
+complexName name;
+unsigned    age;
+string      title;
+        end;
+
+in2rec := record
+idRec;
+complexName name;
+real8       age;
+boolean     dead;
+        end;
+
+
+in1 := dataset([
+        {1,'Gavin','Hawthorn','',33,'Mr'},
+        {2,'Mia','Hawthorn','',33,'Dr'},
+        {3,'Elizabeth','Windsor',99,'Queen'}
+        ], in1rec);
+
+
+in2 := dataset([
+        {1,'Gavin','Hawthorn','',33,false},
+        {2,'Mia','','Jean',33,false},
+        {3,'Elizabeth','Windsor',99.1,false}
+        ], in2rec);
+
+outrec :=
+        record
+unsigned        id;
+string35        diff1;
+string35        diff2;
+        end;
+
+outrec t1(in1 l, in2 r) := transform
+//      self.id := if(l = r, SKIP, l.id);
+        self.id := l.id;
+        self.diff1 := rowdiff(l, r);
+        self.diff2 := rowdiff(l.name, r.name);
+    end;
+
+output(join(in1, in2, left.id = right.id, t1(left, right)));
+
+
+outrec t2(in1 l, in2 r) := transform
+        self.id := l.id;
+        self.diff1 := rowdiff(l, r, count);
+        self.diff2 := rowdiff(l.name, r.name, count);
+    end;
+
+output(join(in1, in2, left.id = right.id, t2(left, right)));

testing/regress/ecl/streame.ecl

@@ -50,8 +50,8 @@ ENDEMBED;
 
 dataset(namesRecord) streamedNames(data d, utf8 u) := EMBED(Python)
   return [  \
-     ("Gavin", "Halliday", [("a", 1),("b", 2),("c", 3)], [("aa", 11)], ("aaa", 111), 250, -1,  U'là',  U'là',  U'là', 0x01000000, d, False, set(["1","2"])), \
-     ("John", "Smith", [], [], ("c", 3), 250, -1,  U'là',  U'là',  u, 0x02000000, d, True, []) \
+     ("Gavin", "Halliday", [("a", 1),("b", 2),("c", 3)], [("aa", 11)], ("aaa", 111), 250, -1,  U'là',  U'là',  U'là', 1, d, False, set(["1","2"])), \
+     ("John", "Smith", [], [], ("c", 3), 250, -1,  U'là',  U'là',  u, 2, d, True, []) \
      ]
 ENDEMBED;
 
@@ -80,3 +80,13 @@ childrec tnamed(string s) := EMBED(Python)
 ENDEMBED;
 
 output(tnamed('Yo').name);
+
+// Test passing records into Python
+
+dataset(namesRecord) streamInOut(dataset(namesRecord) recs) := EMBED(Python)
+  for rec in recs:
+    if rec.name1 == 'Gavin':
+       yield rec
+ENDEMBED;
+
+output(streamInOut(streamedNames(d'AA', u'là')));