Merge branch 'candidate-5.4.0'

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

dali/base/dadfs.cpp

@@ -4666,7 +4666,12 @@ class CDistributedSuperFile: public CDistributedFileBase<IDistributedSuperFile>
                     return false;
                 }
                 if (!transaction->isSubFile(parent, subfile, true))
+                {
                     WARNLOG("removeSubFile: File %s is not a subfile of %s", subfile.get(), parent->queryLogicalName());
+                    parent.clear();
+                    sub.clear();
+                    return true; // NB: sub was not a member of super, issue warning and continue without locking
+                }
             }
             // Try to lock all files
             addFileLock(parent);

docs/ECLStandardLibraryReference/SLR-Mods/DayOfYear.xml

@@ -0,0 +1,48 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE sect1 PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN"
+"http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd">
+<sect1 id="DayOfYear">
+  <title>DayOfYear</title>
+
+  <para><emphasis role="bold">STD.Date.DayOfYear<indexterm>
+      <primary>STD.Date.DayOfYear</primary>
+    </indexterm><indexterm>
+      <primary>DayOfYear</primary>
+    </indexterm>(</emphasis> <emphasis role="bold">date)</emphasis></para>
+
+  <informaltable colsep="1" frame="all" rowsep="1">
+    <tgroup cols="2">
+      <colspec colwidth="80.50pt" />
+
+      <colspec />
+
+      <tbody>
+        <row>
+          <entry><emphasis>date</emphasis></entry>
+
+          <entry>A date value in the Date_t format.</entry>
+        </row>
+
+        <row>
+          <entry>Return:</entry>
+
+          <entry>DayofYear returns an INTEGER value in the range of 1 through
+          366.</entry>
+        </row>
+      </tbody>
+    </tgroup>
+  </informaltable>
+
+  <para>The <emphasis role="bold">DayOfYear</emphasis> function returns a
+  number representing the day of the year for the given date. The date must be
+  in the Gregorian calendar after the year 1600.</para>
+
+  <para>Example:</para>
+
+  <programlisting format="linespecific">IMPORT STD;
+D1 := STD.Date.DayOfYear(STD.Date.Today());
+      // D1 contains the day of the year for today's date
+</programlisting>
+
+  <para></para>
+</sect1>

docs/ECLStandardLibraryReference/SLR-Mods/Today.xml

@@ -0,0 +1,42 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE sect1 PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN"
+"http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd">
+<sect1 id="Today">
+  <title>Today</title>
+
+  <para><emphasis role="bold">STD.Date.Today<indexterm>
+      <primary>STD.Date.Today</primary>
+    </indexterm><indexterm>
+      <primary>Today</primary>
+    </indexterm>(</emphasis> <emphasis role="bold">)</emphasis></para>
+
+  <informaltable colsep="1" frame="all" rowsep="1">
+    <tgroup cols="2">
+      <colspec colwidth="80.50pt" />
+
+      <colspec />
+
+      <tbody>
+        <row>
+          <entry>Return:</entry>
+
+          <entry>Today returns date_t (an UNSIGNED4 containing a date value in
+          YYYYMMDD format) representing the current date.</entry>
+        </row>
+      </tbody>
+    </tgroup>
+  </informaltable>
+
+  <para>The <emphasis role="bold">Today</emphasis> function returns the
+  current date in the local time zone. </para>
+
+  <para>Example:</para>
+
+  <programlisting format="linespecific">IMPORT STD;
+  
+D1 := STD.Date.Today();
+     //D1 contains today's date
+</programlisting>
+
+  <para></para>
+</sect1>

ecl/ecl-bundle/ecl-bundle.cpp

@@ -949,7 +949,6 @@ public:
         if (optBundle.isEmpty() && bundleCompulsory)
         {
             printf("Missing bundle name\n");
-            usage();
             return false;
         }
         if (!EclCmdCommon::finalizeOptions(globals))
@@ -1527,7 +1526,6 @@ public:
         if (optVersion.isEmpty())
         {
             printf("Version must be specified\n");
-            usage();
             return false;
         }
         return EclCmdBundleBaseWithVersion::finalizeOptions(globals);

ecl/ecl-package/ecl-package.cpp

@@ -55,10 +55,7 @@ public:
     virtual bool parseCommandLineOptions(ArgvIterator &iter)
     {
         if (iter.done())
-        {
-            usage();
             return false;
-        }
 
         for (; !iter.done(); iter.next())
         {
@@ -89,14 +86,12 @@ public:
             return false;
         if (optTarget.isEmpty())
         {
-            fprintf(stdout, "\n ... Missing target name\n\n");
-            usage();
+            fprintf(stdout, "\n ... Missing target name\n");
             return false;
         }
         if (optPackageMap.isEmpty())
         {
-            fprintf(stdout, "\n ... Missing package map name\n\n");
-            usage();
+            fprintf(stdout, "\n ... Missing package map name\n");
             return false;
         }
         if (optProcess.isEmpty())
@@ -151,10 +146,7 @@ public:
     virtual bool parseCommandLineOptions(ArgvIterator &iter)
     {
         if (iter.done())
-        {
-            usage();
             return false;
-        }
 
         for (; !iter.done(); iter.next())
         {
@@ -185,14 +177,12 @@ public:
             return false;
         if (optTarget.isEmpty())
         {
-            fprintf(stdout, "\n ... Missing target name\n\n");
-            usage();
+            fprintf(stdout, "\n ... Missing target name\n");
             return false;
         }
         if (optPackageMap.isEmpty())
         {
-            fprintf(stdout, "\n ... Missing package map name\n\n");
-            usage();
+            fprintf(stdout, "\n ... Missing package map name\n");
             return false;
         }
         if (optProcess.isEmpty())
@@ -332,10 +322,7 @@ public:
     virtual bool parseCommandLineOptions(ArgvIterator &iter)
     {
         if (iter.done())
-        {
-            usage();
             return false;
-        }
 
         for (; !iter.done(); iter.next())
         {
@@ -402,10 +389,7 @@ public:
     virtual bool parseCommandLineOptions(ArgvIterator &iter)
     {
         if (iter.done())
-        {
-            usage();
             return false;
-        }
 
         for (; !iter.done(); iter.next())
         {
@@ -439,14 +423,13 @@ public:
         }
         StringBuffer err;
         if (optPackageMap.isEmpty())
-            err.append("\n ... Missing package map name\n\n");
+            err.append("\n ... Missing package map name\n");
         else if (optTarget.isEmpty())
-            err.append("\n ... Specify a target cluster name\n\n");
+            err.append("\n ... Specify a target cluster name\n");
 
         if (err.length())
         {
             fprintf(stdout, "%s", err.str());
-            usage();
             return false;
         }
 
@@ -504,10 +487,7 @@ public:
     virtual bool parseCommandLineOptions(ArgvIterator &iter)
     {
         if (iter.done())
-        {
-            usage();
             return false;
-        }
 
         for (; !iter.done(); iter.next())
         {
@@ -555,14 +535,13 @@ public:
         }
         StringBuffer err;
         if (optFileName.isEmpty())
-            err.append("\n ... Missing package file name\n\n");
+            err.append("\n ... Missing package file name\n");
         else if (optTarget.isEmpty())
-            err.append("\n ... Specify a cluster name\n\n");
+            err.append("\n ... Specify a cluster name\n");
 
         if (err.length())
         {
             fprintf(stdout, "%s", err.str());
-            usage();
             return false;
         }
 
@@ -661,10 +640,7 @@ public:
     virtual bool parseCommandLineOptions(ArgvIterator &iter)
     {
         if (iter.done())
-        {
-            usage();
             return false;
-        }
 
         for (; !iter.done(); iter.next())
         {
@@ -717,16 +693,15 @@ public:
         if (optValidateActive)
             pcount++;
         if (pcount==0)
-            err.append("\n ... Package file name, --pmid, or --active required\n\n");
+            err.append("\n ... Package file name, --pmid, or --active required\n");
         else if (pcount > 1)
-            err.append("\n ... Package file name, --pmid, and --active are mutually exclusive\n\n");
+            err.append("\n ... Package file name, --pmid, and --active are mutually exclusive\n");
         if (optTarget.isEmpty())
             err.append("\n ... Specify a cluster name\n\n");
 
         if (err.length())
         {
             fprintf(stdout, "%s", err.str());
-            usage();
             return false;
         }
         return true;
@@ -854,10 +829,7 @@ public:
     virtual bool parseCommandLineOptions(ArgvIterator &iter)
     {
         if (iter.done())
-        {
-            usage();
             return false;
-        }
 
         for (; !iter.done(); iter.next())
         {
@@ -893,14 +865,13 @@ public:
         }
         StringBuffer err;
         if (optTarget.isEmpty())
-            err.append("\n ... A target cluster must be specified\n\n");
+            err.append("\n ... A target cluster must be specified\n");
         if (optQueryId.isEmpty())
-            err.append("\n ... A query must be specified\n\n");
+            err.append("\n ... A query must be specified\n");
 
         if (err.length())
         {
             fprintf(stdout, "%s", err.str());
-            usage();
             return false;
         }
         return true;

ecl/eclcmd/eclcmd_common.cpp

@@ -261,6 +261,12 @@ eclObjParameterType EclObjectParameter::set(const char *param)
         loadFile();
     else if (looksLikeOnlyAWuid(param))
         type = eclObjWuid;
+    else if (accept & eclObjQuery)
+    {
+        query.set(value.get());
+        type = eclObjQuery;
+        value.clear();
+    }
     return type;
 }
 
@@ -278,6 +284,8 @@ const char *EclObjectParameter::queryTypeName()
         return "ECL Manifest";
     case eclObjSharedObject:
         return "ECL Shared Object";
+    case eclObjQuery:
+        return "ECL Query";
     default:
         return "Unknown Type";
     }
@@ -551,29 +559,46 @@ bool matchVariableOption(ArgvIterator &iter, const char prefix, IArrayOf<IEspNam
     addNamedValue(arg, values);
     return true;
 }
+
+bool EclCmdWithEclTarget::setTarget(const char *target)
+{
+    if (!optTargetCluster.isEmpty())
+        return streq(optTargetCluster, target);
+    optTargetCluster.set(target);
+    return true;
+ }
+
+bool EclCmdWithEclTarget::setParam(const char *in, bool final)
+{
+    bool success = true;
+    if (++paramCount>2)
+        success = false;
+    else if (!param.isEmpty())
+        success = setTarget(param);
+    if (!success)
+    {
+        fprintf(stderr, "\nunrecognized argument %s\n", in);
+        return false;
+    }
+    if (final) //a 'final' parameter is always last
+        paramCount=2;
+    param.set(in);
+    return true;
+ }
+
 eclCmdOptionMatchIndicator EclCmdWithEclTarget::matchCommandLineOption(ArgvIterator &iter, bool finalAttempt)
 {
     const char *arg = iter.query();
     if (streq(arg, "-"))
     {
-        optObj.set("stdin");
+        if (!setParam("stdin", true))
+            return EclCmdOptionCompletion;
         return EclCmdOptionMatch;
     }
     if (*arg!='-')
     {
-        if (optObj.value.length())
-        {
-            if (optObj.type==eclObjTypeUnknown && (optObj.accept & eclObjQuery) && !optObj.query.length())
-            {
-                optObj.type=eclObjQuery;
-                optObj.query.set(arg);
-                return EclCmdOptionMatch;
-            }
-
-            fprintf(stderr, "\nunrecognized argument %s\n", arg);
+        if (!setParam(arg, false))
             return EclCmdOptionCompletion;
-        }
-        optObj.set(arg);
         return EclCmdOptionMatch;
     }
     if (matchVariableOption(iter, 'f', debugValues))
@@ -600,8 +625,16 @@ eclCmdOptionMatchIndicator EclCmdWithEclTarget::matchCommandLineOption(ArgvItera
         return EclCmdOptionMatch;
     if (iter.matchOption(optTargetCluster, ECLOPT_CLUSTER_DEPRECATED)||iter.matchOption(optTargetCluster, ECLOPT_CLUSTER_DEPRECATED_S))
         return EclCmdOptionMatch;
-    if (iter.matchOption(optTargetCluster, ECLOPT_TARGET)||iter.matchOption(optTargetCluster, ECLOPT_TARGET_S))
+    StringAttr target;
+    if (iter.matchOption(target, ECLOPT_TARGET)||iter.matchOption(target, ECLOPT_TARGET_S))
+    {
+        if (!setTarget(target))
+        {
+            fprintf(stderr, "\nTarget names do not match %s != %s\n", optTargetCluster.str(), target.str());
+            return EclCmdOptionCompletion;
+        }
         return EclCmdOptionMatch;
+    }
 
     return EclCmdCommon::matchCommandLineOption(iter, finalAttempt);
 }
@@ -610,7 +643,8 @@ bool EclCmdWithEclTarget::finalizeOptions(IProperties *globals)
 {
     if (!EclCmdCommon::finalizeOptions(globals))
         return false;
-
+    if (!param.isEmpty())
+        optObj.set(param);
     if (optObj.type == eclObjTypeUnknown)
     {
         if (optAttributePath.length())
@@ -634,7 +668,7 @@ bool EclCmdWithEclTarget::finalizeOptions(IProperties *globals)
     if (optResultLimit == (unsigned)-1)
         extractEclCmdOption(optResultLimit, globals, ECLOPT_RESULT_LIMIT_ENV, ECLOPT_RESULT_LIMIT_INI, 0);
 
-    if (optObj.value.isEmpty() && optAttributePath.isEmpty())
+    if (optObj.value.isEmpty() && optObj.query.isEmpty() && optAttributePath.isEmpty())
     {
         fprintf(stderr, "\nMust specify a Query, WUID, ECL File, Archive, or shared object\n");
         return false;
@@ -709,10 +743,7 @@ bool EclCmdWithQueryTarget::finalizeOptions(IProperties *globals)
 bool EclCmdWithQueryTarget::parseCommandLineOptions(ArgvIterator &iter)
 {
     if (iter.done())
-    {
-        usage();
         return false;
-    }
 
     for (; !iter.done(); iter.next())
     {

ecl/eclcmd/eclcmd_common.hpp

@@ -249,11 +249,13 @@ public:
 class EclCmdWithEclTarget : public EclCmdCommon
 {
 public:
-    EclCmdWithEclTarget() : optLegacy(false), optNoArchive(false), optResultLimit((unsigned)-1), optDebug(false)
+    EclCmdWithEclTarget() : optLegacy(false), optNoArchive(false), optResultLimit((unsigned)-1), optDebug(false), paramCount(0)
     {
     }
     virtual eclCmdOptionMatchIndicator matchCommandLineOption(ArgvIterator &iter, bool finalAttempt=false);
     virtual bool finalizeOptions(IProperties *globals);
+    bool setTarget(const char *target);
+    bool setParam(const char *in, bool final);
 
     virtual void usage()
     {
@@ -292,6 +294,7 @@ public:
         }
     }
 public:
+    StringAttr param;
     StringAttr optTargetCluster;
     EclObjectParameter optObj;
     StringBuffer optLibPath;
@@ -302,6 +305,7 @@ public:
     IArrayOf<IEspNamedValue> debugValues;
     IArrayOf<IEspNamedValue> definitions;
     unsigned optResultLimit;
+    unsigned paramCount;
     bool optNoArchive;
     bool optLegacy;
     bool optDebug;

ecl/eclcmd/eclcmd_core.cpp

@@ -222,10 +222,7 @@ public:
     virtual bool parseCommandLineOptions(ArgvIterator &iter)
     {
         if (iter.done())
-        {
-            usage();
             return false;
-        }
 
         for (; !iter.done(); iter.next())
         {
@@ -262,15 +259,15 @@ public:
             "text, file, archive, shared object, or dll.  The workunit will be created in\n"
             "the 'compiled' state.\n"
             "\n"
-            "ecl deploy --target=<val> --name=<val> <ecl_file|->\n"
-            "ecl deploy --target=<val> --name=<val> <archive|->\n"
-            "ecl deploy [--target=<val>] [--name=<val>] <so|dll|->\n\n"
+            "ecl deploy <target> <file> [--name=<val>]\n"
+            "ecl deploy <target> <archive> [--name=<val>]\n"
+            "ecl deploy <target> <so|dll> [--name=<val>]\n"
+            "ecl deploy <target> - [--name=<val>]\n\n"
             "   -                      specifies object should be read from stdin\n"
-            "   <ecl_file|->           ecl text file to deploy\n"
-            "   <archive|->            ecl archive to deploy\n"
-            "   <so|dll|->             workunit dll or shared object to deploy\n"
+            "   <file>                 ecl text file to deploy\n"
+            "   <archive>              ecl archive to deploy\n"
+            "   <so|dll>               workunit dll or shared object to deploy\n"
             " Options:\n"
-            "   -t, --target=<val>     target cluster to associate workunit with\n"
             "   -n, --name=<val>       workunit job name\n",
             stdout);
         EclCmdWithEclTarget::usage();
@@ -292,10 +289,7 @@ public:
     virtual bool parseCommandLineOptions(ArgvIterator &iter)
     {
         if (iter.done())
-        {
-            usage();
             return false;
-        }
 
         for (; !iter.done(); iter.next())
         {
@@ -360,7 +354,7 @@ public:
         }
         if (optNoActivate && (optSuspendPrevious || optDeletePrevious))
         {
-            fputs("invalid --suspend-prev and --delete-prev require activation.\n\n", stderr);
+            fputs("invalid --suspend-prev and --delete-prev require activation.\n", stderr);
             return false;
         }
         if (!optSuspendPrevious && !optDeletePrevious)
@@ -371,17 +365,17 @@ public:
         }
         if (optSuspendPrevious && optDeletePrevious)
         {
-            fputs("invalid --suspend-prev and --delete-prev are mutually exclusive options.\n\n", stderr);
+            fputs("invalid --suspend-prev and --delete-prev are mutually exclusive options.\n", stderr);
             return false;
         }
         if (optMemoryLimit.length() && !isValidMemoryValue(optMemoryLimit))
         {
-            fprintf(stderr, "invalid --memoryLimit value of %s.\n\n", optMemoryLimit.get());
+            fprintf(stderr, "invalid --memoryLimit value of %s.\n", optMemoryLimit.get());
             return false;
         }
         if (optPriority.length() && !isValidPriorityValue(optPriority))
         {
-            fprintf(stderr, "invalid --priority value of %s.\n\n", optPriority.get());
+            fprintf(stderr, "invalid --priority value of %s.\n", optPriority.get());
             return false;
         }
         return true;
@@ -456,18 +450,17 @@ public:
             "If the query is being created from an ECL file, archive, shared object, dll,\n"
             "or text, a workunit is first created and then published to the queryset.\n"
             "\n"
-            "ecl publish [--target=<val>] [--name=<val>] [--activate] <wuid>\n"
-            "ecl publish [--target=<val>] [--name=<val>] [--activate] <so|dll|->\n"
-            "ecl publish --target=<val> --name=<val> [--activate] <archive|->\n"
-            "ecl publish --target=<val> --name=<val> [--activate] <ecl_file|->\n\n"
+            "ecl publish <target> <wuid> --name=<val>\n"
+            "ecl publish <target> <so|dll> --name=<val>\n"
+            "ecl publish <target> <archive> --name=<val>\n"
+            "ecl publish <target> <file> --name=<val>\n"
+            "ecl publish <target> - --name=<val>\n\n"
             "   -                      specifies object should be read from stdin\n"
             "   <wuid>                 workunit to publish\n"
-            "   <archive|->            archive to publish\n"
-            "   <ecl_file|->           ECL text file to publish\n"
-            "   <so|dll|->             workunit dll or shared object to publish\n"
+            "   <archive>              archive to publish\n"
+            "   <file>                 ECL text file to publish\n"
+            "   <so|dll>               workunit dll or shared object to publish\n"
             " Options:\n"
-            "   -t, --target=<val>     target cluster to publish workunit to\n"
-            "                          (defaults to cluster defined inside workunit)\n"
             "   -n, --name=<val>       query name to use for published workunit\n"
             "   -A, --activate         Activate query when published (default)\n"
             "   -sp, --suspend-prev    Suspend previously active query\n"
@@ -520,10 +513,7 @@ public:
     virtual bool parseCommandLineOptions(ArgvIterator &iter)
     {
         if (iter.done())
-        {
-            usage();
             return false;
-        }
 
         for (; !iter.done(); iter.next())
         {
@@ -572,7 +562,6 @@ public:
 
         StringBuffer wuid;
         StringBuffer wuCluster;
-        StringBuffer queryset;
         StringBuffer query;
 
         if (optObj.type==eclObjWuid)
@@ -583,10 +572,10 @@ public:
         }
         else if (optObj.type==eclObjQuery)
         {
-            req->setQuerySet(queryset.set(optObj.value.get()).str());
+            req->setQuerySet(optTargetCluster);
             req->setQuery(query.set(optObj.query.get()).str());
             if (optVerbose)
-                fprintf(stdout, "Running query %s/%s\n", queryset.str(), query.str());
+                fprintf(stdout, "Running query %s/%s\n", optTargetCluster.str(), query.str());
         }
         else
         {
@@ -631,25 +620,24 @@ public:
     {
         fputs("\nUsage:\n"
             "\n"
-            "The 'run' command exectues an ECL workunit, text, file, archive, shared\n"
-            "object, or dll on the specified HPCC target cluster.\n"
+            "The 'run' command exectues an ECL workunit, text, file, archive, query,\n"
+            "shared object, or dll on the specified HPCC target cluster.\n"
             "\n"
             "Query input can be provided in xml form via the --input parameter.  Input\n"
             "xml can be provided directly or by refrencing a file\n"
             "\n"
-            "ecl run [--target=<val>][--input=<file|xml>][--wait=<ms>] <wuid>\n"
-            "ecl run [--target=<c>][--input=<file|xml>][--wait=<ms>] <queryset> <query>\n"
-            "ecl run [--target=<c>][--name=<nm>][--input=<file|xml>][--wait=<i>] <dll|->\n"
-            "ecl run --target=<c> --name=<nm> [--input=<file|xml>][--wait=<i>] <archive|->\n"
-            "ecl run --target=<c> --name=<nm> [--input=<file|xml>][--wait=<i>] <eclfile|->\n\n"
+            "ecl run <target> <wuid> [--input=<file|xml>][--wait=<ms>]\n"
+            "ecl run <target> <query> [--input=<file|xml>][--wait=<ms>]\n"
+            "ecl run <target> <dll> [--name=<nm>][--input=<file|xml>][--wait=<i>]\n"
+            "ecl run <target> <archive> --name=<nm> [--input=<file|xml>][--wait=<i>]\n"
+            "ecl run <target> <eclfile> --name=<nm> [--input=<file|xml>][--wait=<i>]\n"
+            "ecl run <target> - --name=<nm> [--input=<file|xml>][--wait=<i>]\n\n"
             "   -                      specifies object should be read from stdin\n"
             "   <wuid>                 workunit to publish\n"
-            "   <archive|->            archive to publish\n"
-            "   <ecl_file|->           ECL text file to publish\n"
-            "   <so|dll|->             workunit dll or shared object to publish\n"
+            "   <archive>              archive to publish\n"
+            "   <eclfile>              ECL text file to publish\n"
+            "   <so|dll>               workunit dll or shared object to publish\n"
             " Options:\n"
-            "   -t, --target=<val>        target cluster to run job on\n"
-            "                             (defaults to cluster defined inside workunit)\n"
             "   -n, --name=<val>          job name\n"
             "   -in,--input=<file|xml>    file or xml content to use as query input\n"
             "   -X<name>=<value>          sets the stored input value (stored('name'))\n"
@@ -721,9 +709,9 @@ public:
             "The 'activate' command assigns a query to the active alias with the same\n"
             "name as the query.\n"
             "\n"
-            "ecl activate <queryset> <query_id>\n"
+            "ecl activate <target> <query_id>\n"
             " Options:\n"
-            "   <queryset>             name of queryset containing query to activate\n"
+            "   <target>               name of target queryset containing query to activate\n"
             "   <query_id>             query to activate\n",
             stdout);
         EclCmdWithQueryTarget::usage();
@@ -765,11 +753,11 @@ public:
     {
         fputs("\nUsage:\n"
             "\n"
-            "The 'unpublish' command removes a query from a queryset.\n"
+            "The 'unpublish' command removes a query from a target queryset.\n"
             "\n"
-            "ecl unpublish <queryset> <query_id>\n"
+            "ecl unpublish <target> <query_id>\n"
             " Options:\n"
-            "   <queryset>             name of queryset containing the query to remove\n"
+            "   <target>               name of target queryset containing the query to remove\n"
             "   <query_id>             query to remove from the queryset\n",
             stdout);
         EclCmdWithQueryTarget::usage();
@@ -817,12 +805,12 @@ public:
     {
         fputs("\nUsage:\n"
             "\n"
-            "The 'deactivate' command removes an active query alias from the given queryset.\n"
+            "The 'deactivate' command removes an active query alias from the given target.\n"
             "\n"
-            "ecl deactivate <queryset> <active_alias>\n"
+            "ecl deactivate <target> <active_alias>\n"
             "\n"
             " Options:\n"
-            "   <queryset>             queryset containing alias to deactivate\n"
+            "   <target>               target queryset containing alias to deactivate\n"
             "   <active_alias>         active alias to be removed from the queryset\n",
             stdout);
         EclCmdWithQueryTarget::usage();
@@ -840,10 +828,7 @@ public:
     {
         bool retVal = false;
         if (iter.done())
-        {
-            usage();
-            return retVal;
-        }
+            return false;
 
         for (; !iter.done(); iter.next())
         {
@@ -973,10 +958,7 @@ public:
     {
         bool retVal = false;
         if (iter.done())
-        {
-            usage();
-            return retVal;
-        }
+            return false;
 
         for (; !iter.done(); iter.next())
         {
@@ -1059,10 +1041,7 @@ public:
     {
         bool retVal = false;
         if (iter.done())
-        {
-            usage();
-            return retVal;
-        }
+            return false;
 
         for (; !iter.done(); iter.next())
         {
@@ -1145,10 +1124,7 @@ public:
     {
         bool retVal = false;
         if (iter.done())
-        {
-            usage();
-            return retVal;
-        }
+            return false;
 
         for (; !iter.done(); iter.next())
         {

ecl/eclcmd/eclcmd_shell.cpp

@@ -97,10 +97,15 @@ int EclCMDShell::processCMD(ArgvIterator &iter)
         return 0;
     }
     if (!c->parseCommandLineOptions(iter))
+    {
+        c->usage();
         return 0;
-
+    }
     if (!c->finalizeOptions(globals))
+    {
+        c->usage();
         return 0;
+    }
 
     return c->processCMD();
 }

ecl/eclcmd/queries/ecl-queries.cpp

@@ -95,10 +95,7 @@ public:
     virtual bool parseCommandLineOptions(ArgvIterator &iter)
     {
         if (iter.done())
-        {
-            usage();
             return false;
-        }
 
         for (; !iter.done(); iter.next())
         {
@@ -134,7 +131,7 @@ public:
                         flags |= QUERYLIST_SHOW_UNFLAGGED;
                         break;
                     default:
-                        fprintf(stderr, "Unrecognized --show flag = %c", *ch);
+                        fprintf(stderr, "Unrecognized --show flag = %c\n", *ch);
                         return false;
                     }
                 }
@@ -151,7 +148,7 @@ public:
         {
             if (flags)
             {
-                fputs("--show and --inactive should not be used together.\n\n", stderr);
+                fputs("--show and --inactive should not be used together.\n", stderr);
                 return false;
             }
 
@@ -309,10 +306,7 @@ public:
     virtual bool parseCommandLineOptions(ArgvIterator &iter)
     {
         if (iter.done())
-        {
-            usage();
             return false;
-        }
 
         for (; !iter.done(); iter.next())
         {
@@ -339,12 +333,12 @@ public:
     {
         if (optTarget.isEmpty())
         {
-            fputs("Target must be specified.\n\n", stderr);
+            fputs("Target must be specified.\n", stderr);
             return false;
         }
         if (optTarget.isEmpty())
         {
-            fputs("Query must be specified.\n\n", stderr);
+            fputs("Query must be specified.\n", stderr);
             return false;
         }
         if (!EclCmdCommon::finalizeOptions(globals))
@@ -413,10 +407,7 @@ public:
     virtual bool parseCommandLineOptions(ArgvIterator &iter)
     {
         if (iter.done())
-        {
-            usage();
             return false;
-        }
 
         for (; !iter.done(); iter.next())
         {
@@ -477,17 +468,17 @@ public:
             return false;
         if (optSourceQueryPath.isEmpty() && optTargetCluster.isEmpty())
         {
-            fputs("source and target must both be specified.\n\n", stderr);
+            fputs("source and target must both be specified.\n", stderr);
             return false;
         }
         if (optMemoryLimit.length() && !isValidMemoryValue(optMemoryLimit))
         {
-            fprintf(stderr, "invalid --memoryLimit value of %s.\n\n", optMemoryLimit.get());
+            fprintf(stderr, "invalid --memoryLimit value of %s.\n", optMemoryLimit.get());
             return false;
         }
         if (optPriority.length() && !isValidPriorityValue(optPriority))
         {
-            fprintf(stderr, "invalid --priority value of %s.\n\n", optPriority.get());
+            fprintf(stderr, "invalid --priority value of %s.\n", optPriority.get());
             return false;
         }
 
@@ -600,10 +591,7 @@ public:
     virtual bool parseCommandLineOptions(ArgvIterator &iter)
     {
         if (iter.done())
-        {
-            usage();
             return false;
-        }
 
         for (; !iter.done(); iter.next())
         {
@@ -646,7 +634,7 @@ public:
             return false;
         if (optSourceQuerySet.isEmpty() || optDestQuerySet.isEmpty())
         {
-            fputs("source and destination querysets must both be specified.\n\n", stderr);
+            fputs("source and destination querysets must both be specified.\n", stderr);
             return false;
         }
         return true;
@@ -742,10 +730,7 @@ public:
     virtual bool parseCommandLineOptions(ArgvIterator &iter)
     {
         if (iter.done())
-        {
-            usage();
             return false;
-        }
 
         for (; !iter.done(); iter.next())
         {
@@ -788,17 +773,17 @@ public:
             return false;
         if (optTargetCluster.isEmpty() || optQueryId.isEmpty())
         {
-            fputs("Target and QueryId must both be specified.\n\n", stderr);
+            fputs("Target and QueryId must both be specified.\n", stderr);
             return false;
         }
         if (optMemoryLimit.length() && !isValidMemoryValue(optMemoryLimit))
         {
-            fprintf(stderr, "invalid --memoryLimit value of %s.\n\n", optMemoryLimit.get());
+            fprintf(stderr, "invalid --memoryLimit value of %s.\n", optMemoryLimit.get());
             return false;
         }
         if (optPriority.length() && !isValidPriorityValue(optPriority))
         {
-            fprintf(stderr, "invalid --priority value of %s.\n\n", optPriority.get());
+            fprintf(stderr, "invalid --priority value of %s.\n", optPriority.get());
             return false;
         }
         return true;

ecl/eclcmd/roxie/ecl-roxie.cpp

@@ -177,7 +177,7 @@ public:
             return false;
         if (optProcess.isEmpty())
         {
-            fputs("process cluster must be specified.\n\n", stderr);
+            fputs("process cluster must be specified.\n", stderr);
             return false;
         }
         return true;
@@ -283,7 +283,7 @@ public:
             return false;
         if (optProcess.isEmpty())
         {
-            fputs("process cluster must be specified.\n\n", stderr);
+            fputs("process cluster must be specified.\n", stderr);
             return false;
         }
         return true;
@@ -401,7 +401,7 @@ public:
             return false;
         if (optProcess.isEmpty())
         {
-            fputs("process cluster must be specified.\n\n", stderr);
+            fputs("process cluster must be specified.\n", stderr);
             return false;
         }
         if (optDeleteRecursive)

ecl/hql/hqlexpr.cpp

@@ -1953,7 +1953,6 @@ childDatasetType getChildDatasetType(IHqlExpression * expr)
     case no_split:
     case no_spill:
     case no_activerow:
-    case no_executewhen:  //second argument is independent of the other arguments
     case no_selectnth:
     case no_readspill:
     case no_commonspill:
@@ -1964,6 +1963,7 @@ childDatasetType getChildDatasetType(IHqlExpression * expr)
     case no_setgraphloopresult:
     case no_spillgraphresult:
         return childdataset_dataset_noscope;
+    case no_executewhen:  //second argument is independent of the other arguments
     case no_setresult:
     case no_sizeof:
     case no_offsetof:
@@ -2232,12 +2232,12 @@ inline unsigned doGetNumChildTables(IHqlExpression * dataset)
     case no_extractresult:
     case no_filtergroup:
     case no_forcegraph:
-    case no_executewhen:
     case no_normalizegroup:
     case no_owned_ds:
     case no_dataset_alias:
     case no_ensureresult:
         return 1;
+    case no_executewhen:
     case no_deserialize:
     case no_serialize:
         if (dataset->queryChild(0)->isDataset())

ecl/hql/hqlgram.y

@@ -6440,11 +6440,15 @@ primexpr1
                             $$.setExpr(createCompound($3.getExpr(), $5.getExpr()));
                             $$.setPosition($1);
                         }
-    | WHEN '(' expression ',' action ')'
+    | WHEN '(' expression ',' action sideEffectOptions ')'
                         {
                             parser->normalizeExpression($3);
-                            $$.setExpr(createCompound($5.getExpr(), $3.getExpr()));
-                            $$.setPosition($1);
+                            OwnedHqlExpr options = $6.getExpr();
+                            OwnedHqlExpr expr = $3.getExpr();
+                            if (options)
+                                $$.setExpr(createValueF(no_executewhen, expr->getType(), LINK(expr), $5.getExpr(), options.getClear(), NULL), $1);
+                            else
+                                $$.setExpr(createCompound($5.getExpr(), expr), $1);
                         }
     | __COMMON__ '(' expression ')'
                         {
@@ -7335,9 +7339,13 @@ simpleDataRow
                             parser->normalizeExpression($5, type_stringorunicode, false);
                             $$.setExpr(createRow(no_fromjson, $3.getExpr(), createComma($5.getExpr(), $6.getExpr())), $1);
                         }
-    | WHEN '(' dataRow ',' action ')'
+    | WHEN '(' dataRow ',' action sideEffectOptions ')'
                         {
-                            $$.setExpr(createCompound($5.getExpr(), $3.getExpr()), $1);
+                            OwnedHqlExpr options = $6.getExpr();
+                            if (options)
+                                $$.setExpr(createRow(no_executewhen, $3.getExpr(), createComma($5.getExpr(), options.getClear())), $1);
+                            else
+                                $$.setExpr(createCompound($5.getExpr(), $3.getExpr()), $1);
                         }
     ;
 
@@ -12037,8 +12045,12 @@ pattern0
                             parser->checkPattern($$, args);
                             $$.setExpr(parser->createPatternOr(args, $3));
                         }
-    | TOK_PATTERN '(' stringOrUnicodeConstExpr ')'
+    | TOK_PATTERN '(' expr ')'
                         {
+                            parser->normalizeExpression($3, type_stringorunicode, true);
+                            if (!$3.queryExpr()->isConstant())
+                                parser->reportError(ERR_EXPECTED_CONST, $3, "Pattern requires a constant argument");
+
                             IHqlExpression * pattern = parser->convertPatternToExpression($3);
                             $$.setExpr(createValue(no_pat_pattern, makePatternType(), $3.getExpr(), pattern));
                             parser->checkPattern($$, false);

ecl/hql/hqlgram2.cpp

@@ -4580,6 +4580,8 @@ IHqlExpression * HqlGram::convertPatternToExpression(attribute & text)
     try
     {
         IValue * value = expr->queryValue();
+        if (!value)
+            return NULL;
         unsigned len = value->queryType()->getStringLen();
         const void * data = value->queryValue();
         switch (expr->queryType()->getTypeCode())

ecl/hqlcpp/hqlcpp.cpp

@@ -2363,6 +2363,9 @@ void HqlCppTranslator::buildExprAssign(BuildCtx & ctx, const CHqlBoundTarget & t
         buildStmt(ctx, expr->queryChild(0));
         buildExprAssign(ctx, target, expr->queryChild(1));
         break;
+    case no_executewhen:
+        doBuildAssignExecuteWhen(ctx, target, expr);
+        break;
     case no_concat:
         doBuildAssignConcat(ctx, target, expr);
         break;
@@ -3179,6 +3182,7 @@ void HqlCppTranslator::buildExpr(BuildCtx & ctx, IHqlExpression * expr, CHqlBoun
     case no_loopcounter:
     case no_toxml:
     case no_tojson:
+    case no_executewhen:
         buildTempExpr(ctx, expr, tgt);
         return;
     case no_asstring:
@@ -7213,6 +7217,34 @@ void HqlCppTranslator::doBuildAssignConcat(BuildCtx & ctx, const CHqlBoundTarget
     }
 }
 
+void HqlCppTranslator::doBuildAssignExecuteWhen(BuildCtx & ctx, const CHqlBoundTarget & target, IHqlExpression * expr)
+{
+    IHqlExpression * value = expr->queryChild(0);
+    IHqlExpression * action = expr->queryChild(1);
+
+    if (expr->hasAttribute(beforeAtom))
+    {
+        buildStmt(ctx, action);
+        buildExprAssign(ctx, target, value);
+    }
+    else if (expr->hasAttribute(failureAtom))
+    {
+        BuildCtx tryctx(ctx);
+        tryctx.addTry();
+        buildExprAssign(tryctx, target, value);
+
+        BuildCtx catchctx(ctx);
+        catchctx.addCatch(NULL);
+        buildStmt(catchctx, action);
+        catchctx.addThrow(NULL);
+    }
+    else
+    {
+        buildExprAssign(ctx, target, value);
+        buildStmt(ctx, action);
+    }
+}
+
 //---------------------------------------------------------------------------
 //-- no_div --
 // also used for no_modulus

ecl/hqlcpp/hqlcpp.ipp

@@ -1190,6 +1190,7 @@ public:
     void doBuildAssignConcat(BuildCtx & ctx, const CHqlBoundTarget & target, IHqlExpression * expr);
     void doBuildAssignCount(BuildCtx & ctx, const CHqlBoundTarget & target, IHqlExpression * expr);
     void doBuildAssignDivide(BuildCtx & ctx, const CHqlBoundTarget & target, IHqlExpression * expr);
+    void doBuildAssignExecuteWhen(BuildCtx & ctx, const CHqlBoundTarget & target, IHqlExpression * expr);
     void doBuildAssignEventExtra(BuildCtx & ctx, const CHqlBoundTarget & target, IHqlExpression * expr);
     void doBuildAssignEventName(BuildCtx & ctx, const CHqlBoundTarget & target, IHqlExpression * expr);
     void doBuildAssignFailMessage(BuildCtx & ctx, const CHqlBoundTarget & target, IHqlExpression * expr);

ecl/hqlcpp/hqlhtcpp.cpp

@@ -3609,8 +3609,12 @@ unsigned HqlCppTranslator::buildRtlField(StringBuffer * instanceName, IHqlExpres
         IHqlExpression *defaultValue = queryAttributeChild(field, defaultAtom, 0);
         if (defaultValue)
         {
+            LinkedHqlExpr targetField = field;
+            if (fieldType->getTypeCode() == type_bitfield)
+                targetField.setown(createField(field->queryId(), LINK(fieldType->queryChildType()), NULL));
+
             MemoryBuffer target;
-            if (createConstantField(target, field, defaultValue))
+            if (createConstantField(target, targetField, defaultValue))
                 appendStringAsQuotedCPP(defaultInitializer, target.length(), target.toByteArray(), false);
             else
                 throwError1(HQLERR_CouldNotGenerateDefault, str(field->queryName()));

ecl/hqlcpp/hqliproj.cpp

@@ -1393,7 +1393,7 @@ void ImplicitProjectTransformer::analyseExpr(IHqlExpression * expr)
             Parent::analyseExpr(expr);
             break;
         case no_executewhen:
-            if (expr->isDataset())
+            if (expr->isDataset() || expr->isDatarow())
             {
                 assertex(extra->activityKind() == SimpleActivity);
                 Parent::analyseExpr(expr);

ecl/hqlcpp/hqlstmt.cpp

@@ -220,6 +220,17 @@ IHqlStmt * BuildCtx::addCase(IHqlStmt * _owner, IHqlExpression * source)
 }
 
 
+IHqlStmt * BuildCtx::addCatch(IHqlExpression * caught)
+{
+    if (ignoreInput)
+        return NULL;
+    HqlCompoundStmt * next = new HqlCompoundStmt(catch_stmt, curStmts);
+    if (caught)
+        next->addExpr(LINK(caught));
+    return appendCompound(next);
+}
+
+
 IHqlStmt * BuildCtx::addConditionalGroup(IHqlStmt * stmt)
 {
     if (ignoreInput)
@@ -476,6 +487,26 @@ IHqlStmt * BuildCtx::addSwitch(IHqlExpression * source)
 }
 
 
+IHqlStmt * BuildCtx::addThrow(IHqlExpression * thrown)
+{
+    if (ignoreInput)
+        return NULL;
+    HqlStmt * next = new HqlStmt(throw_stmt, curStmts);
+    if (thrown)
+        next->addExpr(LINK(thrown));
+    return appendSimple(next);
+}
+
+
+IHqlStmt * BuildCtx::addTry()
+{
+    if (ignoreInput)
+        return NULL;
+    HqlCompoundStmt * next = new HqlCompoundStmt(try_stmt, curStmts);
+    return appendCompound(next);
+}
+
+
 HqlStmt * BuildCtx::appendCompound(HqlCompoundStmt * next)
 {
     assertThrow(!ignoreInput);
@@ -941,6 +972,8 @@ IHqlStmt * BuildCtx::recursiveGetBestContext(HqlStmts * searchStmts, HqlExprCopy
     case quote_compound_stmt:
     case quote_compoundopt_stmt:
     case indirect_stmt:
+    case catch_stmt:
+    case try_stmt:
         return NULL;
     case filter_stmt:
     case switch_stmt:

ecl/hqlcpp/hqlstmt.hpp

@@ -99,6 +99,7 @@ public:
     IHqlStmt *                  addBlock();
     IHqlStmt *                  addBreak();
     IHqlStmt *                  addCase(IHqlStmt * owner, IHqlExpression * condition);
+    IHqlStmt *                  addCatch(IHqlExpression * caught);
     IHqlStmt *                  addConditionalGroup(IHqlStmt * stmt); // generated if stmt->isIncluded() is true
     IHqlStmt *                  addContinue();
     IHqlStmt *                  addDeclare(IHqlExpression * name, IHqlExpression * value=NULL);
@@ -125,6 +126,8 @@ public:
     IHqlStmt *                  addQuoted(StringBuffer & text)              { return addQuoted(text.str()); }
     IHqlStmt *                  addQuotedCompound(StringBuffer & text, const char * extra = NULL){ return addQuotedCompound(text.str(), extra); }
     IHqlStmt *                  addSwitch(IHqlExpression * condition);
+    IHqlStmt *                  addThrow(IHqlExpression * thrown);
+    IHqlStmt *                  addTry();
     void                        associate(HqlExprAssociation & next);
     void                        associateOwn(HqlExprAssociation & next);
     HqlExprAssociation *        associateExpr(IHqlExpression * represents, IHqlExpression * expr);
@@ -199,6 +202,9 @@ enum StmtKind {
              continue_stmt,
              function_stmt,
              assign_link_stmt,
+             try_stmt,
+             catch_stmt,
+             throw_stmt,
 };
 
 

ecl/hqlcpp/hqlttcpp.cpp

@@ -4963,9 +4963,9 @@ IHqlExpression * GlobalAttributeInfo::queryFilename(IHqlExpression * value, ICon
             if (storedName)
             {
                 if (persistOp == no_stored)
-                    prefix.append("spill::stored");
+                    prefix.append("jobtemp::stored");
                 else if (persistOp == no_checkpoint)
-                    prefix.append("spill::checkpoint");
+                    prefix.append("jobtemp::checkpoint");
             }
             if (persistOp == no_once)
                 prefix.append("once::");
@@ -5604,7 +5604,7 @@ unsigned WorkflowTransformer::ensureWorkflowAction(IHqlExpression * expr)
 
 unsigned WorkflowTransformer::splitValue(IHqlExpression * value)
 {
-    GlobalAttributeInfo info("spill::wf", "wf", value);
+    GlobalAttributeInfo info("jobtemp::wf", "wf", value);
     info.sequence.setown(getLocalSequenceNumber());
     info.setOp = no_setresult;
     info.persistOp = no_global;
@@ -5663,7 +5663,7 @@ void WorkflowTransformer::extractDependentInputs(UnsignedArray & visited, Depend
 IHqlExpression * WorkflowTransformer::extractWorkflow(IHqlExpression * untransformed, IHqlExpression * expr)
 {
     IHqlExpression * value = expr->queryChild(0);
-    GlobalAttributeInfo info("spill::wf", "wf", value);
+    GlobalAttributeInfo info("jobtemp::wf", "wf", value);
     info.sequence.setown(getLocalSequenceNumber());
     OwnedHqlExpr scheduleActions;
 
@@ -6016,7 +6016,7 @@ IHqlExpression * WorkflowTransformer::extractCommonWorkflow(IHqlExpression * exp
     DBGLOG("%s", s.str());
     translator.addWorkunitException(SeverityInformation, 0, s.str(), location);
 
-    GlobalAttributeInfo info("spill::wfa", "wfa", transformed);
+    GlobalAttributeInfo info("jobtemp::wfa", "wfa", transformed);
     info.extractGlobal(NULL, translator.getTargetClusterType());       // should really be a slightly different function
 
     OwnedHqlExpr setValue;
@@ -7537,7 +7537,7 @@ IHqlExpression * ExplicitGlobalTransformer::createTransformed(IHqlExpression * e
                 }
                 else
                 {
-                    GlobalAttributeInfo info("spill::global","gl", transformed);
+                    GlobalAttributeInfo info("jobtemp::global","gl", transformed);
                     if (op == no_nothor)
                         info.extractGlobal(NULL, RoxieCluster);
                     else
@@ -8134,7 +8134,7 @@ IHqlExpression * AutoScopeMigrateTransformer::createTransformed(IHqlExpression *
         s.append("as an item to hoist");
         DBGLOG("%s", s.str());
 
-        GlobalAttributeInfo info("spill::auto","auto", transformed);
+        GlobalAttributeInfo info("jobtemp::auto","auto", transformed);
         info.extractGlobal(NULL, translator.getTargetClusterType());
         if (translator.targetThor() && extra->globalInsideChild)
             info.preventDiskSpill();

ecl/hqlcpp/hqlwcpp.cpp

@@ -1741,6 +1741,26 @@ void HqlCppWriter::generateStmt(IHqlStmt * stmt)
         case switch_stmt:
             generateStmtSwitch(stmt);
             break;
+        case try_stmt:
+            indent().append("try");
+            generateChildren(stmt, true);
+            break;
+        case catch_stmt:
+            generateStmtCatch(stmt);
+            break;
+        case throw_stmt:
+            {
+                IHqlExpression * value = stmt->queryExpr(0);
+                if (value)
+                {
+                    indent().append("throw ");
+                    generateExprCpp(stmt->queryExpr(0)).append(';');
+                }
+                else
+                    indent().append("throw;");
+                newline();
+                break;
+            }
         case assigninc_stmt:
         case assigndec_stmt:
             generateStmtAssignModify(stmt);
@@ -1918,6 +1938,21 @@ void HqlCppWriter::generateStmtCase(IHqlStmt * stmt)
     }
 }
 
+void HqlCppWriter::generateStmtCatch(IHqlStmt * stmt)
+{
+    IHqlExpression * caught = stmt->queryExpr(0);
+    indent().append("catch (");
+    if (caught)
+    {
+        generateParamCpp(caught, NULL);
+    }
+    else
+        out.append("...");
+
+    out.append(")");
+    generateChildren(stmt, true);
+}
+
 void HqlCppWriter::generateStmtDeclare(IHqlStmt * declare)
 {
     IHqlExpression * name = declare->queryExpr(0);

ecl/hqlcpp/hqlwcpp.ipp

@@ -91,6 +91,7 @@ protected:
     void generateStmtAssign(IHqlStmt * assign, bool link);
     void generateStmtAssignModify(IHqlStmt * assign);
     void generateStmtCase(IHqlStmt * stmt);
+    void generateStmtCatch(IHqlStmt * stmt);
     void generateStmtDeclare(IHqlStmt * declare);
     void generateStmtFilter(IHqlStmt * stmt);
     void generateStmtFunction(IHqlStmt * stmt);

esp/scm/ws_dfu.ecm

@@ -76,13 +76,16 @@ ESPStruct DFUFileStat
     string MaxSkew;
 };
 
-ESPStruct DFUFilePartsOnCluster
+ESPStruct [nil_remove] DFUFilePartsOnCluster
 {
     string Cluster;
+    [min_ver("1.31")] string BaseDir;
+    [min_ver("1.31")] string ReplicateDir;
+    [min_ver("1.31")] bool Replicate;
     ESParray<ESPstruct DFUPart> DFUFileParts;
 };
 
-ESPStruct DFUFileDetail
+ESPStruct [nil_remove] DFUFileDetail
 {
     string Name;
     string Filename;
@@ -688,7 +691,7 @@ ESPresponse [exceptions_inline, nil_remove, http_encode(0)] DFUGetFileMetaDataRe
 
 //  ===========================================================================
 ESPservice [
-    version("1.30"),
+    version("1.31"),
     noforms, 
     exceptions_inline("./smc_xslt/exceptions.xslt")] WsDfu
 {

esp/scm/ws_machine.ecm

@@ -309,6 +309,64 @@ ESPresponse [encode(0), exceptions_inline] GetTargetClusterInfoResponse
     string TimeStamp;
     [min_ver("1.12")] string AcceptLanguage;
 };
+
+ESPstruct [nil_remove] StatusReport
+{
+    int StatusID;
+    string Status;
+    string StatusDetails;
+    string Reporter;
+    int64 TimeReported;
+    string TimeReportedStr;
+    string TimeCached;
+    string URL;
+};
+
+ESPstruct [nil_remove] ComponentStatus
+{
+    int ComponentTypeID;
+    string ComponentType;
+    string EndPoint;
+    int StatusID;
+    string Status;
+    int64 TimeReported;
+    string TimeReportedStr;
+    string Reporter;
+    ESParray<ESPstruct StatusReport> StatusReports;
+};
+
+ESPrequest [nil_remove] GetComponentStatusRequest
+{
+};
+
+ESPresponse [encode(0), nil_remove, exceptions_inline] GetComponentStatusResponse 
+{
+    int StatusCode;
+    string Status;
+
+    string ComponentType;
+    string EndPoint;
+    int ComponentStatusID;
+    string ComponentStatus;
+    int64 TimeReported;
+    string TimeReportedStr;
+    string Reporter;
+    ESPstruct StatusReport StatusReport;
+    ESParray<ESPstruct ComponentStatus, ComponentStatus> ComponentStatusList;
+};
+
+ESPrequest [nil_remove] UpdateComponentStatusRequest
+{
+    string Reporter;
+    ESParray<ESPstruct ComponentStatus, ComponentStatus> ComponentStatusList;
+};
+
+ESPresponse [encode(0), nil_remove, exceptions_inline] UpdateComponentStatusResponse 
+{
+    int StatusCode;
+    string Status;
+};
+
 //-------- service ---------
 ESPservice [version("1.13")] ws_machine
 {
@@ -330,6 +388,8 @@ ESPservice [version("1.13")] ws_machine
     ESPmethod [resp_xsl_default("./smc_xslt/ws_machine/machines.xslt"), exceptions_inline("./smc_xslt/exceptions.xslt")] 
        GetMachineInfoEx(GetMachineInfoRequestEx, GetMachineInfoResponseEx);
 
+    ESPmethod GetComponentStatus(GetComponentStatusRequest, GetComponentStatusResponse);
+    ESPmethod UpdateComponentStatus(UpdateComponentStatusRequest, UpdateComponentStatusResponse);
 
     ESPmethod [resp_xsl_default("./smc_xslt/ws_machine/metrics.xslt"), exceptions_inline("./smc_xslt/exceptions.xslt")] 
        GetMetrics(MetricsRequest, MetricsResponse);

esp/services/ws_dfu/ws_dfuService.cpp

@@ -1739,6 +1739,7 @@ bool CWsDfuEx::getUserFilePermission(IEspContext &context, IUserDescriptor* udes
 void CWsDfuEx::getFilePartsOnClusters(IEspContext &context, const char* clusterReq, StringArray& clusters, IDistributedFile* df, IEspDFUFileDetail& FileDetails,
     offset_t& mn, offset_t& mx, offset_t& sum, offset_t& count)
 {
+    double version = context.getClientVersion();
     IArrayOf<IConstDFUFilePartsOnCluster>& partsOnClusters = FileDetails.getDFUFilePartsOnClusters();
     ForEachItemIn(i, clusters)
     {
@@ -1788,6 +1789,24 @@ void CWsDfuEx::getFilePartsOnClusters(IEspContext &context, const char* clusterR
             }
         }
 
+        if (version >= 1.31)
+        {
+            IClusterInfo* clusterInfo = fdesc->queryCluster(clusterName);
+            if (clusterInfo) //Should be valid. But, check it just in case.
+            {
+                partsOnCluster->setReplicate(clusterInfo->queryPartDiskMapping().isReplicated());
+                const char* defaultDir = fdesc->queryDefaultDir();
+                if (defaultDir && *defaultDir)
+                {
+                    DFD_OS os = SepCharBaseOs(getPathSepChar(defaultDir));
+                    StringBuffer baseDir, repDir;
+                    clusterInfo->getBaseDir(baseDir, os);
+                    clusterInfo->getReplicateDir(repDir, os);
+                    partsOnCluster->setBaseDir(baseDir.str());
+                    partsOnCluster->setReplicateDir(baseDir.str());
+                }
+            }
+        }
         partsOnClusters.append(*partsOnCluster.getClear());
     }
 }

esp/services/ws_ecl/ws_ecl_service.cpp

@@ -782,6 +782,25 @@ void buildSampleDataset(StringBuffer &xml, IPropertyTree *xsdtree, const char *s
 
 }
 
+void buildSampleJsonDataset(StringBuffer &json, IPropertyTree *xsdtree, const char *service, const char *method, const char *resultname)
+{
+    StringBuffer schemaXml;
+    toXML(xsdtree, schemaXml);
+
+    Owned<IXmlSchema> schema = createXmlSchemaFromString(schemaXml);
+    if (schema.get())
+    {
+        IXmlType* type = schema->queryElementType("Dataset");
+        if (type)
+        {
+            StringArray parentTypes;
+            delimitJSON(json, true);
+            JsonHelpers::buildJsonMsg(parentTypes, type, json, resultname, NULL, REQSF_SAMPLE_DATA);
+        }
+    }
+
+}
+
 void CWsEclBinding::buildSampleResponseXml(StringBuffer& msg, IEspContext &context, CHttpRequest* request, WsEclWuInfo &wsinfo)
 {
     StringBuffer element;
@@ -1556,6 +1575,39 @@ void CWsEclBinding::getWsEclJsonRequest(StringBuffer& jsonmsg, IEspContext &cont
     }
 }
 
+void CWsEclBinding::buildSampleResponseJSON(StringBuffer& msg, IEspContext &context, CHttpRequest* request, WsEclWuInfo &wsinfo)
+{
+    StringBuffer element;
+    element.append(wsinfo.queryname.str()).append("Response");
+
+    StringBuffer xsds;
+    wsinfo.getSchemas(xsds);
+
+    const char *jsonp = context.queryRequestParameters()->queryProp("jsonp");
+    if (jsonp && *jsonp)
+        msg.append(jsonp).append('(');
+    msg.append('{');
+    appendJSONName(msg, element);
+    msg.append('{');
+    appendJSONName(msg, "Results");
+    msg.append('{');
+
+    Owned<IPropertyTree> xsds_tree;
+    if (xsds.length())
+        xsds_tree.setown(createPTreeFromXMLString(xsds.str()));
+
+    if (xsds_tree)
+    {
+        Owned<IPropertyTreeIterator> result_xsds =xsds_tree->getElements("Result");
+        ForEach (*result_xsds)
+            buildSampleJsonDataset(msg, result_xsds->query().queryPropTree("xs:schema"), wsinfo.qsetname.str(), wsinfo.queryname.str(), result_xsds->query().queryProp("@name"));
+    }
+
+    msg.append("}}}");
+    if (jsonp && *jsonp)
+        msg.append(");");
+}
+
 void CWsEclBinding::getSoapMessage(StringBuffer& soapmsg, IEspContext &context, CHttpRequest* request, WsEclWuInfo &wsinfo, unsigned flags, bool validate)
 {
     soapmsg.append(
@@ -2261,26 +2313,46 @@ int CWsEclBinding::getWsEclExample(CHttpRequest* request, CHttpResponse* respons
     StringBuffer qs;
     StringBuffer qid;
     splitLookupInfo(parms, thepath, wuid, qs, qid);
+
+    StringBuffer format;
+    nextPathNode(thepath, format);
+
+    ESPSerializationFormat fmt = strieq(format, "json") ? ESPSerializationJSON : ESPSerializationXML;
+
     WsEclWuInfo wsinfo(wuid.str(), qs.str(), qid.str(), context->queryUserId(), context->queryPassword());
 
     context->setBindingValue(&wsinfo);
+
+    StringBuffer output;
+    const char *contentType = HTTP_TYPE_APPLICATION_XML;
     if (!stricmp(exampletype.str(), "request"))
-        return onGetReqSampleXml(*context, request, response, qs.str(), qid.str());
+    {
+        if (fmt==ESPSerializationXML)
+            return onGetReqSampleXml(*context, request, response, qs.str(), qid.str());
+
+        getWsEclJsonRequest(output, *context, request, wsinfo, "json", NULL, REQSF_ROOT | REQSF_SAMPLE_DATA, true);
+        contentType = HTTP_TYPE_TEXT_PLAIN;
+    }
     else if (!stricmp(exampletype.str(), "response"))
     {
-        StringBuffer output;
-        buildSampleResponseXml(output, *context, request, wsinfo);
-        if (output.length())
-            {
-                response->setStatus("200 OK");
-                response->setContent(output.str());
-                response->setContentType(HTTP_TYPE_APPLICATION_XML);
-                response->send();
-            }
+        if (fmt==ESPSerializationXML)
+            buildSampleResponseXml(output, *context, request, wsinfo);
+        else
+        {
+            buildSampleResponseJSON(output, *context, request, wsinfo);
+            contentType = HTTP_TYPE_TEXT_PLAIN;
+        }
     }
     else if (!stricmp(exampletype.str(), "url"))
         return getRestURL(context, request, response, wsinfo, parms);
 
+    if (output.length())
+    {
+        response->setStatus("200 OK");
+        response->setContent(output.str());
+        response->setContentType(contentType);
+        response->send();
+    }
     return 0;
 }
 
@@ -2460,7 +2532,13 @@ int CWsEclBinding::onGet(CHttpRequest* request, CHttpResponse* response)
             response->redirect(*request, url);
             return 0;
         }
-
+        else if (strieq(methodName.str(), "json"))
+        {
+            StringBuffer url;
+            url.append("/WsEcl/forms/json/").append(thepath);
+            response->redirect(*request, url);
+            return 0;
+        }
     }
     catch (IMultiException* mex)
     {

esp/services/ws_ecl/ws_ecl_service.hpp

@@ -205,7 +205,8 @@ public:
 
     int getJsonTestForm(IEspContext &context, CHttpRequest* request, CHttpResponse* response, WsEclWuInfo &wsinfo, const char *formtype);
     void getWsEclJsonRequest(StringBuffer& soapmsg, IEspContext &context, CHttpRequest* request, WsEclWuInfo &wsinfo, const char *xmltype, const char *ns, unsigned flags, bool validate);
-    
+    void buildSampleResponseJSON(StringBuffer& msg, IEspContext &context, CHttpRequest* request, WsEclWuInfo &wsinfo);
+
     void sendRoxieRequest(const char *process, StringBuffer &req, StringBuffer &resp, StringBuffer &status, const char *query, bool trim, const char *contentType);
 };
 

esp/services/ws_machine/CMakeLists.txt

@@ -33,6 +33,7 @@ set (    SRCS
          ws_machineService.cpp 
          ws_machineServiceMetrics.cpp 
          ws_machineServiceRexec.cpp 
+         componentstatus.cpp
     )
 
 include_directories ( 

esp/services/ws_machine/componentstatus.cpp

@@ -0,0 +1,342 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2015 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#include "jiface.hpp"
+#include "componentstatus.hpp"
+#include "jlib.hpp"
+#include "esp.hpp"
+#include "ws_machine_esp.ipp"
+#include "ws_machine.hpp"
+
+static unsigned csCacheMinutes = 30; //30 minutes as default
+static MapStringTo<int> componentTypeMap;
+static MapStringTo<int> componentStatusTypeMap;
+static unsigned componentTypeIDCount;
+
+const char* formatTimeStamp(time_t tNow, StringAttr& out)
+{
+    char timeStr[32];
+#ifdef _WIN32
+    struct tm *ltNow;
+    ltNow = localtime(&tNow);
+    strftime(timeStr, 32, "%Y-%m-%d %H:%M:%S", ltNow);
+#else
+    struct tm ltNow;
+    localtime_r(&tNow, &ltNow);
+    strftime(timeStr, 32, "%Y-%m-%d %H:%M:%S", &ltNow);
+#endif
+    out.set(timeStr);
+    return out.get();
+}
+
+CESPComponentStatusInfo::CESPComponentStatusInfo(const char* _reporter)
+{
+    componentStatusID = -1;
+    addToCache = _reporter? true : false;
+    if (_reporter && *_reporter)
+        reporter.set(_reporter);
+
+    expireTimeStamp = 0;
+    if (csCacheMinutes > 0)
+    { //A status report should be expired if it was reported >csCacheMinutes ago.
+        CDateTime timeNow;
+        timeNow.setNow();
+        timeNow.adjustTime(-csCacheMinutes);
+        expireTimeStamp = timeNow.getSimple();
+    }
+}
+
+void CESPComponentStatusInfo::init(IPropertyTree* cfg)
+{
+    StringArray statusTypeMap;
+    Owned<IPropertyTreeIterator> statusTypes = cfg->getElements("StatusType");
+    ForEach(*statusTypes)
+    {
+        IPropertyTree& statusType = statusTypes->query();
+        const char* name = statusType.queryProp("@name");
+        if (name && *name)
+            componentStatusTypeMap.setValue(name, statusType.getPropInt("@id"));
+    }
+
+    if (componentStatusTypeMap.count() < 1)
+    {
+        componentStatusTypeMap.setValue("normal", 1);
+        componentStatusTypeMap.setValue("warning", 2);
+        componentStatusTypeMap.setValue("error", 3);
+    }
+    componentTypeIDCount = 0;
+
+    cfg->getPropInt("CSCacheMinutes", csCacheMinutes);
+}
+
+bool CESPComponentStatusInfo::isSameComponent(const char* ep, int componentTypeID, IConstComponentStatus& status)
+{
+    const char* ep1 = status.getEndPoint();
+    if (!ep1 || !*ep1 || !ep || !*ep)
+        return false;
+    bool hasPort = strchr(ep, ':');
+    if (hasPort)
+        return streq(ep1, ep);
+    //If no port, for now, only one componentType is reported per IP.
+    return ((componentTypeID == status.getComponentTypeID()) && streq(ep1, ep));
+}
+
+void CESPComponentStatusInfo::addStatusReport(const char* reporterIn, const char* timeCachedIn, IConstComponentStatus& csIn, IEspComponentStatus& csOut, bool firstReport)
+{
+    IArrayOf<IConstStatusReport>& statusReports = csOut.getStatusReports();
+    IArrayOf<IConstStatusReport>& reportsIn = csIn.getStatusReports();
+    ForEachItemIn(i, reportsIn)
+    {
+        IConstStatusReport& report = reportsIn.item(i);
+        const char* status = report.getStatus();
+        if (!status || !*status)
+            continue;
+
+        if ((expireTimeStamp > 0) && (report.getTimeReported() < expireTimeStamp))
+            continue;
+
+        int statusID;
+        if (addToCache) //from the update status request
+            statusID = queryComponentStatusID(status);
+        else //from a cache report which StatusID has been set.
+            statusID = report.getStatusID();
+
+        Owned<IEspStatusReport> statusReport = createStatusReport();
+        statusReport->setStatusID(statusID);
+        statusReport->setStatus(status);
+
+        const char* details = report.getStatusDetails();
+        if (details && *details)
+            statusReport->setStatusDetails(details);
+
+        const char* url = report.getURL();
+        if (url && *url)
+            statusReport->setURL(url);
+
+        statusReport->setReporter(reporterIn);
+        statusReport->setTimeCached(timeCachedIn);
+        statusReport->setTimeReported(report.getTimeReported());
+
+        if (!addToCache)
+        {//We need to add more info for a user-friendly output
+            StringAttr timeStr;
+            statusReport->setTimeReportedStr(formatTimeStamp(report.getTimeReported(), timeStr));
+
+            if (firstReport || (statusID > csOut.getStatusID())) //worst case for component
+            {
+                csOut.setStatusID(statusID);
+                csOut.setStatus(status);
+                csOut.setTimeReportedStr(timeStr.get());
+                csOut.setReporter(reporterIn);
+            }
+            if (statusID > componentStatusID) //worst case for whole system
+            {
+                componentStatusID = statusID;
+                componentStatus.set(status);
+                timeReported = report.getTimeReported();
+                timeReportedStr.set(timeStr.get());
+                reporter.set(reporterIn);
+                componentTypeID = csIn.getComponentTypeID();
+                componentType.set(csIn.getComponentType());
+                endPoint.set(csIn.getEndPoint());
+                componentStatusReport.setown(statusReport.getLink());
+            }
+        }
+        statusReports.append(*statusReport.getClear());
+    }
+}
+
+void CESPComponentStatusInfo::addComponentStatus(const char* reporterIn, const char* timeCachedIn, IConstComponentStatus& st)
+{
+    Owned<IEspComponentStatus> cs = createComponentStatus();
+    cs->setEndPoint(st.getEndPoint());
+    cs->setComponentType(st.getComponentType());
+
+    int componentTypeID;
+    if (addToCache) //from the update status request
+        componentTypeID = queryComponentTypeID(st.getComponentType());
+    else
+        componentTypeID = st.getComponentTypeID();
+    cs->setComponentTypeID(componentTypeID);
+
+    IArrayOf<IConstStatusReport> statusReports;
+    cs->setStatusReports(statusReports);
+    addStatusReport(reporterIn, timeCachedIn, st, *cs, true);
+
+    statusList.append(*cs.getClear());
+}
+
+void CESPComponentStatusInfo::appendUnchangedComponentStatus(IEspComponentStatus& statusOld)
+{
+    bool componentFound = false;
+    const char* ep = statusOld.getEndPoint();
+    int componentTypeID = statusOld.getComponentTypeID();
+    ForEachItemIn(i, statusList)
+    {
+        if (isSameComponent(ep, componentTypeID, statusList.item(i)))
+        {
+            componentFound =  true;
+            break;
+        }
+    }
+    if (!componentFound)
+        addComponentStatus(reporter.get(), timeCached, statusOld);
+}
+
+bool CESPComponentStatusInfo::cleanExpiredStatusReports(IArrayOf<IConstStatusReport>& reports)
+{
+    bool expired = true;
+    ForEachItemInRev(i, reports)
+    {
+        IConstStatusReport& report = reports.item(i);
+        if ((expireTimeStamp > 0) && (report.getTimeReported() < expireTimeStamp))
+            reports.remove(i);
+        else
+            expired = false;
+    }
+    return expired;
+}
+
+int CESPComponentStatusInfo::queryComponentTypeID(const char *key)
+{
+    int* id = componentTypeMap.getValue(key);
+    if (id)
+        return *id;
+
+    componentTypeMap.setValue(key, ++componentTypeIDCount);
+    return componentTypeIDCount;
+}
+
+int CESPComponentStatusInfo::queryComponentStatusID(const char *key)
+{
+    StringBuffer buf(key);
+    int* value = componentStatusTypeMap.getValue(buf.toLowerCase().str());
+    if (!value)
+        return 0;
+    return *value;
+}
+
+bool CESPComponentStatusInfo::cleanExpiredComponentReports(IESPComponentStatusInfo& statusInfo)
+{
+    bool expired = true;
+    IArrayOf<IEspComponentStatus>& statusList = statusInfo.getComponentStatusList();
+    ForEachItemInRev(i, statusList)
+    {
+        IEspComponentStatus& status = statusList.item(i);
+        if (cleanExpiredStatusReports(status.getStatusReports()))
+            statusList.remove(i);
+        else
+            expired = false;
+    }
+    return expired;
+}
+
+void CESPComponentStatusInfo::mergeComponentStatusInfoFromReports(IESPComponentStatusInfo& statusInfo)
+{
+    const char* reporterIn = statusInfo.getReporter();
+    const char* timeCachedIn = statusInfo.getTimeCached();
+    IArrayOf<IEspComponentStatus>& statusListIn = statusInfo.getComponentStatusList();
+    ForEachItemIn(i, statusListIn)
+    {
+        IEspComponentStatus& statusIn = statusListIn.item(i);
+
+        bool newCompoment = true;
+        const char* ep = statusIn.getEndPoint();
+        int componentTypeID = statusIn.getComponentTypeID();
+        ForEachItemIn(ii, statusList)
+        {
+            IEspComponentStatus& statusOut = statusList.item(ii);
+            if (isSameComponent(ep, componentTypeID, statusOut))
+            {// multiple reports from different reporters.
+                addStatusReport(reporterIn, timeCachedIn, statusIn, statusOut, false);
+                newCompoment =  false;
+                break;
+            }
+        }
+        if (newCompoment)
+            addComponentStatus(reporterIn, timeCachedIn, statusIn);
+    }
+}
+
+void CESPComponentStatusInfo::setComponentStatus(IArrayOf<IConstComponentStatus>& statusListIn)
+{
+    time_t tNow;
+    time(&tNow);
+    formatTimeStamp(tNow, timeCached);
+
+    statusList.kill();
+    ForEachItemIn(i, statusListIn)
+        addComponentStatus(reporter, timeCached, statusListIn.item(i));
+}
+
+void CESPComponentStatusInfo::mergeCachedComponentStatus(IESPComponentStatusInfo& statusInfo)
+{
+    IArrayOf<IEspComponentStatus>& csList = statusInfo.getComponentStatusList();
+    ForEachItemIn(i, csList)
+        appendUnchangedComponentStatus(csList.item(i));
+}
+
+static CriticalSection componentStatusSect;
+
+IESPComponentStatusInfo* CComponentStatusFactory::getComponentStatus()
+{
+    CriticalBlock block(componentStatusSect);
+    Owned<IESPComponentStatusInfo> status = new CESPComponentStatusInfo(NULL);
+    ForEachItemInRev(i, cache)
+    {
+        IESPComponentStatusInfo& statusInfo = cache.item(i);
+        if (status->cleanExpiredComponentReports(statusInfo))
+            cache.remove(i);
+        else
+            status->mergeComponentStatusInfoFromReports(statusInfo);
+    }
+    return status.getClear();
+}
+
+void CComponentStatusFactory::updateComponentStatus(const char* reporter, IArrayOf<IConstComponentStatus>& statusList)
+{
+    CriticalBlock block(componentStatusSect);
+
+    Owned<IESPComponentStatusInfo> status = new CESPComponentStatusInfo(reporter);
+    status->setComponentStatus(statusList);
+
+    ForEachItemIn(i, cache)
+    {
+        IESPComponentStatusInfo& cachedStatus = cache.item(i);
+        if (strieq(reporter, cachedStatus.getReporter()))
+        {
+            status->mergeCachedComponentStatus(cachedStatus);
+            cache.remove(i);
+            break;
+        }
+    }
+    cache.append(*status.getClear());
+}
+
+static CComponentStatusFactory *csFactory = NULL;
+
+static CriticalSection getComponentStatusSect;
+
+extern COMPONENTSTATUS_API IComponentStatusFactory* getComponentStatusFactory()
+{
+    CriticalBlock block(getComponentStatusSect);
+
+    if (!csFactory)
+        csFactory = new CComponentStatusFactory();
+
+    return LINK(csFactory);
+}

esp/services/ws_machine/componentstatus.hpp

@@ -0,0 +1,97 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2015 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#include "jiface.hpp"
+#include "jlib.hpp"
+#include "esp.hpp"
+#include "componentstatus.ipp"
+#include "ws_machine_esp.ipp"
+
+class CESPComponentStatusInfo : public CInterface, implements IESPComponentStatusInfo
+{
+    StringAttr reporter;
+    StringAttr timeCached;
+    IArrayOf<IEspComponentStatus> statusList;
+
+    bool addToCache; //this CESPComponentStatusInfo object is created by ws_machine.UpdateComponentStatus
+    __int64 expireTimeStamp;
+    int componentStatusID; //the worst component status in the system
+    int componentTypeID; //the worst component status in the system
+    StringAttr componentStatus; //the worst component status in the system
+    StringAttr componentType; //the worst component status in the system
+    StringAttr endPoint; //the worst component status in the system
+    StringAttr timeReportedStr; //the worst component status in the system
+    __int64 timeReported; //the worst component status in the system
+    Owned<IEspStatusReport> componentStatusReport; //the worst component status in the system
+
+    bool isSameComponent(const char* ep, int componentTypeID, IConstComponentStatus& status);
+    void addStatusReport(const char* reporterIn, const char* timeCachedIn, IConstComponentStatus& csIn,
+        IEspComponentStatus& csOut, bool firstReport);
+    void addComponentStatus(const char* reporterIn, const char* timeCachedIn, IConstComponentStatus& st);
+    void appendUnchangedComponentStatus(IEspComponentStatus& statusOld);
+    bool cleanExpiredStatusReports(IArrayOf<IConstStatusReport>& reports);
+
+public:
+    IMPLEMENT_IINTERFACE;
+
+    CESPComponentStatusInfo(const char* _reporter);
+
+    static void init(IPropertyTree* cfg);
+
+    inline const char* getReporter() { return reporter.get(); };
+    inline const char* getTimeCached() { return timeCached.get(); };
+    inline int getComponentStatusID() { return componentStatusID; };
+    inline const char* getComponentStatus() { return componentStatus.get(); };
+    inline const char* getTimeReportedStr() { return timeReportedStr.get(); };
+    inline __int64 getTimeReported() { return timeReported; };
+    inline const int getComponentTypeID() { return componentTypeID; };
+    inline const char* getComponentType() { return componentType.get(); };
+    inline const char* getEndPoint() { return endPoint.get(); };
+    inline IEspStatusReport* getStatusReport() { return componentStatusReport; };
+    inline IArrayOf<IEspComponentStatus>& getComponentStatusList() { return statusList; };
+
+    int queryComponentTypeID(const char *key);
+    int queryComponentStatusID(const char *key);
+    virtual bool cleanExpiredComponentReports(IESPComponentStatusInfo& statusInfo);
+    virtual void mergeComponentStatusInfoFromReports(IESPComponentStatusInfo& statusInfo);
+    virtual void setComponentStatus(IArrayOf<IConstComponentStatus>& statusListIn);
+    void mergeCachedComponentStatus(IESPComponentStatusInfo& statusInfo);
+};
+
+class CComponentStatusFactory : public CInterface, implements IComponentStatusFactory
+{
+    IArrayOf<IESPComponentStatusInfo> cache; //multiple caches from different reporter
+public:
+    IMPLEMENT_IINTERFACE;
+
+    CComponentStatusFactory() { };
+
+    virtual ~CComponentStatusFactory()
+    {
+        cache.kill();
+    };
+
+    virtual void init(IPropertyTree* cfg)
+    {
+        CESPComponentStatusInfo::init(cfg);
+    };
+
+    virtual IESPComponentStatusInfo* getComponentStatus();
+    virtual void updateComponentStatus(const char* reporter, IArrayOf<IConstComponentStatus>& statusList);
+};
+
+extern "C" COMPONENTSTATUS_API IComponentStatusFactory* getComponentStatusFactory();

esp/services/ws_machine/componentstatus.ipp

@@ -0,0 +1,76 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2015 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#pragma warning (disable : 4786)
+
+#ifndef _COMPONENTSTATUS_HPP__
+#define _COMPONENTSTATUS_HPP__
+
+#include "jiface.hpp"
+
+class StringBuffer;
+class IPropertyTree;
+class IPropertyTreeIterator;
+
+class IEspStatusReport;
+class IEspComponentStatus;
+class IConstComponentStatus;
+template<class IEspComponentStatus> class IArrayOf;
+
+
+#ifdef WIN32
+    #ifdef SMCLIB_EXPORTS
+        #define COMPONENTSTATUS_API __declspec(dllexport)
+    #else
+        #define COMPONENTSTATUS_API __declspec(dllimport)
+    #endif
+#else
+    #define COMPONENTSTATUS_API
+#endif
+
+class IESPComponentStatusInfo : public IInterface
+{
+public:
+    virtual int queryComponentTypeID(const char *key) = 0;
+    virtual int queryComponentStatusID(const char *key) = 0;
+
+    virtual const char* getReporter() = 0;
+    virtual const char* getTimeCached() = 0;
+    virtual const char* getComponentStatus() = 0;
+    virtual int getComponentStatusID() = 0;
+    virtual const char* getTimeReportedStr() = 0;
+    virtual __int64 getTimeReported() = 0;
+    virtual const char* getEndPoint() = 0;
+    virtual const char* getComponentType() = 0;
+    virtual const int getComponentTypeID() = 0;
+    virtual IEspStatusReport* getStatusReport() = 0;
+    virtual IArrayOf<IEspComponentStatus>& getComponentStatusList() = 0;
+    virtual void setComponentStatus(IArrayOf<IConstComponentStatus>& StatusList) = 0;
+    virtual void mergeCachedComponentStatus(IESPComponentStatusInfo& statusInfo) = 0;
+    virtual void mergeComponentStatusInfoFromReports(IESPComponentStatusInfo& statusInfo) = 0;
+    virtual bool cleanExpiredComponentReports(IESPComponentStatusInfo& statusInfo) = 0;
+};
+
+class IComponentStatusFactory : public IInterface
+{
+public:
+    virtual void init(IPropertyTree* cfg) = 0;
+    virtual IESPComponentStatusInfo* getComponentStatus() = 0;
+    virtual void updateComponentStatus(const char* reporter, IArrayOf<IConstComponentStatus>& StatusList) = 0;
+};
+
+#endif  //_COMPONENTSTATUS_HPP__

esp/services/ws_machine/ws_machineService.cpp

@@ -21,6 +21,7 @@
 #include "exception_util.hpp"
 #include "workunit.hpp"
 #include "roxiecommlibscm.hpp"
+#include "componentstatus.hpp"
 
 #ifndef eqHoleCluster
 #define eqHoleCluster  "HoleCluster"
@@ -198,6 +199,9 @@ void Cws_machineEx::init(IPropertyTree *cfg, const char *process, const char *se
         NULL, m_threadPoolSize, 10000, m_threadPoolStackSize)); //10 sec timeout for available thread; use stack size of 2MB
 
     setupLegacyFilters();
+
+    Owned<IComponentStatusFactory> factory = getComponentStatusFactory();
+    factory->init(pServiceNode);
 }
 
 StringBuffer& Cws_machineEx::getAcceptLanguage(IEspContext& context, StringBuffer& acceptLanguage)
@@ -2255,3 +2259,69 @@ void Cws_machineEx::getAccountAndPlatformInfo(const char* address, StringBuffer&
 
     bLinux = machine->getOS() == MachineOsLinux;
 }
+
+bool Cws_machineEx::onGetComponentStatus(IEspContext &context, IEspGetComponentStatusRequest &req, IEspGetComponentStatusResponse &resp)
+{
+    try
+    {
+        if (!context.validateFeatureAccess(FEATURE_URL, SecAccess_Read, false))
+            throw MakeStringException(ECLWATCH_MACHINE_INFO_ACCESS_DENIED, "Failed to Get Component Status. Permission denied.");
+
+        Owned<IComponentStatusFactory> factory = getComponentStatusFactory();
+        Owned<IESPComponentStatusInfo> status = factory->getComponentStatus();
+        if (!status) //Should never happen
+            return false;
+
+        int statusID = status->getComponentStatusID();
+        if (statusID < 0)
+        {
+            resp.setStatus("Not reported");
+        }
+        else
+        {
+            resp.setComponentType(status->getComponentType());
+            resp.setEndPoint(status->getEndPoint());
+            resp.setReporter(status->getReporter());
+            resp.setComponentStatus(status->getComponentStatus());
+            resp.setTimeReportedStr(status->getTimeReportedStr());
+
+            IConstStatusReport* componentStatus = status->getStatusReport();
+            if (componentStatus)
+                resp.setStatusReport(*componentStatus);
+
+            resp.setComponentStatusList(status->getComponentStatusList());
+        }
+        resp.setComponentStatusID(statusID);
+        resp.setStatusCode(0);
+    }
+    catch(IException* e)
+    {
+        FORWARDEXCEPTION(context, e,  ECLWATCH_INTERNAL_ERROR);
+    }
+
+    return true;
+}
+
+bool Cws_machineEx::onUpdateComponentStatus(IEspContext &context, IEspUpdateComponentStatusRequest &req, IEspUpdateComponentStatusResponse &resp)
+{
+    try
+    {
+        if (!context.validateFeatureAccess(FEATURE_URL, SecAccess_Write, false))
+            throw MakeStringException(ECLWATCH_MACHINE_INFO_ACCESS_DENIED, "Failed to Update Component Status. Permission denied.");
+
+        const char* reporter = req.getReporter();
+        if (!reporter || !*reporter)
+            throw MakeStringException(ECLWATCH_INVALID_INPUT, "Report not specified.");
+        
+        Owned<IComponentStatusFactory> factory = getComponentStatusFactory();
+        factory->updateComponentStatus(reporter, req.getComponentStatusList());
+        resp.setStatusCode(0);
+    }
+    catch(IException* e)
+    {
+        FORWARDEXCEPTION(context, e,  ECLWATCH_INTERNAL_ERROR);
+    }
+
+    return true;
+}
+

esp/services/ws_machine/ws_machineService.hpp

@@ -732,6 +732,8 @@ public:
     bool onGetMachineInfo(IEspContext &context, IEspGetMachineInfoRequest &req, IEspGetMachineInfoResponse &resp);
     bool onGetTargetClusterInfo(IEspContext &context, IEspGetTargetClusterInfoRequest &req, IEspGetTargetClusterInfoResponse &resp);
     bool onGetMachineInfoEx(IEspContext &context, IEspGetMachineInfoRequestEx &req, IEspGetMachineInfoResponseEx &resp);
+    bool onGetComponentStatus(IEspContext &context, IEspGetComponentStatusRequest &req, IEspGetComponentStatusResponse &resp);
+    bool onUpdateComponentStatus(IEspContext &context, IEspUpdateComponentStatusRequest &req, IEspUpdateComponentStatusResponse &resp);
 
     bool onGetMetrics(IEspContext &context, IEspMetricsRequest &req, IEspMetricsResponse &resp);
     bool onStartStop( IEspContext &context, IEspStartStopRequest &req,  IEspStartStopResponse &resp);

esp/services/ws_workunits/ws_workunitsService.cpp

@@ -3326,11 +3326,17 @@ bool CWsWorkunitsEx::onWUProcessGraph(IEspContext &context,IEspWUProcessGraphReq
             throw MakeStringException(ECLWATCH_CANNOT_OPEN_WORKUNIT,"Cannot open workunit %s.",wuid.str());
         ensureWsWorkunitAccess(context, *cw, SecAccess_Read);
 
-        Owned <IConstWUGraph> graph = cw->getGraph(req.getName());
-        Owned <IPropertyTree> xgmml = graph->getXGMMLTree(true); // merge in graph progress information
+        if (isEmpty(req.getName()))
+            throw MakeStringException(ECLWATCH_GRAPH_NOT_FOUND, "Please specify a graph name.");
+
+        Owned<IConstWUGraph> graph = cw->getGraph(req.getName());
+        if (!graph)
+            throw MakeStringException(ECLWATCH_GRAPH_NOT_FOUND, "Invalid graph name: %s for %s", req.getName(), wuid.str());
 
         StringBuffer xml;
-        resp.setTheGraph(toXML(xgmml.get(), xml).str());
+        Owned<IPropertyTree> xgmml = graph->getXGMMLTree(true); // merge in graph progress information
+        toXML(xgmml.get(), xml);
+        resp.setTheGraph(xml.str());
     }
 
     catch(IException* e)
@@ -3429,7 +3435,8 @@ bool CWsWorkunitsEx::onWUGetGraph(IEspContext& context, IEspWUGetGraphRequest& r
         else
         {
             Owned<IConstWUGraph> graph = cw->getGraph(req.getGraphName());
-            readGraph(context, req.getSubGraphId(), id, running, graph, graphs);
+            if (graph)
+                readGraph(context, req.getSubGraphId(), id, running, graph, graphs);
         }
         resp.setGraphs(graphs);
     }

esp/src/eclwatch/ESPTopology.js

@@ -109,6 +109,17 @@ define([
             }
         },
 
+        getNetaddress: function () {
+            if (this.Netaddress) {
+                return this.Netaddress;
+            } else if (this.__hpcc_parent) {
+                if (this.__hpcc_parent.Netaddress) {
+                    return this.__hpcc_parent.Netaddress;
+                }
+            }
+            return "";
+        },
+
         getLogDirectory: function () {
             if (this.LogDirectory) {
                 return this.LogDirectory;
@@ -424,7 +435,27 @@ define([
                 });
             };
             retVal.hasLogs = function () {
-                return this.__hpcc_treeItem.getLogDirectory && this.__hpcc_treeItem.getLogDirectory();
+                return this.getNetaddress() && this.getLogDirectory();
+            };
+            retVal.getOS = function () {
+                return this.__hpcc_treeItem.OS;
+            };
+            retVal.getNetaddress = function () {
+                var retVal = null;
+                if (this.__hpcc_treeItem.getNetaddress) {
+                    retVal = this.__hpcc_treeItem.getNetaddress();
+                }
+                if (!retVal && parentNode && parentNode.__hpcc_treeItem.getNetaddress) {
+                    retVal = parentNode.__hpcc_treeItem.getNetaddress();
+                }
+                return retVal;
+            };
+            retVal.getLogDirectory = function () {
+                var retVal = null;
+                if (this.__hpcc_treeItem.getLogDirectory) {
+                    retVal = this.__hpcc_treeItem.getLogDirectory();
+                }
+                return retVal;
             };
             return retVal;
         },

esp/src/eclwatch/LFDetailsWidget.js

@@ -335,6 +335,9 @@ define([
                 this.refreshActionState();
                 //  Force Icon to Show (I suspect its not working due to Circular Reference Loading)
                 this.queriesWidget.set("iconClass", "dijitInline dijitIcon dijitTabButtonIcon iconFind");
+            } else if (name === "DFUFilePartsOnClusters") {
+            	// Currently only checking first cluster may add loop through clusters and add a tab at a later date
+            	this.updateInput("DFUFilePartsOnClusters", oldValue, newValue.DFUFilePartsOnCluster[0].Replicate);
             }
         },
 

esp/src/eclwatch/LogWidget.js

@@ -164,7 +164,7 @@ define([
             this.logTargetSelect.init({
                 Logs: true,
                 includeBlank: false,
-                params: this.params
+                treeNode: this.params
             });
 
             this.initLogGrid();

esp/src/eclwatch/TargetSelectClass.js

@@ -346,9 +346,9 @@ define([
             FileSpray.FileList({
                 request: {
                     Mask: "*.log",
-                    Netaddr: params.params.Netaddress,
-                    OS: params.params.OS,
-                    Path: params.params.getLogDirectory()
+                    Netaddr: params.treeNode.getNetaddress(),
+                    OS: params.treeNode.getOS(),
+                    Path: params.treeNode.getLogDirectory()
                 }
             }).then(function (response) {
                 if (lang.exists("FileListResponse.files.PhysicalFileStruct", response)) {

esp/src/eclwatch/TopologyDetailsWidget.js

@@ -189,7 +189,7 @@ define([
                 });
             } else if (currSel.id == this.widget._Logs.id && !this.widget._Logs.__hpcc_initalized) {
                 this.widget._Logs.__hpcc_initalized = true;
-                this.widget._Logs.init(this.params.__hpcc_treeItem);
+                this.widget._Logs.init(this.params);
             }
         },
 

esp/src/eclwatch/nls/hpcc.js

@@ -206,6 +206,7 @@ define({root:
     IP: "IP",
     IPAddress: "IP Address",
     IsLibrary: "Is Library",
+    IsReplicated: "Is Replicated",
     Jobname: "Jobname",
     JobName: "Job Name",
     jsmi: "jsmi*",

esp/src/eclwatch/templates/LFDetailsWidget.html

@@ -150,6 +150,10 @@
                                 <label for="${id}RecordCount">${i18n.RecordCount}: </label>
                                 <div id="${id}RecordCount"></div>
                             </li>
+                            <li>
+                                <label for="${id}DFUFilePartsOnClusters">${i18n.IsReplicated}: </label>
+                                <div id="${id}DFUFilePartsOnClusters"></div>
+                            </li>
                         </ul>
                     </form>
                 </div>

esp/xslt/wsecl3_links.xslt

@@ -30,10 +30,19 @@
                     Sample REST URL:&nbsp;&nbsp;<a  target="_blank" href="/WsEcl/example/url/query/{$pathval}/{$queryname}">link</a>
                 </li>
                 <li>
-                    Sample Request:&nbsp;&nbsp;<a href="/WsEcl/example/request/query/{$pathval}/{$queryname}?display">display</a>&nbsp;&nbsp;<a  target="_blank" href="/WsEcl/example/request/query/{$pathval}/{$queryname}">link</a>
+                    Sample SOAP Request:&nbsp;&nbsp;<a href="/WsEcl/example/request/query/{$pathval}/{$queryname}?display">display</a>&nbsp;&nbsp;<a  target="_blank" href="/WsEcl/example/request/query/{$pathval}/{$queryname}">link</a>
                 </li>
                 <li>
-                    Sample Response:&nbsp;&nbsp;<a href="/WsEcl/example/response/query/{$pathval}/{$queryname}?display">display</a>&nbsp;&nbsp;<a  target="_blank" href="/WsEcl/example/response/query/{$pathval}/{$queryname}">link</a>
+                    Sample SOAP Response:&nbsp;&nbsp;<a href="/WsEcl/example/response/query/{$pathval}/{$queryname}?display">display</a>&nbsp;&nbsp;<a  target="_blank" href="/WsEcl/example/response/query/{$pathval}/{$queryname}">link</a>
+                </li>
+                <li>
+                    Sample JSON Request:&nbsp;&nbsp;<a href="/WsEcl/example/request/query/{$pathval}/{$queryname}/json?display">display</a>&nbsp;&nbsp;<a  target="_blank" href="/WsEcl/example/request/query/{$pathval}/{$queryname}/json">link</a>
+                </li>
+                <li>
+                    Sample JSON Response:&nbsp;&nbsp;<a href="/WsEcl/example/response/query/{$pathval}/{$queryname}/json?display">display</a>&nbsp;&nbsp;<a  target="_blank" href="/WsEcl/example/response/query/{$pathval}/{$queryname}/json">link</a>
+                </li>
+                <li>
+                    Sample JSONP Response:&nbsp;&nbsp;<a href="/WsEcl/example/response/query/{$pathval}/{$queryname}/json?display&amp;jsonp=methodname">display</a>&nbsp;&nbsp;<a  target="_blank" href="/WsEcl/example/response/query/{$pathval}/{$queryname}/json?jsonp=methodname">link</a>
                 </li>
                 <li>
                     Parameter XML:&nbsp;&nbsp;<a href="/WsEcl/definitions/query/{$pathval}/{$queryname}/resource/soap/{$queryname}.xml?display">display</a>
@@ -42,6 +51,9 @@
                     SOAP (Post SOAP messages to this URL):&nbsp;&nbsp;<a href="/WsEcl/soap/query/{$pathval}/{$queryname}">/WsEcl/soap/query/<xsl:value-of select="$pathval"/>/<xsl:value-of select="$queryname"/></a>
                 </li>
                 <li>
+                    JSON (Post JSON messages to this URL):&nbsp;&nbsp;<a href="/WsEcl/json/query/{$pathval}/{$queryname}">/WsEcl/json/query/<xsl:value-of select="$pathval"/>/<xsl:value-of select="$queryname"/></a>
+                </li>
+                <li>
                     WSDL:&nbsp;&nbsp;<a href="/WsEcl/definitions/query/{$pathval}/{$queryname}/main/{$queryname}.wsdl?display">display</a>&nbsp;&nbsp;<a  target="_blank" href="/WsEcl/definitions/query/{$pathval}/{$queryname}/main/{$queryname}.wsdl">link</a>
                 </li>
                 <li>

initfiles/componentfiles/configxml/CMakeLists.txt

@@ -31,6 +31,7 @@ FOREACH( iFILES
     ${CMAKE_CURRENT_BINARY_DIR}/eclagent_config.xsd
     ${CMAKE_CURRENT_BINARY_DIR}/esp.xsd
     ${CMAKE_CURRENT_BINARY_DIR}/espsmcservice.xsd
+    ${CMAKE_CURRENT_BINARY_DIR}/ftslave_linux.xsd
     ${CMAKE_CURRENT_BINARY_DIR}/roxie.xsd
     ${CMAKE_CURRENT_BINARY_DIR}/thor.xsd
     ${CMAKE_CURRENT_BINARY_DIR}/buildset.xml

plugins/proxies/lib_saltlib.ecllib

@@ -16,7 +16,12 @@
 ############################################################################## */
 
 /* Proxy service header for (EE-only) saltlib plugin version SALTLIB 1.0.05 */
-
+EXPORT GeohashDecodeResultRecord := RECORD
+    REAL latitude;
+    REAL longitude;
+    REAL latitude_err;
+    REAL longitude_err;
+END;
 export SaltLib := SERVICE : plugin('saltlib')
   boolean UnicodeLocaleWithinEditN(const unicode left, const unicode right, unsigned4 distance,  const varstring localename) : c, pure,entrypoint='ulUnicodeLocaleWithinEditN', hole;
   boolean UnicodeWithinEditN(const unicode left, const unicode right, unsigned4 distance) : c, pure,entrypoint='ulUnicodeWithinEditN', hole;
@@ -27,4 +32,7 @@ export SaltLib := SERVICE : plugin('saltlib')
   integer4 UnicodeMatchBagofwords2(const unicode left, const unicode right, unsigned4 mode, unsigned4 score_mode) : c, pure,entrypoint='ulUnicodeMatchBagofwords2', hole;
   integer4 UnicodeLocaleMatchBagofwords2(const unicode left, const unicode right, const varstring localename, unsigned4 mode, unsigned4 score_mode) : c, pure,entrypoint='ulUnicodeLocaleMatchBagofwords2', hole;
   integer4 StringMatchBagofwords2(const string left, const string right, unsigned4 mode, unsigned4 score_mode) : c, pure,entrypoint='ulStringMatchBagofwords2', hole;
+  STRING GeohashLatLongEncode(REAL latitude, REAL longitude, UNSIGNED precision=12) : c, pure,entrypoint='saltGeohashLatLongEncode';
+  STRING GeohashNeighbor(const STRING geohash, INTEGER direction0, INTEGER direction1) : c, pure,entrypoint='saltGeohashNeighbor';
+  DATASET(GeohashDecodeResultRecord) GeohashDecode(const STRING geohash) : c, pure,entrypoint='saltGeohashDecode';
 END;

roxie/roxiemem/roxierowbuff.cpp

@@ -138,7 +138,7 @@ void RoxieOutputRowArray::transferRows(rowidx_t & outNumRows, const void * * & o
 
 bool DynamicRoxieOutputRowArray::ensure(rowidx_t requiredRows)
 {
-    unsigned newSize = maxRows;
+    rowidx_t newSize = maxRows;
     //This condition must be <= at least 1/scaling factor below otherwise you'll get an infinite loop.
     if (newSize <= 4)
         newSize = requiredRows;
@@ -150,7 +150,17 @@ bool DynamicRoxieOutputRowArray::ensure(rowidx_t requiredRows)
         //   reduce fragmentation.
         //Use 25% for the moment.  It should possibly be configurable - e.g., higher for thor global sort.
         while (newSize < requiredRows)
-            newSize += newSize/4;
+        {
+            rowidx_t nextSize = newSize + newSize / 4;
+            //check to see if it has wrapped
+            if (nextSize < newSize)
+            {
+                newSize = requiredRows;
+                break;
+            }
+            else
+                newSize = nextSize;
+        }
     }
 
     const void * * newRows;
@@ -162,7 +172,7 @@ bool DynamicRoxieOutputRowArray::ensure(rowidx_t requiredRows)
     }
     catch (IException * e)
     {
-        //Pahological cases - not enough memory to reallocate the target row buffer, or no contiguous pages available.
+        //Pathological cases - not enough memory to reallocate the target row buffer, or no contiguous pages available.
         unsigned code = e->errorCode();
         if ((code == ROXIEMM_MEMORY_LIMIT_EXCEEDED) || (code == ROXIEMM_MEMORY_POOL_EXHAUSTED))
         {

testing/regress/ecl/childds2.ecl

@@ -49,5 +49,5 @@ cntBad := COUNT(ds(assertTrue(seq > 10, 'seq > 10'))) + NOFOLD(100000);
 //Problem1: Converting this to a filtered disk read means that cntBad is evaluated always
 cond1 := IF(trueValue, ds, NOFOLD(ds)(seq != cntBad));
 cond2 := IF(falseValue, NOFOLD(ds)(seq != cntBad), ds);
-output(cond1+cond2);
+output(cond1 & cond2);
 

testing/regress/ecl/key/when10.xml

@@ -0,0 +1,12 @@
+<Dataset name='Result 1'>
+ <Row><Result_1>Ready</Result_1></Row>
+</Dataset>
+<Dataset name='Result 2'>
+ <Row><Result_2>Done</Result_2></Row>
+</Dataset>
+<Dataset name='Result 3'>
+ <Row><Result_3>Doing</Result_3></Row>
+</Dataset>
+<Dataset name='Result 5'>
+ <Row><Result_5>7</Result_5></Row>
+</Dataset>

testing/regress/ecl/key/when11.xml

@@ -0,0 +1,12 @@
+<Dataset name='Result 1'>
+ <Row><Result_1>Ready</Result_1></Row>
+</Dataset>
+<Dataset name='Result 2'>
+ <Row><Result_2>Done</Result_2></Row>
+</Dataset>
+<Dataset name='Result 3'>
+ <Row><Result_3>Doing</Result_3></Row>
+</Dataset>
+<Dataset name='Result 5'>
+ <Row><f1>9</f1><f2>3</f2><f3>4</f3><f4>5</f4></Row>
+</Dataset>

testing/regress/ecl/when10.ecl

@@ -0,0 +1,38 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2015 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+r := {unsigned f1, unsigned f2, unsigned f3, unsigned f4 };
+
+r t(unsigned a, unsigned b, unsigned c, unsigned d) := TRANSFORM
+    SELF.f1 := a;
+    SELF.f2 := b;
+    SELF.f3 := c;
+    SELF.f4 := d;
+END;
+
+ds := dataset([
+        t(1,2,3,4),
+        t(1,4,2,5),
+        t(9,3,4,5),
+        t(3,4,2,9)]);
+
+
+p := WHEN(COUNT(NOFOLD(ds)), OUTPUT('Ready'), BEFORE);
+p2 := WHEN(p, OUTPUT('Done'), SUCCESS);
+p3 := WHEN(p2, OUTPUT('Doing'), PARALLEL);
+p4 := WHEN(p3, OUTPUT('Failed'), FAILURE);
+output(p4+3);

testing/regress/ecl/when11.ecl

@@ -0,0 +1,38 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2015 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+r := {unsigned f1, unsigned f2, unsigned f3, unsigned f4 };
+
+r t(unsigned a, unsigned b, unsigned c, unsigned d) := TRANSFORM
+    SELF.f1 := a;
+    SELF.f2 := b;
+    SELF.f3 := c;
+    SELF.f4 := d;
+END;
+
+ds := dataset([
+        t(1,2,3,4),
+        t(1,4,2,5),
+        t(9,3,4,5),
+        t(3,4,2,9)]);
+
+
+p := WHEN(ds[3], OUTPUT('Ready'), BEFORE);
+p2 := WHEN(p, OUTPUT('Done'), SUCCESS);
+p3 := WHEN(p2, OUTPUT('Doing'), PARALLEL);
+p4 := WHEN(p3, OUTPUT('Failed'), FAILURE);
+output(DATASET(p4));

testing/regress/hpcc/regression/regress.py

@@ -223,26 +223,34 @@ class Regression:
                     timeout = query.getTimeout()
                     oldCnt = cnt
 
+                started = False
                 for threadId in range(self.maxthreads):
-                    if not self.exitmutexes[threadId].locked():
-                        # Start a new test case with a reused thread id
-                        self.taskParam[threadId]['taskId']=cnt
-                        cnt += 1
-                        if timeout != 0:
-                            self.timeouts[threadId] = timeout
-                        else:
-                            self.timeouts[threadId] = self.timeout
-
-                        self.taskParam[threadId]['timeoutValue'] = self.timeout
-                        query = suiteItems[self.taskParam[threadId]['taskId']]
-                        query.setTimeout(self.timeout)
-                        #logging.debug("self.timeout[%d]:%d", threadId, self.timeouts[threadId])
-                        sysThreadId = thread.start_new_thread(self.runQuery, (cluster, query, report, cnt, suite.testPublish(query.ecl),  threadId))
-                        time.sleep(0.4)
-                        self.taskParam[threadId]['jobName'] = query.getJobname()
-                        self.taskParam[threadId]['retryCount'] = int(self.config.maxAttemptCount)
+
+                    for startThreadId in range(self.maxthreads):
+                        if not self.exitmutexes[startThreadId].locked():
+                            # Start a new test case with a reused thread id
+                            self.taskParam[startThreadId]['taskId']=cnt
+                            cnt += 1
+                            if timeout != 0:
+                                self.timeouts[startThreadId] = timeout
+                            else:
+                                self.timeouts[startThreadId] = self.timeout
+
+                            self.taskParam[startThreadId]['timeoutValue'] = self.timeout
+                            query = suiteItems[self.taskParam[startThreadId]['taskId']]
+                            query.setTimeout(self.timeout)
+                            #logging.debug("self.timeout[%d]:%d", startThreadId, self.timeouts[startThreadId])
+                            self.taskParam[startThreadId]['jobName'] = query.getJobname()
+                            self.taskParam[startThreadId]['retryCount'] = int(self.config.maxAttemptCount)
+                            self.exitmutexes[startThreadId].acquire()
+                            sysThreadId = thread.start_new_thread(self.runQuery, (cluster, query, report, cnt, suite.testPublish(query.ecl),  startThreadId))
+                            started = True
+                            break
+
+                    if started:
                         break
-                    else:
+
+                    if self.exitmutexes[threadId].locked():
                         if self.timeouts[threadId] % 10 == 0:
                             self.loggermutex.acquire()
                             logging.debug("%3d. timeout counter:%d" % (self.taskParam[threadId]['taskId']+1, self.timeouts[threadId]),  extra={'taskId':self.taskParam[threadId]['taskId']+1})
@@ -300,7 +308,8 @@ class Regression:
                                 self.timeouts[threadId] = -1
 
                 # give some time to other threads
-                time.sleep(0.4)
+                if not started:
+                    time.sleep(0.2)
 
             # All tasks are scheduled
             #Some of them finished, others are not yet, but should check the still running tasks' timeout and retry state
@@ -402,6 +411,7 @@ class Regression:
                     self.timeouts[th] = self.timeout
                 self.retryCount = int(self.config.maxAttemptCount)
                 query.setTimeout(self.timeouts[th])
+                self.exitmutexes[th].acquire()
                 thread.start_new_thread(self.runQuery, (cluster, query, report, cnt, suite.testPublish(query.ecl),  th))
                 time.sleep(0.1)
                 self.CheckTimeout(cnt, th,  query)
@@ -449,6 +459,7 @@ class Regression:
             else:
                 self.timeouts[threadId] = self.timeout
             self.retryCount = int(self.config.maxAttemptCount)
+            self.exitmutexes[threadId].acquire()
             sysThreadId = thread.start_new_thread(self.runQuery, (cluster, eclfile, report, cnt, eclfile.testPublish(),  threadId))
             time.sleep(0.1)
             self.CheckTimeout(cnt, threadId,  eclfile)
@@ -470,7 +481,6 @@ class Regression:
     def runQuery(self, cluster, query, report, cnt=1, publish=False,  th = 0):
         startTime = time.time()
         self.loggermutex.acquire()
-        self.exitmutexes[th].acquire()
 
         logging.debug("runQuery(cluster: '%s', query: '%s', cnt: %d, publish: %s, thread id: %d" % ( cluster, query.ecl, cnt, publish,  th))
         logging.warn("%3d. Test: %s" % (cnt, query.getBaseEclRealName()),  extra={'taskId':cnt})

tools/esdlcomp/esdl_utils.cpp

@@ -48,8 +48,11 @@ void es_createDirectory(const char* dir)
             if (mkdir(dir,0755)!=0)
 #endif
             {
-                fprintf(stderr,"Create directory %s failed", dir);
-                exit(1);
+                if (!es_checkDirExists(dir))
+                {
+                   fprintf(stderr,"Create directory %s failed", dir);
+                   exit(1);
+                }
             }
         }
     }

tools/hidl/hidl_utils.cpp

@@ -44,8 +44,11 @@ void createDirectory(const char* dir)
             if (mkdir(dir,0755)!=0)
 #endif
             {
-                fprintf(stderr,"Create directory %s failed", dir);
-                exit(1);
+                if (!checkDirExists(dir))
+                {
+                    fprintf(stderr,"Create directory %s failed", dir);
+                    exit(1);
+                }
             }
         }
     }