Merge branch 'candidate-5.4.0'

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

common/dllserver/thorplugin.cpp

@@ -82,7 +82,7 @@ public:
     virtual const char * queryVersion() const;
     virtual const char * queryName() const;
     virtual const byte * getResource(unsigned id) const;
-    virtual bool getResource(size32_t & len, const void * & data, const char * type, unsigned id) const;
+    virtual bool getResource(size32_t & len, const void * & data, const char * type, unsigned id, bool trace) const;
 
     bool load(bool isGlobal, bool raiseOnError);
     bool loadCurrentExecutable();
@@ -184,7 +184,7 @@ const byte resourceHeaderVersion=1;
 const size32_t resourceHeaderLength = sizeof(byte) + sizeof(byte) + sizeof(bool) + sizeof(size32_t);
 
 
-bool HelperDll::getResource(size32_t & len, const void * & data, const char * type, unsigned id) const
+bool HelperDll::getResource(size32_t & len, const void * & data, const char * type, unsigned id, bool trace) const
 {
 #ifdef _WIN32
     HINSTANCE dllHandle = so.getInstanceHandle();
@@ -200,7 +200,8 @@ bool HelperDll::getResource(size32_t & len, const void * & data, const char * ty
     data = (const void *) getEntry(symName.str());
     if (!data)
     {
-        printf("Failed to locate symbol %s", symName.str());
+        if (trace)
+            printf("Failed to locate symbol %s", symName.str());
         return false;
     }
     byte bom;
@@ -468,7 +469,7 @@ extern DLLSERVER_API bool checkEmbeddedWorkUnitXML(ILoadedDllEntry *dll)
 {
     size32_t len = 0;
     const void * data = NULL;
-    return dll->getResource(len, data, "WORKUNIT", 1000);
+    return dll->getResource(len, data, "WORKUNIT", 1000, false);
 }
 
 extern DLLSERVER_API bool getResourceXMLFromFile(const char *filename, const char *type, unsigned id, StringBuffer &xml)

common/dllserver/thorplugin.hpp

@@ -30,7 +30,7 @@ interface ILoadedDllEntry : extends IInterface
     virtual const char * queryVersion() const = 0;
     virtual const char * queryName() const = 0;
     virtual const byte * getResource(unsigned id) const = 0;
-    virtual bool getResource(size32_t & len, const void * & data, const char * type, unsigned id) const = 0;
+    virtual bool getResource(size32_t & len, const void * & data, const char * type, unsigned id, bool trace=true) const = 0;
 };
 
 extern DLLSERVER_API ILoadedDllEntry * createDllEntry(const char *name, bool isGlobal, const IFileIO *dllFile);

common/thorhelper/roxiehelper.cpp

@@ -2421,9 +2421,9 @@ static const char *skipLfnForeign(const char *lfn)
     return lfn;
 }
 
-StringBuffer & expandLogicalFilename(StringBuffer & logicalName, const char * fname, IConstWorkUnit * wu, bool resolveLocally)
+StringBuffer & expandLogicalFilename(StringBuffer & logicalName, const char * fname, IConstWorkUnit * wu, bool resolveLocally, bool ignoreForeignPrefix)
 {
-    const char *native = skipLfnForeign(fname); //foreign location should already be reflected in local dali dfs meta data
+    const char *native = (ignoreForeignPrefix) ? skipLfnForeign(fname) : fname; //for published roxie queries foreign location already reflected in local dfs meta data
     if (fname[0]=='~')
         logicalName.append(native);
     else if (resolveLocally)

common/thorhelper/roxiehelper.hpp

@@ -299,6 +299,6 @@ private:
 
 THORHELPER_API StringBuffer & mangleHelperFileName(StringBuffer & out, const char * in, const char * wuid, unsigned int flags);
 THORHELPER_API StringBuffer & mangleLocalTempFilename(StringBuffer & out, char const * in);
-THORHELPER_API StringBuffer & expandLogicalFilename(StringBuffer & logicalName, const char * fname, IConstWorkUnit * wu, bool resolveLocally);
+THORHELPER_API StringBuffer & expandLogicalFilename(StringBuffer & logicalName, const char * fname, IConstWorkUnit * wu, bool resolveLocally, bool ignoreForeignPrefix);
 
 #endif // ROXIEHELPER_HPP

dali/base/dasds.cpp

@@ -4516,7 +4516,7 @@ void CSDSTransactionServer::processMessage(CMessageBuffer &mb)
                 mb.clear().append((int) DAMP_SDSREPLY_OK);
                 if (queryTransactionLogging())
                 {
-                    CServerRemoteTree *idTree = (CServerRemoteTree *) SDSManager->queryRegisteredTree(serverId);
+                    Owned<CServerRemoteTree> idTree = (CServerRemoteTree *) SDSManager->getRegisteredTree(serverId);
                     transactionLog.log("%s", idTree?idTree->queryName():"???");
                 }
                 SDSManager->getExternalValueFromServerId(serverId, mb);
@@ -7034,7 +7034,7 @@ void CCovenSDSManager::getExternalValue(__int64 index, MemoryBuffer &mb)
 
 void CCovenSDSManager::getExternalValueFromServerId(__int64 serverId, MemoryBuffer &mb)
 {
-    CServerRemoteTree *idTree = (CServerRemoteTree *) SDSManager->queryRegisteredTree(serverId);
+    Owned<CServerRemoteTree> idTree = (CServerRemoteTree *) SDSManager->getRegisteredTree(serverId);
     if (idTree)
     {
         CHECKEDCRITICALBLOCK(extCrit, fakeCritTimeout);

dali/dfuplus/dfuplus.cpp

@@ -367,15 +367,29 @@ bool CDfuPlusHelper::variableSpray(const char* srcxml,const char* srcip,const ch
 
     const char* encoding = globals->queryProp("encoding");
     const char* rowtag = globals->queryProp("rowtag");
-    if(stricmp(format, "xml") == 0)
+    const char* rowpath = globals->queryProp("rowpath");
+    if(strieq(format, "json"))
+    {
+        req->setIsJSON(true);
+        if (!encoding)
+            encoding = "utf8";
+        else if (strieq(encoding, "ascii"))
+            throw MakeStringExceptionDirect(-1, "json format only accepts utf encodings");
+        if(rowtag && *rowtag)
+            throw MakeStringExceptionDirect(-1, "You can't use rowtag option with json format");
+        if (rowpath && *rowpath)
+            req->setSourceRowPath(rowpath);
+    }
+    else if(stricmp(format, "xml") == 0)
     {
         if(encoding == NULL)
             encoding = "utf8";
         else if(stricmp(encoding, "ascii") == 0)
             throw MakeStringException(-1, "xml format only accepts utf encodings");
-        
         if(rowtag == NULL || *rowtag == '\0')
             throw MakeStringException(-1, "rowtag not specified.");
+        if(rowpath && *rowpath)
+            throw MakeStringException(-1, "You can't use rowpath option with xml format");
     }
     else if(stricmp(format, "csv") == 0)
     {
@@ -384,6 +398,8 @@ bool CDfuPlusHelper::variableSpray(const char* srcxml,const char* srcip,const ch
 
         if(rowtag != NULL && *rowtag != '\0')
             throw MakeStringException(-1, "You can't use rowtag option with csv/delimited format");
+        if(rowpath && *rowpath)
+            throw MakeStringException(-1, "You can't use rowpath option with csv/delimited format");
 
         const char* separator = globals->queryProp("separator");
         if(separator)
@@ -527,7 +543,7 @@ int CDfuPlusHelper::spray()
     bool ok;
     if ((stricmp(format, "fixed") == 0)||(stricmp(format, "recfmvb") == 0)||(stricmp(format, "recfmv") == 0)||(stricmp(format, "variablebigendian") == 0))
         ok = fixedSpray(srcxml,srcip,srcfile,xmlbuf,dstcluster,dstname,format,wuid,errmsg);
-    else if((stricmp(format, "csv") == 0)||(stricmp(format, "xml") == 0)||(stricmp(format, "variable") == 0))
+    else if((stricmp(format, "csv") == 0)||(stricmp(format, "xml") == 0)||(stricmp(format, "json") == 0)||(stricmp(format, "variable") == 0))
         ok = variableSpray(srcxml,srcip,srcfile,xmlbuf,dstcluster,dstname,format,wuid, errmsg);
     else
         throw MakeStringException(-1, "format %s not supported", format);

dali/dfuplus/main.cpp

@@ -65,7 +65,7 @@ void handleSyntax()
     out.append("        srcxml=<xml-file> -- replaces srcip and srcfile\n");
     out.append("        dstname=<destination-logical-name>\n");
     out.append("        dstcluster=<cluster-name>\n");
-    out.append("        format=fixed|csv|delimited|xml|variable|recfmv|recfmvb\n");
+    out.append("        format=fixed|csv|delimited|xml|json|variable|recfmv|recfmvb\n");
     out.append("        prefix=filename{:length},filesize{:[B|L][1-8]}\n");
     out.append("        options for fixed:\n");
     out.append("            recordsize=<record-size>\n");
@@ -82,6 +82,9 @@ void handleSyntax()
     out.append("            rowtag=rowTag -- required\n");
     out.append("            encoding=utf8|utf8n|utf16|utf16le|utf16be|utf32|utf32le|utf32be -- optional, default is utf8\n");
     out.append("            maxrecordsize=<max-record-size> -- optional, default is 8192\n");
+    out.append("        options for json:\n");
+    out.append("            rowpath=rowPath - optional, default is \"/\"\n");
+    out.append("            maxrecordsize=<max-record-size> -- optional, default is 8192\n");
     out.append("    replicate options:\n");
     out.append("        srcname=<source-logical-name>\n");
     out.append("        cluster=<cluster-name>     -- replicates to (additional) cluster\n");

docs/ECLLanguageReference/ECLR_mods/Basics-Constants.xml

@@ -4,7 +4,7 @@
 <sect1 id="Constants">
   <title>Constants</title>
 
-  <sect2 id="String">
+  <sect2 id="Const_String">
     <title>String</title>
 
     <para>All string literals must be contained within single quotation marks

docs/ECLLanguageReference/ECLR_mods/ParSppt-PARSPattrn.xml

@@ -83,7 +83,7 @@
     </indexterm><emphasis> </emphasis><emphasis role="bold">[
   (</emphasis><emphasis> recstruct </emphasis><emphasis role="bold">)
   ]</emphasis><emphasis> ruleid </emphasis><emphasis role="bold">:=
-  </emphasis><emphasis>parsepattern</emphasis><emphasis
+  </emphasis><emphasis>rulePattern</emphasis><emphasis
   role="bold">;</emphasis></para>
 
   <informaltable colsep="1" frame="all" rowsep="1">
@@ -108,12 +108,12 @@
         </row>
 
         <row>
-          <entry><emphasis>parsepattern</emphasis></entry>
+          <entry><emphasis>rulePattern</emphasis></entry>
 
-          <entry>The token pattern, very similar to regular expressions. This
-          may contain PATTERN attributes but no TOKEN or RULE attributes. See
-          <emphasis role="bold">ParsePattern Definitions</emphasis>
-          below.</entry>
+          <entry>The rule pattern, very similar to regular expressions. This
+          may contain PATTERN attributes, TOKEN attributes, or RULE
+          attributes. See <emphasis role="bold">ParsePattern
+          Definitions</emphasis> below.</entry>
         </row>
       </tbody>
     </tgroup>
@@ -126,7 +126,7 @@
 
   <para>If the PARSE option is present on the PARSE function (thereby
   implementing tomita parsing for the operation), each alternative RULE
-  <emphasis>parsepattern</emphasis> may have an associated TRANSFORM function.
+  <emphasis>rulePattern</emphasis> may have an associated TRANSFORM function.
   The different input patterns can be referred to using $1, $2 etc. If the
   pattern has an associated <emphasis>recstruct</emphasis> then $1 is a row,
   otherwise it is a string. Default TRANSFORM functions are created in two
@@ -411,7 +411,8 @@ the following supported syntax elements:
  (?=…) (?!...)     Look ahead assertion
  (?&lt;=…) (?&lt;!...)   Look behind assertion
 
-Escape sequences can be used to define UNICODE Character ranges. The encoding is UTF-16 Big Endian.  
+Escape sequences can be used to define UNICODE Character ranges. 
+The encoding is UTF-16 Big Endian.  
 For example:
 PATTERN AnyChar    := PATTERN(U'[\u0001-\u7fff]');
 </programlisting><!--*** Note this and the following row entries have been monospace optimized for PDF/HTML*** --></entry>

docs/ECLLanguageReference/ECLR_mods/RecordStructure.xml

@@ -82,7 +82,7 @@
           <emphasis>#OPTION(maxLength,####)</emphasis> to change the default).
           The maximum record size should be set as conservatively as possible,
           and is better set on a per-field basis (see the <emphasis
-          role="bold">Field Modifiers</emphasis>section below). </entry>
+          role="bold">Field Modifiers</emphasis>section below).</entry>
         </row>
 
         <row>
@@ -624,11 +624,10 @@ END;</programlisting>
             <emphasis>value</emphasis> will be used: <para>1. When a
             DICTIONARY lookup returns no match.</para><para>2. When an
             out-of-range record is fetched using ds[n] (as in ds[5] when ds
-            contains only 4 records). </para><para>3. In the default record
-            passed to TRANSFORM functions in LEFT ONLY or RIGHT ONLY JOINs
-            where there is no corresponding row. </para><para>4. When
-            defaulting field values in a TRANSFORM using SELF = [
-            ].</para></entry>
+            contains only 4 records). </para><para>3. In the default records
+            passed to TRANSFORM functions in non-INNER JOINS where there is no
+            corresponding row. </para><para>4. When defaulting field values in
+            a TRANSFORM using SELF = [ ].</para></entry>
           </row>
 
           <row>

docs/ECLStandardLibraryReference/SLR-Mods/FindCount.xml

@@ -51,8 +51,8 @@
    'Success',
    'Failure - 1');  //success
     
-B := IF(STD.Str.FindCount('ABCDEABCDE', 'BC') = 2,
+B := IF(STD.Str.FindCount('ABCDEABCDE', 'BC') = 1,
    'Success',
-   'Failure - 2');  //failure
+   'Failure - 1');  //failure
 </programlisting>
 </sect1>

docs/ECLWatch/ECLWa_mods/ECLWatchSrc.xml

@@ -416,7 +416,7 @@
 
         <!-- added WU Details stuff here -->
 
-        <sect3>
+        <sect3 role="brk">
           <title>Publish Action Button</title>
 
           <para>Click on the Publish action button to publish a query. <figure>
@@ -483,7 +483,7 @@
           </itemizedlist>
         </sect3>
 
-        <sect3 id="ZAP">
+        <sect3 id="ZAP" role="brk">
           <title>Z.A.P. Utility</title>
 
           <para>The Zipped Analysis Package (Z.A.P.) button is a utility for
@@ -530,7 +530,7 @@
           personally identifiable information (PII).</para>
         </sect3>
 
-        <sect3>
+        <sect3 role="brk">
           <title>Outputs tab</title>
 
           <para>Click on the Outputs tab to see all results.<figure>
@@ -578,7 +578,7 @@
           </itemizedlist>
         </sect3>
 
-        <sect3>
+        <sect3 role="brk">
           <title>Visualizations</title>
 
           <para>You can see visual representations of select workunits.
@@ -604,7 +604,7 @@
           menu, to change the parameters.</para>
         </sect3>
 
-        <sect3>
+        <sect3 role="brk">
           <title>Inputs tab</title>
 
           <para>Click on the <emphasis role="bold">Inputs</emphasis> tab to
@@ -627,7 +627,7 @@
           how many times a file was used in the workunit.</para>
         </sect3>
 
-        <sect3>
+        <sect3 role="brk">
           <title>Timers tab</title>
 
           <para>Click on the <emphasis role="bold">Timers</emphasis> tab to
@@ -653,7 +653,7 @@
 
           <para>Click on the <emphasis role="bold">Graphs</emphasis> tab to
           see the graphs produced by the workunit. <figure>
-              <title>GRAPHS***</title>
+              <title>GRAPHS</title>
 
               <mediaobject>
                 <imageobject>
@@ -667,7 +667,7 @@
           the graph.</para>
         </sect3>
 
-        <sect3>
+        <sect3 role="brk">
           <title>Timers tab</title>
 
           <para>Click on the <emphasis role="bold">Timers</emphasis> tab to
@@ -689,6 +689,27 @@
 
           <para><emphasis role="bold">Usage:</emphasis> how many times a file
           was used in the work unit.</para>
+
+          <sect4>
+            <title>Stats</title>
+
+            <para>On the workunit Timers tab is another tab for Stats. The
+            Stats tab is a visual representation of all the workunit timings.
+            </para>
+
+            <figure>
+              <title>Stats</title>
+
+              <mediaobject>
+                <imageobject>
+                  <imagedata fileref="../../images/ECLWa201.jpg" />
+                </imageobject>
+              </mediaobject>
+            </figure>
+
+            <para>Select the timer values from the drop list on the Stats tab
+            to view the various charts and graphs. </para>
+          </sect4>
         </sect3>
 
         <sect3>

ecl/ecl-bundle/CMakeLists.txt

@@ -51,6 +51,7 @@ include_directories (
          ${HPCC_SOURCE_DIR}/system/include
          ${HPCC_SOURCE_DIR}/system/xmllib
          ${HPCC_SOURCE_DIR}/ecl/eclcmd
+         ${HPCC_SOURCE_DIR}/ecl/eclcc
     )
 
 ADD_DEFINITIONS( -D_CONSOLE )

ecl/ecl-package/CMakeLists.txt

@@ -50,6 +50,7 @@ include_directories (
          ${HPCC_SOURCE_DIR}/system/include
          ${HPCC_SOURCE_DIR}/system/xmllib
          ${HPCC_SOURCE_DIR}/ecl/eclcmd
+         ${HPCC_SOURCE_DIR}/ecl/eclcc
     )
 
 ADD_DEFINITIONS( -D_CONSOLE )

ecl/eclagent/eclagent.cpp

@@ -1372,7 +1372,7 @@ bool EclAgent::expandLogicalName(StringBuffer & fullname, const char * logicalNa
 ILocalOrDistributedFile *EclAgent::resolveLFN(const char *fname, const char *errorTxt, bool optional, bool noteRead, bool isWrite, StringBuffer * expandedlfn)
 {
     StringBuffer lfn;
-    expandLogicalFilename(lfn, fname, queryWorkUnit(), resolveFilesLocally);
+    expandLogicalFilename(lfn, fname, queryWorkUnit(), resolveFilesLocally, false);
     if (resolveFilesLocally && *fname != '~')
     {
         StringBuffer name;

ecl/eclcc/CMakeLists.txt

@@ -28,6 +28,7 @@ project( eclcc )
 set (    SRCS
          reservedwords.hpp
          reservedwords.cpp
+         eclcc.hpp
          eclcc.cpp
     )
 

ecl/eclcc/eclcc.cpp

@@ -50,6 +50,7 @@
 #include "rmtfile.hpp"
 
 #include "reservedwords.hpp"
+#include "eclcc.hpp"
 
 #ifdef _USE_CPPUNIT
 #include <cppunit/extensions/TestFactoryRegistry.h>
@@ -1935,7 +1936,7 @@ bool EclCC::parseCommandLineOptions(int argc, const char* argv[])
         {
             debugOptions.append(tempArg);
         }
-        else if (iter.matchFlag(tempBool, "-g"))
+        else if (iter.matchFlag(tempBool, "-g") || iter.matchFlag(tempBool, "--debug"))
         {
             if (tempBool)
             {
@@ -2164,123 +2165,32 @@ bool EclCC::parseCommandLineOptions(int argc, const char* argv[])
 
 //=========================================================================================
 
-// Exclamation in the first column indicates it is only part of the verbose output
-const char * const helpText[] = {
-    "",
-    "Usage:",
-    "    eclcc <options> queryfile.ecl",
-    "",
-    "General options:",
-    "    -I <path>     Add path to locations to search for ecl imports",
-    "    -L <path>     Add path to locations to search for system libraries",
-    "    -o <file>     Specify name of output file (default a.out if linking to",
-    "                  executable, or stdout)",
-    "    -manifest     Specify path to manifest file listing resources to add",
-    "    -foption[=value] Set an ecl option (#option)",
-    "    -main <ref>   Compile definition <ref> from the source collection",
-    "    -syntax       Perform a syntax check of the ECL",
-    "    -platform=hthor Generate code for hthor executable (default)",
-    "    -platform=roxie Generate code for roxie cluster",
-    "    -platform=thor  Generate code for thor cluster",
-    "",
-    "Output control options",
-    "    -E            Output preprocessed ECL in xml archive form",
-    "!   -M            Output meta information for the ecl files",
-    "!   -Md           Output dependency information",
-    "!   -Me           eclcc should evaluate supplied ecl code rather than generating a workunit",
-    "    -q            Save ECL query text as part of workunit",
-    "    -wu           Only generate workunit information as xml file",
-    "",
-    "c++ options",
-    "    -S            Generate c++ output, but don't compile",
-    "!   -c            compile only (don't link)",
-    "    -g            Enable debug symbols in generated code",
-    "    -Wc,xx        Pass option xx to the c++ compiler",
-    "!   -Wl,xx        Pass option xx to the linker",
-    "!   -Wa,xx        Passed straight through to c++ compiler",
-    "!   -Wp,xx        Passed straight through to c++ compiler",
-    "!   -save-cpps    Do not delete generated c++ files (implied if -g)",
-    "!   -save-temps   Do not delete intermediate files",
-    "    -shared       Generate workunit shared object instead of a stand-alone exe",
-    "",
-    "Other options:",
-    "!   -aoption[=value] Set an application option",
-    "!   --allow=str   Allow use of named feature",
-    "!   -b            Batch mode.  Each source file is processed in turn.  Output",
-    "!                 name depends on the input filename",
-    "!   -checkVersion Enable/disable ecl version checking from archives",
-    "!   --component   Set the name of the component this is executing on behalf of",
-#ifdef _WIN32
-    "!   -brk <n>      Trigger a break point in eclcc after nth allocation",
-#endif
-    "!   -Dname=value  Override the definition of a global attribute 'name'",
-    "!   --deny=all    Disallow use of all named features not specifically allowed using --allow",
-    "!   --deny=str    Disallow use of named feature",
-    "    -help, --help Display this message",
-    "    -help -v      Display verbose help message",
-    "!   -internal     Run internal tests",
-    "!   -legacy       Use legacy import and when semantics (deprecated)",
-    "!   --keywords    Outputs the list of ECL reserved words to ECLKeywords.xml",
-    "!   -legacyimport Use legacy import semantics (deprecated)",
-    "!   -legacywhen   Use legacy when/side-effects semantics (deprecated)",
-    "    --logfile <file> Write log to specified file",
-    "!   --logdetail=n Set the level of detail in the log file",
-    "!   --nologfile   Do not write any logfile",
-#ifdef _WIN32
-    "!   -m            Enable leak checking",
-#endif
-    "    --nosourcepath Compile as if the source came from stdin",
-#ifndef _WIN32
-    "!   -pch          Generate precompiled header for eclinclude4.hpp",
-#endif
-    "!   -P <path>     Specify the path of the output files (only with -b option)",
-    "!   -showpaths    Print information about the searchpaths eclcc is using",
-    "    -specs file   Read eclcc configuration from specified file",
-    "!   -split m:n    Process a subset m of n input files (only with -b option)",
-    "    -v --verbose  Output additional tracing information while compiling",
-    "    -wxxxx=level  Set the severity for a particular warning code or category",
-    "!                 -wall sets default severity for all warnings",
-    "!                 level=ignore|log|warning|error|fail",
-    "    --version     Output version information",
-    "!   --timings     Output additional timing information",
-    "!",
-    "!#options",
-    "! -factivitiesPerCpp      Number of activities in each c++ file",
-    "!                         (requires -fspanMultipleCpp)",
-    "! -fapplyInstantEclTransformations Limit non file outputs with a CHOOSEN",
-    "! -fapplyInstantEclTransformationsLimit Number of records to limit to",
-    "! -fcheckAsserts          Check ASSERT() statements",
-    "! -fexportDependencies    Generate information about inter-definition dependencies",
-    "! -fmaxCompileThreads     Number of compiler instances to compile the c++",
-    "! -fnoteRecordSizeInGraph Add estimates of record sizes to the graph",
-    "! -fpickBestEngine        Allow simple thor queries to be passed to thor",
-    "! -freportCppWarnings     Report warnings from c++ compilation",
-    "! -fsaveCppTempFiles      Retain the generated c++ files",
-    "! -fshowActivitySizeInGraph Show estimates of generated c++ size in the graph",
-    "! -fshowMetaInGraph       Add distribution/sort orders to the graph",
-    "! -fshowRecordCountInGraph Show estimates of record counts in the graph",
-    "! -fspanMultipleCpp       Generate a work unit in multiple c++ files",
-    "",
-};
+
 
 void EclCC::usage()
 {
     for (unsigned line=0; line < _elements_in(helpText); line++)
     {
         const char * text = helpText[line];
+        StringBuffer wsPrefix;
+        if (*text == '?')  //NOTE: '?' indicates eclcmd usage so don't print.
+        {
+            text = text+1;
+            if (*text == ' ' || ( *text == '!' && text[1] == ' '))
+                wsPrefix.append(' ');
+        }
         if (*text == '!')
         {
             if (logVerbose)
             {
-                //Allow conditional headers
-                if (text[1] == ' ')
-                    fprintf(stdout, " %s\n", text+1);
-                else
-                    fprintf(stdout, "%s\n", text+1);
+                text = text+1;
+                if (*text == ' ')
+                    wsPrefix.append(' ');
+                fprintf(stdout, "%s%s\n", wsPrefix.str(), text);
             }
         }
         else
-            fprintf(stdout, "%s\n", text);
+            fprintf(stdout, "%s%s\n", wsPrefix.str(), text);
     }
 }
 

ecl/eclcc/eclcc.hpp

@@ -0,0 +1,127 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2015 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#ifndef ECLCC_HPP
+#define ECLCC_HPP
+
+// The following is used by eclcc and eclcmd (ecl <command>).
+// The primary intention is for eclcc and only a selection are also used by eclcmd for its eclcc options.
+// The markers '!' and '?', present in either the first or first two columns, indicate that lines inclusion.
+// e.g. '?' indicates inclusion in both eclcmd and eclcc).
+//      '?!' indicates inclusion in the verbose usage for both.
+//      '!' indicates inclusion in the verbose usage for eclcc only.
+//      None of the above indicates inclusion in eclcc only.
+
+const char * const helpText[] = {
+    "",
+    "Usage:",
+    "    eclcc <options> queryfile.ecl",
+    "",
+    "General options:",
+    "?   -I <path>     Add path to locations to search for ecl imports",
+    "?   -L <path>     Add path to locations to search for system libraries",
+    "    -o <file>     Specify name of output file (default a.out if linking to",
+    "                  executable, or stdout)",
+    "?   -manifest     Specify path to manifest file listing resources to add",
+    "    -foption[=value] Set an ecl option (#option)",
+    "    -main <ref>   Compile definition <ref> from the source collection",
+    "    -syntax       Perform a syntax check of the ECL",
+    "    -platform=hthor Generate code for hthor executable (default)",
+    "    -platform=roxie Generate code for roxie cluster",
+    "    -platform=thor  Generate code for thor cluster",
+    "",
+    "Output control options",
+    "    -E            Output preprocessed ECL in xml archive form",
+    "!   -M            Output meta information for the ecl files",
+    "!   -Md           Output dependency information",
+    "!   -Me           eclcc should evaluate supplied ecl code rather than generating a workunit",
+    "    -q            Save ECL query text as part of workunit",
+    "    -wu           Only generate workunit information as xml file",
+    "",
+    "c++ options",
+    "    -S            Generate c++ output, but don't compile",
+    "!   -c            compile only (don't link)",
+    "?   -g, --debug   Enable debug symbols in generated code",
+    "    -Wc,xx        Pass option xx to the c++ compiler",
+    "!   -Wl,xx        Pass option xx to the linker",
+    "!   -Wa,xx        Passed straight through to c++ compiler",
+    "!   -Wp,xx        Passed straight through to c++ compiler",
+    "!   -save-cpps    Do not delete generated c++ files (implied if -g)",
+    "!   -save-temps   Do not delete intermediate files",
+    "    -shared       Generate workunit shared object instead of a stand-alone exe",
+    "",
+    "Other options:",
+    "!   -aoption[=value] Set an application option",
+    "!   --allow=str   Allow use of named feature",
+    "!   -b            Batch mode.  Each source file is processed in turn.  Output",
+    "!                 name depends on the input filename",
+    "!   -checkVersion Enable/disable ecl version checking from archives",
+    "!   --component   Set the name of the component this is executing on behalf of",
+#ifdef _WIN32
+    "!   -brk <n>      Trigger a break point in eclcc after nth allocation",
+#endif
+    "!   -Dname=value  Override the definition of a global attribute 'name'",
+    "!   --deny=all    Disallow use of all named features not specifically allowed using --allow",
+    "!   --deny=str    Disallow use of named feature",
+    "    -help, --help Display this message",
+    "    -help -v      Display verbose help message",
+    "!   -internal     Run internal tests",
+    "?!  -legacy       Use legacy import and when semantics (deprecated)",
+    "!   --keywords    Outputs the list of ECL reserved words to ECLKeywords.xml",
+    "!   -legacyimport Use legacy import semantics (deprecated)",
+    "!   -legacywhen   Use legacy when/side-effects semantics (deprecated)",
+    "    --logfile <file> Write log to specified file",
+    "!   --logdetail=n Set the level of detail in the log file",
+    "!   --nologfile   Do not write any logfile",
+#ifdef _WIN32
+    "!   -m            Enable leak checking",
+#endif
+    "    --nosourcepath Compile as if the source came from stdin",
+#ifndef _WIN32
+    "!   -pch          Generate precompiled header for eclinclude4.hpp",
+#endif
+    "!   -P <path>     Specify the path of the output files (only with -b option)",
+    "!   -showpaths    Print information about the searchpaths eclcc is using",
+    "    -specs file   Read eclcc configuration from specified file",
+    "!   -split m:n    Process a subset m of n input files (only with -b option)",
+    "    -v --verbose  Output additional tracing information while compiling",
+    "    -wxxxx=level  Set the severity for a particular warning code or category",
+    "!                 -wall sets default severity for all warnings",
+    "!                 level=ignore|log|warning|error|fail",
+    "    --version     Output version information",
+    "!   --timings     Output additional timing information",
+    "!",
+    "?!#options",
+    "?!  -factivitiesPerCpp      Number of activities in each c++ file",
+    "?!                          (requires -fspanMultipleCpp)",
+    "?!  -fapplyInstantEclTransformations Limit non file outputs with a CHOOSEN",
+    "?!  -fapplyInstantEclTransformationsLimit Number of records to limit to",
+    "?!  -fcheckAsserts          Check ASSERT() statements",
+    "?!  -fexportDependencies    Generate information about inter-definition dependencies",
+    "?!  -fmaxCompileThreads     Number of compiler instances to compile the c++",
+    "?!  -fnoteRecordSizeInGraph Add estimates of record sizes to the graph",
+    "?!  -fpickBestEngine        Allow simple thor queries to be passed to thor",
+    "?!  -freportCppWarnings     Report warnings from c++ compilation",
+    "?!  -fsaveCppTempFiles      Retain the generated c++ files",
+    "?!  -fshowActivitySizeInGraph Show estimates of generated c++ size in the graph",
+    "?!  -fshowMetaInGraph       Add distribution/sort orders to the graph",
+    "?!  -fshowRecordCountInGraph Show estimates of record counts in the graph",
+    "?!  -fspanMultipleCpp       Generate a work unit in multiple c++ files",
+    "",
+};
+
+#endif

ecl/eclcmd/CMakeLists.txt

@@ -58,6 +58,7 @@ include_directories (
          ./../../system/include
          ./../../esp/bindings
          ./../../ecl/hql
+         ./../../ecl/eclcc
          ./../../dali/base
          ./../../esp/platform
          ./../../system/jlib

ecl/eclcmd/eclcmd_common.cpp

@@ -296,6 +296,12 @@ eclCmdOptionMatchIndicator EclCmdCommon::matchCommandLineOption(ArgvIterator &it
     bool boolValue;
     if (iter.matchFlag(boolValue, ECLOPT_HELP))
     {
+        if (!optVerbose && iter.next())
+        {
+            iter.matchFlag(optVerbose, ECLOPT_VERBOSE);
+            if (!optVerbose)
+                iter.matchFlag(optVerbose, ECLOPT_VERBOSE_S);
+        }
         usage();
         return EclCmdOptionCompletion;
     }

ecl/eclcmd/eclcmd_common.hpp

@@ -19,6 +19,7 @@
 #define ECLCMD_COMMON_HPP
 
 #include "ws_workunits.hpp"
+#include "eclcc.hpp"
 
 //=========================================================================================
 
@@ -261,13 +262,31 @@ public:
             "   --limit=<limit>        Sets the result limit for the query, defaults to 100\n"
             "   -f<option>[=value]     Set an ECL option (equivalent to #option)\n"
             "   -Dname=value           Override the definition of a global attribute 'name'\n"
-            " eclcc options:\n"
-            "   -Ipath                 Add path to locations to search for ecl imports\n"
-            "   -Lpath                 Add path to locations to search for system libraries\n"
-            "   --manifest             Specify path to manifest file\n"
-            "   --legacy               Use legacy import semantics (deprecated)\n"
-            "   --debug, -g            Enable debug symbols in generated code\n"
+            " eclcc options (everything following):\n"
         );
+        for (unsigned line=0; line < _elements_in(helpText); line++)
+        {
+            const char * text = helpText[line];
+            StringBuffer wsPrefix;
+            if (*text == '?')
+            {
+                text = text+1;
+                if (*text != ' ')
+                    wsPrefix.append(' ');
+            }
+            else
+                continue;
+            if (*text == '!')
+            {
+                if (optVerbose)
+                {
+                    text = text+1;
+                    fprintf(stdout, "%s%s\n", wsPrefix.str(), text);
+                }
+            }
+            else
+                fprintf(stdout, "%s%s\n", wsPrefix.str(), text);
+        }
     }
 public:
     StringAttr optTargetCluster;

ecl/eclcmd/queries/CMakeLists.txt

@@ -38,6 +38,7 @@ include_directories (
          ${HPCC_SOURCE_DIR}/system/include
          ${HPCC_SOURCE_DIR}/system/xmllib
          ${HPCC_SOURCE_DIR}/ecl/eclcmd
+         ${HPCC_SOURCE_DIR}/ecl/eclcc
     )
 
 ADD_DEFINITIONS( -D_CONSOLE )

ecl/eclcmd/roxie/CMakeLists.txt

@@ -40,6 +40,7 @@ include_directories (
          ${HPCC_SOURCE_DIR}/system/include
          ${HPCC_SOURCE_DIR}/system/xmllib
          ${HPCC_SOURCE_DIR}/ecl/eclcmd
+         ${HPCC_SOURCE_DIR}/ecl/eclcc
     )
 
 ADD_DEFINITIONS( -D_CONSOLE )

ecl/hqlcpp/hqlcerrors.hpp

@@ -215,6 +215,7 @@
 #define HQLERR_DistributionVariableLengthX      4195
 #define HQLERR_DistributionUnsupportedTypeXX    4196
 #define HQLERR_InconsistentEmbedded             4197
+#define HQLERR_UnsupportedRowDiffType           4198
 
 //Warnings....
 #define HQLWRN_PersistDataNotLikely             4500
@@ -503,6 +504,7 @@
 #define HQLERR_VariableRowMustBeLinked_Text     "External function '%s' cannot return a non-linked variable length row"
 #define HQLERR_UserCodeNotAllowed_Text          "Workunit-supplied code is not permitted on this system"
 #define HQLERR_StreamInputUsedDirectly_Text     "Library input used directly in a child query"
+#define HQLERR_UnsupportedRowDiffType_Text      "ROWDIFF: Does not support type '%s' for field %s"
 
 //Warnings.
 #define HQLWRN_CannotRecreateDistribution_Text  "Cannot recreate the distribution for a persistent dataset"

ecl/hqlcpp/hqlhtcpp.cpp

@@ -7998,7 +7998,11 @@ void HqlCppTranslator::doBuildExprRowDiff(BuildCtx & ctx, const CHqlBoundTarget
             case type_dictionary:
             case type_table:
             case type_groupedtable:
-                UNIMPLEMENTED;
+                {
+                    StringBuffer typeName;
+                    getFriendlyTypeStr(leftType, typeName);
+                    throwError2(HQLERR_UnsupportedRowDiffType, typeName.str(), expr->queryId()->str());
+                }
             }
 
             StringBuffer fullName;

ecl/hqlcpp/hqlinline.cpp

@@ -284,7 +284,7 @@ static unsigned calcInlineFlags(BuildCtx * ctx, IHqlExpression * expr)
             return 0;
         return getInlineFlags(ctx, expr->queryChild(0));
     case no_getgraphresult:
-        if (expr->hasAttribute(_distributed_Atom))
+        if (expr->hasAttribute(_distributed_Atom) || expr->hasAttribute(_streaming_Atom))
             return 0;
         return expr->isDatarow() ? RETevaluate : RETassign;
     case no_temptable:

ecl/hqlcpp/hqlttcpp.cpp

@@ -3458,6 +3458,10 @@ IHqlExpression * ThorHqlTransformer::normalizeMergeAggregate(IHqlExpression * ex
 
     //If locally distributed then don't do anything
     OwnedHqlExpr noMerge = removeAttribute(expr, mergeAtom);
+    //This transformation only works for grouped aggregation
+    if (!groupBy || groupBy->isAttribute())
+        return noMerge.getClear();
+
     if (!translator.targetThor() || expr->hasAttribute(localAtom) || isPartitionedForGroup(dataset, groupBy, true))
         return noMerge.getClear();
 

ecl/hthor/hthor.cpp

@@ -664,7 +664,7 @@ void CHThorDiskWriteActivity::publish()
     properties.setPropInt("@formatCrc", helper.getFormatCrc());
 
     StringBuffer lfn;
-    expandLogicalFilename(lfn, mangledHelperFileName.str(), agent.queryWorkUnit(), agent.queryResolveFilesLocally());
+    expandLogicalFilename(lfn, mangledHelperFileName.str(), agent.queryWorkUnit(), agent.queryResolveFilesLocally(), false);
     CDfsLogicalFileName logicalName;
     if (agent.queryResolveFilesLocally())
         logicalName.allowOsPath(true);
@@ -992,7 +992,7 @@ CHThorIndexWriteActivity::CHThorIndexWriteActivity(IAgentContext &_agent, unsign
     incomplete = false;
     StringBuffer lfn;
     OwnedRoxieString fname(helper.getFileName());
-    expandLogicalFilename(lfn, fname, agent.queryWorkUnit(), agent.queryResolveFilesLocally());
+    expandLogicalFilename(lfn, fname, agent.queryWorkUnit(), agent.queryResolveFilesLocally(), false);
     if (!agent.queryResolveFilesLocally())
     {
         Owned<IDistributedFile> f = queryDistributedFileDirectory().lookup(lfn, agent.queryCodeContext()->queryUserDescriptor(), true);
@@ -1210,7 +1210,7 @@ void CHThorIndexWriteActivity::execute()
     {
         dfile.setown(queryDistributedFileDirectory().createNew(desc));
         OwnedRoxieString fname(helper.getFileName());
-        expandLogicalFilename(lfn, fname, agent.queryWorkUnit(), agent.queryResolveFilesLocally());
+        expandLogicalFilename(lfn, fname, agent.queryWorkUnit(), agent.queryResolveFilesLocally(), false);
         dfile->attach(lfn.str(),agent.queryCodeContext()->queryUserDescriptor());
         agent.logFileAccess(dfile, "HThor", "CREATED");
     }

esp/platform/espprotocol.cpp

@@ -112,7 +112,7 @@ const StringBuffer &CEspApplicationPort::getAppFrameHtml(time_t &modified, const
 
     if (needRefresh || embedded_url || !appFrameHtml.length())
     {
-        int passwordDaysRemaining = -1;//-1 means dont display change password screen
+        int passwordDaysRemaining = scPasswordExpired;//-1 means dont display change password screen
 #ifdef _USE_OPENLDAP
         ISecUser* user = ctx->queryUser();
         ISecManager* secmgr = ctx->querySecManager();
@@ -120,8 +120,8 @@ const StringBuffer &CEspApplicationPort::getAppFrameHtml(time_t &modified, const
         {
             passwordDaysRemaining = user->getPasswordDaysRemaining();//-1 if expired, -2 if never expires
             int passwordExpirationDays = (int)secmgr->getPasswordExpirationWarningDays();
-            if (passwordDaysRemaining==-2 || passwordDaysRemaining > passwordExpirationDays)
-                passwordDaysRemaining = -1;
+            if (passwordDaysRemaining == scPasswordNeverExpires || passwordDaysRemaining > passwordExpirationDays)
+                passwordDaysRemaining = scPasswordExpired;
         }
 #endif
         StringBuffer xml;

esp/services/ws_access/ws_accessService.cpp

@@ -440,10 +440,10 @@ bool Cws_accessEx::onUsers(IEspContext &context, IEspUserRequest &req, IEspUserR
                     StringBuffer sb;
                     switch (usr->getPasswordDaysRemaining())//-1 if expired, -2 if never expires
                     {
-                    case -1:
+                    case scPasswordExpired:
                         sb.set("Expired");
                         break;
-                    case -2:
+                    case scPasswordNeverExpires:
                         sb.set("Never");
                         break;
                     default:

esp/src/eclwatch/HPCCPlatformWidget.js

@@ -29,6 +29,7 @@ define([
     "dijit/Tooltip",
 
     "dojox/widget/UpgradeBar",
+    "dojox/widget/ColorPicker",
 
     "hpcc/_TabContainerWidget",
     "hpcc/ESPRequest",
@@ -62,7 +63,7 @@ define([
 
 ], function (declare, lang, i18n, nlsHPCC, arrayUtil, dom, domForm, domStyle, domGeo, cookie,
                 registry, Tooltip,
-                UpgradeBar,
+                UpgradeBar, ColorPicker,
                 _TabContainerWidget, ESPRequest, ESPActivity, WsAccount, WsAccess, WsSMC, WsTopology, GraphWidget, DelayLoadWidget,
                 template) {
     return declare("HPCCPlatformWidget", [_TabContainerWidget], {

esp/src/eclwatch/TargetSelectWidget.js

@@ -66,7 +66,7 @@ define([
                 });
             }
             if (params.Groups === true) {
-                this.loadGroups();
+                this.loadClusterGroups();
             } else if (params.SprayTargets === true) {
                 this.loadSprayTargets();
             } else if (params.DropZones === true) {
@@ -170,17 +170,23 @@ define([
             }
         },
 
-        loadGroups: function () {
+        loadClusterGroups: function () {
             var context = this;
             WsTopology.TpGroupQuery({
                 load: function (response) {
                     if (lang.exists("TpGroupQueryResponse.TpGroups.TpGroup", response)) {
                         var targetData = response.TpGroupQueryResponse.TpGroups.TpGroup;
                         for (var i = 0; i < targetData.length; ++i) {
-                            context.options.push({
-                                label: targetData[i].Name,
-                                value: targetData[i].Name
-                            });
+                            switch(targetData[i].Kind) {
+                                case "Thor":
+                                case "hthor":
+                                case "Roxie":
+                                    context.options.push({
+                                        label: targetData[i].Name,
+                                        value: targetData[i].Name
+                                    });
+                                break;
+                            }
                         }
                         context._postLoad();
                     }

esp/src/eclwatch/nls/pt-br/hpcc.js

@@ -8,63 +8,130 @@
     Activate: "Ativar",
     Activated: "Ativa",
     Active: "Ativa",
+    ActivePackageMap: "Package Map Ativo",
     ActiveWorkunit: "Tarefa Ativa",
     Activities: "Atividades",
     Activity: "Atividade",
     ActualSize: "Tamanho Efectivo",
     Add: "Adicionar",
+    AddFile: "Adicionar Arquivo",
+    AddGroup: "Adicionar Grupo",
+    AdditionalResources: "Recursos Adicionais",
+    AddProcessMap: "Adicionar Package Map",
     AddToSuperfile: "Adicionar ao Superarquivo",
+    AddUser: "Adicionar Usuário",
     Advanced: "Avançado",
+    All: "Todos",
+    AllowAccess: "Permitir Acesso",
+    AllowForeignFiles: "Permitir Arquivos Remotos",
+    AllowFull: "Permissão Total",
+    AllowRead: "Permissão de Leitura",
+    AllowWrite: "Permissão de Escrita",
+    ANY: "Qualquer",
     Append: "Anexar",
+    Apply: "Aplicar",
     ArchivedOnly: "Apenas Arquivadas",
+    ArchivedWarning: "Aviso: Por favor especifique intervalos de datas menore. Caso contrário, a operação para buscar workunits pode demorar causando um erro no navegador.",
+    AutoRefresh: "Recarregar Automático",
+    Back: "voltar",
+    BannerColor: "Cor da Faixa",
+    BannerMessage: "Mensagem da Faixa",
+    BannerScroll: "Faixa de Paginação",
+    BannerSize: "Tamanho da Faixa",
     BinaryInstalls: "Instalação Binária",
+    Blob: "BLOB",
+    BlobPrefix: "Prefixo do BLOB",
+    Bottom: "fundo",
     Cancel: "Cancelar",
+    Clear: "Limpar",
+    ClearPermissionsCache: "Limpar Cache de Permissões",
+    ClearPermissionsCacheConfirm: "Tem certeza que você quer limpar todos os caches de permissões dos servidores DALI e ESP? Workunits em execução podem ter a performance significantemente prejudicadas até que os caches tenham sido recarregados.",
     Clone: "Clonar",
+    ClonedWUID: "WUID clonado",
     Close: "Fechar",
     Cluster: "Aglomerado (Cluster)",
     Cluster: "Cluster (Aglomerado)",
     ClusterName: "Nome do Aglomerado (Cluster)",
+    ClusterPlaceholder: "r?x*",
     ClusterProcesses: "Processos de Aglomerados",
     Code: "Código",
+    CodeGenerator: "Gerador de Código",
     Col: "Col",
     Command: "Comando",
     Comment: "Comentário",
     Completed: "Completo",
+    ComplexityWarning: "Mais de {limite} atividades ({Contador de atividades}) - suprimir amostragem inicial?",
     Component: "Componente",
     Compress: "Compactar",
+    Compressed: "Comprimido",
+    CompressedFileSize: "Tamanho do Arquivo Comprimido",
+    Configuration: "Configuração",
+    ConfirmPassword: "Confirmar Senha",
+    ConfirmRemoval: "Tem certeza que quer fazer isso?",
+    Content: "Conteúdo",
     Contents: "Conteúdo",
+    ContentType: "Tipo de Conteúdo",
     Copy: "Copiar",
     Count: "Conta",
     Created: "Criado",
     Dali: "Dali",
     dataset: ":=dataset*",
     Date: "Data",
+    Day: "Dia",
     Deactivate: "Desativar",
+    Debug: "Debug",
     DEF: "DEF",
+    Delete: "Remover",
+    Deleted: "Removido",
     DeleteSelectedFiles: "Apagar Arquivos Selecionados?",
+    DeleteSelectedGroups: "Remover Grupos Selecionados",
+    DeleteSelectedPermissions: "Remover Permissões Selecionadas",
     DeleteSelectedQueries: "Apagar Consultas Selecionadas?",
+    DeleteSelectedUsers: "Remover Usuários Selecionados",
     DeleteSelectedWorkunits: "Apagar Tarefas Selecionadas?",
+    DeleteSuperfile2: "Remover Super Arquivo",
     DeleteSuperfile: "Apagar Super Arquivo?",
+    DeleteThisPackage: "Remover Este Pacote",
     Delimited: "Delimitado",
+    DenyAccess: "Bloquear",
+    DenyFull: "Bloquear Completamente",
+    DenyRead: "Bloquear Leitura",
+    DenyWrite: "Bloquear Escrita",
     Depth: "Profundidade",
+    DepthTooltip: "Profundidade Máxima de Subgrafo",
     Deschedule: "Desagendar",
+    DescheduleSelectedWorkunits: "Desagendar Unidades de Trabalho Selecionadas",
     Description: "Descrição",
     Despray: "Despray (Consolidar Dados dos Nós)",
+    Details: "Detalhes",
     DFUServerName: "Nome do Servidor DFU",
+    DFUWorkunit: "DFU Unidade de Trabalho",
     Directory: "Diretório",
+    DisableScopeScanConfirm: "Tem certeza que quer desabilitar busca de escopo? Alterações de configuração serão aplicadas quando DALI reiniciar.",
+    DisableScopeScans: "Desabilitar busca de escopo",
+    DiskUsage: "Uso de disco",
     Distance: "Distancia",
+    DistanceTooltip: "Distância a vizinhança de máxima atividade",
     Dll: "Dll (Biblioteca de Vínculo Dinâmico)",
     DOT: "DOT",
     DOTAttributes: "Propriedades de DOT",
+    Down: "Pra baixo",
     Download: "Baixar",
     DropZone: "Zona de entrada de arquivos",
     Duration: "Duração",
     EBCDIC: "EBCDIC",
     ECL: "ECL",
+    ECLWorkunit: "ECL Unidade de Trabalho",
     Edges: "Bordas",
+    Edit: "Alterar",
     EditDOT: "Editar DOT",
     EditGraphAttributes: "Editar Propriedades do Gráfico",
     EditXGMML: "Editar XGMML",
+    Empty: "Vazio",
+    Enable: "Ativar",
+    EnableScopeScans: "Ativar busca de escopo",
+    EnableScopeScansConfirm: "Tem certeza que gostaria de ativar busca de escopo? Alterações serão aplicadas quando DALI reiniciar.",
+    EnglishQ: "Inglês?",
     Error: "Error",
     Errorparsingserverresult: "Erro interpretando o resultado do servidor",
     Errors: "Erros",
@@ -73,36 +140,70 @@
     Escape: "Escape",
     ESPBuildVersion: "Versão‎ do ESP",
     ESPNetworkAddress: "Endereço de Rede do ESP",
+    EventName: "Nome do Evento",
+    EventNamePH: "Nome do Evento",
+    EventScheduler: "Gerenciador de Evento",
+    EventText: "Texto do Evento",
+    EventTextPH: "Texto do Evento",
+    Exception: "Exceção",
+    ExpandAll: "Expandir Tudo",
+    Export: "Exportar",
+    FailIfNoSourceFile: "Falha em caso de arquivo sem fonte",
     FetchingData: "Buscando dados...",
     fetchingresults: "Buscando Resultados",
     File: "Arquivo",
+    FileCounts: "Número de Arquivos",
     FileName: "Nome do Arquivo",
     FileParts: "Partes do Arquivo",
     FilePath: "Caminho e nome do Arquivo",
     Files: "Arquivos",
+    FileScopes: "Escopo de arquivos",
     FileSize: "Tamanho do Arquivo",
+    FilesNoPackage: "Arquivos sem definição de pacotes.",
+    FilesWithUnknownSize: "Arquivos com tamanho desconhecido",
+    FileType: "Tipo de Arquivo",
     FileUploader: "Uploader(Enviador) de Arquivo",
+    FileUploadStillInProgress: "Carregamento de Arquivo em andamento",
+    Filter: "Filtro",
     Find: "Busca",
     FindNext: "Próxima Busca",
     FindPrevious: "Busca Anterior",
+    Finished: "Completo",
+    FirstName: "Primeiro Nome",
+    FirstNRows: "Primeiras N Linhas",
     Fixed: "Fixo",
+    Folder: "Pasta",
     Format: "Formato",
+    Forward: "Avançar",
     FromDate: "Data Inicial",
     FromSizes: "Tamanho Mínimo",
+    FromTime: "Tempo inicial",
+    FullName: "Nome Completo",
+    Graph: "Grafo",
     Graphs: "Gráficos",
     GraphView: "Visão Gráfica",
     Group: "Grupo",
+    GroupBy: "Agrupado por",
+    GroupDetails: "Detalhe de Grupo",
+    Grouping: "Agrupar",
     GroupName: "Nome do Grupo",
+    GroupPermissions: "Permissões de Grupo",
     Groups: "Grupos",
     GZip: "GZip",
     help: "Essa area exibi o mapas de arvores (hierarquias) para o gráfico dessa tarefa. O tamanho e cor indicam a duração de cada gráfico (Quanto maior e mais escuro mais tempo levou)",
     Helpers: "Ajudantes",
+    Hex: "Hex",
     HexPreview: "Pré-visualização Hex",
+    High: "Alto",
     History: "História",
+    HPCCSystems: "HPCCSystems",
     ID: "ID",
+    Inactive: "Inativo",
+    Index: "Índice",
     Info: "Info",
     InfoDialog: "Caixa de Informação",
     Inputs: "Insumos",
+    InvalidResponse: "(resposta inválida)",
     IP: "IP",
     IPAddress: "Endereço IP",
     IsLibrary: "Biblioteca?",
@@ -114,81 +215,177 @@
     Label: "Rótulo de Texto",
     LandingZone: "Zona de Entrada de Arquivos",
     LandingZones: "Zona de entrada de arquivos",
+    LanguageFiles: "Arquivo de Línguas",
+    Largest: "Maior",
+    LargestFile: "Maior Arquivo",
+    LargestSize: "Maior Tamanho",
+    LastName: "Sobrenome",
     LastNDays: "Ultimos N Dias",
+    LastNHours: "Últimas N Horas",
+    LastNRows: "Últimas N Filas",
+    LDAPWarning: "<b>Error do serviço de LDAP</b> 'Número de usuários excedido'. Por favor use um filtro.",
     LegacyForm: "Formato Antigo",
     LibrariesUsed: "Usa Biblioteca?",
+    LibraryName: "Nome da biblioteca",
     Line: "Linha",
     LineTerminators: "Terminador de linhas",
     Links: "Links",
+    Loading: "Carregando",
+    LoadingCachedLayout: "Carregando diagrama do cache",
     LoadingData: "Carragando Dados...",
+    loadingMessage: "...carregando...",
+    LoadPackageContentHere: "(Carregue conteúdo de pacote aqui)",
+    LoadPackageFromFile: "Carregar pacote de arquivo",
     Local: "Local",
+    Log: "Log",
+    LogFile: "Arquivo de log",
     LoggedInAs: "Entrou no sistema como",
     LogicalFile: "Arquivo Lógico",
     LogicalFiles: "Arquivos Lógicos",
+    LogicalFilesAndSuperfiles: "Arquivos (super e lógicos)",
+    LogicalFilesOnly: "Somente arquivos lógicos",
     LogicalFileType: "Tipo de Arquivo Lógico",
     LogicalName: "Nome Lógico",
+    Logs: "Logs",
     log_analysis_1: "log_analysis_1*",
+    Low: "Baixo",
+    ManualCopy: "Pressione Ctrl+C",
+    ManualOverviewSelection: "(Seleção manual será necessária)",
+    ManualTreeSelection: "(Seleção manual de árvore será requerida)",
     Mappings: "Mapeamento",
     Mask: "Mask (bloqueador de caracteres)",
     Max: "Max",
     MaxRecordLength: "Maximo tamanho do registro",
+    MemberOf: "Membro de",
+    Members: "Membros",
     Message: "Mensagem",
     Min: "Min",
+    Missing: "Não encontrado",
+    MixedNodeStates: "Estados de Nós misturados",
     Modification: "Modificação",
+    Modified: "Modificado",
     ModifiedUTCGMT: "Modificado (UTC/GMT)",
     Modify: "Modificar",
     MonitorEventName: "Nome do Evento do Monitorador",
     MonitorShotLimit: "Limite do Monitorador",
     MonitorSub: "Sub do Monitorador",
+    Month: "Mês",
+    More: "Mais",
+    MustContainUppercaseAndSymbol: "Deve conter símbolo e caracteres maiúsculos",
     NA: "N/A",
     Name: "Nome",
+    NamePrefix: "Prefixo de nome",
+    NamePrefixPlaceholder: "algum::prefixo",
+    Newest: "Mais recente",
+    NewPassword: "Senha Nova",
+    NoContent: "(sem conteúdo)",
+    noDataMessage: "... Zero linhas...",
     Node: "Nó (Node)",
+    NodeGroup: "Grupo de Nós",
     NoFilterCriteriaSpecified: "Filtro não especificado",
+    None: "Nenhum",
+    Normal: "Normal",
+    NoScheduledEvents: "Eventos agendados não encontrados",
     NoSplit: "Sem Separador",
+    NotActive: "Inativo",
+    NothingSelected: "Nada Selecionado",
+    NotInSuperfiles: "Não encontrado em super arquivo",
+    NotSuspendedbyUser: "Não suspenso por usuário",
     NumberofParts: "Numero de Partes",
     OK: "Aceitar",
+    Oldest: "Mais Antigo",
+    OldPassword: "Senha antiga",
     OmitSeparator: "Omitir Separador",
+    Only1PackageFileAllowed: "Somente one arquivo de pacote permitido",
+    Open: "Abrir",
+    OpenInNewPage: "Abrir em página nova",
+    OpenInNewPageNoFrame: "Abrir em página nova (sem quadro)",
     OpenLegacyECLWatch: "Abrir Antigo ECL Watch",
     OpenSafeMode: "Abrir em Modo de Segurança",
+    OpenSource: "Código Público",
+    OpenTreeMode: "Abrir (modo árvore)",
     Operations: "Operações",
     Options: "Opções",
     Outputs: "Resultados",
     Overview: "Visão Geral",
     Overwrite: "Overwrite",
+    OverwriteMessage: "Alguns arquivos já existem. Selecione opção de escrever por cima para continuar.",
     Owner: "Proprietário",
+    PackageContent: "Conteúdo de pacote",
+    PackageContentNotSet: "Conteúdo de pacote não definido",
+    PackageMap: "Mapa de Pacotes",
     PackageMaps: "Mapas do Pacote de Dados",
+    PackagesNoQuery: "Pacotes sem queries",
     ParameterXML: "Parametro XML",
+    Part: "Parte",
     Parts: "Partes",
+    Password: "Senha",
+    PasswordExpiration: "Validade da Senha",
+    PasswordExpired: "Sua senha não é mais válida. Por favor altere agora.",
+    PasswordExpirePostfix: "dia(s). Quer trocar agora?",
+    PasswordExpirePrefix: "Senha válida até",
+    PasswordOpenZAP: "Senha para abrir ZAP (opicional)",
+    PasswordsDoNotMatch: "Senhas não são as mesmas",
     Path: "Caminho",
     PathMask: "Bloqueador do Caminho",
+    Pause: "Pausar",
+    PauseNow: "Pausar agora",
     PctComplete: "% Completo",
     PercentDone: "Percentual Completo",
     PerformingLayout: "Executando Layout...",
+    PermissionName: "Nome da Permissão",
     Permissions: "Permissões",
     PlaceholderFindText: "Wuid, Usuário‎, Mais...",
+    PlaceholderFirstName: "João",
+    PlaceholderLastName: "da Silva",
     Playground: "Area de Recreio",
+    Plugins: "Plugins",
+    Port: "Port",
     Prefix: "Prefixo",
+    PrefixPlaceholder: "'nomedoarquivo{:comprimento}, tamanho{:[B|L][1-8]}'",
+    Preview: "Prévia",
     Priority: "Prioridade",
+    Process: "Processo",
+    ProcessFilter: "Filtro de Processo",
     ProgressMessage: "Mensagem de Progresso",
     Properties: "Propriedades",
     Property: "Propriedade",
     Protect: "Proteger",
     Protected: "Protegido",
     Publish: "Publicado",
+    Published: "Publicado",
     PublishedBy: "Publicado por",
     PublishedQueries: "Consultas de ECL Publicadas",
+    PushEvent: "Evento de publicação",
+    Quarter: "Quarto",
     Queries: "Consultas ECL",
+    QueriesNoPackage: "Queries sem pacote",
+    Query: "Query",
     QueryDetailsfor: "Detalhes para",
+    QueryID: "Query ID",
+    QueryIDPlaceholder: "some?q*ry.1",
+    QueryName: "Nome da Query",
+    QueryNamePlaceholder: "Minha?Su?erQ*ry",
     QuerySet: "Grupo de Consultas",
     Queue: "Fila",
     Quote: "Aspas",
+    RawTextPage: "Texto original (página atual)",
     RecordCount: "Numero de Registros",
     RecordLength: "Tamanho do Registro",
     Records: "Registros",
     RecordSize: "Tamanho do Registro",
+    RecordStructurePresent: "Estrutura de registro disponível",
+    Recover: "Recuperar",
+    Refresh: "Recarregar",
     ReleaseNotes: "Notas de Versão",
+    Reload: "Recarregar",
+    Remaining: "Faltando",
+    RemoteCopy: "Cópia Remota",
     RemoteDali: "Dali Remota",
+    RemoteDaliIP: "IP Dali Remoto",
     Remove: "Remover",
+    RemoveSubfiles: "Remover sub-arquivo(s)",
+    RemoveUser: "Você está próximo de remover-se do grupo",
     Rename: "Renomear",
     RenderedSVG: "Rendered SVG",
     RenderSVG: "Renderização SVG",
@@ -196,12 +393,22 @@
     RequestSchema: "Schema da Busca",
     Reschedule: "Reagendar",
     Reset: "Limpar",
+    ResetThisQuery: "Restaurar esta Pesquisa?",
+    ResetViewToSelection: "Restaurar Visão da seleção",
+    Resource: "Recurso",
     Resources: "Recursos",
     ResponseSchema: "Schema da Resposta",
     Restart: "Restart",
+    Restarted: "Recomeçar",
+    Restore: "Restaurar",
     Resubmit: "Resubmeter",
+    Resubmitted: "Submeter de novo",
     Results: "Resultado(s)",
+    Resume: "Continue",
     RetainSuperfileStructure: "Preservar Estrutura do Super Arquivo",
+    RetypePassword: "Digite senha de novo",
+    Reverse: "Reverter",
+    RowPath: "linha de acesso",
     Rows: "Linhas",
     RowTag: "Nome da Linha",
     RoxieCluster: "Roxie Cluster (Aglomerado)",
@@ -212,8 +419,11 @@
     Scope: "Escopo",
     SearchResults: "Resultado da Busca",
     SecondsRemaining: "Segundos Restantes",
+    Security: "Segurança",
+    SelectPackageFile: "Selecione pacote que arquivos",
     Separators: "Seperadores",
     SetBanner: "Fixar Banner",
+    SetTextError: "Erro para mostrar texto(muito longo?)  Use 'ajuda' para baixar",
     SetToFailed: "Indicar como Falhado",
     Severity: "Severidade",
     Show: "Exibir",
@@ -221,16 +431,23 @@
     Size: "Tamanho",
     Skew: "Distorção",
     Slaves: "Slaves",
+    Smallest: "Menor",
+    SmallestFile: "Arquivo Menor",
+    SmallestSize: "Menor tamanho",
     SOAP: "SOAP",
+    SomeDescription: "Alguma descrição",
     somefile: "*::algum_arquivo*",
     somefile: "*::somefile",
     Source: "Fonte",
     SourceCode: "Código Fonte",
+    SourcePath: "fonte de acesso",
+    SourceProcess: "fonte de processo",
     SplitPrefix: "Prefixo de separação",
     Spray: "Spray (Distribuir aos Nós)",
     Start: "Começar",
     Started: "Início",
     State: "Estado",
+    Status: "estado",
     Stopped: "Parado",
     Subgraphs: "Subgráfico",
     Submit: "Submeter",
@@ -239,26 +456,38 @@
     Suspend: "Suspender",
     Suspended: "Suspendido",
     SuspendedBy: "Suspendido por",
+    SuspendedByCluster: "Interrompido pelo Bloco",
+    SuspendedByUser: "Interrompido pelo Usuário",
+    SuspendedReason: "Razão para interrupção",
     SVGSource: "Fonte de SVG",
     Sync: "Sync",
+    SyncSelection: "Sincronizar seleção",
     SystemServers: "Servidores do Sistema",
     tag: "tag",
     Target: "Destino",
     TargetClusters: "Aglomeragos de Destino(Alvos)",
     TargetName: "Nome do Destino",
+    TargetNamePlaceholder: "algum::lógico::arquivo",
+    TargetWuid: "Unidade de trabalho alvo",
     Terminators: "Terminadores",
     TestPages: "Páginas de Teste",
+    Text: "Texto",
+    ThorMasterAddress: "Endereço de metre thor",
     ThorNetworkAddress: "Endereço de Rede do Thor",
     Time: "Tempo",
+    Timers: "Controladores de tempo",
     TimeSeconds: "Tempo (Segundos)",
     TimeStarted: "Início",
     TimeStopped: "Fim",
     Timings: "Medidas de Tempo",
     TimingsMap: "Mapa das Medidas de Tempo",
     title_Activity: "Atividate",
+    title_ClusterInfo: "grupos",
     title_DFUQuery: "Arquivos Lógicos",
     title_DFUWUDetails: "Tarefa DFU",
+    title_DiskUsage: "uso de disco",
     title_ECLPlayground: "ECL Playground",
+    title_EventScheduleWorkunit: "Escalonador de eventos",
     title_GetDFUWorkunits: "Tarefas DFU",
     title_Graph: "Gráficos",
     title_GraphPage: "Título",
@@ -267,10 +496,17 @@
     title_HPCCPlatformECL: "ECL Watch - Página Inicial",
     title_HPCCPlatformFiles: "ECL Watch - Arquivos",
     title_HPCCPlatformMain: "ECL Watch - Página Inicial",
+    title_HPCCPlatformOps: "ECL Watch - Operações",
     title_HPCCPlatformOpsWidget: "ECL Watch - Operações",
     title_HPCCPlatformRoxie: "ECL Watch - Roxie",
+    title_HPCCPlatformServicesPlugin: "ECL Watch - Adaptadores",
+    title_Inputs: "Entradas",
     title_LFDetails: "Detalhes dos Arquivos Lógicos",
+    title_LibrariesUsed: "Bibliotecas usadas",
+    title_Log: "arquivo de registro",
     title_LZBrowse: "Zona de Entrada de Arquivos",
+    title_MemberOf: "Membros de",
+    title_Members: "Membros",
     title_QuerySetDetails: "Detalhes da Consulta ECL",
     title_QuerySetErrors: "Erros",
     title_QuerySetLogicalFiles: "Arquivos Lógicos",
@@ -279,29 +515,64 @@
     title_QueryTest: "Super Arquivos",
     title_Result: "Resultados",
     title_Results: "Atividade",
+    title_SearchResults: "Resultados de busca",
+    title_Topology: "Topologia",
+    title_TpThorStatus: "Estado de Thor",
+    title_UserPermissions: "Permissões de usuário",
+    title_UserQuery: "Permissões",
     title_WUDetails: "Detalhes da Tarefa ECL",
     title_WUQuery: "Tarefas de ECL",
+    To: "Para",
     ToDate: "Data Final",
     Toenablegraphviews: "Para ativar visão gráfica, por favor instalar o plugin",
+    Top: "Topo",
+    Topology: "Topologia",
     ToSizes: "Tamanho Máximo",
+    TotalClusterTime: "Tempo total do bloco",
+    TotalSize: "Tamanho total",
     TotalThorTime: "Tempo Total do Thor",
+    TransitionGuide: "Guia de transição",
+    Tree: "Árvore",
     Type: "Tipo",
     undefined: "undefined",
     Unknown: "Unknown",
     Unprotect: "Desprotege",
+    UnsupportedIE9FF: "Não Supportado",
     Unsuspend: "Reativar",
     Unsuspended: "Reativado",
+    Up: "Cima",
     Upload: "Enviar",
+    Usage: "Uso",
     Used: "Usado",
     User: "Usuário",
+    UserDetails: "Detalhes do usuário",
+    UserID: "Identificação do usurario",
+    UserName: "Nome do usuário",
+    Username: "Nome do usuário",
+    UserPermissions: "Permissão do Usuário",
     Users: "Usuários",
     UseSingleConnection: "Usar Conexão Única",
+    Validate: "Validar",
+    ValidateActivePackageMap: "Validar pacote de mapa ativo",
+    ValidatePackageContent: "Validar pacote de conteúdo",
+    ValidatePackageMap: "Validar pacote de mapa",
+    ValidateResultHere: "Validar resultados",
     Value: "Valor",
+    Variable: "Variável",
+    VariableBigendian: "Variavel Big-edian",
+    Variables: "Variáveis",
+    VariableSourceType: "Tipo de fonte",
     Version: "Versão",
+    ViewByScope: "Visão por escopo",
     Views: "Visões",
     Visualize: "Visualize",
     Warnings: "Avisos",
+    WarnOldGraphControl: "Alerta: Controle de grafico velho",
+    What: "O que",
+    Where: "Onde",
+    Who: "Quem",
     Width: "Largura",
+    Workflows: "Fluxo de Trabalho",
     Workunit: "Tarefa",
     Workunits: "Tarefas",
     Wrap: "Wrap",
@@ -313,6 +584,8 @@
     XLS: "XLS",
     XML: "XML",
     XRef: "XRef",
+    Year: "Ano",
+    YourBrowserMayNotSupport: "O seu navegador provavelmente não supporta arquivo(s) deste tamanho",
     ZAP: "Z.A.P",
     ZeroLogicalFilesCheckFilter: "Nenhum Arquivo Lógico (verifique filtro)",
     Zip: "Zip",

esp/src/eclwatch/templates/HPCCPlatformWidget.html

@@ -71,13 +71,13 @@
             <button id="${id}Close" type="submit" data-dojo-attach-event="onClick:_onAboutClose" data-dojo-type="dijit.form.Button">${i18n.Close}</button>
         </div>
     </div>
-    <div id="${id}SetBannerDialog" title="Set Banner" style="width: 480px;" data-dojo-type="dijit.Dialog">
+    <div id="${id}SetBannerDialog" title="Set Banner" style="width: 580px;" data-dojo-type="dijit.Dialog">
         <div class="dijitDialogPaneContentArea">
             <div id="${id}SetBannerForm" onsubmit="return false;" data-dojo-type="dijit.form.Form">
                 <div data-dojo-type="hpcc.TableContainer">
                     <input id="${id}ShowBanner" title="${i18n.Enable}:" name="BannerAction" data-dojo-type="dijit.form.CheckBox" />
                     <input id="${id}BannerContent" title="${i18n.BannerMessage}:" name="BannerContent" style="width:100%;" data-dojo-props="trim: true" data-dojo-type="dijit.form.Textarea" />
-                    <input id="${id}BannerColor" title="${i18n.BannerColor}:" name="BannerColor" style="width:100%;" value="red" data-dojo-props="trim: true" data-dojo-type="dijit.form.Textarea" />
+                    <input id="${id}BannerColor" title="${i18n.BannerColor}:" name="BannerColor" style="width:100%;" value="red" data-dojo-props="trim: true" data-dojo-type="dojox.widget.ColorPicker" />
                     <input id="${id}BannerSize" title="${i18n.BannerSize}:" name="BannerSize" style="width:100%;" value="4" data-dojo-props="trim: true" data-dojo-type="dijit.form.Textarea" />
                     <input id="${id}BannerScroll" title="${i18n.BannerScroll}:" name="BannerScroll" style="width:100%;" value="2" data-dojo-props="trim: true" data-dojo-type="dijit.form.Textarea" />
                 </div>

esp/src/eclwatch/templates/LZBrowseWidget.html

@@ -85,7 +85,7 @@
                                             <option value="9">UTF-32BE</option>
                                         </select>
                                         <input id="${id}SprayDelimitedMaxRecordLength" title="${i18n.MaxRecordLength}:" style="width: 95%;" name="sourceMaxRecordSize" value="8192" required="true" colspan="2" data-dojo-props="trim: true, placeHolder:'8192'" data-dojo-type="dijit.form.ValidationTextBox" />
-                                        <input id="${id}SprayDelimitedSeparators" title="${i18n.Separators}:" style="width: 95%;" name="sourceCsvSeparate" value="," data-dojo-props="trim: true, placeHolder:'\,'" colspan="2" data-dojo-type="dijit.form.ValidationTextBox" />
+                                        <input id="${id}SprayDelimitedSeparators" title="${i18n.Separators}:" style="width: 95%;" name="sourceCsvSeparate" value="\," data-dojo-props="trim: true, placeHolder:'\,'" colspan="2" data-dojo-type="dijit.form.ValidationTextBox" />
                                         <input title="${i18n.OmitSeparator}:" name="NoSourceCsvSeparator" colspan="2" data-dojo-type="dijit.form.CheckBox" />
                                         <input id="${id}SprayDelimitedEscape" title="${i18n.Escape}:" style="width: 95%;" name="sourceCsvEscape" data-dojo-props="trim: true" colspan="2" data-dojo-type="dijit.form.TextBox" />
                                         <input id="${id}SprayDelimitedTerminators" title="${i18n.LineTerminators}:" name="sourceCsvTerminate" style="width: 95%;" value="\n,\r\n" required="true" colspan="2" data-dojo-props="trim: true, placeHolder:'\\n,\\r\\n'" data-dojo-type="dijit.form.ValidationTextBox" />

esp/src/stub.htm

@@ -37,6 +37,7 @@
     <link href="/esp/files/dojox/widget/UpgradeBar/UpgradeBar.css" rel="stylesheet">
     <link href="/esp/files/dojox/form/resources/FileUploader.css" rel="stylesheet">
     <link href="/esp/files/dojox/form/resources/UploaderFileList.css" rel="stylesheet">
+    <link href="/esp/files/dojox/widget/ColorPicker/ColorPicker.css" rel="stylesheet">
     <link href="/esp/files/dgrid/css/dgrid.css" rel="stylesheet">
     <link href="/esp/files/dgrid/css/skins/claro.css" rel="stylesheet">
     <link href="/esp/files/eclwatch/css/hpcc.css" rel="stylesheet">

initfiles/sbin/hpcc/cluster/host.py

@@ -28,9 +28,6 @@ class Host(object):
     Several static help methods are provided to create cluster hast list.
     '''
     # Default user name and passwords
-    user_name = 'hpcc'
-    user_passowrd = 'hpcc'
-    admin_password = 'hpcc'
 
     logger = logging.getLogger("hpcc.cluster.Host")
 
@@ -62,41 +59,28 @@ class Host(object):
 
     @property
     def user_name(self):
-        if self._user_name:
-            return self._user_name
-        else:
-            return Host.user_name
+        return self._user_name
 
     @user_name.setter
     def user_name(self, value):
         self._user_name = value
 
-
     @property
     def user_password(self):
-        if self._user_password:
-            return self._user_password
-        else:
-            return Host.user_password
+        return self._user_password
 
     @user_password.setter
     def user_password(self, value):
         self._user_password = value
 
-
-
     @property
     def admin_password(self):
-        if self._admin_password:
-            return self._admin_password
-        else:
-            return Host.admin_password
+        return self._admin_password
 
     @admin_password.setter
     def admin_password(self, value):
         self._admin_password = value
 
-
     @classmethod
     def get_hosts_from_env(cls, env_xml="/etc/HPCCSystems/environment.xml",
                            hpcc_home="/opt/HPCCSystems", exclude_local=False):
@@ -134,7 +118,6 @@ class Host(object):
         else:
             return hosts
 
-
     @classmethod
     def get_hosts_from_file(cls, file_name, exclude_local=False):
         hosts = []

initfiles/sbin/hpcc_setenv.in

@@ -47,7 +47,7 @@ function pidwait_fn () {
       return 0
     fi
 
-    kill $WATCH_PID
+    kill $WATCH_PID > /dev/null 2>&1
     while [[ -d /proc/$WATCH_PID ]] && [[ $TIMEOUT -gt 0 ]]; do
       sleep 0.2
       local TIMEOUT=$(($TIMEOUT-200))
@@ -64,7 +64,7 @@ function pidwait_fn () {
       fi
     fi
 
-    rm -f $PID
+    rm -f $PID > /dev/null 2>&1
     return 0
 }
 

initfiles/sbin/install-cluster.sh.in

@@ -39,7 +39,8 @@ export CMDPREFIX="sudo"
 
 print_usage(){
     echo ""
-    echo "usage: install-cluster.sh [-h|--help] [-k|--newkey] [-n|--concurrent <number>] <Platform Package>"
+    echo "usage: install-cluster.sh [-h|--help] [-p|--pushkeydir <directory>] [-k|--newkey] [-n|--concurrent <number>] <Platform Package>"
+    echo "   -p:  copy previously generated keys from a directory"
     echo "   -k:  generate a new ssh key pair."
     echo "   -n:  how many concurrent execution allowd. The default is 5"
     echo "   <Platform Package>: HPCCSystems package file."
@@ -108,6 +109,9 @@ createPayload(){
     if [ ${NEW_KEY} -eq 1 ]; then
         mkdir -p ${NEW}
         cp -r ${GENKEY}/* ${NEW}/
+    elif [[ ${COPY_KEY} -eq 1 && -n $KEY_DIR ]]; then
+        mkdir -p ${NEW}
+        cp -r ${KEY_DIR}/* ${NEW}/
     fi
     cp -r ${PKG} ${REMOTE_INSTALL}
     cp -r ${CONFIG_DIR}/${ENV_XML_FILE} ${REMOTE_INSTALL}
@@ -141,15 +145,29 @@ fi
 
 
 NEW_KEY=0
+COPY_KEY=0
 OPTIONS="-l DEBUG -e ${CONFIG_DIR}/${ENV_CONF_FILE} -s ${SECTION:-DEFAULT}"
 
-TEMP=`/usr/bin/getopt -o n:s:kh --long help,newkey,concurrent: -n 'install-cluster' -- "$@"`
+TEMP=`/usr/bin/getopt -o p:n:s:kh --long help,pushkeydir,newkey,concurrent: -n 'install-cluster' -- "$@"`
 if [ $? != 0 ] ; then echo "Failure to parse commandline." >&2 ; exit 1 ; fi
 eval set -- "$TEMP"
 while true ; do
     case "$1" in
-        -k|--newkey) NEW_KEY=1
+        -k|--newkey)
+            NEW_KEY=1
+            if [[ $COPY_KEY -eq 1 ]]; then
+                echo "Cannot use both [-p|--pushkeydir] and [-k|--newkey] options"
+                exit 1
+            fi
             shift ;;
+        -p|--pushkeydir)
+            COPY_KEY=1
+            KEY_DIR=$2
+            if [[ $NEW_KEY -eq 1 ]]; then
+                echo "Cannot use both [-p|--pushkeydir] and [-k|--newkey] options"
+                exit 1
+            fi
+            shift 2 ;;
         -n|--concurrent)
             if [ -n "$2" ] && [[ $2 =~ ^[0-9]+$ ]]
             then
@@ -181,7 +199,9 @@ fi
 if [ ${NEW_KEY} -eq 1 ]; then
     generateKey
 fi
+
 export NEW_KEY
+export KEY_DIR
 
 createPayload;
 

initfiles/sbin/remote-install-engine.sh.in

@@ -150,8 +150,8 @@ checkKeys(){
                 fi
                 rm -rf ${MD5}
                 rm -rf ${CURR}
-    else
-        CNT=1
+        else
+                CNT=1
         fi
         if [ "$CNT" == "1" ]; then
                 installKeys ${NEW} $1

roxie/ccd/ccdcontext.cpp

@@ -299,7 +299,7 @@ protected:
         SCMStringBuffer name;
         const char *logicalName = item.getPersistName(name).str();
         StringBuffer whenName;
-        expandLogicalFilename(whenName, logicalName, workunit, false);
+        expandLogicalFilename(whenName, logicalName, workunit, false, false);
         whenName.append("$when");
         if (!isResult(whenName, ResultSequencePersist))
             return false;
@@ -354,7 +354,7 @@ private:
     bool checkPersistUptoDate(IRuntimeWorkflowItem & item, const char * logicalName, unsigned eclCRC, unsigned __int64 allCRC, bool isFile, StringBuffer &errText)
     {
         StringBuffer lfn, crcName, eclName;
-        expandLogicalFilename(lfn, logicalName, workunit, false);
+        expandLogicalFilename(lfn, logicalName, workunit, false, false);
         crcName.append(lfn).append("$crc");
         eclName.append(lfn).append("$eclcrc");
 
@@ -416,7 +416,7 @@ private:
     IRemoteConnection *getPersistReadLock(const char * logicalName)
     {
         StringBuffer lfn;
-        expandLogicalFilename(lfn, logicalName, workunit, false);
+        expandLogicalFilename(lfn, logicalName, workunit, false, false);
         if (!lfn.length())
             throw MakeStringException(0, "Invalid persist name used : '%s'", logicalName);
 
@@ -507,7 +507,7 @@ private:
     void updatePersist(IRemoteConnection *persistLock, const char * logicalName, unsigned eclCRC, unsigned __int64 allCRC)
     {
         StringBuffer lfn, crcName, eclName, whenName;
-        expandLogicalFilename(lfn, logicalName, workunit, false);
+        expandLogicalFilename(lfn, logicalName, workunit, false, false);
         crcName.append(lfn).append("$crc");
         eclName.append(lfn).append("$eclcrc");
         whenName.append(lfn).append("$when");
@@ -532,7 +532,7 @@ private:
     void checkPersistMatches(const char * logicalName, unsigned eclCRC)
     {
         StringBuffer lfn, eclName;
-        expandLogicalFilename(lfn, logicalName, workunit, true);
+        expandLogicalFilename(lfn, logicalName, workunit, true, false);
         eclName.append(lfn).append("$eclcrc");
 
         if (!isResult(lfn, ResultSequencePersist))
@@ -565,7 +565,7 @@ private:
     void deleteLRUPersists(const char * logicalName, unsigned keep)
     {
         StringBuffer lfn;
-        expandLogicalFilename(lfn, logicalName, workunit, false);
+        expandLogicalFilename(lfn, logicalName, workunit, false, false);
         logicalName = lfn.str();
         const char *tail = strrchr(logicalName, '_');     // Locate the trailing double-underbar
         assertex(tail);
@@ -3489,7 +3489,7 @@ public:
     {
         CriticalBlock b(contextCrit);
         StringBuffer expandedName;
-        expandLogicalFilename(expandedName, fileName, workUnit, false);
+        expandLogicalFilename(expandedName, fileName, workUnit, false, !workUnit);
         Linked<const IResolvedFile> ret = fileCache.getValue(expandedName);
         if (!ret)
         {
@@ -3648,7 +3648,7 @@ public:
     virtual char * getExpandLogicalName(const char * logicalName)
     {
         StringBuffer lfn;
-        expandLogicalFilename(lfn, logicalName, workUnit, false);
+        expandLogicalFilename(lfn, logicalName, workUnit, false, false);
         return lfn.detach();
     }
     virtual void setWorkflowCondition(bool value) { if(workflow) workflow->setCondition(value); }
@@ -3695,7 +3695,7 @@ public:
     virtual unsigned __int64 getDatasetHash(const char * logicalName, unsigned __int64 crc)
     {
         StringBuffer fullname;
-        expandLogicalFilename(fullname, logicalName, workUnit, false);
+        expandLogicalFilename(fullname, logicalName, workUnit, false, false);
         Owned<IDistributedFile> file = queryDistributedFileDirectory().lookup(fullname.str(),queryUserDescriptor());
         if (file)
         {

roxie/ccd/ccddali.cpp

@@ -572,7 +572,7 @@ public:
 
     virtual void commitCache()
     {
-        if (isConnected)
+        if (isConnected && cache)
         {
             CriticalBlock b(cacheCrit);
             if (!recursiveCreateDirectory(queryDirectory))

roxie/ccd/ccdserver.cpp

@@ -10032,7 +10032,7 @@ protected:
         {
             OwnedRoxieString rawLogicalName = helper.getFileName();
             StringBuffer lfn;   // logical filename
-            expandLogicalFilename(lfn, rawLogicalName, wu, false);
+            expandLogicalFilename(lfn, rawLogicalName, wu, false, false);
             if (lfn.length())
             {
                 unsigned flags = helper.getFlags();
@@ -10510,7 +10510,7 @@ class CRoxieServerIndexWriteActivity : public CRoxieServerInternalSinkActivity,
         {
             OwnedRoxieString rawLogicalName = helper.getFileName();
             StringBuffer lfn;   // logical filename
-            expandLogicalFilename(lfn, rawLogicalName, wu, false);
+            expandLogicalFilename(lfn, rawLogicalName, wu, false, false);
             if (lfn.length())
             {
                 if (helper.getSequence() >= 0)

roxie/ccd/ccdstate.cpp

@@ -669,7 +669,7 @@ public:
     virtual const IResolvedFile *lookupFileName(const char *_fileName, bool opt, bool useCache, bool cacheResult, IConstWorkUnit *wu) const
     {
         StringBuffer fileName;
-        expandLogicalFilename(fileName, _fileName, wu, false);
+        expandLogicalFilename(fileName, _fileName, wu, false, !wu);
         if (traceLevel > 5)
             DBGLOG("lookupFileName %s", fileName.str());
 
@@ -690,7 +690,7 @@ public:
     virtual IRoxieWriteHandler *createFileName(const char *_fileName, bool overwrite, bool extend, const StringArray &clusters, IConstWorkUnit *wu) const
     {
         StringBuffer fileName;
-        expandLogicalFilename(fileName, _fileName, wu, false);
+        expandLogicalFilename(fileName, _fileName, wu, false, false);
         Owned<IResolvedFile> resolved = lookupFile(fileName, false, false, true, true);
         if (!resolved)
             resolved.setown(resolveLFNusingDaliOrLocal(fileName, false, false, true, true, resolveLocally()));

system/security/LdapSecurity/ldapconnection.cpp

@@ -1409,7 +1409,7 @@ public:
                 {
                     //This path is typical if running ESP on Windows. We have no way
                     //to determine if password entered is valid but expired
-                    if (user.getPasswordDaysRemaining() == -1)
+                    if (user.getPasswordDaysRemaining() == scPasswordExpired)
                     {
                         DBGLOG("LDAP: Password Expired(2) for user %s", username);
                         user.setAuthenticateStatus(AS_PASSWORD_EXPIRED);

system/security/LdapSecurity/ldapsecurity.ipp

@@ -110,13 +110,13 @@ public:
    virtual int getPasswordDaysRemaining()
    {
        if (m_passwordExpiration.isNull())
-           return -2;//-2 if never expires
+           return scPasswordNeverExpires;//-2 if never expires
        CDateTime expiry(m_passwordExpiration);
        CDateTime now;
        now.setNow();
        now.adjustTime(now.queryUtcToLocalDelta());
        if (expiry <= now)
-           return -1;//-1 if already expired
+           return scPasswordExpired;//-1 if already expired
        expiry.setTime(0,0,0,0);
        now.setTime(23,59,59);
        int numDays = 0;

system/security/shared/SecureUser.hpp

@@ -200,7 +200,7 @@ public:
 
     virtual CDateTime & getPasswordExpiration(CDateTime& expirationDate){ return expirationDate; }
     virtual bool setPasswordExpiration(CDateTime& expirationDate) { return true; }
-    virtual int getPasswordDaysRemaining() {return -1;}
+    virtual int getPasswordDaysRemaining() {return scPasswordNeverExpires;}//never expires
     virtual authStatus getAuthenticateStatus() {return m_authenticateStatus;}
     virtual void setAuthenticateStatus(authStatus status){m_authenticateStatus = status;}
 

system/security/shared/seclib.hpp

@@ -116,6 +116,8 @@ enum SecUserStatus
 };
 
 
+const static int scPasswordExpired = -1;
+const static int scPasswordNeverExpires = -2;
 
 interface ISecCredentials : extends IInterface
 {

testing/regress/ecl/issue13081.ecl

@@ -0,0 +1,64 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2015 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//Test only makes any sense on thor, test roxie to ensure attributes are ignored
+//nohthor
+
+DataRec := RECORD
+    UNSIGNED2   f1;
+END;
+
+dsTemp := DATASET
+    (
+        [1, 2, 3, 4, 5],
+        DataRec
+    );
+
+ds := DISTRIBUTE(dsTemp);
+
+TableRec := RECORD
+    REAL8       average := AVE(GROUP, ds.f1);
+END;
+
+r1 := TABLE(ds, TableRec, FEW);
+OUTPUT(r1, NAMED('Few'));
+
+r2 := TABLE(ds, TableRec, FEW, MERGE);
+OUTPUT(r2, NAMED('FewMerge'));
+
+r1x := TABLE(ds, TableRec, f1 DIV 2, FEW);
+OUTPUT(r1x, NAMED('FewX'));
+
+r2x := TABLE(ds, TableRec, f1 DIV 2, FEW, MERGE);
+OUTPUT(r2x, NAMED('FewMergeX'));
+
+dsTemp2 := NOFOLD(DATASET
+    (
+        [1, 1, 1],
+        DataRec
+    ));
+
+TableRec2 := RECORD
+    REAL8       average := AVE(GROUP, dsTemp2.f1);
+END;
+
+r3 := TABLE(dsTemp2, TableRec2, MERGE);
+OUTPUT(r3, NAMED('Merge2'));
+
+r3x := TABLE(dsTemp2, TableRec2, f1, MERGE);
+OUTPUT(r3x, NAMED('Merge2x'));
+

testing/regress/ecl/key/issue13081.xml

@@ -0,0 +1,22 @@
+<Dataset name='Few'>
+ <Row><average>3.0</average></Row>
+</Dataset>
+<Dataset name='FewMerge'>
+ <Row><average>3.0</average></Row>
+</Dataset>
+<Dataset name='FewX'>
+ <Row><average>1.0</average></Row>
+ <Row><average>2.5</average></Row>
+ <Row><average>4.5</average></Row>
+</Dataset>
+<Dataset name='FewMergeX'>
+ <Row><average>1.0</average></Row>
+ <Row><average>2.5</average></Row>
+ <Row><average>4.5</average></Row>
+</Dataset>
+<Dataset name='Merge2'>
+ <Row><average>1.0</average></Row>
+</Dataset>
+<Dataset name='Merge2x'>
+ <Row><average>1.0</average></Row>
+</Dataset>

thorlcr/activities/hashdistrib/thhashdistribslave.cpp

@@ -778,7 +778,7 @@ protected:
                         loop
                         {
                             if (timer.elapsedCycles() >= queryOneSecCycles()*10)
-                                owner.ActPrintLog("HD sender, waiting for space, inactive writers = %d, totalSz = %d", queryInactiveWriters(), queryTotalSz());
+                                owner.ActPrintLog("HD sender, waiting for space, inactive writers = %d, totalSz = %d, numFinished = %d", queryInactiveWriters(), queryTotalSz(), atomic_read(&numFinished));
                             timer.reset();
 
                             if (senderFullSem.wait(10000))

thorlcr/activities/nsplitter/thnsplitterslave.cpp

@@ -40,6 +40,7 @@ public:
 class CSplitterOutput : public CSplitterOutputBase
 {
     NSplitterSlaveActivity &activity;
+    Semaphore writeBlockSem;
 
     unsigned output;
     rowcount_t rec, max;
@@ -67,7 +68,8 @@ class NSplitterSlaveActivity : public CSlaveActivity, implements ISharedSmartBuf
     bool eofHit;
     bool inputsConfigured;
     bool writeBlocked, pagedOut;
-    CriticalSection startLock, writeAheadCrit, writeBlockedCrit;
+    CriticalSection startLock, writeAheadCrit;
+    PointerArrayOf<Semaphore> stalledWriters;
     unsigned nstopped;
     rowcount_t recsReady;
     IThorDataLink *input;
@@ -92,8 +94,9 @@ class NSplitterSlaveActivity : public CSlaveActivity, implements ISharedSmartBuf
         ~CWriter() { stop(); }
         virtual void main()
         {
+            Semaphore writeBlockSem;
             while (!stopped && !parent.eofHit)
-                current = parent.writeahead(current, stopped);
+                current = parent.writeahead(current, stopped, writeBlockSem);
         }
         void start()
         {
@@ -261,6 +264,7 @@ public:
         eofHit = false;
         recsReady = 0;
         writeBlocked = false;
+        stalledWriters.kill();
         if (inputsConfigured)
         {
             // ensure old inputs cleared, to avoid being reused before re-setup on subsequent executions
@@ -334,7 +338,7 @@ public:
             throw LINK(writeAheadException);
         return row.getClear();
     }
-    rowcount_t writeahead(rowcount_t current, const bool &stopped)
+    rowcount_t writeahead(rowcount_t current, const bool &stopped, Semaphore &writeBlockSem)
     {
         // NB: readers call writeahead, which will block others
         CriticalBlock b(writeAheadCrit);
@@ -344,16 +348,11 @@ public:
                 return recsReady;
             if (current < recsReady)
                 return recsReady;
-            else if (writeBlocked)
+            else if (writeBlocked) // NB: only used by 'balanced' splitter, which blocks write when too far ahead
             {
-                // NB: When here, writeAheadCrit has been released _and_ writeBlockedCrit will be held until the writer is unblocked
-                {
-                    CriticalUnblock ub(writeAheadCrit);
-                    {
-                        CriticalBlock b(writeBlockedCrit); // This will block until the thread that is writing ahead has done
-                        // Once writeBlockCrit gained, writeBlocked is always 'false'
-                    }
-                }
+                stalledWriters.append(&writeBlockSem);
+                CriticalUnblock ub(writeAheadCrit);
+                writeBlockSem.wait(); // when active writer unblocks, signals all stalledWriters
                 // recsReady or eofHit will have been updated by the blocking thread by now, loop and re-check
             }
             else
@@ -374,7 +373,7 @@ public:
                     row.setown(input->nextRow());
                     if (row)
                     {
-                        smartBuf->putRow(NULL, this);
+                        smartBuf->putRow(NULL, this); // may call blocked() (see ISharedSmartBufferCallback impl. below)
                         ++recsReady;
                     }
                 }
@@ -421,7 +420,6 @@ public:
     virtual void paged() { pagedOut = true; }
     virtual void blocked()
     {
-        writeBlockedCrit.enter(); // enter before unblocking writeahead
         writeBlocked = true; // Prevent other users getting beyond checking recsReady in writeahead()
         writeAheadCrit.leave();
     }
@@ -429,7 +427,11 @@ public:
     {
         writeAheadCrit.enter();
         writeBlocked = false;
-        writeBlockedCrit.leave(); // leave after re-blocking writer
+        if (stalledWriters.ordinality())
+        {
+            ForEachItemInRev(s, stalledWriters)
+                stalledWriters.popGet()->signal();
+        }
     }
 friend class CInputWrapper;
 friend class CSplitterOutput;
@@ -465,7 +467,7 @@ void CSplitterOutput::stop()
 const void *CSplitterOutput::nextRow()
 {
     if (rec == max)
-        max = activity.writeahead(max, activity.queryAbortSoon());
+        max = activity.writeahead(max, activity.queryAbortSoon(), writeBlockSem);
     ActivityTimer t(totalCycles, activity.queryTimeActivities());
     const void *row = activity.nextRow(output); // pass ptr to max if need more
     ++rec;

tools/esdlcmd-xml/esdl2xml.cpp

@@ -28,7 +28,7 @@ int gArgc = 0;
 void static usage(const char* programName)
 {
     printf("\nESDL Compiler\n\n");
-    printf("Usage:        %s [options] filename.ecm [<outdir>]\n", programName);
+    printf("Usage:        %s [options] filename.(ecm|esdl) [<outdir>]\n", programName);
     printf("Output:       (srcdir|<outdir>)/filename.xml\n\n");
 
     puts("Available options:");

tools/esdlcmd-xml/esdl2xml.hpp

@@ -45,7 +45,7 @@ public:
     void setVerbose(bool verbose){optVerbose = verbose;};
     bool getVerbose(){return optVerbose;};
 
-    void transform(const char * source, const char * outdir="", StringBuffer * out=NULL, bool outputIncludes=true)
+    void transform(const char * source, const char * outdir="", StringBuffer * out=NULL, bool outputIncludes=true, bool includedESDL=false)
     {
         if (added.getValue(source) == false)
         {
@@ -56,7 +56,7 @@ public:
                     fprintf(stdout, "Output directory not specified\n");
             }
 
-            ESDLcompiler hc(source, out==NULL, outdir, outputIncludes);
+            ESDLcompiler hc(source, out==NULL, outdir, outputIncludes, includedESDL);
             hc.Process();
             if (out != NULL)
                 out->append(hc.getEsxdlContent());
@@ -71,8 +71,8 @@ public:
                 IncludeInfo * ii;
                 for (ii=hc.includes;ii;ii=ii->next)
                 {
-                   subfile.setf("%s%s.ecm", srcDir.str(), ii->pathstr.str());
-                   transform(subfile, outdir, out, outputIncludes);
+                   subfile.setf("%s%s%s", srcDir.str(), ii->pathstr.str(), ESDL_FILE_EXTENSION);
+                   transform(subfile, outdir, out, outputIncludes, true);
                 }
             }
         }

tools/esdlcmd/esdl-publish.cpp

@@ -160,11 +160,11 @@ public:
     void usage()
     {
         printf("\nUsage:\n\n"
-                "esdl publish <servicename> <filename.ecm> [command options]\n\n"
-                "   <servicename>        The ESDL defined ESP service to publish\n"
-                "   <filename.ecm>       The ESDL file containing service definition\n"
+                "esdl publish <servicename> <filename.(ecm|esdl)> [command options]\n\n"
+                "   <servicename>               The ESDL defined ESP service to publish\n"
+                "   <filename.(ecm|esdl|xml)>   The ESDL file containing service definition in esdl format (.ecm |.esdl) or in esxdl format (.xml) \n"
                 "Options (use option flag followed by appropriate value):\n"
-                "   --overwrite                    Overwrite the latest version of this ESDL Definition\n"
+                "   --overwrite                 Overwrite the latest version of this ESDL Definition\n"
                 );
 
         EsdlPublishCmdCommon::usage();
@@ -180,7 +180,7 @@ public:
 
        //First x parameter's order is fixed.
        //<servicename>
-       //<filename.ecm>       The ESDL file containing service definition\n"
+       //<filename.(ecm|esdl|xml)>       The ESDL file containing service definition\n"
        for (int cur = 0; cur < 2 && !iter.done(); cur++)
        {
           const char *arg = iter.query();

tools/esdlcmd/esdl2ecl.cpp

@@ -88,30 +88,30 @@ public:
         return qname;
     }
 
-    void loadFile(const char *srcpath, const char *srcfile, const char *srcext="", IProperties *versions=NULL, bool loadincludes=false)
+    void loadFile(const char *srcpath, const char *srcfile, const char *srcext="", IProperties *versions=NULL, bool loadincludes=false, bool isIncludedESDL=false)
     {
         if (!srcfile || !*srcfile)
             throw MakeStringException(-1, "EsdlInclude no file name");
 
         if (!included.getValue(srcfile))
         {
-            DBGLOG("ESDL Loading include: %s", srcfile);
+            DBGLOG("ESDL Loading: %s", srcfile);
 
             StringBuffer fileName(srcpath);
             fileName.append(srcfile);
 
             IPropertyTree *src = NULL;
 
-            if (stricmp(srcext,".ecm")==0)
+            if (stricmp(srcext,LEGACY_FILE_EXTENSION)==0 || stricmp(srcext,ESDL_FILE_EXTENSION)==0)
             {
-                fileName.append(".ecm");
+                fileName.append(srcext);
                 StringBuffer esxml;
-                EsdlCmdHelper::convertECMtoESXDL(fileName.str(), srcfile, esxml, false, true, true);
+                EsdlCmdHelper::convertECMtoESXDL(fileName.str(), srcfile, esxml, false, true, true, isIncludedESDL);
                 src = createPTreeFromXMLString(esxml, 0);
             }
-            else if (!srcext || !*srcext || stricmp(srcext, ".xml")==0)
+            else if (!srcext || !*srcext || stricmp(srcext, XML_FILE_EXTENSION)==0)
             {
-                fileName.append(".xml");
+                fileName.append(XML_FILE_EXTENSION);
                 src = createPTreeFromXMLFile(fileName.str(), false);
             }
             else
@@ -178,7 +178,7 @@ public:
                 ForEachItemIn(idx, add_includes)
                 {
                     const char *file=add_includes.item(idx);
-                    loadFile(srcpath, file, srcext, versions, loadincludes);
+                    loadFile(srcpath, file, srcext, versions, loadincludes, true);
                 }
             }
         }
@@ -281,7 +281,7 @@ public:
 
         unsigned start = msTick();
         EsdlIndexedPropertyTrees trees;
-        trees.loadFile(srcPath.str(), srcName.str(), srcExt.str(), NULL, optGenerateAllIncludes);
+        trees.loadFile(srcPath.str(), srcName.str(), srcExt.str(), NULL, optGenerateAllIncludes, false);
         DBGLOG("Time taken to load ESDL files %u", msTick() - start);
 
         StringBuffer idxxml("<types><type name=\"StringArrayItem\" src=\"share\"/>");
@@ -325,7 +325,6 @@ public:
             Owned<IPropertyTreeIterator> files = trees.all->getElements("esxdl");
             ForEach(*files)
             {
-
                 IPropertyTree &file = files->query();
                 const char * filename = file.queryProp("@name");
                 StringBuffer xmlfile;
@@ -347,11 +346,12 @@ public:
                     toXML(file, xmlfile, 0,0);
 
                     //expandEsxdlForEclGeneration(trees, srcName.str());
-                    outputEcl(srcPath.str(), srcName.str(), optOutDirPath.get(), idxxml.str(), xmlfile); //rodrigo
+                    outputEcl(srcPath.str(), srcName.str(), optOutDirPath.get(), idxxml.str(), xmlfile);
                 }
             }
         }
 
+        DBGLOG("Time taken to translate ESDL files %u", msTick() - start);
         return 0;
     }
 

tools/esdlcmd/esdlcmd_common.hpp

@@ -157,10 +157,10 @@ public:
             StringBuffer extension;
             StringBuffer filename;
             splitFilename(sourceFileName, NULL, NULL, &filename, &extension);
-            if (stricmp(extension.str(),".ecm")==0)
+            if (stricmp(extension.str(),LEGACY_FILE_EXTENSION)==0 || stricmp(extension.str(),ESDL_FILE_EXTENSION)==0)
             {
                 StringBuffer esxml;
-                EsdlCmdHelper::convertECMtoESXDL(sourceFileName, filename.str(), esxml, true, true, false);
+                EsdlCmdHelper::convertECMtoESXDL(sourceFileName, filename.str(), esxml, true, true, false, true);
                 esdlDef->addDefinitionFromXML(esxml, serviceName, (int)version);
             }
             else
@@ -184,14 +184,14 @@ public:
         }
     }
 
-    static void convertECMtoESXDL(const char * filepath, const char * esxdlname, StringBuffer & esxml, bool recursive, bool verbose, bool outputincludes)
+    static void convertECMtoESXDL(const char * filepath, const char * esxdlname, StringBuffer & esxml, bool recursive, bool verbose, bool outputincludes, bool isIncludedESDL)
     {
         if (verbose)
-            fprintf(stdout,"Converting ecm file %s to XML\n", filepath);
+            fprintf(stdout,"Converting ESDL file %s to XML\n", filepath);
 
         Owned<Esdl2Esxdl> cmd = new Esdl2Esxdl(recursive, verbose);
         esxml.setf( "<esxdl name=\"%s\">", esxdlname);
-        cmd->transform(filepath, "", &esxml, outputincludes); //output to buffer
+        cmd->transform(filepath, "", &esxml, outputincludes, isIncludedESDL); //output to buffer
         esxml.append("</esxdl>");
     }
 

tools/esdlcomp/esdlcomp.cpp

@@ -31,8 +31,6 @@
 inline bool es_strieq(const char* s,const char* t) { return stricmp(s,t)==0; }
 
 //-------------------------------------------------------------------------------------------------------------
-#define ESDL "ESDL"
-
 extern FILE *yyin;
 extern int yyparse();
 
@@ -1088,7 +1086,7 @@ char* getTargetBase(const char* outDir, const char* src)
         return strdup(src);
 }
 
-ESDLcompiler::ESDLcompiler(const char * sourceFile, bool generatefile, const char *outDir, bool outputIncludes_)
+ESDLcompiler::ESDLcompiler(const char * sourceFile, bool generatefile, const char *outDir, bool outputIncludes_, bool isIncludedEsdl)
 {
     outputIncludes = outputIncludes_;
     modules = NULL;
@@ -1100,8 +1098,8 @@ ESDLcompiler::ESDLcompiler(const char * sourceFile, bool generatefile, const cha
     methods=NULL;
     versions = NULL;
 
-    splitFilename(sourceFile, NULL, &srcDir, &name, NULL);
-
+    StringBuffer ext;
+    splitFilename(sourceFile, NULL, &srcDir, &name, &ext);
 
     filename = strdup(sourceFile);
     size_t l = strlen(filename);
@@ -1109,8 +1107,29 @@ ESDLcompiler::ESDLcompiler(const char * sourceFile, bool generatefile, const cha
     yyin = fopen(sourceFile, "rt");
     if (!yyin)
     {
-        fprintf(stderr, "Fatal Error: Cannot read %s\n",sourceFile);
-        exit(1);
+        if (isIncludedEsdl)
+        {
+            StringBuffer alternateExtFilename(srcDir);
+            alternateExtFilename.append(name.str());
+
+            if (stricmp(ext.str(), ESDL_FILE_EXTENSION)==0)
+                alternateExtFilename.append(LEGACY_FILE_EXTENSION);
+            else
+                alternateExtFilename.append(ESDL_FILE_EXTENSION);
+
+            yyin = fopen(alternateExtFilename.str(), "rt");
+            if (!yyin)
+            {
+                fprintf(stderr, "Fatal Error: Could not load included ESDL grammar %s\n", filename);
+                exit(1);
+            }
+        }
+        else
+        {
+            fprintf(stderr, "Fatal Error: Could not load ESDL grammar %s\n", sourceFile);
+            exit(1);
+        }
+
     }
 
     packagename = es_gettail(sourceFile);

tools/esdlcomp/esdlcomp.h

@@ -55,6 +55,9 @@ inline bool es_streq(const char* s, const char* t) {  return strcmp(s,t)==0; }
 #define PF_ESPSTRUCT    0x800
 #define PF_MASK        0x7fff
 
+#define LEGACY_FILE_EXTENSION ".ecm"
+#define ESDL_FILE_EXTENSION   ".esdl"
+#define XML_FILE_EXTENSION    ".xml"
 
 enum type_kind
 {
@@ -1286,7 +1289,7 @@ public:
 class esdlcomp_decl ESDLcompiler
 {
 public:
-    ESDLcompiler(const char * sourceFile, bool generatefile, const char * outDir, bool outputIncludes);
+    ESDLcompiler(const char * sourceFile, bool generatefile, const char * outDir, bool outputIncludes, bool includedEsdl);
     ~ESDLcompiler();
 
     void Process();