Merge branch 'candidate-5.4.0'

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

dali/base/dafdesc.hpp

@@ -37,6 +37,10 @@ interface INamedGroupStore;
 
 #define MAX_REPLICATION_LEVELS 4
 
+#define DEFAULTXMLROWTAG "Row"
+#define DEFAULTXMLHEADER "<Dataset>"
+#define DEFAULTXMLFOOTER "</Dataset>"
+
 enum DFD_OS
 {
     DFD_OSdefault,

dali/ft/filecopy.cpp

@@ -41,6 +41,7 @@
 #include "dasds.hpp"
 #include "jlog.hpp"
 #include "dalienv.hpp"
+#include "ftbase.ipp"
 
 #define DEFAULT_MAX_CONNECTIONS 25
 #define PARTITION_RECOVERY_LIMIT 1000
@@ -1054,7 +1055,7 @@ void FileSprayer::calculateMany2OnePartition()
         setCanAccessDirectly(curFilename);
         if (partSeparator)
         {
-            offset_t contentLength = cur.size - cur.xmlHeaderLength - cur.xmlFooterLength;
+            offset_t contentLength = (cur.size > cur.xmlHeaderLength + cur.xmlFooterLength ? cur.size - cur.xmlHeaderLength - cur.xmlFooterLength : 0);
             if (contentLength)
             {
                 if (lastContentLength)
@@ -1449,7 +1450,7 @@ void FileSprayer::analyseFileHeaders(bool setcurheadersize)
             }
             io.setown(createCompressedFileReader(io,expander)); 
         }
-        
+
         if (defaultFormat != FFTunknown)
         {
             FileFormatType thisType;
@@ -1501,15 +1502,43 @@ void FileSprayer::analyseFileHeaders(bool setcurheadersize)
         {
             try
             {
-                if (srcFormat.headerLength == (unsigned)-1 || srcFormat.footerLength == (unsigned)-1)
-                    locateContentHeader(io, cur.headerSize, cur.xmlHeaderLength, cur.xmlFooterLength);
+                if (distributedSource)
+                {
+                    // Despray from distributed file
+
+                    // Check XMLheader/footer in file level
+                    DistributedFilePropertyLock lock(distributedSource);
+                    IPropertyTree &curProps = lock.queryAttributes();
+                    if (curProps.hasProp(FPheaderLength) && curProps.hasProp(FPfooterLength))
+                    {
+                        cur.xmlHeaderLength = curProps.getPropInt(FPheaderLength, 0);
+                        cur.xmlFooterLength = curProps.getPropInt(FPfooterLength, 0);
+                    }
+                    else
+                    {
+                        // Try it in file part level
+                        Owned<IDistributedFilePart> curPart = distributedSource->getPart(idx);
+                        IPropertyTree& curPartProps = curPart->queryAttributes();
+                        cur.xmlHeaderLength = curPartProps.getPropInt(FPheaderLength, 0);
+                        cur.xmlFooterLength = curPartProps.getPropInt(FPfooterLength, 0);
+                    }
+                }
                 else
                 {
-                    cur.xmlHeaderLength = srcFormat.headerLength;
-                    cur.xmlFooterLength = srcFormat.footerLength;
+                    // Spray from file
+                    if (srcFormat.headerLength == (unsigned)-1 || srcFormat.footerLength == (unsigned)-1)
+                        locateContentHeader(io, cur.headerSize, cur.xmlHeaderLength, cur.xmlFooterLength);
+                    else
+                    {
+                        cur.xmlHeaderLength = srcFormat.headerLength;
+                        cur.xmlFooterLength = srcFormat.footerLength;
+                    }
                 }
                 cur.headerSize += (unsigned)cur.xmlHeaderLength;
-                cur.size -= (cur.xmlHeaderLength + cur.xmlFooterLength);
+                if (cur.size >= cur.xmlHeaderLength + cur.xmlFooterLength)
+                    cur.size -= (cur.xmlHeaderLength + cur.xmlFooterLength);
+                else
+                    throwError3(DFTERR_InvalidXmlPartSize, cur.size, cur.xmlHeaderLength, cur.xmlFooterLength);
             }
             catch (IException * e)
             {
@@ -1557,12 +1586,38 @@ void FileSprayer::locateXmlHeader(IFileIO * io, unsigned headerSize, offset_t &
 
     reader.set(in);
     reader.seek(headerSize);
-    xmlHeaderLength = splitter.getHeaderLength(reader);
+    if (xmlHeaderLength == 0)
+    {
+        try
+        {
+            xmlHeaderLength = splitter.getHeaderLength(reader);
+        }
+        catch (IException * e)
+        {
+            if (e->errorCode() != DFTERR_CannotFindFirstXmlRecord)
+                throw;
+            e->Release();
+            xmlHeaderLength = 0;
+        }
+    }
 
     offset_t size = io->size();
     offset_t endOffset = (size > srcFormat.maxRecordSize*2 + headerSize) ? size - srcFormat.maxRecordSize*2 : headerSize;
     reader.seek(endOffset);
-    xmlFooterLength = splitter.getFooterLength(reader, size);
+    if (xmlFooterLength == 0)
+    {
+        try
+        {
+            xmlFooterLength = splitter.getFooterLength(reader, size);
+        }
+        catch (IException * e)
+        {
+            if (e->errorCode() != DFTERR_CannotFindLastXmlRecord)
+                throw;
+            e->Release();
+            xmlFooterLength= 0;
+        }
+    }
 }
 
 void FileSprayer::locateJsonHeader(IFileIO * io, unsigned headerSize, offset_t & headerLength, offset_t & footerLength)
@@ -1943,8 +1998,10 @@ void FileSprayer::cloneHeaderFooter(unsigned idx, bool isHeader)
     PartitionPoint & next = * new PartitionPoint;
     //NB: headerSize include the size of the xmlHeader; size includes neither header or footers.
     if (isHeader)
+        // Set offset to the XML header
         next.inputOffset = curSrc.headerSize - curSrc.xmlHeaderLength;
     else
+        //Set offset to the XML footer
         next.inputOffset = curSrc.headerSize + curSrc.size;
     next.inputLength = isHeader ? curSrc.xmlHeaderLength : curSrc.xmlFooterLength;
     next.outputLength = needToCalcOutput() ? next.inputLength : 0;
@@ -2837,6 +2894,46 @@ void FileSprayer::spray()
     cleanupRecovery();
 }
 
+bool FileSprayer::isSameSizeHeaderFooter()
+{
+    bool retVal = true;
+    unsigned whichHeaderInput = 0;
+    bool isEmpty = true;
+    headerSize = 0;
+    footerSize = 0;
+
+    ForEachItemIn(idx, partition)
+    {
+        PartitionPoint & cur = partition.item(idx);
+        if (idx+1 == partition.ordinality() || partition.item(idx+1).whichOutput != cur.whichOutput)
+        {
+            if (isEmpty)
+            {
+                headerSize = sources.item(whichHeaderInput).xmlHeaderLength;
+                footerSize = sources.item(cur.whichInput).xmlFooterLength;
+                isEmpty = false;
+                continue;
+            }
+
+            if (headerSize != sources.item(whichHeaderInput).xmlHeaderLength)
+            {
+                retVal = false;
+                break;
+            }
+
+            if (footerSize != sources.item(cur.whichInput).xmlFooterLength)
+            {
+                retVal = false;
+                break;
+            }
+
+            if ( idx+1 != partition.ordinality() )
+                whichHeaderInput = partition.item(idx+1).whichInput;
+        }
+
+    }
+    return retVal;
+}
 
 void FileSprayer::updateTargetProperties()
 {
@@ -2849,6 +2946,8 @@ void FileSprayer::updateTargetProperties()
         CRC32Merger totalCRC;
         offset_t totalLength = 0;
         offset_t totalCompressedSize = 0;
+        unsigned whichHeaderInput = 0;
+        bool sameSizeHeaderFooter = isSameSizeHeaderFooter();
         ForEachItemIn(idx, partition)
         {
             PartitionPoint & cur = partition.item(idx);
@@ -2873,6 +2972,20 @@ void FileSprayer::updateTargetProperties()
                 // TODO: Create DistributedFilePropertyLock for parts
                 curPart->lockProperties();
                 IPropertyTree& curProps = curPart->queryAttributes();
+
+                if (!sameSizeHeaderFooter)
+                {
+                    FilePartInfo & curHeaderSource = sources.item(whichHeaderInput);
+                    curProps.setPropInt(FPheaderLength, curHeaderSource.xmlHeaderLength);
+
+                    FilePartInfo & curFooterSource = sources.item(cur.whichInput);
+                    curProps.setPropInt(FPfooterLength, curFooterSource.xmlFooterLength);
+
+                    if ( idx+1 != partition.ordinality() )
+                        whichHeaderInput = partition.item(idx+1).whichInput;
+                }
+
+
                 if (calcCRC())
                 {
                     curProps.setPropInt(FAcrc, partCRC.get());
@@ -2967,6 +3080,13 @@ void FileSprayer::updateTargetProperties()
             curProps.setPropInt64(FArecordCount,totalLength/(offset_t)rs);
             gotrc = true;
         }
+
+        if (sameSizeHeaderFooter)
+        {
+            curProps.setPropInt(FPheaderLength, headerSize);
+            curProps.setPropInt(FPfooterLength, footerSize);
+        }
+
         if (srcAttr.get() && !mirroring) {
             StringBuffer s;
             // copy some attributes (do as iterator in case we want to change to *exclude* some

dali/ft/filecopy.ipp

@@ -262,6 +262,7 @@ protected:
     void updateSizeRead();
     void waitForTransferSem(Semaphore & sem);
     void addPrefix(size32_t len, const void * data, unsigned idx, PartitionPointArray & partitionWork);
+    bool isSameSizeHeaderFooter();
     
 private:
     bool calcUsePull();
@@ -317,6 +318,8 @@ protected:
     StringAttr              encryptKey;
     StringAttr              decryptKey;
     bool                    preserveCompression;
+    offset_t                headerSize;
+    offset_t                footerSize;
 };
 
 

dali/ft/ftbase.cpp

@@ -34,33 +34,6 @@
 #include "dalienv.hpp"
 #include "rmtspawn.hpp"
 
-#define DEFAULT_MAX_CSV_SIZE     8096
-
-//Use hash defines for properties so I can't mis-spell them....
-#define ANinput             "@input"
-#define ANinputDirect       "@inputDirect"
-#define ANinputLength       "@inputLength"
-#define ANinputOffset       "@inputOffset"
-#define ANlength            "@length"
-#define ANoutput            "@output"
-#define ANoutputDirect      "@outputDirect"
-#define ANoutputLength      "@outputLength"
-#define ANoutputOffset      "@outputOffset"
-
-//File attributes
-#define FPrecordSize        "@recordSize"
-#define FPformat            "@format"
-#define FPmaxRecordSize     "@maxRecordSize"
-#define FPcsvSeparate       "@csvSeparate"
-#define FPcsvQuote          "@csvQuote"
-#define FPcsvTerminate      "@csvTerminate"
-#define FPcsvEscape         "@csvEscape"
-#define FProwTag            "@rowTag"
-#define FPkind              "@kind"
-#define FPheaderLength      "@headerLength"
-#define FPfooterLength      "@footerLength"
-
-
 
 //----------------------------------------------------------------------------
 

dali/ft/ftbase.ipp

@@ -23,6 +23,32 @@
 #include "daft.hpp"
 #include "ftbase.hpp"
 
+#define DEFAULT_MAX_CSV_SIZE     8096
+
+//Use hash defines for properties so I can't mis-spell them....
+#define ANinput             "@input"
+#define ANinputDirect       "@inputDirect"
+#define ANinputLength       "@inputLength"
+#define ANinputOffset       "@inputOffset"
+#define ANlength            "@length"
+#define ANoutput            "@output"
+#define ANoutputDirect      "@outputDirect"
+#define ANoutputLength      "@outputLength"
+#define ANoutputOffset      "@outputOffset"
+
+//File attributes
+#define FPrecordSize        "@recordSize"
+#define FPformat            "@format"
+#define FPmaxRecordSize     "@maxRecordSize"
+#define FPcsvSeparate       "@csvSeparate"
+#define FPcsvQuote          "@csvQuote"
+#define FPcsvTerminate      "@csvTerminate"
+#define FPcsvEscape         "@csvEscape"
+#define FProwTag            "@rowTag"
+#define FPkind              "@kind"
+#define FPheaderLength      "@headerLength"
+#define FPfooterLength      "@footerLength"
+
 class DALIFT_API PartitionPoint : public CInterface
 {
 public:

dali/ft/fterror.hpp

@@ -80,6 +80,7 @@
 #define DFTERR_WrongRECFMvRecordSize            8107
 #define DFTERR_WrongSplitRecordSize             8108
 #define DFTERR_CannotFindFirstJsonRecord        8109
+#define DFTERR_InvalidXmlPartSize               8110
 
 //Internal errors
 #define DFTERR_UnknownFormatType                8190
@@ -147,6 +148,7 @@
 #define DFTERR_WrongRECFMvbBlockSize_Text       "Invalid RECFMvb file Block Size (%d) or the file is not RECFMvb format!"
 #define DFTERR_WrongRECFMvRecordSize_Text       "Invalid RECFMv file Record Size (%d) or the file is not RECFMv format!"
 #define DFTERR_WrongSplitRecordSize_Text        "Invalid Record Size (%d, 0x%08x)!"
+#define DFTERR_InvalidXmlPartSize_Text          "Invalid XML part size:%" I64F "d! Size is less than XML Header (%" I64F "d) + Footer (%" I64F "d)) size!"
 
 #define DFTERR_UnknownFormatType_Text           "INTERNAL: Save unknown format type"
 #define DFTERR_OutputOffsetMismatch_Text        "INTERNAL: Output offset does not match expected (%" I64F "d expected %" I64F "d) at %s of block %d"

docs/ECLStandardLibraryReference/SLR-Mods/FromStringToDate.xml

@@ -0,0 +1,85 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE sect1 PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN"
+"http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd">
+<sect1 id="FromStringToDate">
+  <title>FromStringToDate</title>
+
+  <para><emphasis role="bold">STD.Date.FromStringToDate<indexterm>
+      <primary>STD.Date.FromStringToDate</primary>
+    </indexterm><indexterm>
+      <primary>Date.FromStringToDate</primary>
+    </indexterm><indexterm>
+      <primary>FromStringToDate</primary>
+    </indexterm>(</emphasis> <emphasis>date_text, format</emphasis> <emphasis
+  role="bold">)</emphasis></para>
+
+  <informaltable colsep="1" frame="all" rowsep="1">
+    <tgroup cols="2">
+      <colspec colwidth="80.50pt" />
+
+      <colspec />
+
+      <tbody>
+        <row>
+          <entry><emphasis>date_text</emphasis></entry>
+
+          <entry>The string to be converted</entry>
+        </row>
+
+        <row>
+          <entry><emphasis>format</emphasis></entry>
+
+          <entry>The format of the input string. See strftime documentation
+          for details (<ulink
+          url="http://strftime.org/">http://strftime.org/</ulink>)</entry>
+        </row>
+
+        <row>
+          <entry>return</entry>
+
+          <entry>The date that was matched in the string. Returns 0 if failed
+          to match or if the date components match but the result is an
+          invalid date.</entry>
+        </row>
+      </tbody>
+    </tgroup>
+  </informaltable>
+
+  <para>The <emphasis role="bold">FromStringToDate</emphasis> function
+  converts a string to a Date_t using the relevant string format. The
+  resulting date must be representable within the Gregorian calendar after the
+  year 1600.</para>
+
+  <para><programlisting>Supported characters:
+
+    %B          Full month name
+    %b or %h    Abbreviated month name
+    %d          Day of month (two digits)
+    %e          Day of month (two digits, or a space followed by a single digit)
+    %m          Month (two digits)
+    %t          Whitespace
+    %y          year within century (00-99)
+    %Y          Full year (yyyy)
+    %j          Julian day (1-366)
+
+Common date formats
+
+    American    '%m/%d/%Y'  mm/dd/yyyy
+    Euro        '%d/%m/%Y'  dd/mm/yyyy
+    Iso format  '%Y-%m-%d'  yyyy-mm-dd
+    Iso basic   'Y%m%d'     yyyymmdd
+                '%d-%b-%Y'  dd-mon-yyyy    e.g., '21-Mar-1954'
+</programlisting></para>
+
+  <para>Example:</para>
+
+  <programlisting format="linespecific">IMPORT STD;
+ 
+D1 := STD.Date.FromStringToDate('19720607', '%Y%m%d');
+    //D1 contains 19720607
+D2 := STD.Date.FromStringToDate('19720007', '%Y%m%d');
+    //D2 contains 0
+ </programlisting>
+
+  <para></para>
+</sect1>

docs/ECLStandardLibraryReference/SLR-Mods/ToGregorianYMD.xml

@@ -27,7 +27,8 @@
         <row>
           <entry>Return:</entry>
 
-          <entry>ToGregorianYMD returns a Date_t value.</entry>
+          <entry>ToGregorianYMD returns separate values for Year, Month, and
+          Day.</entry>
         </row>
       </tbody>
     </tgroup>
@@ -35,7 +36,8 @@
 
   <para>The <emphasis role="bold">ToGregorianYMD</emphasis> function converts
   the number days since 31st December 1BC to a date in the Gregorian calendar.
-  It returns three separate values: Year, or Month, or Day.</para>
+  It returns It returns a module with three exported values: Year, Month, and
+  Day.</para>
 
   <para>Example:</para>
 
@@ -46,10 +48,13 @@ UNSIGNED1 MyDay   := 1;
   
 J := STD.Date.FromGregorianYMD(MyYear,MyMonth,MyDay);
     //J contains 734503
-    
-Y := STD.Date.ToGregorianYMD(J).Year;   //Y contains 2012
-M := STD.Date.ToGregorianYMD(J).Month;  //M contains 1
-D := STD.Date.ToGregorianYMD(J).Day;    //D contains 1
+
+X := STD.Date.ToGregorianYMD(J);
+    // X is a module with exported values
+
+Y := X.Year;   //Y contains 2012
+M := X.Month;  //M contains 1
+D := X.Day;    //D contains 1
 </programlisting>
 
   <para></para>

docs/ECLStandardLibraryReference/SLR-includer.xml

@@ -396,6 +396,8 @@
 
     <xi:include href="ECLStandardLibraryReference/SLR-Mods/ToGregorianYMD.xml"
                 xmlns:xi="http://www.w3.org/2001/XInclude" />
+    <xi:include href="ECLStandardLibraryReference/SLR-Mods/FromStringToDate.xml"
+                xmlns:xi="http://www.w3.org/2001/XInclude" />            
   </chapter>
 
   <chapter id="Cluster_Handling">

docs/HPCCClientTools/CT_Mods/CT_Comm_Line_ECL.xml

@@ -28,7 +28,7 @@
         <title><emphasis>eclplus.exe</emphasis></title>
 
         <para><emphasis role="bold">eclplus </emphasis><emphasis> action=
-        owner= user= password= cluster= server= queue= timeout=
+        owner= user= password= cluster= server= queue= graph= timeout=
         ecl=</emphasis><emphasis> file= format= output= jobname= -debugparam1=
         _applicationparam1=</emphasis><emphasis>
         /variablename1=</emphasis></para>
@@ -86,6 +86,12 @@
                 </row>
 
                 <row>
+                  <entry><emphasis>graph=</emphasis></entry>
+
+                  <entry>The name of graph.</entry>
+                </row>
+
+                <row>
                   <entry><emphasis>timeout=</emphasis></entry>
 
                   <entry>Query timeout in seconds (0 for
@@ -323,11 +329,14 @@ Richard Chapman</programlisting>
         </sect3>
 
         <sect3>
-          <title>See the thor graph of a workunit:</title>
+          <title>Return graph data for a workunit:</title>
+
+          <para>This action returns the XML data for one or more workunit
+          graphs.</para>
 
-          <para>The thor graph</para>
+          <programlisting>eclplus action=graph graph=graph1 wuid=W20090226-100958-85552898</programlisting>
 
-          <programlisting>eclplus action=graph wuid=W20090226-100958-85552898</programlisting>
+          <para>Graph name must be supplied in the graph= parameter.</para>
         </sect3>
 
         <sect3>

ecl/hql/hqlgram.y

@@ -3586,11 +3586,22 @@ outputWuFlag
                         }
     ;
 
-optCommaTrim
+fromXmlOptions
     :                   {   $$.setExpr(NULL); }
-    | ',' TRIM          {
+    | fromXmlOptions ',' fromXmlOption
+                        {
+                            $$.setExpr(createComma($1.getExpr(), $3.getExpr()));
+                        }
+    ;
+
+fromXmlOption
+    : TRIM              {
                             $$.setExpr(createAttribute(trimAtom), $1);
                         }
+    | ONFAIL '(' transform ')'
+                        {
+                            $$.setExpr(createExprAttribute(onFailAtom, $3.getExpr()), $1);
+                        }
     ;
     
 applyActions
@@ -7329,14 +7340,20 @@ simpleDataRow
 
                             $$.setExpr(createRow(no_matchrow, LINK(record), $3.getExpr())); //, parser->createUniqueId())));
                         }
-    | FROMXML '(' recordDef ',' expression optCommaTrim ')'
+    | FROMXML '(' recordDef ',' expression fromXmlOptions ')'
                         {
                             parser->normalizeExpression($5, type_stringorunicode, false);
+                            IHqlExpression * onFail = queryAttributeInList(onFailAtom, $6.queryExpr());
+                            if (onFail)
+                                parser->checkRecordTypesMatch($3.queryExpr(), onFail->queryChild(0), $6);
                             $$.setExpr(createRow(no_fromxml, $3.getExpr(), createComma($5.getExpr(), $6.getExpr())), $1);
                         }
-    | FROMJSON '(' recordDef ',' expression optCommaTrim ')'
+    | FROMJSON '(' recordDef ',' expression fromXmlOptions ')'
                         {
                             parser->normalizeExpression($5, type_stringorunicode, false);
+                            IHqlExpression * onFail = queryAttributeInList(onFailAtom, $6.queryExpr());
+                            if (onFail)
+                                parser->checkRecordTypesMatch($3.queryExpr(), onFail->queryChild(0), $6);
                             $$.setExpr(createRow(no_fromjson, $3.getExpr(), createComma($5.getExpr(), $6.getExpr())), $1);
                         }
     | WHEN '(' dataRow ',' action sideEffectOptions ')'

ecl/hqlcpp/hqlcppds.cpp

@@ -321,6 +321,8 @@ void HqlCppTranslator::buildNullRow(BuildCtx & ctx, IHqlExpression * expr, CHqlB
 
 IReferenceSelector * HqlCppTranslator::doBuildRowFromXMLorJSON(BuildCtx & ctx, IHqlExpression * expr)
 {
+    IHqlExpression * onFail = expr->queryAttribute(onFailAtom);
+
 //  assertex(supportsLinkCountedRows);
     Owned<ITypeInfo> overrideType = setLinkCountedAttr(expr->queryType(), true);
     Owned<ITypeInfo> utf8Type = makeUtf8Type(UNKNOWN_LENGTH, NULL);
@@ -362,18 +364,39 @@ IReferenceSelector * HqlCppTranslator::doBuildRowFromXMLorJSON(BuildCtx & ctx, I
     else
         function.setown(bindFunctionCall(createRowFromXmlId, args, overrideType));
 
-    CHqlBoundExpr bound;
-    buildExpr(ctx, function, bound);
+    Owned<BoundRow> result;
+    if (onFail)
+    {
+        result.setown(declareTempRow(ctx, ctx, expr));
+        BuildCtx tryctx(ctx);
+        tryctx.addTry();
+        buildRowAssign(tryctx, result, function);
 
-    Owned<ITypeInfo> rowType = makeReferenceModifier(LINK(overrideType));
-    OwnedHqlExpr rowExpr = ctx.getTempDeclare(rowType, NULL);
-    Owned<BoundRow> row = createBoundRow(expr, rowExpr);
-    ctx.associate(*row);                                    // associate here because it is compared inside the loop
+        HqlExprArray dummyArgs;
+        OwnedHqlExpr exception = createParameter(createIdAtom("e"), 0, makePointerType(makeClassType("IException")), dummyArgs);
+        BuildCtx catchctx(ctx);
+        catchctx.addCatch(exception);
+        catchctx.addQuoted("Owned<IException> _e = e;");    // ensure that the exception is released
+
+        associateLocalFailure(catchctx, "e");
+        OwnedHqlExpr row = createRow(no_createrow, LINK(onFail->queryChild(0)));
+        buildRowAssign(catchctx, result, row);
+    }
+    else
+    {
+        CHqlBoundExpr bound;
+        buildExpr(ctx, function, bound);
 
-    OwnedHqlExpr defaultRowPtr = getPointer(bound.expr);
-    ctx.addAssign(rowExpr, defaultRowPtr);
+        Owned<ITypeInfo> rowType = makeReferenceModifier(LINK(overrideType));
+        OwnedHqlExpr rowExpr = ctx.getTempDeclare(rowType, NULL);
+        result.setown(createBoundRow(expr, rowExpr));
 
-    return createReferenceSelector(row);
+        OwnedHqlExpr defaultRowPtr = getPointer(bound.expr);
+        ctx.addAssign(rowExpr, defaultRowPtr);
+    }
+
+    ctx.associate(*result);
+    return createReferenceSelector(result);
 }
 
 

ecl/hqlcpp/hqlwcpp.cpp

@@ -1796,6 +1796,7 @@ void HqlCppWriter::generateStmtAssign(IHqlStmt * assign, bool link)
         case type_pointer:
         case type_enumerated:
         case type_record:
+        case type_class:
             generateSimpleAssign(target, source);
             break;
         case type_varstring:

ecl/hthor/CMakeLists.txt

@@ -66,6 +66,7 @@ include_directories (
          ${HPCC_SOURCE_DIR}/ecl/schedulectrl
          ${CMAKE_BINARY_DIR}
          ${CMAKE_BINARY_DIR}/oss
+         ${HPCC_SOURCE_DIR}/dali/ft
     )
 
 ADD_DEFINITIONS( -D_USRDLL -DHTHOR_EXPORTS -DSTARTQUERY_EXPORTS )

ecl/hthor/hthor.cpp

@@ -45,6 +45,7 @@
 #include "thorstep.hpp"
 #include "eclagent.ipp"
 #include "roxierowbuff.hpp"
+#include "ftbase.ipp"
 
 #define EMPTY_LOOP_LIMIT 1000
 
@@ -882,7 +883,7 @@ CHThorXmlWriteActivity::CHThorXmlWriteActivity(IAgentContext &_agent, unsigned _
 {
     OwnedRoxieString xmlpath(helper.getXmlIteratorPath());
     if (!xmlpath)
-        rowTag.append("Row");
+        rowTag.append(DEFAULTXMLROWTAG);
     else
     {
         const char *path = xmlpath;
@@ -899,15 +900,14 @@ void CHThorXmlWriteActivity::execute()
     StringBuffer header;
     OwnedRoxieString suppliedHeader(helper.getHeader());
     if (kind==TAKjsonwrite)
-    {
         buildJsonHeader(header, suppliedHeader, rowTag);
-        headerLength = header.length();
-    }
     else if (suppliedHeader)
         header.set(suppliedHeader);
     else
-        header.set("<Dataset>\n");
-    diskout->write(header.length(), header.str());
+        header.append(DEFAULTXMLHEADER).newline();
+
+    headerLength = header.length();
+    diskout->write(headerLength, header.str());
 
     Owned<IXmlWriterExt> writer = createIXmlWriterExt(helper.getXmlFlags(), 0, NULL, (kind==TAKjsonwrite) ? WTJSON : WTStandard);
     writer->outputBeginArray(rowTag); //need to set up the array
@@ -941,16 +941,14 @@ void CHThorXmlWriteActivity::execute()
     OwnedRoxieString suppliedFooter(helper.getFooter());
     StringBuffer footer;
     if (kind==TAKjsonwrite)
-    {
         buildJsonFooter(footer.newline(), suppliedFooter, rowTag);
-        footerLength=footer.length();
-    }
     else if (suppliedFooter)
         footer.append(suppliedFooter);
     else
-        footer.append("</Dataset>");
+        footer.append(DEFAULTXMLFOOTER).newline();
 
-    diskout->write(footer.length(), footer);
+    footerLength=footer.length();
+    diskout->write(footerLength, footer);
 }
 
 void CHThorXmlWriteActivity::setFormat(IFileDescriptor * desc)
@@ -958,10 +956,9 @@ void CHThorXmlWriteActivity::setFormat(IFileDescriptor * desc)
     desc->queryProperties().setProp("@format","utf8n");
     desc->queryProperties().setProp("@rowTag",rowTag.str());
     desc->queryProperties().setProp("@kind", (kind==TAKjsonwrite) ? "json" : "xml");
-    if (headerLength)
-        desc->queryProperties().setPropInt("@headerLength", headerLength);
-    if (footerLength)
-        desc->queryProperties().setPropInt("@footerLength", footerLength);
+
+    desc->queryProperties().setPropInt(FPheaderLength, headerLength);
+    desc->queryProperties().setPropInt(FPfooterLength, footerLength);
 
     const char *recordECL = helper.queryRecordECL();
     if (recordECL && *recordECL)
@@ -6070,9 +6067,9 @@ void CHThorWorkUnitWriteActivity::execute()
             else if (agent.queryOutputFmt() == ofXML)
             {
                 CommonXmlWriter xmlwrite(0,1);
-                xmlwrite.outputBeginNested("Row", false);
+                xmlwrite.outputBeginNested(DEFAULTXMLROWTAG, false);
                 helper.serializeXml((byte *) nextrec.get(), xmlwrite);
-                xmlwrite.outputEndNested("Row");
+                xmlwrite.outputEndNested(DEFAULTXMLROWTAG);
                 agent.queryOutputSerializer()->fwrite(seq, (const void*)xmlwrite.str(), 1, xmlwrite.length());
             }
         }
@@ -6092,7 +6089,7 @@ void CHThorWorkUnitWriteActivity::execute()
         if (agent.queryOutputFmt() == ofXML)
         {
             StringBuffer sb;
-            sb.appendf("</Dataset>\n");
+            sb.appendf(DEFAULTXMLFOOTER).newline();
             agent.queryOutputSerializer()->fwrite(seq, (const void*)sb.str(), 1, sb.length());
         }
         else if (agent.queryOutputFmt() != ofSTD)

ecl/regress/fromxml.ecl

@@ -1,6 +1,6 @@
 /*##############################################################################
 
-    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+    HPCC SYSTEMS software Copyright (C) 2015 HPCC Systems.
 
     This program is free software: you can redistribute it and/or modify
     you may not use this file except in compliance with the License.
@@ -59,4 +59,4 @@ namesTable := dataset([
     ], { utf8 text; });
 
 output(namesTable, { FROMXML(personRecord, text, TRIM); });
-output(1);
+output(1);

ecl/regress/fromxml5err.ecl

@@ -0,0 +1,69 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2015 HPCC Systems.
+
+    This program is free software: you can redistribute it and/or modify
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+phoneRecord :=
+            RECORD
+string5         areaCode{xpath('@areaCode')};
+udecimal12      number{xpath('@number')};
+            END;
+
+contactrecord :=
+            RECORD
+phoneRecord     phone;
+boolean         hasemail{xpath('@hasEmail')};
+                ifblock(self.hasemail)
+string              email;
+                end;
+            END;
+
+bookRec :=
+    RECORD
+unicode     title;
+string      author;
+    END;
+
+personRecord :=
+            RECORD
+unicode20       surname;
+unicode10       forename;
+phoneRecord     homePhone;
+boolean         hasMobile;
+                ifblock(self.hasMobile)
+phoneRecord         mobilePhone;
+                end;
+contactRecord   contact;
+dataset(bookRec) books;
+set of string    colours;
+string2         endmarker := '$$';
+            END;
+
+namesTable := dataset([
+    u8'<Row><surname>Hälliday</surname><forename>Gavin</forename><homephone areaCode="09876" number="123987"/><hasmobile>true</hasmobile><mobilephone areaCode="07967" number="123987"/>' +
+        u8'<contact hasEmail="true"><phone areaCode="n/a" number="0"/><email>gavin@edata.com</email></contact>' +
+        u8'<books><Row><title>εν αρχη ην ο λογος</title><author>john</author></Row><Row><title>To kill a mocking bird</title><author>Lee</author></Row><Row><title>Zen and the art of motorcycle maintainence</title><author>Pirsig</author></Row></books><colours><All/></colours><endmarker>$$</endmarker></Row>',
+    u8'<Row><unmatchedtag></Row>',
+    u8'<Row><surname>Halliday</surname><forename>Abigäil</forename><homephone areaCode="09876" number="123987"/><hasmobile>false</hasmobile><contact hasEmail="false"><phone areaCode="" number="0"/></contact><books><Row><title>The cat in the hat</title><author>Suess</author></Row><Row><title>Wolly the sheep</title><author></author></Row></books><colours><Item>Red</Item><Item>Yellow</Item></colours><endmarker>$$</endmarker></Row>'
+    ], { utf8 text; });
+
+contactRecord createFailure() := TRANSFORM
+    SELF.hasemail := TRUE;
+    SELF.email := FAILMESSAGE;
+    SELF := [];
+END;
+
+output(namesTable, { FROMXML(personRecord, text, TRIM, ONFAIL(createFailure())); });
+output(1);

ecllibrary/std/Date.ecl

@@ -1,4 +1,4 @@
-/*##############################################################################
+/*##############################################################################
 ## HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.  All rights reserved.
 ############################################################################## */
 
@@ -601,7 +601,8 @@ EXPORT Seconds_t SecondsFromDateTimeRec(DateTime_rec datetime, BOOLEAN is_local_
  * @param date_text     The string to be converted.
  * @param format        The format of the input string.
  *                      (See documentation for strftime)
- * @return              The date that was matched in the string.  Returns 0 if failed to match.
+ * @return              The date that was matched in the string.  Returns 0 if failed to match 
+ *                      or if the date components match but the result is an invalid date.
  *
  * Supported characters:
     %B          Full month name

esp/services/ws_workunits/ws_workunitsService.cpp

@@ -1973,6 +1973,7 @@ void doWUQueryFromArchive(IEspContext &context, const char* sashaServerIP, unsig
 
         void setFilterString()
         {
+            addToFilterString("wuid", req.getWuid());
             addToFilterString("cluster", req.getCluster());
             addToFilterString("owner", req.getOwner());
             addToFilterString("jobName", req.getJobname());
@@ -1996,6 +1997,8 @@ void doWUQueryFromArchive(IEspContext &context, const char* sashaServerIP, unsig
             cmd->setArchived(true);
             cmd->setStart(pageFrom);
             cmd->setLimit(pageSize+1); //read an extra WU to check hasMoreWU
+            if (notEmpty(req.getWuid()))
+                cmd->addId(req.getWuid());
             if (notEmpty(req.getCluster()))
                 cmd->setCluster(req.getCluster());
             if (notEmpty(req.getOwner()))

esp/src/eclwatch/HPCCPlatformWidget.js

@@ -20,10 +20,12 @@ define([
     "dojo/i18n!./nls/hpcc",
     "dojo/_base/array",
     "dojo/dom",
+    "dojo/dom-class",
     "dojo/dom-form",
     "dojo/dom-style",
     "dojo/dom-geometry",
     "dojo/cookie",
+    "dojo/topic",
 
     "dijit/registry",
     "dijit/Tooltip",
@@ -40,6 +42,7 @@ define([
     "hpcc/WsTopology",
     "hpcc/GraphWidget",
     "hpcc/DelayLoadWidget",
+    "hpcc/ws_machine",
 
     "dojo/text!../templates/HPCCPlatformWidget.html",
 
@@ -61,10 +64,10 @@ define([
     "hpcc/TableContainer",
     "hpcc/InfoGridWidget"
 
-], function (declare, lang, i18n, nlsHPCC, arrayUtil, dom, domForm, domStyle, domGeo, cookie,
+], function (declare, lang, i18n, nlsHPCC, arrayUtil, dom, domClass, domForm, domStyle, domGeo, cookie, topic,
                 registry, Tooltip,
                 UpgradeBar, ColorPicker,
-                _TabContainerWidget, ESPRequest, ESPActivity, WsAccount, WsAccess, WsSMC, WsTopology, GraphWidget, DelayLoadWidget,
+                _TabContainerWidget, ESPRequest, ESPActivity, WsAccount, WsAccess, WsSMC, WsTopology, GraphWidget, DelayLoadWidget, WsMachine,
                 template) {
     return declare("HPCCPlatformWidget", [_TabContainerWidget], {
         templateString: template,
@@ -83,6 +86,7 @@ define([
             this.mainPage = registry.byId(this.id + "_Main");
             this.errWarnPage = registry.byId(this.id + "_ErrWarn");
             this.pluginsPage = registry.byId(this.id + "_Plugins");
+            this.operationsPage = registry.byId(this.id + "_OPS");
             registry.byId(this.id + "SetBanner").set("disabled", true);
 
             this.upgradeBar = new UpgradeBar({
@@ -140,6 +144,16 @@ define([
                 return;
 
             var context = this;
+
+            WsMachine.GetComponentStatus({
+                request: {}
+            }).then(function (response) {
+                if (lang.exists("GetComponentStatusResponse.ComponentStatus", response)) {
+                    var status = response.GetComponentStatusResponse.ComponentStatus
+                    context.checkMonitoring(status);
+                }
+            });
+
             WsAccount.MyAccount({
             }).then(function (response) {
                 if (lang.exists("MyAccountResponse.username", response)) {
@@ -197,6 +211,10 @@ define([
             this.createStackControllerTooltip(this.id + "_OPS", this.i18n.Operations);
             this.createStackControllerTooltip(this.id + "_Plugins", this.i18n.Plugins);
             this.initTab();
+
+            topic.subscribe("hpcc/monitoring_component_update", function (topic) {
+                context.checkMonitoring(topic.status);
+            });
         },
 
         initTab: function () {
@@ -212,10 +230,18 @@ define([
             return "ECL Watch";
         },
 
+        checkMonitoring: function (status) {
+            if (status) {
+                domClass.remove("MonitorStatus", status);
+                domClass.add("MonitorStatus", status);
+            }
+        },
+
         checkIfAdmin: function (user) {
             var context = this;
             if(user == null){
                 registry.byId(context.id + "SetBanner").set("disabled", false);
+                dojo.destroy(this.monitorStatus);
             }else{
                 WsAccess.UserEdit({
                     suppressExceptionToaster: true,
@@ -309,7 +335,7 @@ define([
             }
             this.stackContainer.selectChild(this.widget._Config);
         },
-        
+
         _onOpenErrWarn: function (evt) {
             this.stackContainer.selectChild(this.errWarnPage);
         },
@@ -382,6 +408,13 @@ define([
             this.aboutDialog.hide();
         },
 
+        _onMonitoring: function (evt) {
+            this.stackContainer.selectChild(this.operationsPage);
+            this.operationsPage.ensureWidget().then(function (operationsPage) {
+                operationsPage.widget.TabContainer.selectChild(operationsPage.widget._Monitoring);
+            });
+        },
+
         _onSetBanner: function (evt) {
             registry.byId(this.id + "ShowBanner").set("value", this.activity.ShowBanner);
             dom.byId(this.id + "BannerContent").value = this.activity.BannerContent;

esp/src/eclwatch/MonitoringWidget.js

@@ -0,0 +1,169 @@
+/*##############################################################################
+#    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+############################################################################## */
+define([
+    "dojo/_base/declare",
+    "dojo/_base/lang",
+    "dojo/i18n",
+    "dojo/i18n!./nls/hpcc",
+    "dojo/_base/array",
+    "dojo/on",
+    "dojo/dom-class",
+    "dojo/topic",
+
+    "dijit/registry",
+    "dijit/form/ToggleButton",
+    "dijit/ToolbarSeparator",
+
+    "dgrid/tree",
+    "dgrid/extensions/ColumnHider",
+
+    "hpcc/GridDetailsWidget",
+    "hpcc/ws_machine",
+    "hpcc/ESPWorkunit",
+    "hpcc/DelayLoadWidget",
+    "hpcc/ESPUtil"
+
+], function (declare, lang, i18n, nlsHPCC, arrayUtil, on, domClass, topic,
+                registry, ToggleButton, ToolbarSeparator,
+                tree, ColumnHider,
+                GridDetailsWidget, WsMachine, ESPWorkunit, DelayLoadWidget, ESPUtil) {
+    return declare("MonitoringWidget", [GridDetailsWidget], {
+        i18n: nlsHPCC,
+
+        gridTitle: nlsHPCC.PrimaryMonitoring,
+        idProperty: "__hpcc_id",
+
+        init: function (params) {
+            var context = this;
+            if (this.inherited(arguments))
+                return;
+            this._refreshActionState();
+            this.refreshGrid();
+            this.startTimer();
+
+            topic.subscribe("hpcc/monitoring_component_update", function (topic) {
+                context.refreshGrid();
+            });
+        },
+
+        createGrid: function (domID) {
+            this.store.mayHaveChildren = function (item) {
+                if (item.StatusReports && item.StatusReports.StatusReport && item.StatusReports.StatusReport.length) {
+                    return true;
+                }
+                return false;
+            };
+
+            this.store.getChildren = function (parent, options) {
+               var retVal =  this.query({__hpcc_parentName: parent.__hpcc_id}, options);
+               return retVal;
+            };
+
+            var retVal = new declare([ESPUtil.Grid(false, true)])({
+                store: this.store,
+                sort: [{ attribute: "StatusID", descending: true },{ attribute: "Status", descending: true }],
+                columns: {
+                    StatusID: {label: "", width: 0, sortable: false, hidden: true},
+                    ComponentType: tree({
+                        label: "Name", sortable: true, width:200,
+                        formatter: function (Name, row) {
+                            switch (row.Status) {
+                                case "Normal":
+                                    return dojoConfig.getImageHTML("normal.png") + Name;
+                                case "Warning":
+                                    return dojoConfig.getImageHTML("warning.png") + Name;
+                                case "Error":
+                                    return dojoConfig.getImageHTML("error.png") + Name;
+                            }
+                            return "";
+                        }
+                    }),
+                    StatusDetails: {label: "Details", sortable: false},
+                    URL: {label: "URL", width: 200, sortable: false,
+                        formatter: function (Name, row) {
+                            if (Name) {
+                                return "<a href=http://"+Name+" target='_blank'>" + Name + "</a>";
+                            } else {
+                                return "";
+                            }
+                        }},
+                    EndPoint: {label: "IP", sortable: true, width:140},
+                    TimeReportedStr: {label: "Time Reported", width:140, sortable: true},
+                    Status: { label: this.i18n.Severity, width: 130, sortable: false,
+                        renderCell: function (object, value, node, options) {
+                            switch (value) {
+                                case "Error":
+                                    domClass.add(node, "ErrorCell");
+                                    break;
+                                case "Warning":
+                                    domClass.add(node, "WarningCell");
+                                    break;
+                                case "Normal":
+                                    domClass.add(node, "NormalCell");
+                                    break;
+                            }
+                            node.innerText = value;
+                        }
+                    }
+                }
+            }, domID);
+
+            return retVal;
+        },
+
+        refreshGrid: function () {
+            var context = this;
+
+                WsMachine.GetComponentStatus({
+                    request: {}
+                }).then(function (response) {
+                    var results = [];
+                    var newRows = [];
+                    if (lang.exists("GetComponentStatusResponse.ComponentStatusList.ComponentStatus", response)) {
+                        results = response.GetComponentStatusResponse.ComponentStatusList.ComponentStatus;
+                    }
+                    arrayUtil.forEach(results, function (row, idx) {
+                        lang.mixin(row, {
+                            __hpcc_parentName: null,
+                            __hpcc_id: row.ComponentType + row.EndPoint
+                        });
+
+                        arrayUtil.forEach(row.StatusReports.StatusReport, function (statusReport, idx) {
+                            newRows.push({
+                                __hpcc_parentName: row.ComponentType + row.EndPoint,
+                                __hpcc_id: row.ComponentType + row.EndPoint + "_" + idx,
+                                ComponentType: statusReport.Reporter,
+                                Status:statusReport.Status,
+                                StatusDetails:statusReport.StatusDetails,
+                                URL: statusReport.URL
+                            });
+                        });
+                    });
+
+                    arrayUtil.forEach(newRows, function (newRow) {
+                        results.push(newRow);
+                    });
+
+                    context.store.setData(results);
+                    context.grid.set("query", {__hpcc_parentName: null });
+                });
+        },
+
+        startTimer: function () {
+            WsMachine.MonitorComponentStatus({request: {}})
+        }
+    });
+});

esp/src/eclwatch/QuerySetDetailsWidget.js

@@ -21,6 +21,7 @@ define([
     "dojo/dom",
     "dojo/dom-attr",
     "dojo/promise/all",
+    "dojo/_base/array",
 
     "dijit/registry",
 
@@ -48,7 +49,7 @@ define([
     "dijit/TooltipDialog",
     "dijit/TitlePane"
 
-], function (declare, lang, i18n, nlsHPCC, dom, domAttr, all,
+], function (declare, lang, i18n, nlsHPCC, dom, domAttr, all, arrayUtil,
                 registry,
                 OnDemandGrid, Keyboard, Selection, selector, ColumnResizer, DijitRegistry,
                 ESPQuery, _TabContainerWidget, DelayLoadWidget,
@@ -187,6 +188,7 @@ define([
         },
 
         updateInput: function (name, oldValue, newValue) {
+           var context = this;
            var registryNode = registry.byId(this.id + name);
             if (registryNode) {
                 registryNode.set("value", newValue);
@@ -231,17 +233,6 @@ define([
                 this.graphsTab.set("tooltip", tooltip);
             } else if (name === "ResourceURLCount" && newValue) {
                 this.widget._Resources.set("title", this.i18n.Resources + " (" + newValue + ")");
-            } else if (name === "LogicalFiles") {
-                if (lang.exists("Item.length", newValue)) {
-                    this.logicalFilesTab.set("title", this.i18n.LogicalFiles + " (" + newValue.Item.length + ")");
-                    var tooltip = "";
-                    for (var i = 0; i < newValue.Item.length; ++i) {
-                        if (tooltip != "")
-                            tooltip += "\n";
-                        tooltip += newValue.Item[i];
-                    }
-                    this.logicalFilesTab.set("tooltip", tooltip);
-                }
             } else if (name === "SuperFiles") {
                 if (lang.exists("SuperFile.length", newValue)) {
                     this.superFilesTab.set("title", this.i18n.SuperFiles + " (" + newValue.SuperFile.length + ")");
@@ -253,6 +244,13 @@ define([
                     }
                     this.superFilesTab.set("tooltip", tooltip);
                 }
+                var count = 0;
+                arrayUtil.forEach(context.query.SuperFiles.SuperFile, function (item, idx) {
+                    arrayUtil.forEach(item.SubFiles.File, function (item, idx) {
+                        count++
+                    });
+                });
+                this.logicalFilesTab.set("title", this.i18n.LogicalFiles + " (" + count + ")");
             } else if (name === "LibrariesUsed") {
                 this.librariesUsedTab.set("title", this.i18n.LibrariesUsed + " (" + newValue.Item.length + ")");
                 var tooltip = "";

esp/src/eclwatch/QuerySetLogicalFilesWidget.js

@@ -54,8 +54,7 @@ define([
                 store: this.store,
                 columns: {
                     col1: selector({ width: 27, selectorType: 'checkbox' }),
-                    Name: {label: this.i18n.LogicalFiles
-                    }
+                    File: {label: this.i18n.LogicalFiles}
                 }
             }, domID);
             return retVal;
@@ -64,13 +63,13 @@ define([
         createDetail: function (id, row, params) {
             return new DelayLoadWidget({
                 id: id,
-                title: row.Name,
+                title: row.File,
                 closable: true,
                 delayWidget: "LFDetailsWidget",
                 hpcc: {
                     type: "LFDetailsWidget",
                     params: {
-                        Name: row.Name
+                        Name: row.File
                     }
                 }
             });
@@ -80,12 +79,14 @@ define([
             var context = this;
             this.query.refresh().then(function (response) {
                 var logicalFiles = [];
-                if (lang.exists("LogicalFiles.Item", context.query)) {
-                    arrayUtil.forEach(context.query.LogicalFiles.Item, function (item, idx) {
-                        var file = {
-                            Name: item
-                        }
-                        logicalFiles.push(file);
+                if (lang.exists("SuperFiles.SuperFile", context.query)) {
+                    arrayUtil.forEach(context.query.SuperFiles.SuperFile, function (item, idx) {
+                        arrayUtil.forEach(item.SubFiles.File, function (item, idx) {
+                            var file = {
+                                File: item
+                            }
+                            logicalFiles.push(file);
+                        });
                     });
                 }
                 context.store.setData(logicalFiles);

esp/src/eclwatch/css/hpcc.css

@@ -472,6 +472,50 @@ hr.dashedLine {
     width: 16px;
     height: 16px;
 }
+
+.None{
+    background: url("../img/nodata.png") no-repeat; 
+    width: 16px;
+    height: 16px;
+    display: block;
+    float: left;
+    margin-top: 17px;
+    padding-right: 0px;
+    cursor: pointer;
+}
+
+.Normal{
+    background: url("../img/normal.png") no-repeat; 
+    width: 16px;
+    height: 16px;
+    display: block;
+    float: left;
+    margin-top: 17px;
+    padding-right: 0px;
+    cursor: pointer;
+}
+
+.Warning{
+    background: url("../img/warning.png") no-repeat; 
+    width: 16px;
+    height: 16px;
+    display: block;
+    float: left;
+    margin-top: 17px;
+    padding-right: 0px;
+    cursor: pointer;
+}
+
+.Error{
+    background: url("../img/error.png") no-repeat; 
+    width: 16px;
+    height: 16px;
+    display: block;
+    float: left;
+    margin-top: 17px;
+    padding-right: 0px;
+    cursor: pointer;
+}
 /*###############################*/
 #appLayout {
     height: 100%;
@@ -1006,6 +1050,10 @@ margin-left:-20px;
     background: yellow;
 }
 
+.NormalCell{
+    background: rgba(0,181,0,1);
+}
+
 .Prompt{
     font-weight:bold;
     white-space: nowrap;

esp/src/eclwatch/nls/hpcc.js

@@ -266,6 +266,7 @@ define({root:
     Modification: "Modification",
     ModifiedUTCGMT: "Modified (UTC/GMT)",
     Modify: "Modify",
+    Monitoring: "Monitoring",
     MonitorEventName: "Monitor Event Name",
     MonitorShotLimit: "Monitor Shot Limit",
     MonitorSub: "Monitor Sub",
@@ -347,6 +348,7 @@ define({root:
     PrefixPlaceholder: "filename{:length}, filesize{:[B|L][1-8]}",
     PreserveCompression: "Preserve Compression",
     Preview: "Preview",
+    PrimaryMonitoring: "Primary Monitoring",
     Priority: "Priority",
     Process: "Process",
     ProcessFilter: "Process Filter",
@@ -480,6 +482,7 @@ define({root:
     TargetClusters: "Target Clusters",
     TargetName: "Target Name",
     TargetNamePlaceholder: "some::logical::name",
+    TargetScope: "Target Scope",
     TargetWuid: "Target/Wuid",
     Terminators: "Terminators",
     TestPages: "Test Pages",
@@ -614,4 +617,4 @@ define({root:
 "pt-br": true,
 "sr": true,
 "zh": true
-});
+});

esp/src/eclwatch/templates/HPCCPlatformOpsWidget.html

@@ -18,6 +18,8 @@
             </div>
             <div id="${id}_Resources" title="${i18n.Resources}" style="padding: 0px; border:0px; border-color:none; overflow: hidden" data-dojo-type="dijit.layout.ContentPane">
             </div>
+            <div id="${id}_Monitoring" title="${i18n.Monitoring}" style="padding: 0px; border:0px; border-color:none; overflow: hidden" data-dojo-props="delayWidget: 'MonitoringWidget'" data-dojo-type="DelayLoadWidget">
+            </div>
         </div>
     </div>
 </div>

esp/src/eclwatch/templates/HPCCPlatformWidget.html

@@ -3,8 +3,8 @@
         <div id="${id}Titlebar" class="hpccTitlebar" data-dojo-props="region: 'top'" data-dojo-type="dijit.layout.ContentPane">
             <div id="${id}StackController" class="left glow" data-dojo-props="containerId:'${id}TabContainer'" data-dojo-type="dijit.layout.StackController"></div>
             <div class="searchUserMoreComponents">
-                <div id="${id}plugin_32-32" class="left" style="margin-top:9px;width:32px;height:32px;overflow:hidden">
-                </div>
+                <div id="${id}plugin_32-32" class="left" style="margin-top:9px;width:32px;height:32px;overflow:hidden"></div>
+                <div id="MonitorStatus" class="None" data-dojo-attach-event="onClick:_onMonitoring"></div>
                 <div class="seperator grey"></div>
                 <form id="search-form" onsubmit="return false;">
                     <input id="${id}FindText" class="roundForm" data-dojo-props="placeHolder: '${i18n.PlaceholderFindText}', trim: true" data-dojo-type="dijit.form.TextBox" />

esp/src/eclwatch/templates/LZBrowseWidget.html

@@ -36,7 +36,7 @@
                                     <legend>${i18n.Target}</legend>
                                     <div data-dojo-type="hpcc.TableContainer">
                                         <input id="${id}SprayFixedDestination" title="${i18n.Group}:" style="width: 95%;" name="destGroup" data-dojo-type="TargetSelectWidget" />
-                                        <input id="${id}SprayFixedDestinationNamePrefix" title="${i18n.NamePrefix}:" style="width: 95%;" name="namePrefix" data-dojo-props="trim: true, placeHolder:'${i18n.NamePrefixPlaceholder}'" data-dojo-type="dijit.form.TextBox" />
+                                        <input id="${id}SprayFixedDestinationNamePrefix" title="${i18n.TargetScope}:" style="width: 95%;" name="namePrefix" data-dojo-props="trim: true, placeHolder:'${i18n.NamePrefixPlaceholder}'" data-dojo-type="dijit.form.TextBox" />
                                     </div>
                                     <div id="${id}SprayFixedGrid" data-dojo-type="SelectionGridWidget">
                                     </div>
@@ -65,7 +65,7 @@
                                     <legend>${i18n.Target}</legend>
                                     <div data-dojo-type="hpcc.TableContainer">
                                         <input id="${id}SprayDelimitedDestination" title="${i18n.Group}:" style="width: 95%;" name="destGroup" data-dojo-type="TargetSelectWidget" />
-                                        <input title="${i18n.NamePrefix}:" style="width: 95%;" name="namePrefix" data-dojo-props="trim: true, placeHolder:'${i18n.NamePrefixPlaceholder}'" data-dojo-type="dijit.form.TextBox" />
+                                        <input title="${i18n.TargetScope}:" style="width: 95%;" name="namePrefix" data-dojo-props="trim: true, placeHolder:'${i18n.NamePrefixPlaceholder}'" data-dojo-type="dijit.form.TextBox" />
                                     </div>
                                     <div id="${id}SprayDelimitedGrid" data-dojo-type="SelectionGridWidget">
                                     </div>
@@ -113,7 +113,7 @@
                                     <legend>${i18n.Target}</legend>
                                     <div data-dojo-type="hpcc.TableContainer">
                                         <input id="${id}SprayXmlDestinationSelect" title="${i18n.Group}:" style="width: 95%;" name="destGroup" data-dojo-type="TargetSelectWidget" />
-                                        <input title="${i18n.NamePrefix}:" style="width: 95%;" name="namePrefix" data-dojo-props="trim: true, placeHolder:'${i18n.NamePrefixPlaceholder}'" data-dojo-type="dijit.form.TextBox" />
+                                        <input title="${i18n.TargetScope}:" style="width: 95%;" name="namePrefix" data-dojo-props="trim: true, placeHolder:'${i18n.NamePrefixPlaceholder}'" data-dojo-type="dijit.form.TextBox" />
                                     </div>
                                     <div id="${id}SprayXmlGrid" data-dojo-type="SelectionGridWidget">
                                     </div>
@@ -153,7 +153,7 @@
                                     <legend>${i18n.Target}</legend>
                                     <div data-dojo-type="hpcc.TableContainer">
                                         <input id="${id}SprayJsonDestinationSelect" title="${i18n.Group}:" style="width: 95%;" name="destGroup" data-dojo-type="TargetSelectWidget" />
-                                        <input title="${i18n.NamePrefix}:" style="width: 95%;" name="namePrefix" data-dojo-props="trim: true, placeHolder:'${i18n.NamePrefixPlaceholder}'" data-dojo-type="dijit.form.TextBox" />
+                                        <input title="${i18n.TargetScope}:" style="width: 95%;" name="namePrefix" data-dojo-props="trim: true, placeHolder:'${i18n.NamePrefixPlaceholder}'" data-dojo-type="dijit.form.TextBox" />
                                     </div>
                                     <div id="${id}SprayJsonGrid" data-dojo-type="SelectionGridWidget">
                                     </div>
@@ -193,7 +193,7 @@
                                     <legend>${i18n.Target}</legend>
                                     <div data-dojo-type="hpcc.TableContainer">
                                         <input id="${id}SprayVariableDestination" title="${i18n.Group}:" style="width: 95%;" name="destGroup" data-dojo-type="TargetSelectWidget" />
-                                        <input title="${i18n.NamePrefix}:" style="width: 95%;" name="namePrefix" data-dojo-props="trim: true, placeHolder:'${i18n.NamePrefixPlaceholder}'" data-dojo-type="dijit.form.TextBox" />
+                                        <input title="${i18n.TargetScope}:" style="width: 95%;" name="namePrefix" data-dojo-props="trim: true, placeHolder:'${i18n.NamePrefixPlaceholder}'" data-dojo-type="dijit.form.TextBox" />
                                     </div>
                                     <div id="${id}SprayVariableGrid" data-dojo-type="SelectionGridWidget">
                                     </div>

esp/src/eclwatch/templates/MonitoringWidget.html

@@ -0,0 +1,17 @@
+<div class="${baseClass}">
+    <div id="${id}BorderContainer" class="${baseClass}BorderContainer" style="width: 100%; height: 100%;" data-dojo-type="dijit.layout.BorderContainer">
+        <div id="${id}TabContainer" data-dojo-props="region: 'center', tabPosition: 'top'" style="width: 100%; height: 100%" data-dojo-type="dijit.layout.TabContainer">
+            <div id="${id}_Monitoring" style="width: 100%; height: 100%" data-dojo-type="dijit.layout.BorderContainer">
+                <div id="${id}Toolbar" class="topPanel" data-dojo-props="region: 'top'" data-dojo-type="dijit.Toolbar">
+                    <div id="${id}Refresh" data-dojo-attach-event="onClick:_onRefresh" data-dojo-props="iconClass:'iconRefresh'" data-dojo-type="dijit.form.Button">${i18n.Refresh}</div>
+                    <span data-dojo-type="dijit.ToolbarSeparator"></span>
+                    <div id="${id}NewPage" class="right" data-dojo-attach-event="onClick:_onNewPage" data-dojo-props="iconClass:'iconNewPage', showLabel:false" data-dojo-type="dijit.form.Button">${i18n.OpenInNewPage}</div>
+                </div>
+                <div id="${id}LandingZonesGridCP" style="border:0px; padding: 0px; border-color:none" data-dojo-props="region: 'center'" data-dojo-type="dijit.layout.ContentPane">
+                    <div id="${id}LandingZonesGrid">
+                    </div>
+                </div>
+            </div>
+        </div>
+    </div>
+</div>

esp/src/eclwatch/ws_machine.js

@@ -0,0 +1,70 @@
+/*##############################################################################
+#    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+############################################################################## */
+define([
+    "dojo/_base/declare",
+    "dojo/_base/lang",
+    "dojo/_base/array",
+    "dojo/topic",
+    "dojo/store/JsonRest",
+    "dojo/store/Memory",
+    "dojo/store/Cache",
+    "dojo/store/Observable",
+
+    "hpcc/ESPRequest",
+    "hpcc/ws_machine"
+], function (declare, lang, arrayUtil, topic, JsonRest, Memory, Cache, Observable,
+    ESPRequest, WsMachine) {
+
+    var NagiosStore = declare([ESPRequest.Store], {
+        service: "ws_machine",
+        action: "GetComponentStatus",
+        responseQualifier: "GetComponentStatusResponse.ComponentStatusList.ComponentStatus",
+        idProperty: "__hpcc_id"
+    });
+
+    var monitorHandle;
+    return {
+        GetComponentStatus: function (params) {
+            return ESPRequest.send("ws_machine", "GetComponentStatus", params);
+        },
+
+        MonitorComponentStatus: function (params) {
+            var prevResponse = null;
+            if (!monitorHandle) {
+                var context = this;
+                monitorHandle = setInterval(function() {
+                    context.GetComponentStatus(params).then(function(response) {
+                        if (response && response.GetComponentStatusResponse.ComponentStatus) {
+                            response.GetComponentStatusResponse.ComponentStatusList.ComponentStatus.forEach(function(row) {
+                                topic.publish("hpcc/monitoring_component_update", {
+                                    response: response,
+                                    status: response.GetComponentStatusResponse.ComponentStatus
+                                });
+                            });
+                        }
+                        prevResponse = response;
+                    });
+                }, 60000);
+            }
+            return prevResponse;
+        },
+
+        CreateNagiosStore: function (options) {
+            var store = new NagiosStore(options);
+            return Observable(store);
+        }
+    };
+});

initfiles/componentfiles/configxml/thor.xsd.in

@@ -396,6 +396,13 @@
           </xs:appinfo>
         </xs:annotation>
       </xs:attribute>
+      <xs:attribute name="fileCacheLimit" type="xs:nonNegativeInteger" default="1800">
+        <xs:annotation>
+          <xs:appinfo>
+            <tooltip>File Cache limit (in MB). It will default to 1800 if unset</tooltip>
+          </xs:appinfo>
+        </xs:annotation>
+      </xs:attribute>
       <xs:attribute name="globalMemorySize" type="xs:nonNegativeInteger" use="optional">
         <xs:annotation>
           <xs:appinfo>

plugins/fileservices/fileservices.cpp

@@ -521,7 +521,6 @@ static void blockUntilComplete(const char * label, IClientFileSpray &server, ICo
 
     while(true)
     {
-        Owned<IWorkUnit> wu = ctx->updateWorkUnit(); // may return NULL
 
         Owned<IClientGetDFUWorkunit> req = server.createGetDFUWorkunitRequest();
         req->setWuid(wuid);
@@ -536,8 +535,10 @@ static void blockUntilComplete(const char * label, IClientFileSpray &server, ICo
         }
 
         IConstDFUWorkunit & dfuwu = result->getResult();
-
+        bool aborting = false;
+        Owned<IWorkUnit> wu = ctx->updateWorkUnit(); // may return NULL
         if (wu.get()) { // if updatable (e.g. not hthor with no agent context)
+            aborting = wu->aborting();
             StringBuffer wuScope, ElapsedLabel, RemainingLabel;
             wuScope.appendf("%s-%s", label, dfuwu.getID());
             ElapsedLabel.append(wuScope).append(" (Elapsed) ");
@@ -548,6 +549,7 @@ static void blockUntilComplete(const char * label, IClientFileSpray &server, ICo
             updateWorkunitTimeStat(wu, SSTdfuworkunit, wuScope, StTimeRemaining, RemainingLabel, milliToNano(dfuwu.getSecsLeft()*1000));
             wu->setApplicationValue(label, dfuwu.getID(), dfuwu.getSummaryMessage(), true);
             wu->commit();
+            wu.clear();
         }
 
         DFUstate state = (DFUstate)dfuwu.getState();
@@ -570,29 +572,23 @@ static void blockUntilComplete(const char * label, IClientFileSpray &server, ICo
         case DFUstate_aborted:
         case DFUstate_failed:
             throw MakeStringException(0, "DFUServer Error %s", dfuwu.getSummaryMessage());
-            return;
 
         case DFUstate_finished:
             return;
         }
 
-        if (wu.get()&&wu->aborting())
+        if (aborting)
         {
             Owned<IClientAbortDFUWorkunit> abortReq = server.createAbortDFUWorkunitRequest();
             abortReq->setWuid(wuid);
             Linked<IClientAbortDFUWorkunitResponse> abortResp = server.AbortDFUWorkunit(abortReq);
 
-            {   //  Add warning of DFU Abort Request - should this be information  ---
-                StringBuffer s("DFU Workunit Abort Requested for ");
-                s.append(wuid);
-                WUmessage(ctx,SeverityWarning,"blockUntilComplete",s.str());
-            }
-
-            wu->setState(WUStateAborting);
+            //  Add warning of DFU Abort Request - should this be information  ---
+            StringBuffer s("DFU Workunit Abort Requested for ");
+            s.append(wuid);
+            WUmessage(ctx,SeverityWarning,"blockUntilComplete",s.str());
             throw MakeStringException(0, "Workunit abort request received");
-
         }
-        wu.clear();
 
         if (time.timedout()) {
             unsigned left = dfuwu.getSecsLeft();

roxie/ccd/CMakeLists.txt

@@ -83,6 +83,7 @@ include_directories (
          ./../../testing/unittests
          ${CMAKE_BINARY_DIR}
          ${CMAKE_BINARY_DIR}/oss
+         ${HPCC_SOURCE_DIR}/dali/ft
     )
 
 ADD_DEFINITIONS( -D_USRDLL -DCCD_EXPORTS -DSTARTQUERY_EXPORTS )

roxie/ccd/ccdserver.cpp

@@ -42,6 +42,7 @@
 
 #include "dafdesc.hpp"
 #include "dautils.hpp"
+#include "ftbase.ipp"
 
 namespace ccdserver_hqlhelper
 {
@@ -10404,7 +10405,7 @@ public:
         CRoxieServerDiskWriteActivity::start(parentExtractSize, parentExtract, paused);
         OwnedRoxieString xmlpath(xmlHelper.getXmlIteratorPath());
         if (!xmlpath)
-            rowTag.set("Row");
+            rowTag.set(DEFAULTXMLROWTAG);
         else
         {
             const char *path = xmlpath;
@@ -10419,14 +10420,13 @@ public:
         StringBuffer header;
         OwnedRoxieString suppliedHeader(xmlHelper.getHeader());
         if (kind==TAKjsonwrite)
-        {
             buildJsonHeader(header, suppliedHeader, rowTag);
-            headerLength = header.length();
-        }
         else if (suppliedHeader)
             header.set(suppliedHeader);
         else
-            header.set("<Dataset>\n");
+            header.append(DEFAULTXMLHEADER).newline();
+
+        headerLength = header.length();
         diskout->write(header.length(), header.str());
 
         Owned<IXmlWriterExt> writer = createIXmlWriterExt(xmlHelper.getXmlFlags(), 0, NULL, (kind==TAKjsonwrite) ? WTJSON : WTStandard);
@@ -10447,15 +10447,13 @@ public:
         OwnedRoxieString suppliedFooter(xmlHelper.getFooter());
         StringBuffer footer;
         if (kind==TAKjsonwrite)
-        {
             buildJsonFooter(footer.newline(), suppliedFooter, rowTag);
-            footerLength=footer.length();
-        }
         else if (suppliedFooter)
             footer.append(suppliedFooter);
         else
-            footer.append("</Dataset>");
+            footer.append(DEFAULTXMLFOOTER);
 
+        footerLength=footer.length();
         diskout->write(footer.length(), footer);
     }
 
@@ -10471,10 +10469,9 @@ public:
         desc->queryProperties().setProp("@format","utf8n");
         desc->queryProperties().setProp("@rowTag",rowTag.get());
         desc->queryProperties().setProp("@kind", (kind==TAKjsonwrite) ? "json" : "xml");
-        if (headerLength)
-            desc->queryProperties().setPropInt("@headerLength", headerLength);
-        if (footerLength)
-            desc->queryProperties().setPropInt("@footerLength", footerLength);
+
+        desc->queryProperties().setPropInt(FPheaderLength, headerLength);
+        desc->queryProperties().setPropInt(FPfooterLength, footerLength);
     }
 
     virtual bool isOutputTransformed() const { return true; }
@@ -19177,7 +19174,7 @@ public:
                     if (response->mlFmt==MarkupFmt_JSON)
                         writeFlags |= XWFnoindent;
                     writer.setown(createIXmlWriterExt(writeFlags, 1, response, (response->mlFmt==MarkupFmt_JSON) ? WTJSON : WTStandard));
-                    writer->outputBeginArray("Row");
+                    writer->outputBeginArray(DEFAULTXMLROWTAG);
                 }
             }
 
@@ -19254,9 +19251,9 @@ public:
                 }
                 else if (writer)
                 {
-                    writer->outputBeginNested("Row", false);
+                    writer->outputBeginNested(DEFAULTXMLROWTAG, false);
                     helper.serializeXml((byte *) row, *writer);
-                    writer->outputEndNested("Row");
+                    writer->outputEndNested(DEFAULTXMLROWTAG);
                 }
                 else
                 {
@@ -19283,7 +19280,7 @@ public:
             }
         }
         if (writer)
-            writer->outputEndArray("Row");
+            writer->outputEndArray(DEFAULTXMLROWTAG);
         if (saveInContext)
             serverContext->appendResultDeserialized(storedName, sequence, builder.getcount(), builder.linkrows(), (helper.getFlags() & POFextend) != 0, LINK(meta.queryOriginal()));
         if (workunit)

roxie/ccd/ccdsnmp.cpp

@@ -887,10 +887,10 @@ class CQueryStatsAggregator : public CInterface, implements IQueryStatsAggregato
     }
     
     static CQueryStatsAggregator globalStatsAggregator;
-    static CIArrayOf<CQueryStatsAggregator> queryStatsAggregators;
     static SpinLock queryStatsCrit;
 
 public:
+    static CIArrayOf<CQueryStatsAggregator> queryStatsAggregators;
     virtual void Link(void) const { CInterface::Link(); }
     virtual bool Release(void) const    
     {
@@ -912,7 +912,13 @@ public:
         expirySeconds = _expirySeconds;
         queryStatsAggregators.append(*LINK(this));
     }
-
+    ~CQueryStatsAggregator()
+    {
+        while (recent.ordinality())
+        {
+            recent.dequeue()->Release();
+        }
+    }
     static IPropertyTree *getAllQueryStats(bool includeQueries, time_t from, time_t to)
     {
         Owned<IPTree> result = createPTree("QueryStats");
@@ -1002,8 +1008,8 @@ public:
     }
 };
 
-CQueryStatsAggregator CQueryStatsAggregator::globalStatsAggregator(NULL, SLOT_LENGTH);
 CIArrayOf<CQueryStatsAggregator> CQueryStatsAggregator::queryStatsAggregators;
+CQueryStatsAggregator CQueryStatsAggregator::globalStatsAggregator(NULL, SLOT_LENGTH);
 SpinLock CQueryStatsAggregator::queryStatsCrit;
 
 IQueryStatsAggregator *queryGlobalQueryStatsAggregator()

system/jlib/jstats.cpp

@@ -1046,6 +1046,7 @@ public:
         {
             Statistic * next = Statistic::deserialize(in, version);
             stats.append(*next);
+            next->Release();
         }
 
         unsigned numChildren;
@@ -1149,7 +1150,8 @@ public:
 //other public interface functions
     void addStatistic(StatisticKind kind, unsigned __int64 value)
     {
-        stats.append(*new Statistic(kind, value));
+        Statistic s(kind, value);
+        stats.append(s);
     }
 
     void updateStatistic(StatisticKind kind, unsigned __int64 value, StatsMergeAction mergeAction)
@@ -1166,7 +1168,8 @@ public:
                 }
             }
         }
-        stats.append(*new Statistic(kind, value));
+        Statistic s(kind, value);
+        stats.append(s);
     }
 
     CStatisticCollection * ensureSubScope(const StatsScopeId & search, bool hasChildren)
@@ -1357,6 +1360,7 @@ public:
     }
     virtual void endScope()
     {
+        scopes.tos().Release();
         scopes.pop();
     }
     virtual void addStatistic(StatisticKind kind, unsigned __int64 value)

testing/regress/ecl/fromxml5.ecl

@@ -0,0 +1,71 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    This program is free software: you can redistribute it and/or modify
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+phoneRecord :=
+            RECORD
+string5         areaCode{xpath('@areaCode')};
+udecimal12      number{xpath('@number')};
+            END;
+
+contactrecord :=
+            RECORD
+phoneRecord     phone;
+boolean         hasemail{xpath('@hasEmail')};
+                ifblock(self.hasemail)
+string              email;
+                end;
+            END;
+
+bookRec :=
+    RECORD
+unicode     title;
+string      author;
+    END;
+
+personRecord :=
+            RECORD
+unicode20       surname;
+unicode10       forename;
+phoneRecord     homePhone;
+boolean         hasMobile;
+                ifblock(self.hasMobile)
+phoneRecord         mobilePhone;
+                end;
+contactRecord   contact;
+dataset(bookRec) books;
+set of string    colours;
+string2         endmarker := '$$';
+            END;
+
+namesTable := dataset([
+    u8'<Row><surname>Hälliday</surname><forename>Gavin</forename><homephone areaCode="09876" number="123987"/><hasmobile>true</hasmobile><mobilephone areaCode="07967" number="123987"/>' +
+        u8'<contact hasEmail="true"><phone areaCode="n/a" number="0"/><email>gavin@edata.com</email></contact>' +
+        u8'<books><Row><title>εν αρχη ην ο λογος</title><author>john</author></Row><Row><title>To kill a mocking bird</title><author>Lee</author></Row><Row><title>Zen and the art of motorcycle maintainence</title><author>Pirsig</author></Row></books><colours><All/></colours><endmarker>$$</endmarker></Row>',
+    u8'<Row><unmatchedtag></Row>',
+    u8'<Row><surname>Halliday</surname><forename>Abigäil</forename><homephone areaCode="09876" number="123987"/><hasmobile>false</hasmobile><contact hasEmail="false"><phone areaCode="" number="0"/></contact><books><Row><title>The cat in the hat</title><author>Suess</author></Row><Row><title>Wolly the sheep</title><author></author></Row></books><colours><Item>Red</Item><Item>Yellow</Item></colours><endmarker>$$</endmarker></Row>'
+    ], { utf8 text; });
+
+personRecord createFailure() := TRANSFORM
+    SELF.surname := '**FAIL**';
+    SELF.forename := (UNICODE)FAILCODE;
+    SELF.contact.hasemail := TRUE;
+    SELF.contact.email := FAILMESSAGE;
+    SELF := [];
+END;
+
+output(namesTable, { FROMXML(personRecord, text, TRIM, ONFAIL(createFailure())); });
+output(1);

thorlcr/activities/activitymasters_lcr.cmake

@@ -89,7 +89,8 @@ include_directories (
          ./../mfilemanager 
          ./../../common/thorhelper 
          ./../activities 
-         ./../../rtl/eclrtl 
+         ./../../rtl/eclrtl
+         ${HPCC_SOURCE_DIR}/dali/ft
     )
 
 HPCC_ADD_LIBRARY( activitymasters_lcr SHARED ${SRCS} )

thorlcr/activities/activityslaves_lcr.cmake

@@ -106,6 +106,7 @@ include_directories (
          ./../activities 
          ./../../rtl/eclrtl 
          ./../../roxie/roxiemem
+         ${HPCC_SOURCE_DIR}/dali/ft
     )
 
 HPCC_ADD_LIBRARY( activityslaves_lcr SHARED ${SRCS} )

thorlcr/activities/xmlwrite/thxmlwrite.cpp

@@ -23,6 +23,7 @@
 #include "eclhelper.hpp"
 #include "deftype.hpp"
 #include "thxmlwrite.ipp"
+#include "ftbase.ipp"
 
 class CXmlWriteActivityMaster : public CWriteMasterBase
 {
@@ -45,7 +46,7 @@ public:
         StringBuffer rowTag;
         OwnedRoxieString xmlpath(helper->getXmlIteratorPath());
         if (!xmlpath)
-            rowTag.append("Row");
+            rowTag.append(DEFAULTXMLROWTAG);
         else
         {
             const char *path = xmlpath;
@@ -61,10 +62,26 @@ public:
         {
             StringBuffer s;
             OwnedRoxieString supplied(helper->getHeader());
-            props.setPropInt("@headerLength", buildJsonHeader(s, supplied, rowTag).length());
+            props.setPropInt(FPheaderLength, buildJsonHeader(s, supplied, rowTag).length());
 
             supplied.set(helper->getFooter());
-            props.setPropInt("@footerLength", buildJsonFooter(s.clear(), supplied, rowTag).length());
+            props.setPropInt(FPfooterLength, buildJsonFooter(s.clear(), supplied, rowTag).length());
+        }
+        else
+        {
+            StringBuffer supplied(helper->getHeader());
+            size32_t headerLength = supplied.length();
+            if (headerLength == 0 )
+                headerLength = supplied.set(DEFAULTXMLHEADER).newline().length();
+
+            props.setPropInt(FPheaderLength, headerLength);
+
+            supplied.set(helper->getFooter());
+            size32_t footerLength = supplied.length();
+            if (footerLength == 0 )
+                footerLength = supplied.set(DEFAULTXMLFOOTER).newline().length();
+
+            props.setPropInt(FPfooterLength, footerLength);
         }
     }
 };

thorlcr/activities/xmlwrite/thxmlwriteslave.cpp

@@ -44,7 +44,7 @@ public:
         OwnedRoxieString xmlpath(helper->getXmlIteratorPath());
         if (!xmlpath)
         {
-            rowTag.append("Row");
+            rowTag.append(DEFAULTXMLROWTAG);
         }
         else
         {
@@ -63,7 +63,7 @@ public:
             else if (suppliedHeader)
                 out.set(suppliedHeader);
             else
-                out.set("<Dataset>").newline();
+                out.set(DEFAULTXMLHEADER).newline();
             outraw->write(out.length(), out.str());
             if (calcFileCrc)
                 fileCRC.tally(out.length(), out.str());
@@ -91,7 +91,7 @@ public:
             else if (suppliedFooter)
                 out.set(suppliedFooter);
             else
-                out.set("</Dataset>").newline();
+                out.set(DEFAULTXMLFOOTER).newline();
             outraw->write(out.length(), out.str());
             if (calcFileCrc)
                 fileCRC.tally(out.length(), out.str());