Merge remote-tracking branch 'origin/closedown-5.0.x'

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

cmake_modules/FindR.cmake

@@ -40,6 +40,10 @@ IF (NOT R_FOUND)
   FIND_LIBRARY (RCPP_LIBRARY NAMES ${Rcpp_lib} PATHS /usr/lib /usr/share /usr/lib64 /usr/local/lib /usr/local/lib64 PATH_SUFFIXES R/library/Rcpp/lib/ R/site-library/Rcpp/lib/)
   FIND_LIBRARY (RINSIDE_LIBRARY NAMES ${RInside_lib} PATHS /usr/lib /usr/share /usr/lib64 /usr/local/lib /usr/local/lib64 PATH_SUFFIXES R/library/RInside/lib/ R/site-library/RInside/lib/)
 
+  IF (RCPP_LIBRARY STREQUAL "RCPP_LIBRARY-NOTFOUND")
+    SET (RCPP_LIBRARY "")    # Newer versions of Rcpp are header-only, with no associated library.
+  ENDIF()
+
   SET (R_INCLUDE_DIRS ${R_INCLUDE_DIR} ${RINSIDE_INCLUDE_DIR} ${RCPP_INCLUDE_DIR})
   SET (R_LIBRARIES ${R_LIBRARY} ${RINSIDE_LIBRARY} ${RCPP_LIBRARY})
 

cmake_modules/commonSetup.cmake

@@ -84,7 +84,7 @@ IF ("${COMMONSETUP_DONE}" STREQUAL "")
 
   option(USE_OPTIONAL "Automatically disable requested features with missing dependencies" ON)
 
-  if ( USE_PYTHON OR USE_V8 OR USE_JNI OR USE_RINSIDE OR USE_SQLITE3)
+  if ( USE_PYTHON OR USE_V8 OR USE_JNI OR USE_RINSIDE OR USE_SQLITE3 OR USE_MYSQL)
       set( WITH_PLUGINS ON )
   endif()
 
@@ -623,16 +623,16 @@ IF ("${COMMONSETUP_DONE}" STREQUAL "")
         endif()
       endif(USE_OPENSSL)
 
-      if(USE_MYSQL)
+      if(USE_MYSQL_REPOSITORY)
         find_package(MYSQL)
         if (MYSQL_FOUND)
-          add_definitions (-D_USE_MYSQL)
+          add_definitions (-D_USE_MYSQL_REPOSITORY)
         else()
           message(FATAL_ERROR "MYSQL requested but package not found")
         endif()
       else()
-        add_definitions (-D_NO_MYSQL)
-      endif(USE_MYSQL)
+        add_definitions (-D_NO_MYSQL_REPOSITORY)
+      endif(USE_MYSQL_REPOSITORY)
 
       if(USE_APR)
         find_package(APR)

common/fileview2/fileview.hpp

@@ -189,11 +189,11 @@ extern FILEVIEW_API IResultSetFactory * getSecRemoteResultSetFactory(const char
 extern FILEVIEW_API IResultSetFactory * getRemoteResultSetFactory(const char * remoteServer, const char * username, const char * password);
 extern FILEVIEW_API int findResultSetColumn(const INewResultSet * results, const char * columnName);
 
-extern FILEVIEW_API unsigned getResultCursorXml(IStringVal & ret, IResultSetCursor * cursor, const char * name, unsigned start=0, unsigned count=0, const char * schemaName=NULL);
-extern FILEVIEW_API unsigned getResultXml(IStringVal & ret, INewResultSet * cursor,  const char* name, unsigned start=0, unsigned count=0, const char * schemaName=NULL);
+extern FILEVIEW_API unsigned getResultCursorXml(IStringVal & ret, IResultSetCursor * cursor, const char * name, unsigned start=0, unsigned count=0, const char * schemaName=NULL, const IProperties *xmlns=NULL);
+extern FILEVIEW_API unsigned getResultXml(IStringVal & ret, INewResultSet * cursor,  const char* name, unsigned start=0, unsigned count=0, const char * schemaName=NULL, const IProperties *xmlns=NULL);
 extern FILEVIEW_API unsigned getResultJSON(IStringVal & ret, INewResultSet * cursor,  const char* name, unsigned start=0, unsigned count=0, const char * schemaName=NULL);
-extern FILEVIEW_API unsigned writeResultCursorXml(IXmlWriter & writer, IResultSetCursor * cursor, const char * name, unsigned start=0, unsigned count=0, const char * schemaName=NULL);
-extern FILEVIEW_API unsigned writeResultXml(IXmlWriter & writer, INewResultSet * cursor,  const char* name, unsigned start=0, unsigned count=0, const char * schemaName=NULL);
+extern FILEVIEW_API unsigned writeResultCursorXml(IXmlWriter & writer, IResultSetCursor * cursor, const char * name, unsigned start=0, unsigned count=0, const char * schemaName=NULL, const IProperties *xmlns = NULL);
+extern FILEVIEW_API unsigned writeResultXml(IXmlWriter & writer, INewResultSet * cursor,  const char* name, unsigned start=0, unsigned count=0, const char * schemaName=NULL, const IProperties *xmlns = NULL);
 
 extern FILEVIEW_API unsigned getResultCursorBin(MemoryBuffer & ret, IResultSetCursor * cursor, unsigned start=0, unsigned count=0);
 extern FILEVIEW_API unsigned getResultBin(MemoryBuffer & ret, INewResultSet * cursor, unsigned start=0, unsigned count=0);

common/fileview2/fvresultset.cpp

@@ -3168,19 +3168,19 @@ int findResultSetColumn(const INewResultSet * results, const char * columnName)
 }
 
 
-extern FILEVIEW_API unsigned getResultCursorXml(IStringVal & ret, IResultSetCursor * cursor, const char * name, unsigned start, unsigned count, const char * schemaName)
+extern FILEVIEW_API unsigned getResultCursorXml(IStringVal & ret, IResultSetCursor * cursor, const char * name, unsigned start, unsigned count, const char * schemaName, const IProperties *xmlns)
 {
     Owned<CommonXmlWriter> writer = CreateCommonXmlWriter(XWFexpandempty);
-    unsigned rc = writeResultCursorXml(*writer, cursor, name, start, count, schemaName);
+    unsigned rc = writeResultCursorXml(*writer, cursor, name, start, count, schemaName, xmlns);
     ret.set(writer->str());
     return rc;
 
 }
 
-extern FILEVIEW_API unsigned getResultXml(IStringVal & ret, INewResultSet * result, const char* name,unsigned start, unsigned count, const char * schemaName)
+extern FILEVIEW_API unsigned getResultXml(IStringVal & ret, INewResultSet * result, const char* name,unsigned start, unsigned count, const char * schemaName, const IProperties *xmlns)
 {
     Owned<IResultSetCursor> cursor = result->createCursor();
-    return getResultCursorXml(ret, cursor, name, start, count, schemaName);
+    return getResultCursorXml(ret, cursor, name, start, count, schemaName, xmlns);
 }
 
 extern FILEVIEW_API unsigned getResultJSON(IStringVal & ret, INewResultSet * result, const char* name,unsigned start, unsigned count, const char * schemaName)
@@ -3194,7 +3194,7 @@ extern FILEVIEW_API unsigned getResultJSON(IStringVal & ret, INewResultSet * res
     return rc;
 }
 
-extern FILEVIEW_API unsigned writeResultCursorXml(IXmlWriter & writer, IResultSetCursor * cursor, const char * name, unsigned start, unsigned count, const char * schemaName)
+extern FILEVIEW_API unsigned writeResultCursorXml(IXmlWriter & writer, IResultSetCursor * cursor, const char * name, unsigned start, unsigned count, const char * schemaName, const IProperties *xmlns)
 {
     if (schemaName)
     {
@@ -3210,7 +3210,15 @@ extern FILEVIEW_API unsigned writeResultCursorXml(IXmlWriter & writer, IResultSe
     writer.outputBeginDataset(name, true);
     if (schemaName)
         writer.outputCString(schemaName, "@xmlSchema");
-
+    if (xmlns)
+    {
+        Owned<IPropertyIterator> it = const_cast<IProperties*>(xmlns)->getIterator();
+        ForEach(*it)
+        {
+            const char *name = it->getPropKey();
+            writer.outputXmlns(name,const_cast<IProperties*>(xmlns)->queryProp(name));
+        }
+    }
     cursor->beginWriteXmlRows(writer);
     unsigned c=0;
     for(bool ok=cursor->absolute(start);ok;ok=cursor->next())
@@ -3226,10 +3234,10 @@ extern FILEVIEW_API unsigned writeResultCursorXml(IXmlWriter & writer, IResultSe
     return c;
 }
 
-extern FILEVIEW_API unsigned writeResultXml(IXmlWriter & writer, INewResultSet * result, const char* name,unsigned start, unsigned count, const char * schemaName)
+extern FILEVIEW_API unsigned writeResultXml(IXmlWriter & writer, INewResultSet * result, const char* name,unsigned start, unsigned count, const char * schemaName, const IProperties *xmlns)
 {
     Owned<IResultSetCursor> cursor = result->createCursor();
-    return writeResultCursorXml(writer, cursor, name, start, count, schemaName);
+    return writeResultCursorXml(writer, cursor, name, start, count, schemaName, xmlns);
 }
 
 extern FILEVIEW_API unsigned getResultCursorBin(MemoryBuffer & ret, IResultSetCursor * cursor, unsigned start, unsigned count)
@@ -3313,7 +3321,8 @@ extern FILEVIEW_API void writeFullWorkUnitResults(const char *username, const ch
                     SCMStringBuffer name;
                     ds.getResultName(name);
                     Owned<INewResultSet> nr = factory->createNewResultSet(&ds, wuid.str());
-                    writeResultXml(writer, nr.get(), name.str(), 0, 0, (flags & WorkUnitXML_InclSchema) ? name.str() : NULL);
+                    const IProperties *xmlns = ds.queryXmlns();
+                    writeResultXml(writer, nr.get(), name.str(), 0, 0, (flags & WorkUnitXML_InclSchema) ? name.str() : NULL, xmlns);
                 }
             }
         }

common/thorhelper/roxiedebug.cpp

@@ -527,6 +527,10 @@ public:
     {
         // nothing for now
     }
+    virtual void outputXmlns(const char *prefix, const char *uri)
+    {
+        // nothing for now
+    }
 };
 
 class ContainsFieldSearcher : public SimpleFieldSearcher

common/thorhelper/roxiehelper.cpp

@@ -996,7 +996,7 @@ void *FlushingStringBuffer::getPayload(size32_t &length)
     return length ? s.detach() : NULL;
 }
 
-void FlushingStringBuffer::startDataset(const char *elementName, const char *resultName, unsigned sequence, bool _extend)
+void FlushingStringBuffer::startDataset(const char *elementName, const char *resultName, unsigned sequence, bool _extend, const IProperties *xmlns)
 {
     CriticalBlock b(crit);
     extend = _extend;
@@ -1017,6 +1017,20 @@ void FlushingStringBuffer::startDataset(const char *elementName, const char *res
                         s.appendLower(strlen(resultName), resultName).append('\'');
                     else
                         s.append("result_").append(sequence+1).append('\'');
+                    if (xmlns)
+                    {
+                        Owned<IPropertyIterator> it = const_cast<IProperties*>(xmlns)->getIterator(); //should fix IProperties to be const friendly
+                        ForEach(*it)
+                        {
+                            const char *name = it->getPropKey();
+                            s.append(' ');
+                            if (!streq(name, "xmlns"))
+                                s.append("xmlns:");
+                            s.append(name).append("='");
+                            encodeUtf8XML(const_cast<IProperties*>(xmlns)->queryProp(name), s);
+                            s.append("'");
+                        }
+                    }
                 }
                 if (resultName && *resultName)
                     s.appendf(" name='%s'",resultName);

common/thorhelper/roxiehelper.hpp

@@ -160,7 +160,7 @@ public:
     virtual void flush(bool closing) ;
     virtual void addPayload(StringBuffer &s, unsigned int reserve=0);
     virtual void *getPayload(size32_t &length);
-    virtual void startDataset(const char *elementName, const char *resultName, unsigned sequence, bool _extend = false);
+    virtual void startDataset(const char *elementName, const char *resultName, unsigned sequence, bool _extend = false, const IProperties *xmlns=NULL);
     virtual void startScalar(const char *resultName, unsigned sequence);
     virtual void incrementRowCount();
 };

common/thorhelper/thorxmlwrite.cpp

@@ -241,6 +241,14 @@ void CommonXmlWriter::outputUtf8(unsigned len, const char *field, const char *fi
     }
 }
 
+void CommonXmlWriter::outputXmlns(const char *name, const char *uri)
+{
+    StringBuffer fieldname;
+    if (!streq(name, "xmlns"))
+        fieldname.append("xmlns:");
+    outputXmlAttrUtf8(strlen(uri), uri, fieldname.append(name), out);
+}
+
 void CommonXmlWriter::outputBeginDataset(const char *dsname, bool nestChildren)
 {
     outputBeginNested("Dataset", nestChildren, false); //indent row, not dataset for backward compatibility

common/thorhelper/thorxmlwrite.hpp

@@ -71,6 +71,7 @@ public:
     virtual void outputBeginArray(const char *fieldname){}; //repeated elements are inline for xml
     virtual void outputEndArray(const char *fieldname){};
     virtual void outputSetAll();
+    virtual void outputXmlns(const char *name, const char *uri);
 
 protected:
     bool checkForAttribute(const char * fieldname);
@@ -127,6 +128,7 @@ public:
     virtual void outputBeginArray(const char *fieldname);
     virtual void outputEndArray(const char *fieldname);
     virtual void outputSetAll();
+    virtual void outputXmlns(const char *name, const char *uri){}
 
     void outputBeginRoot(){out.append('{');}
     void outputEndRoot(){out.append('}');}
@@ -224,6 +226,8 @@ public:
     virtual void outputEndArray(const char *fieldname){}
     virtual void outputSetAll();
     virtual void outputInlineXml(const char *text){} //for appending raw xml content
+    virtual void outputXmlns(const char *name, const char *uri){}
+
 
 
     void newline();

common/workunit/package.h

@@ -38,6 +38,7 @@ interface IHpccPackage : extends IInterface
 
     virtual const char *queryEnv(const char *varname) const = 0;
     virtual bool getEnableFieldTranslation() const = 0;
+    virtual bool isCompulsory() const = 0;
     virtual const IPropertyTree *queryTree() const = 0;
     virtual hash64_t queryHash() const = 0;
     virtual const char *queryId() const = 0;

common/workunit/pkgimpl.hpp

@@ -101,6 +101,11 @@ protected:
         return (node) ? node->queryProp("@id") : NULL;
     }
 
+    virtual bool isCompulsory() const
+    {
+        return (node) ? node->getPropBool("@compulsory", false) : false;
+    }
+
     virtual bool getSysFieldTranslationEnabled() const {return false;}
     virtual bool getEnableFieldTranslation() const
     {

common/workunit/referencedfilelist.cpp

@@ -187,6 +187,12 @@ public:
         }
     }
 
+    ReferencedFileList(IUserDescriptor *userDesc, bool allowForeignFiles) : allowForeign(allowForeignFiles)
+    {
+        if (userDesc)
+            user.set(userDesc);
+    }
+
     void ensureFile(const char *ln, unsigned flags, const char *pkgid, const char *daliip=NULL, const char *srcCluster=NULL, const char *remotePrefix=NULL);
 
     virtual void addFile(const char *ln, const char *daliip=NULL, const char *srcCluster=NULL, const char *remotePrefix=NULL);
@@ -327,7 +333,7 @@ void ReferencedFile::resolveRemote(IUserDescriptor *user, INode *remote, const c
     if (flags & RefFileInPackage)
         return;
     reset();
-    if (checkLocalFirst)
+    if (checkLocalFirst) //usually means we don't want to overwrite existing file info
     {
         Owned<IDistributedFile> df = queryDistributedFileDirectory().lookup(logicalName.str(), user);
         if(df)
@@ -338,9 +344,17 @@ void ReferencedFile::resolveRemote(IUserDescriptor *user, INode *remote, const c
     }
     Owned<IPropertyTree> tree = getSpecifiedOrRemoteFileTree(user, remote, remotePrefix);
     if (tree)
+    {
         processRemoteFileTree(tree, srcCluster, subfiles);
-    else
-        flags |= RefFileNotFound;
+        return;
+    }
+    else if (!checkLocalFirst && (!srcCluster || !*srcCluster)) //haven't already checked and not told to use a specific copy
+    {
+        resolveLocal(dstCluster, srcCluster, user, subfiles);
+        return;
+    }
+
+    flags |= RefFileNotFound;
 }
 
 void ReferencedFile::resolve(const char *dstCluster, const char *srcCluster, IUserDescriptor *user, INode *remote, const char *remotePrefix, bool checkLocalFirst, StringArray *subfiles, bool resolveForeign)
@@ -703,3 +717,8 @@ IReferencedFileList *createReferencedFileList(const char *user, const char *pw,
 {
     return new ReferencedFileList(user, pw, allowForeignFiles);
 }
+
+IReferencedFileList *createReferencedFileList(IUserDescriptor *user, bool allowForeignFiles)
+{
+    return new ReferencedFileList(user, allowForeignFiles);
+}

common/workunit/referencedfilelist.hpp

@@ -66,6 +66,7 @@ interface IReferencedFileList : extends IInterface
 extern WORKUNIT_API const char *skipForeign(const char *name, StringBuffer *ip=NULL);
 
 extern WORKUNIT_API IReferencedFileList *createReferencedFileList(const char *user, const char *pw, bool allowForeignFiles);
+extern WORKUNIT_API IReferencedFileList *createReferencedFileList(IUserDescriptor *userDesc, bool allowForeignFiles);
 
 extern WORKUNIT_API void splitDfsLocation(const char *address, StringBuffer &cluster, StringBuffer &ip, StringBuffer &prefix, const char *defaultCluster);
 extern WORKUNIT_API void splitDerivedDfsLocation(const char *address, StringBuffer &cluster, StringBuffer &ip, StringBuffer &prefix, const char *defaultCluster, const char *baseCluster, const char *baseIP, const char *basePrefix);

common/workunit/workunit.cpp

@@ -1569,7 +1569,9 @@ class CLocalWUResult : public CInterface, implements IWUResult
 {
     friend class CLocalWorkUnit;
 
+    mutable CriticalSection crit;
     Owned<IPropertyTree> p;
+    Owned<IProperties> xmlns;
     void getSchema(TypeInfoArray &types, StringAttrArray &names, IStringVal * ecl=NULL) const;
 
 public:
@@ -1607,6 +1609,7 @@ public:
     virtual WUResultFormat getResultFormat() const;
     virtual unsigned    getResultHash() const;
     virtual bool        getResultIsAll() const;
+    virtual IProperties *queryXmlns();
 
     // interface IWUResult
     virtual void        setResultStatus(WUResultStatus status);
@@ -1639,6 +1642,7 @@ public:
     virtual void        setResultFormat(WUResultFormat format);
     virtual void        setResultXML(const char *val);
     virtual void        setResultRow(unsigned len, const void * data);
+    virtual void        setXmlns(const char *prefix, const char *uri);
 };
 
 class CLocalWUPlugin : public CInterface, implements IWUPlugin
@@ -4356,8 +4360,11 @@ class CEnvironmentClusterInfo: public CInterface, implements IConstWUClusterInfo
     StringArray thorProcesses;
     StringArray primaryThorProcesses;
     StringAttr prefix;
+    StringAttr ldapUser;
+    StringBuffer ldapPassword;
     ClusterType platform;
     unsigned clusterWidth;
+
 public:
     IMPLEMENT_IINTERFACE;
     CEnvironmentClusterInfo(const char *_name, const char *_prefix, IPropertyTree *agent, IArrayOf<IPropertyTree> &thors, IPropertyTree *roxie)
@@ -4401,6 +4408,10 @@ public:
             platform = RoxieCluster;
             getRoxieProcessServers(roxie, roxieServers);
             clusterWidth = roxieServers.length();
+            ldapUser.set(roxie->queryProp("@ldapUser"));
+            StringBuffer encPassword = roxie->queryProp("@ldapPassword");
+            if (encPassword.length())
+                decrypt(ldapPassword, encPassword);
         }
         else 
         {
@@ -4469,6 +4480,14 @@ public:
     {
         return roxieServers;
     }
+    const char *getLdapUser() const
+    {
+        return ldapUser.get();
+    }
+    virtual const char *getLdapPassword() const
+    {
+        return ldapPassword.str();
+    }
 };
 
 IStringVal &getProcessQueueNames(IStringVal &ret, const char *process, const char *type, const char *suffix)
@@ -6034,7 +6053,6 @@ void CLocalWorkUnit::loadResults() const
         for (r->first(); r->isValid(); r->next())
         {
             IPropertyTree *rp = &r->query();
-
             rp->Link();
             results.append(*new CLocalWUResult(rp));
         }
@@ -7802,6 +7820,28 @@ IStringVal& CLocalWUResult::getResultXml(IStringVal &str) const
     return str;
 }
 
+IProperties *CLocalWUResult::queryXmlns()
+{
+    CriticalBlock block(crit);
+    if (xmlns)
+        return xmlns;
+    xmlns.setown(createProperties());
+    Owned<IAttributeIterator> it = p->getAttributes();
+    unsigned prefixLen = strlen("@xmlns");
+    ForEach(*it)
+    {
+        const char *name = it->queryName();
+        if (!strncmp("@xmlns", name, prefixLen))
+        {
+            if (name[prefixLen]==':') //normal case
+                xmlns->setProp(name+prefixLen+1, it->queryValue());
+            else if (!name[prefixLen]) //special case, unprefixed namespace
+                xmlns->setProp("xmlns", it->queryValue());
+        }
+    }
+    return xmlns;
+}
+
 unsigned CLocalWUResult::getResultFetchSize() const
 {
     return p->getPropInt("fetchSize", 100);
@@ -7890,6 +7930,13 @@ void CLocalWUResult::setResultSchemaRaw(unsigned size, const void *schema)
 {
     p->setPropBin("SchemaRaw", size, schema);
 }
+void CLocalWUResult::setXmlns(const char *prefix, const char *uri)
+{
+    StringBuffer xpath("@xmlns");
+    if (prefix && *prefix)
+        xpath.append(':').append(prefix);
+    p->setProp(xpath, uri);
+}
 void CLocalWUResult::setResultScalar(bool isScalar)
 {
     p->setPropInt("@isScalar", (int) isScalar);

common/workunit/workunit.hpp

@@ -37,6 +37,7 @@
 #include "jtime.hpp"
 #include "jsocket.hpp"
 #include "jstats.h"
+#include "jprop.hpp"
 
 #define CHEAP_UCHAR_DEF
 #ifdef _WIN32
@@ -300,6 +301,7 @@ interface IConstWUResult : extends IInterface
     virtual unsigned getResultHash() const = 0;
     virtual void getResultDecimal(void * val, unsigned length, unsigned precision, bool isSigned) const = 0;
     virtual bool getResultIsAll() const = 0;
+    virtual const IProperties *queryXmlns() = 0;
 };
 
 
@@ -335,6 +337,7 @@ interface IWUResult : extends IConstWUResult
     virtual void setResultFormat(WUResultFormat format) = 0;
     virtual void setResultXML(const char * xml) = 0;
     virtual void setResultRow(unsigned len, const void * data) = 0;
+    virtual void setXmlns(const char *prefix, const char *uri) = 0;
 };
 
 
@@ -573,6 +576,8 @@ interface IConstWUClusterInfo : extends IInterface
     virtual const StringArray & getThorProcesses() const = 0;
     virtual const StringArray & getPrimaryThorProcesses() const = 0;
     virtual const SocketEndpointArray & getRoxieServers() const = 0;
+    virtual const char *getLdapUser() const = 0;
+    virtual const char *getLdapPassword() const = 0;
 };
 
 //! IWorkflowItem

ecl/eclcc/eclcc.cpp

@@ -1014,9 +1014,18 @@ void EclCC::processSingleQuery(EclCompileInstance & instance,
         Owned<IPropertyTreeIterator> iter = instance.srcArchive->getElements("OnWarning");
         ForEach(*iter)
         {
+            const char * name = iter->query().queryProp("@name");
             const char * option = iter->query().queryProp("@value");
-            if (!severityMapper->addCommandLineMapping(option))
-                return;
+            if (name)
+            {
+                if (!severityMapper->addMapping(name, option))
+                    return;
+            }
+            else
+            {
+                if (!severityMapper->addCommandLineMapping(option))
+                    return;
+            }
         }
     }
 
@@ -2066,7 +2075,8 @@ const char * const helpText[] = {
     "    -specs file   Read eclcc configuration from specified file",
     "!   -split m:n    Process a subset m of n input files (only with -b option)",
     "    -v --verbose  Output additional tracing information while compiling",
-    "    -wcode=level  Set the severity for a particular warning code",
+    "    -wxxxx=level  Set the severity for a particular warning code or category",
+    "!                 -wall sets default severity for all warnings",
     "!                 level=ignore|log|warning|error|fail",
     "    --version     Output version information",
     "!   --timings     Output additional timing information",

ecl/eclccserver/eclccserver.cpp

@@ -215,6 +215,9 @@ class EclccCompileThread : public CInterface, implements IPooledThread, implemen
             else
                 optName.set(start);
 
+            if (!optName)
+                return;
+
             if (stricmp(optName, "hook") == 0)
             {
                 if (isLocal)
@@ -227,7 +230,7 @@ class EclccCompileThread : public CInterface, implements IPooledThread, implemen
                 eclccCmd.appendf(" -I%s", value);
             else if (stricmp(optName, "libraryPath") == 0)
                 eclccCmd.appendf(" -L%s", value);
-            else if (stricmp(start, "-allow")==0)
+            else if (stricmp(start, "-allow")==0 || stricmp(optName, "-allow")==0)
             {
                 if (isLocal)
                     throw MakeStringException(0, "eclcc-allow option can not be set per-workunit");  // for security reasons

ecl/eclcmd/eclcmd_common.hpp

@@ -84,6 +84,7 @@ typedef IEclCommand *(*EclCommandFactory)(const char *cmdname);
 #define ECLOPT_DELETE_PREVIOUS_INI "deletePrevDefault"
 #define ECLOPT_DELETE_PREVIOUS_ENV "ACTIVATE_DELETE_PREVIOUS"
 #define ECLOPT_CHECK_DFS "--check-dfs"
+#define ECLOPT_UPDATE_DFS "--update-dfs"
 #define ECLOPT_GLOBAL_SCOPE "--global-scope"
 
 #define ECLOPT_MAIN "--main"

ecl/eclcmd/eclcmd_core.cpp

@@ -195,7 +195,7 @@ class EclCmdPublish : public EclCmdWithEclTarget
 {
 public:
     EclCmdPublish() : optNoActivate(false), optSuspendPrevious(false), optDeletePrevious(false),
-        activateSet(false), optNoReload(false), optDontCopyFiles(false), optMsToWait(10000), optAllowForeign(false)
+        activateSet(false), optNoReload(false), optDontCopyFiles(false), optMsToWait(10000), optAllowForeign(false), optUpdateDfs(false)
     {
         optObj.accept = eclObjWuid | eclObjArchive | eclObjSharedObject;
         optTimeLimit = (unsigned) -1;
@@ -253,6 +253,8 @@ public:
                 continue;
             if (iter.matchFlag(optDeletePrevious, ECLOPT_DELETE_PREVIOUS)||iter.matchFlag(optDeletePrevious, ECLOPT_DELETE_PREVIOUS_S))
                 continue;
+            if (iter.matchFlag(optUpdateDfs, ECLOPT_UPDATE_DFS))
+                continue;
             if (EclCmdWithEclTarget::matchCommandLineOption(iter, true)!=EclCmdOptionMatch)
                 return false;
         }
@@ -328,6 +330,7 @@ public:
         req->setNoReload(optNoReload);
         req->setDontCopyFiles(optDontCopyFiles);
         req->setAllowForeignFiles(optAllowForeign);
+        req->setUpdateDfs(optUpdateDfs);
 
         if (optTimeLimit != (unsigned) -1)
             req->setTimeLimit(optTimeLimit);
@@ -386,6 +389,7 @@ public:
             "   --no-files             Do not copy files referenced by query\n"
             "   --allow-foreign        Do not fail if foreign files are used in query (roxie)\n"
             "   --daliip=<IP>          The IP of the DALI to be used to locate remote files\n"
+            "   --update-dfs           Update local DFS info if remote DALI has changed\n"
             "   --source-process       Process cluster to copy files from\n"
             "   --timeLimit=<ms>       Value to set for query timeLimit configuration\n"
             "   --warnTimeLimit=<ms>   Value to set for query warnTimeLimit configuration\n"
@@ -415,6 +419,7 @@ private:
     bool optSuspendPrevious;
     bool optDeletePrevious;
     bool optAllowForeign;
+    bool optUpdateDfs;
 };
 
 class EclCmdRun : public EclCmdWithEclTarget

ecl/hql/hqlatoms.cpp

@@ -421,6 +421,7 @@ IAtom * workunitAtom;
 IAtom * wuidAtom;
 IAtom * xmlAtom;
 IAtom * xmlDefaultAtom;
+IAtom * xmlnsAtom;
 IAtom * _xmlParse_Atom;
 IAtom * xpathAtom;
 
@@ -835,6 +836,7 @@ MODULE_INIT(INIT_PRIORITY_HQLATOM)
     MAKEATOM(wuid);
     MAKEATOM(xml);
     MAKEATOM(xmlDefault);
+    MAKEATOM(xmlns);
     MAKESYSATOM(xmlParse);
     MAKEATOM(xpath);
 

ecl/hql/hqlatoms.hpp

@@ -425,6 +425,7 @@ extern HQL_API IAtom * workunitAtom;
 extern HQL_API IAtom * wuidAtom;
 extern HQL_API IAtom * xmlAtom;
 extern HQL_API IAtom * xmlDefaultAtom;
+extern HQL_API IAtom * xmlnsAtom;
 extern HQL_API IAtom * _xmlParse_Atom;
 extern HQL_API IAtom * xpathAtom;
 

ecl/hql/hqlerror.cpp

@@ -28,7 +28,7 @@ public:
     virtual StringBuffer &  errorMessage(StringBuffer & ret) const { return ret.append(msg); }
     virtual MessageAudience errorAudience() const { return MSGAUD_user; }
     virtual const char* getFilename() const { return filename; }
-    virtual WarnErrorCategory getCategory() const { return WECunknown; }
+    virtual WarnErrorCategory getCategory() const { return CategoryUnknown; }
     virtual int getLine() const { return lineno; }
     virtual int getColumn() const { return column; }
     virtual int getPosition() const { return position; }

ecl/hql/hqlerror.hpp

@@ -39,7 +39,22 @@ inline bool isFatal(ErrorSeverity severity) { return severity == SeverityFatal;
 //TBD in a separate commit - add support for warnings to be associated with different categories
 enum WarnErrorCategory
 {
-    WECunknown,
+    CategoryCast,       // Suspicious casts between types or out of range values
+    CategoryConfuse,    // Likely to cause confusion
+    CategoryDeprecated, // deprecated features or syntax
+    CategoryEfficiency, // Something that is likely to be inefficient
+    CategoryFuture,     // Likely to cause problems in future versions
+    CategoryIgnored,    // Something that has no effect, or is ignored
+    CategoryIndex,      // Unusual indexing of datasets or strings
+    CategoryMistyped,   // Almost certainly mistyped
+    CategorySyntax,     // Invalid syntax which is painless to recover from
+    CategoryUnusual,    // Not strictly speaking an error, but highly unusual and likely to be a mistake
+    CategoryUnexpected, // Code that could be correct, but has the potential for unexpected behaviour
+
+    CategoryError,
+    CategoryAll,
+    CategoryUnknown,
+    CategoryMax,
 };
 
 interface HQL_API IECLError: public IException

ecl/hql/hqlerrors.hpp

@@ -475,6 +475,7 @@
 #define HQLERR_AtmostFollowUnknownSubstr        3133
 #define HQLERR_AtmostLegacyMismatch             3134
 #define HQLERR_PropertyArgumentNotConstant      3135
+#define HQLERR_InvalidErrorCategory             3136
 
 #define HQLERR_DedupFieldNotFound_Text          "Field removed from dedup could not be found"
 #define HQLERR_CycleWithModuleDefinition_Text   "Module definition contains an illegal cycle/recursive definition %s"
@@ -512,6 +513,7 @@
 #define HQLERR_AtmostFollowUnknownSubstr_Text   "ATMOST [1..*] on an unknown length string must be last in the optional list"
 #define HQLERR_AtmostLegacyMismatch_Text        "Legacy JOIN condition on field[1..*] should be included in the optional fields"
 #define HQLERR_PropertyArgumentNotConstant_Text "The argument to attribute '%s' must be a constant"
+#define HQLERR_InvalidErrorCategory_Text        "Unrecognised ONWARNING category '%s'"
 
 /* parser error */
 #define ERR_PARSER_CANNOTRECOVER    3005  /* The parser can not recover from previous error(s) */

ecl/hql/hqlexpr.cpp

@@ -589,31 +589,16 @@ extern HQL_API void gatherMetaAttributes(HqlExprArray & matches, IAtom * search,
     }
 }
 
-extern HQL_API void gatherMetaAttributes(HqlExprCopyArray & matches, IAtom * search, IHqlExpression * expr)
+extern HQL_API void gatherAttributes(HqlExprArray & matches, IAtom * search, IHqlExpression * expr)
 {
-    loop
+    ForEachChild(i, expr)
     {
-        annotate_kind kind = expr->getAnnotationKind();
-        if (kind == annotate_none)
-            return;
-        if (kind == annotate_meta)
-        {
-            unsigned i=0;
-            IHqlExpression * cur;
-            while ((cur = expr->queryAnnotationParameter(i++)) != NULL)
-            {
-                //It's possible we may want to implement this whole function as a member function and allow
-                //information to be stored in a non expression format, and only create expressions when requested.
-                //may end up less efficient in the end.
-                if (cur->queryName() == search && cur->isAttribute())
-                    matches.append(*cur);
-            }
-        }
-        expr = expr->queryBody(true);
+        IHqlExpression *kid = expr->queryChild(i);
+        if (kid->isAttribute() && kid->queryName()==search)
+            matches.append(*LINK(kid));
     }
 }
 
-
 extern HQL_API IHqlExpression * queryLocation(IHqlExpression * expr)
 {
     IHqlExpression * best = NULL;

ecl/hql/hqlexpr.hpp

@@ -1451,6 +1451,7 @@ extern HQL_API IHqlExpression * queryAttributeInList(IAtom * search, IHqlExpress
 extern HQL_API IHqlExpression * queryAttribute(IAtom * search, const HqlExprArray & exprs);
 extern HQL_API IHqlExpression * queryAnnotation(IHqlExpression * expr, annotate_kind search);       // return first match
 extern HQL_API IHqlNamedAnnotation * queryNameAnnotation(IHqlExpression * expr);
+extern HQL_API void gatherAttributes(HqlExprArray & matches, IAtom * search, IHqlExpression * expr);
 
 inline bool hasAnnotation(IHqlExpression * expr, annotate_kind search){ return queryAnnotation(expr, search) != NULL; }
 inline IHqlExpression * queryNamedSymbol(IHqlExpression * expr) { return queryAnnotation(expr, annotate_symbol); }

ecl/hql/hqlgram.y

@@ -461,6 +461,7 @@ static void eclsyntaxerror(HqlGram * parser, const char * s, short yystate, int
   XMLDECODE
   XMLDEFAULT
   XMLENCODE
+  XMLNS
   XMLPROJECT
   XMLTEXT
   XMLUNICODE
@@ -1506,7 +1507,14 @@ setMetaCommand
                         }
     | HASH_ONWARNING '(' expression ',' warningAction ')'
                         {
-                            parser->normalizeExpression($3, type_int, false);
+                            if (isNumericType($3.queryExprType()))
+                            {
+                                parser->normalizeExpression($3, type_int, false);
+                            }
+                            else
+                            {
+                                parser->normalizeExpression($3, type_string, false);
+                            }
                             $$.setExpr(createValue(no_setmeta, makeVoidType(), createAttribute(onWarningAtom), $3.getExpr(), $5.getExpr()), $1);
                         }
     ;
@@ -3467,6 +3475,13 @@ outputWuFlag
                             $$.setExpr(createAttribute(allAtom));
                             $$.setPosition($1);
                         }
+    | XMLNS '(' expression ',' expression ')'
+                        {
+                            parser->normalizeExpression($3, type_stringorunicode, true);
+                            parser->normalizeExpression($5, type_stringorunicode, true);
+                            $$.setExpr(createExprAttribute(xmlnsAtom, $3.getExpr(), $5.getExpr()));
+                            $$.setPosition($1);
+                        }
     | FIRST '(' constExpression ')'
                         {
                             parser->normalizeExpression($3, type_int, true);

ecl/hql/hqlgram2.cpp

@@ -10609,6 +10609,7 @@ static void getTokenText(StringBuffer & msg, int token)
     case XMLDECODE: msg.append("XMLDECODE"); break;
     case XMLDEFAULT: msg.append("XMLDEFAULT"); break;
     case XMLENCODE: msg.append("XMLENCODE"); break;
+    case XMLNS: msg.append("XMLNS"); break;
     case XMLPROJECT: msg.append("XMLPROJECT"); break;
     case XMLTEXT: msg.append("XMLTEXT"); break;
     case XMLUNICODE: msg.append("XMLUNICODE"); break;

ecl/hql/hqllex.l

@@ -944,6 +944,7 @@ XML                 { RETURNSYM(XML_TOKEN); }
 XMLDECODE           { RETURNSYM(XMLDECODE); }
 XMLDEFAULT          { RETURNSYM(XMLDEFAULT); }
 XMLENCODE           { RETURNSYM(XMLENCODE); }
+XMLNS               { RETURNSYM(XMLNS); }
 XMLPROJECT          { RETURNSYM(XMLPROJECT); }
 XMLTEXT             { RETURNSYM(XMLTEXT); }
 XMLUNICODE          { RETURNSYM(XMLUNICODE); }

ecl/hql/hqlutil.cpp

@@ -3003,7 +3003,7 @@ __int64 getIntValue(IHqlExpression * expr, __int64 dft)
     return dft;
 }
 
-StringBuffer & getStringValue(StringBuffer & out, IHqlExpression * expr, const char * dft)
+StringBuffer & getStringValue(StringBuffer & out, IHqlExpression * expr, const char * dft, bool utf8)
 {
     if (expr)
     {
@@ -3012,7 +3012,10 @@ StringBuffer & getStringValue(StringBuffer & out, IHqlExpression * expr, const c
         IValue * value = expr->queryValue();
         if (value)
         {
-            value->getStringValue(out);
+            if (utf8)
+                value->getUTF8Value(out);
+            else
+                value->getStringValue(out);
             return out;
         }
     }
@@ -3021,6 +3024,16 @@ StringBuffer & getStringValue(StringBuffer & out, IHqlExpression * expr, const c
     return out;
 }
 
+StringBuffer & getStringValue(StringBuffer & out, IHqlExpression * expr, const char * dft)
+{
+    return getStringValue(out, expr, dft, false);
+}
+
+StringBuffer & getUTF8Value(StringBuffer & out, IHqlExpression * expr, const char * dft)
+{
+    return getStringValue(out, expr, dft, true);
+}
+
 bool matchesConstantValue(IHqlExpression * expr, __int64 test)
 {
     if (!expr) return false;
@@ -6465,6 +6478,35 @@ ErrorSeverity getSeverity(IAtom * name)
     return SeverityUnknown;
 }
 
+WarnErrorCategory getCategory(const char * category)
+{
+    if (strieq(category, "all"))
+        return CategoryAll;
+    if (strieq(category, "cast"))
+        return CategoryCast;
+    if (strieq(category, "confuse"))
+        return CategoryConfuse;
+    if (strieq(category, "deprecated"))
+        return CategoryDeprecated;
+    if (strieq(category, "efficiency"))
+        return CategoryEfficiency;
+    if (strieq(category, "future"))
+        return CategoryFuture;
+    if (strieq(category, "ignored"))
+        return CategoryIgnored;
+    if (strieq(category, "index"))
+        return CategoryIndex;
+    if (strieq(category, "mistype"))
+        return CategoryMistyped;
+    if (strieq(category, "syntax"))
+        return CategorySyntax;
+    if (strieq(category, "unusual"))
+        return CategoryUnusual;
+    if (strieq(category, "unexpected"))
+        return CategoryUnexpected;
+    return CategoryUnknown;
+}
+
 static ErrorSeverity getCheckSeverity(IAtom * name)
 {
     ErrorSeverity severity = getSeverity(name);
@@ -6491,6 +6533,8 @@ ErrorSeverityMapper::ErrorSeverityMapper(IErrorReceiver & _errorProcessor) : Ind
 {
     firstActiveMapping = 0;
     activeSymbol = NULL;
+    for (unsigned category = 0; category < CategoryMax; category++)
+        categoryAction[category] = SeverityUnknown;
 }
 
 
@@ -6506,7 +6550,10 @@ bool ErrorSeverityMapper::addCommandLineMapping(const char * mapping)
     const char * equals = strchr(mapping, '=');
     const char * value;
     if (equals)
+    {
         value = equals+1;
+        len = equals-mapping;
+    }
     else if (mapping[len-1] == '+')
     {
         len--;
@@ -6520,24 +6567,46 @@ bool ErrorSeverityMapper::addCommandLineMapping(const char * mapping)
     else
         value = "error";
 
+    StringAttr category(mapping, len);
+    return addMapping(category, value);
+ }
+
+bool ErrorSeverityMapper::addMapping(const char * category, const char * value)
+{
+    if (!category || !*category)
+    {
+        ERRLOG("Error: No warning category supplied");
+        return false;
+    }
+
+    //Ignore mappings with no action
+    if (!value || !*value)
+        return true;
+
     IAtom * action = createAtom(value);
-    if (getSeverity(action) == SeverityUnknown)
+    ErrorSeverity severity = getSeverity(action);
+    if (severity == SeverityUnknown)
     {
-        fprintf(stderr, "invalid warning severity '%s'\n", value);
+        ERRLOG("Error: Invalid warning severity '%s'", value);
         return false;
     }
 
-    if (isdigit(mapping[0]))
+    if (isdigit(*category))
     {
-        unsigned errorCode = atoi(mapping);
+        unsigned errorCode = atoi(category);
         addOnWarning(errorCode, action);
         return true;
     }
-    else
+
+    WarnErrorCategory cat = getCategory(category);
+    if (cat != CategoryUnknown)
     {
-        fprintf(stderr, "mapping doesn't specify a valid warning code '%s'\n", mapping);
-        return false;
+        categoryAction[cat] = severity;
+        return true;
     }
+
+    ERRLOG("Error: Mapping doesn't specify a valid warning code or category '%s'", category);
+    return false;
 }
 
 void ErrorSeverityMapper::addOnWarning(unsigned code, IAtom * action)
@@ -6547,7 +6616,23 @@ void ErrorSeverityMapper::addOnWarning(unsigned code, IAtom * action)
 
 void ErrorSeverityMapper::addOnWarning(IHqlExpression * setMetaExpr)
 {
-    severityMappings.append(*createAttribute(onWarningAtom, LINK(setMetaExpr->queryChild(1)), LINK(setMetaExpr->queryChild(2))));
+    IHqlExpression * code = setMetaExpr->queryChild(1);
+    IHqlExpression * action = setMetaExpr->queryChild(2);
+    if (isStringType(code->queryType()))
+    {
+        StringBuffer text;
+        getStringValue(text, code, NULL);
+        WarnErrorCategory cat = getCategory(text);
+        ErrorSeverity severity = getSeverity(action->queryName());
+        if (cat == CategoryUnknown)
+            throwError1(HQLERR_InvalidErrorCategory, text.str());
+
+        categoryAction[cat] = severity;
+    }
+    else
+    {
+        severityMappings.append(*createAttribute(onWarningAtom, LINK(code), LINK(action)));
+    }
 }
 
 unsigned ErrorSeverityMapper::processMetaAnnotation(IHqlExpression * expr)
@@ -6598,6 +6683,13 @@ IECLError * ErrorSeverityMapper::mapError(IECLError * error)
         ErrorSeverity newSeverity = getWarningAction(mappedError->errorCode(), severityMappings, firstActiveMapping, SeverityUnknown);
         if (newSeverity != SeverityUnknown)
             return mappedError->cloneSetSeverity(newSeverity);
+
+        WarnErrorCategory category = mappedError->getCategory();
+        if (categoryAction[category] != SeverityUnknown)
+            return mappedError->cloneSetSeverity(categoryAction[category]);
+
+        if (categoryAction[CategoryAll] != SeverityUnknown)
+            return mappedError->cloneSetSeverity(categoryAction[CategoryAll]);
     }
     return mappedError.getClear();
 }

ecl/hql/hqlutil.hpp

@@ -113,6 +113,7 @@ extern HQL_API bool matchesConstantString(IHqlExpression * expr, const char * te
 extern HQL_API bool getBoolValue(IHqlExpression * expr, bool dft);
 extern HQL_API __int64 getIntValue(IHqlExpression * expr, __int64 dft = 0);
 extern HQL_API StringBuffer & getStringValue(StringBuffer & out, IHqlExpression * expr, const char * dft = NULL);
+extern HQL_API StringBuffer & getUTF8Value(StringBuffer & out, IHqlExpression * expr, const char * dft = NULL);
 extern HQL_API bool isEmptyList(IHqlExpression * expr);
 extern HQL_API IHqlExpression * queryNextMultiLevelDataset(IHqlExpression * expr, bool followActiveSelectors);
 extern HQL_API bool isMultiLevelDatasetSelector(IHqlExpression * expr, bool followActiveSelectors);
@@ -554,6 +555,7 @@ public:
     virtual IECLError * mapError(IECLError * error);
 
     bool addCommandLineMapping(const char * mapping);
+    bool addMapping(const char * category, const char * value);
     void addOnWarning(unsigned code, IAtom * action);
     void addOnWarning(IHqlExpression * setMetaExpr);
     unsigned processMetaAnnotation(IHqlExpression * expr);
@@ -573,6 +575,7 @@ protected:
     IHqlExpression * activeSymbol;
     HqlExprArray severityMappings;
     unsigned firstActiveMapping;
+    ErrorSeverity categoryAction[CategoryMax];
 };
 
 extern HQL_API bool isGlobalOnWarning(IHqlExpression * expr);

ecl/hqlcpp/hqlcpp.ipp

@@ -1803,7 +1803,7 @@ protected:
     void addSchemaResource(int seq, const char * name, IHqlExpression * record);
     void addSchemaResource(int seq, const char * name, unsigned len, const char * schemaXml);
     void doAddSchemaFields(IHqlExpression * record, MemoryBuffer &schema, IHqlExpression *selector);
-    IWUResult * createDatasetResultSchema(IHqlExpression * sequenceExpr, IHqlExpression * name, IHqlExpression * record, bool createTransformer, bool isFile);
+    IWUResult * createDatasetResultSchema(IHqlExpression * sequenceExpr, IHqlExpression * name, IHqlExpression * record, HqlExprArray &xmlnsAttrs, bool createTransformer, bool isFile);
 
     void buildReturnCsvValue(BuildCtx & ctx, IHqlExpression * _expr);
     void buildCsvListFunc(BuildCtx & classctx, const char * func, IHqlExpression * value, const char * defaultValue);

ecl/hqlcpp/hqlhtcpp.cpp

@@ -1683,6 +1683,7 @@ ActivityInstance::ActivityInstance(HqlCppTranslator & _translator, BuildCtx & ct
         outputDataset = dataset->queryChild(0);
 
     bool removeXpath = dataset->hasAttribute(noXpathAtom) || (op == no_output && translator.queryOptions().removeXpathFromOutput);
+
     LinkedHqlExpr record = queryRecord(outputDataset);
     if (removeXpath)
         record.setown(removeAttributeFromFields(record, xpathAtom));
@@ -5027,6 +5028,92 @@ IWUResult * HqlCppTranslator::createWorkunitResult(int sequence, IHqlExpression
     return result.getClear();
 }
 
+void checkAppendXpathNamePrefix(StringArray &prefixes, const char *xpathName)
+{
+    if (!xpathName || !*xpathName)
+        return;
+    if (*xpathName=='@')
+        xpathName++;
+    if (*xpathName==':')
+        return;
+    const char *colon = strchr(xpathName, ':');
+    if (!colon)
+        return;
+    StringAttr prefix;
+    prefix.set(xpathName, colon-xpathName);
+    if (prefixes.find(prefix.get())==NotFound)
+        prefixes.append(prefix);
+}
+
+void gatherXpathPrefixes(StringArray &prefixes, IHqlExpression * record)
+{
+    ForEachChild(i, record)
+    {
+        IHqlExpression * cur = record->queryChild(i);
+        switch (cur->getOperator())
+        {
+        case no_field:
+        {
+            //don't need to be too picky about xpath field types, worst case if an xpath is too long, we end up with an extra prefix
+            StringBuffer xpathName, xpathItem;
+            extractXmlName(xpathName, &xpathItem, NULL, cur, NULL, false);
+            checkAppendXpathNamePrefix(prefixes, xpathName);
+            checkAppendXpathNamePrefix(prefixes, xpathItem);
+
+            ITypeInfo * type = cur->queryType();
+            switch (type->getTypeCode())
+            {
+            case type_row:
+            case type_dictionary:
+            case type_table:
+            case type_groupedtable:
+                gatherXpathPrefixes(prefixes, cur->queryRecord());
+                break;
+            }
+            break;
+        }
+        case no_ifblock:
+            gatherXpathPrefixes(prefixes, cur->queryChild(1));
+            break;
+        case no_record:
+            gatherXpathPrefixes(prefixes, cur);
+            break;
+        }
+    }
+}
+
+void addDatasetResultXmlNamespaces(IWUResult &result, HqlExprArray &xmlnsAttrs, IHqlExpression *record)
+{
+    StringArray declaredPrefixes;
+    ForEachItemIn(idx, xmlnsAttrs)
+    {
+        IHqlExpression & xmlns = xmlnsAttrs.item(idx);
+        StringBuffer xmlnsPrefix;
+        StringBuffer xmlnsURI;
+        getUTF8Value(xmlnsPrefix, xmlns.queryChild(0));
+        getUTF8Value(xmlnsURI, xmlns.queryChild(1));
+        if (xmlnsURI.length())
+        {
+            result.setXmlns(xmlnsPrefix, xmlnsURI);
+            if (xmlnsPrefix.length() && declaredPrefixes.find(xmlnsPrefix)==NotFound)
+                declaredPrefixes.append(xmlnsPrefix);
+        }
+    }
+    StringArray usedPrefixes;
+    if (record)
+        gatherXpathPrefixes(usedPrefixes, record);
+    ForEachItemIn(p, usedPrefixes)
+    {
+        const char *prefix = usedPrefixes.item(p);
+        if (declaredPrefixes.find(prefix)==NotFound)
+        {
+            StringBuffer uri("urn:hpccsystems:ecl:unknown:");
+            uri.append(prefix);
+            result.setXmlns(prefix, uri);
+        }
+    }
+}
+
 void HqlCppTranslator::buildSetResultInfo(BuildCtx & ctx, IHqlExpression * originalExpr, IHqlExpression * value, ITypeInfo * type, bool isPersist, bool associateResult)
 {
     IHqlExpression * seq = queryAttributeChild(originalExpr, sequenceAtom, 0);
@@ -5185,9 +5272,11 @@ void HqlCppTranslator::buildSetResultInfo(BuildCtx & ctx, IHqlExpression * origi
 
     if(wu())
     {
+        HqlExprArray xmlnsAttrs;
+        gatherAttributes(xmlnsAttrs, xmlnsAtom, originalExpr);
         if (retType == type_row)
         {
-            Owned<IWUResult> result = createDatasetResultSchema(seq, name, ::queryRecord(schemaType), false, false);
+            Owned<IWUResult> result = createDatasetResultSchema(seq, name, ::queryRecord(schemaType), xmlnsAttrs, false, false);
             if (result)
                 result->setResultTotalRowCount(1);
         }
@@ -5211,6 +5300,8 @@ void HqlCppTranslator::buildSetResultInfo(BuildCtx & ctx, IHqlExpression * origi
                         fieldName.append('_');
                 }
 
+                addDatasetResultXmlNamespaces(*result, xmlnsAttrs, NULL);
+
                 MemoryBuffer schema;
                 schema.append(fieldName.str());
                 schemaType->serialize(schema);
@@ -7559,7 +7650,9 @@ void HqlCppTranslator::doBuildStmtSetResult(BuildCtx & ctx, IHqlExpression * exp
                 args.append(*createValue(no_translated, makeSetType(NULL), createValue(no_nullptr, makeSetType(NULL)), getSizetConstant(0)));
                 args.append(*createTranslatedOwned(createValue(no_nullptr, makeBoolType())));
                 buildFunctionCall(subctx, setResultSetId, args);
-                Owned<IWUResult> result = createDatasetResultSchema(seq, name, value->queryRecord(), true, false);
+                HqlExprArray xmlnsAttrs;
+                gatherAttributes(xmlnsAttrs, xmlnsAtom, expr);
+                Owned<IWUResult> result = createDatasetResultSchema(seq, name, value->queryRecord(), xmlnsAttrs, true, false);
                 break;
             }
         default:
@@ -10104,7 +10197,9 @@ ABoundActivity * HqlCppTranslator::doBuildActivityOutputIndex(BuildCtx & ctx, IH
     buildRecordEcl(instance->createctx, dataset, "queryRecordECL");
 
     doBuildSequenceFunc(instance->classctx, querySequence(expr), false);
-    Owned<IWUResult> result = createDatasetResultSchema(querySequence(expr), queryResultName(expr), dataset->queryRecord(), false, true);
+    HqlExprArray xmlnsAttrs;
+    gatherAttributes(xmlnsAttrs, xmlnsAtom, expr);
+    Owned<IWUResult> result = createDatasetResultSchema(querySequence(expr), queryResultName(expr), dataset->queryRecord(), xmlnsAttrs, false, true);
 
     if (expr->hasAttribute(setAtom))
     {
@@ -10419,7 +10514,9 @@ ABoundActivity * HqlCppTranslator::doBuildActivityOutput(BuildCtx & ctx, IHqlExp
 
         IHqlExpression * outputRecord = instance->meta.queryRecord();
         OwnedHqlExpr outputDs = createDataset(no_null, LINK(outputRecord));
-        Owned<IWUResult> result = createDatasetResultSchema(seq, queryResultName(expr), outputRecord, (kind != TAKcsvwrite) && (kind != TAKxmlwrite), true);
+        HqlExprArray xmlnsAttrs;
+        gatherAttributes(xmlnsAttrs, xmlnsAtom, expr);
+        Owned<IWUResult> result = createDatasetResultSchema(seq, queryResultName(expr), outputRecord, xmlnsAttrs, (kind != TAKcsvwrite) && (kind != TAKxmlwrite), true);
         if (expr->hasAttribute(resultAtom))
             result->setResultRowLimit(-1);
 
@@ -10586,8 +10683,7 @@ void HqlCppTranslator::finalizeResources()
 {
 }
 
-
-IWUResult * HqlCppTranslator::createDatasetResultSchema(IHqlExpression * sequenceExpr, IHqlExpression * name, IHqlExpression * record, bool createTransformer, bool isFile)
+IWUResult * HqlCppTranslator::createDatasetResultSchema(IHqlExpression * sequenceExpr, IHqlExpression * name, IHqlExpression * record, HqlExprArray &xmlnsAttrs, bool createTransformer, bool isFile)
 {
     //Some spills have no sequence attached
     if (!sequenceExpr)
@@ -10598,6 +10694,8 @@ IWUResult * HqlCppTranslator::createDatasetResultSchema(IHqlExpression * sequenc
     if (!result)
         return NULL;
 
+    addDatasetResultXmlNamespaces(*result, xmlnsAttrs, record);
+
     MemoryBuffer schema;
     OwnedHqlExpr self = getSelf(record);
     addSchemaFields(record, schema, self);
@@ -11027,12 +11125,13 @@ ABoundActivity * HqlCppTranslator::doBuildActivityOutputWorkunit(BuildCtx & ctx,
         if (maxsize)
             doBuildUnsignedFunction(instance->createctx, "getMaxSize", maxsize->queryChild(0));
 
+        HqlExprArray xmlnsAttrs;
+        gatherAttributes(xmlnsAttrs, xmlnsAtom, expr);
         IHqlExpression * outputRecord = instance->meta.queryRecord();
-        Owned<IWUResult> result = createDatasetResultSchema(seq, name, outputRecord, true, false);
+        Owned<IWUResult> result = createDatasetResultSchema(seq, name, outputRecord, xmlnsAttrs, true, false);
         if (result)
         {
             result->setResultRowLimit(-1);
-
             if (sequence >= 0)
             {
                 OwnedHqlExpr outputDs = createDataset(no_null, LINK(outputRecord));
@@ -11042,6 +11141,7 @@ ABoundActivity * HqlCppTranslator::doBuildActivityOutputWorkunit(BuildCtx & ctx,
 
         if (flags.length())
             doBuildUnsignedFunction(instance->classctx, "getFlags", flags.str()+1);
+
     }
     else
     {
@@ -11083,7 +11183,9 @@ void HqlCppTranslator::doBuildStmtOutput(BuildCtx & ctx, IHqlExpression * expr)
     if (!name)
         name.setown(createQuoted("NULL", LINK(constUnknownVarStringType)));
 
-    Owned<IWUResult> result = createDatasetResultSchema(seq, name, dataset->queryRecord(), true, false);
+    HqlExprArray xmlnsAttrs;
+    gatherAttributes(xmlnsAttrs, xmlnsAtom, expr);
+    Owned<IWUResult> result = createDatasetResultSchema(seq, name, dataset->queryRecord(), xmlnsAttrs, true, false);
 
     CHqlBoundExpr bound;
     buildDataset(ctx, dataset, bound, FormatNatural);

ecl/hqlcpp/hqlttcpp.cpp

@@ -1422,6 +1422,7 @@ IHqlExpression * SequenceNumberAllocator::attachSequenceNumber(IHqlExpression *
                 args.append(*createAttribute(namedAtom, LINK(name)));
             args.append(*createAttribute(outputAtom));
             args.append(*createUniqueId());
+            gatherAttributes(args, xmlnsAtom, expr);
             return createSetResult(args);
         }
     default:

ecl/regress/bind2.ecl

@@ -0,0 +1,25 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#onwarning ('all', error);
+
+f(integer x, integer y) := (x * y);
+g(integer x, integer y, integer z) := f(x, y) + f(x, y) + 1;
+h(integer x, integer y) := g(x, y, 8) + g(x, y, 9);
+
+person := dataset('person', { unsigned8 person_id, string1 per_sex; }, thor);
+output(person,{h(1,3)},'out.d00');      // need to common up if parameters are insignificant

ecl/regress/bind2e.ecl

@@ -0,0 +1,25 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#onwarning ('aloadofrubbish', error);
+
+f(integer x, integer y) := (x * y);
+g(integer x, integer y, integer z) := f(x, y) + f(x, y) + 1;
+h(integer x, integer y) := g(x, y, 8) + g(x, y, 9);
+
+person := dataset('person', { unsigned8 person_id, string1 per_sex; }, thor);
+output(person,{h(1,3)},'out.d00');      // need to common up if parameters are insignificant

ecl/regress/issue9556a.eclxml

@@ -1,7 +1,7 @@
 <Archive legacyImport="0"
          legacyWhen="0">
  <Query attributePath="_local_directory_.temp"/>
- <OnWarning value="1006=ignore"/>
+ <OnWarning name="1006" value="ignore"/>
  <Module key="_local_directory_" name="_local_directory_">
   <Attribute key="temp" name="temp" sourcePath="temp.ecl">
    case(0,&apos;default&apos;);&#10;&#10;&#10;

esp/eclwatch/ws_XSLT/index.xslt

@@ -110,32 +110,35 @@
                                 document.location.href = "/WsSmc/Activity?SortBy=Name";
                         }
 
-                        function commandQueue(action,cluster,clusterType,queue,wuid)
+                        function commandQueue(action,cluster,clusterType,queue,wuid,serverType,ip,port)
                         {
                             document.getElementById("ClusterType").value=clusterType;
                             document.getElementById("Cluster").value=cluster;
                             document.getElementById("QueueName").value=queue;
                             document.getElementById("Wuid").value=wuid || '';
+                            document.getElementById("ServerType").value=serverType;
+                            document.getElementById("NetworkAddress").value=ip;
+                            document.getElementById("Port").value=port;
                             document.forms["queue"].action='/WsSMC/'+action;
                             document.forms["queue"].submit();
                         }
 
                         var oMenu;
 
-                        function queuePopup(cluster,clusterType,queue,paused,stopped,q_rowid)
+                        function queuePopup(cluster,clusterType,queue,serverType,ip,port,paused,stopped,q_rowid)
                         {
                             function clearQueue()
                             {
                                 if(confirm('Do you want to clear the queue for cluster: '+cluster+'?'))
-                                    commandQueue("ClearQueue",cluster,clusterType,queue);
+                                    commandQueue("ClearQueue",cluster,clusterType,queue,'',serverType,ip,port);
                             }
                             function pauseQueue()
                             {
-                                commandQueue("PauseQueue",cluster,clusterType,queue);
+                                commandQueue("PauseQueue",cluster,clusterType,queue,'',serverType,ip,port);
                             }
                             function resumeQueue()
                             {
-                                commandQueue("ResumeQueue",cluster,clusterType,queue);
+                                commandQueue("ResumeQueue",cluster,clusterType,queue,'',serverType,ip,port);
                             }
                             function showUsage()
                             {
@@ -566,6 +569,9 @@
                     <input type="hidden" name="Cluster" id="Cluster" value=""/>
                     <input type="hidden" name="QueueName" id="QueueName" value=""/>
                     <input type="hidden" name="Wuid" id="Wuid" value=""/>
+                    <input type="hidden" name="ServerType" id="ServerType" value=""/>
+                    <input type="hidden" name="NetworkAddress" id="NetworkAddress" value=""/>
+                    <input type="hidden" name="Port" id="Port" value=""/>
                     <xsl:for-each select="ThorClusterList/TargetCluster">
                         <xsl:call-template name="show-queue">
                             <xsl:with-param name="workunits" select="//Running/ActiveWorkunit[(Server='ThorMaster' and TargetClusterName=current()/ClusterName) or (ClusterType='Thor' and ClusterQueueName=current()/QueueName)]"/>
@@ -577,6 +583,7 @@
                             <xsl:with-param name="statusDetails" select="StatusDetails"/>
                             <xsl:with-param name="warning" select="Warning"/>
                             <xsl:with-param name="thorlcr" select="ThorLCR"/>
+                            <xsl:with-param name="serverType" select="'ThorMaster'"/>
                         </xsl:call-template>
                     </xsl:for-each>
 
@@ -590,6 +597,7 @@
                             <xsl:with-param name="clusterStatus" select="ClusterStatus"/>
                             <xsl:with-param name="statusDetails" select="StatusDetails"/>
                             <xsl:with-param name="warning" select="Warning"/>
+                            <xsl:with-param name="serverType" select="'RoxieServer'"/>
                         </xsl:call-template>
                     </xsl:for-each>
 
@@ -603,6 +611,7 @@
                             <xsl:with-param name="clusterStatus" select="ClusterStatus"/>
                             <xsl:with-param name="statusDetails" select="StatusDetails"/>
                             <xsl:with-param name="warning" select="Warning"/>
+                            <xsl:with-param name="serverType" select="'HThorServer'"/>
                         </xsl:call-template>
                     </xsl:for-each>
 
@@ -614,6 +623,9 @@
                             <xsl:with-param name="queue" select="QueueName"/>
                             <xsl:with-param name="queueStatus" select="QueueStatus"/>
                             <xsl:with-param name="statusDetails" select="StatusDetails"/>
+                            <xsl:with-param name="ip" select="NetworkAddress"/>
+                            <xsl:with-param name="port" select="Port"/>
+                            <xsl:with-param name="serverType" select="ServerType"/>
                         </xsl:call-template>
                     </xsl:for-each>
 
@@ -625,6 +637,9 @@
                             <xsl:with-param name="queue" select="QueueName"/>
                             <xsl:with-param name="queueStatus" select="QueueStatus"/>
                             <xsl:with-param name="statusDetails" select="StatusDetails"/>
+                            <xsl:with-param name="ip" select="NetworkAddress"/>
+                            <xsl:with-param name="port" select="Port"/>
+                            <xsl:with-param name="serverType" select="ServerType"/>
                         </xsl:call-template>
                     </xsl:for-each>
 
@@ -636,6 +651,9 @@
                             <xsl:with-param name="queue" select="QueueName"/>
                             <xsl:with-param name="queueStatus" select="QueueStatus"/>
                             <xsl:with-param name="statusDetails" select="StatusDetails"/>
+                            <xsl:with-param name="ip" select="NetworkAddress"/>
+                            <xsl:with-param name="port" select="Port"/>
+                            <xsl:with-param name="serverType" select="ServerType"/>
                         </xsl:call-template>
                     </xsl:for-each>
 
@@ -647,6 +665,7 @@
                             <xsl:with-param name="queue" select="QueueName"/>
                             <xsl:with-param name="queueStatus" select="QueueStatus"/>
                             <xsl:with-param name="statusDetails" select="StatusDetails"/>
+                            <xsl:with-param name="serverType" select="ServerType"/>
                         </xsl:call-template>
                     </xsl:for-each>
                     <xsl:apply-templates select="DFUJobs"/>
@@ -710,6 +729,9 @@
         <xsl:param name="statusDetails" select="''"/>
         <xsl:param name="warning" select="''"/>
         <xsl:param name="thorlcr" select="'0'"/>
+        <xsl:param name="serverType" select="''"/>
+        <xsl:param name="ip" select="''"/>
+        <xsl:param name="port" select="'0'"/>
         <xsl:variable name="showTitle">
             <xsl:choose>
                 <xsl:when test="$clusterType = 'THOR'">1</xsl:when>
@@ -761,8 +783,8 @@
                 <td valign="top">
                     <xsl:if test="$accessRight = 'Access_Full'">
                         <xsl:variable name="popup">return queuePopup('<xsl:value-of select="$cluster"/>','<xsl:value-of select="$clusterType"/>',
-                            '<xsl:value-of select="$queue"/>',<xsl:value-of select="$queueStatus='paused'"/>,
-                            <xsl:value-of select="$queueStatus='stopped'"/>, '<xsl:value-of select="$q_rowid"/>');
+                            '<xsl:value-of select="$queue"/>','<xsl:value-of select="$serverType"/>','<xsl:value-of select="$ip"/>','<xsl:value-of select="$port"/>',
+                            <xsl:value-of select="$queueStatus='paused'"/>,<xsl:value-of select="$queueStatus='stopped'"/>, '<xsl:value-of select="$q_rowid"/>');
                         </xsl:variable>
                         <a id="{$q_rowid}" class="configurecontextmenu" title="Option" onclick="{$popup}">&#160;</a>
                     </xsl:if>
@@ -811,6 +833,7 @@
                             <xsl:when test="$clusterType = 'ROXIE'">RoxieCluster - <xsl:value-of select="$cluster"/></xsl:when>
                             <xsl:when test="$clusterType = 'THOR'">ThorCluster - <xsl:value-of select="$cluster"/></xsl:when>
                             <xsl:when test="$clusterType = 'HTHOR'">HThorCluster - <xsl:value-of select="$cluster"/></xsl:when>
+                            <xsl:when test="$clusterType = 'ECLCCserver' or $clusterType = 'ECLserver'"><xsl:value-of select="$clusterType"/> - <xsl:value-of select="$cluster"/></xsl:when>
                             <xsl:otherwise>
                                 <xsl:value-of select="$clusterType"/> - <xsl:value-of select="$queue"/>
                             </xsl:otherwise>

esp/scm/ws_smc.ecm

@@ -107,6 +107,15 @@ ESPStruct ServerJobQueue
     string ServerType;
     string QueueStatus;
     [min_ver("1.17")] string StatusDetails;
+    [min_ver("1.19")] string NetworkAddress;
+    [min_ver("1.19")] int Port;
+};
+
+ESPstruct [nil_remove] StatusServerInfo
+{
+    ESPstruct TargetCluster TargetClusterInfo;
+    ESPstruct ServerJobQueue ServerInfo;
+    ESParray<ESPstruct ActiveWorkunit> Workunits;
 };
 
 ESPrequest [nil_remove] ActivityRequest
@@ -181,15 +190,18 @@ ESPresponse [] SMCPermissionsResponse
 
 ESPrequest SMCQueueRequest
 {
-    int ClusterType;
     string Cluster;
     string QueueName;
     string Comment;
+    [min_ver("1.19")] string ServerType;
+    [min_ver("1.19")] string NetworkAddress;
+    [min_ver("1.19")] int Port;
 };
 
 
 ESPresponse [exceptions_inline] SMCQueueResponse
 {
+    [min_ver("1.19")] ESPstruct StatusServerInfo StatusServerInfo;
 };
 
 ESPrequest SMCJobRequest
@@ -328,7 +340,24 @@ RoxieControlCmdResponse
     ESParray<ESPstruct RoxieControlEndpointInfo, Endpoint> Endpoints;
 };
 
-ESPservice [noforms, version("1.18"), default_client_version("1.18"), exceptions_inline("./smc_xslt/exceptions.xslt"), use_method_name] WsSMC
+ESPrequest GetStatusServerInfoRequest
+{
+    string ServerName;
+    string ServerType;
+    string NetworkAddress;
+    int Port;
+};
+
+ESPresponse
+[
+    exceptions_inline, nil_remove
+]
+GetStatusServerInfoResponse
+{
+    ESPstruct StatusServerInfo StatusServerInfo;
+};
+
+ESPservice [noforms, version("1.19"), default_client_version("1.19"), exceptions_inline("./smc_xslt/exceptions.xslt"), use_method_name] WsSMC
 {
     ESPmethod Index(SMCIndexRequest, SMCIndexResponse);
     ESPmethod [resp_xsl_default("/esp/xslt/index.xslt")] Activity(ActivityRequest, ActivityResponse);
@@ -353,6 +382,7 @@ ESPservice [noforms, version("1.18"), default_client_version("1.18"), exceptions
     ESPmethod [resp_xsl_default("/esp/xslt/hpccresourcelist.xslt")] BrowseResources(BrowseResourcesRequest, BrowseResourcesResponse);
 
     ESPmethod RoxieControlCmd(RoxieControlCmdRequest, RoxieControlCmdResponse);
+    ESPmethod GetStatusServerInfo(GetStatusServerInfoRequest, GetStatusServerInfoResponse);
 };
 
 SCMexportdef(WSSMC);

esp/scm/ws_workunits.ecm

@@ -1118,6 +1118,7 @@ ESPrequest [nil_remove] WUPublishWorkunitRequest
     bool DontCopyFiles(false);
     string SourceProcess;
     bool AllowForeignFiles(false);
+    bool UpdateDfs(false);
 };
 
 ESPresponse [exceptions_inline] WUPublishWorkunitResponse

esp/services/ws_ecl/ws_ecl_service.cpp

@@ -2209,14 +2209,19 @@ int CWsEclBinding::onSubmitQueryOutput(IEspContext &context, CHttpRequest* reque
     StringBuffer output;
 
     SCMStringBuffer clustertype;
-
+    const char *contentType="application/xml";
     bool isRoxieReq = wsecl->connMap.getValue(wsinfo.qsetname.get())!=NULL;
     bool outputJSON = (format && strieq(format, "json"));
+    const char *jsonp = context.queryRequestParameters()->queryProp("jsonp");
     if (isRoxieReq && outputJSON)
     {
         StringBuffer jsonmsg;
         getWsEclJsonRequest(jsonmsg, context, request, wsinfo, "json", NULL, 0, false);
+        if (jsonp && *jsonp)
+            output.append(jsonp).append('(');
         sendRoxieRequest(wsinfo.qsetname.get(), jsonmsg, output, status, wsinfo.queryname, "application/json");
+        if (jsonp && *jsonp)
+            output.append(");");
     }
     else
     {
@@ -2249,8 +2254,11 @@ int CWsEclBinding::onSubmitQueryOutput(IEspContext &context, CHttpRequest* reque
         }
     }
 
+    if (outputJSON)
+        contentType = (jsonp && *jsonp) ? "application/javascript" : "application/json";
+
     response->setContent(output.str());
-    response->setContentType(outputJSON ? "application/json" : "application/xml");
+    response->setContentType(contentType);
     response->setStatus("200 OK");
     response->send();
 

esp/services/ws_smc/ws_smcService.cpp

@@ -32,6 +32,15 @@
 #include "exception_util.hpp"
 
 #include "roxiecontrol.hpp"
+#include "workunit.hpp"
+
+#define STATUS_SERVER_THOR "ThorMaster"
+#define STATUS_SERVER_HTHOR "HThorServer"
+#define STATUS_SERVER_ROXIE "RoxieServer"
+#define STATUS_SERVER_DFUSERVER "DFUserver"
+#define STATUS_SERVER_ECLSERVER "ECLserver"
+#define STATUS_SERVER_ECLCCSERVER "ECLCCserver"
+#define STATUS_SERVER_ECLAGENT "ECLagent"
 
 static const char* FEATURE_URL = "SmcAccess";
 const char* THORQUEUE_FEATURE = "ThorQueueAccess";
@@ -732,7 +741,7 @@ void CWsSMCEx::getServersAndWUs(IEspContext &context, IEspActivityRequest &req,
                 aws.append(*wu.getLink());
             }
 
-            addServerJobQueue(version, serverJobQueues, queueName, serverName, "ECLCCserver", queueState.str(), queueStateDetails.str());
+            addServerJobQueue(version, serverJobQueues, queueName, serverName, "ECLCCserver", NULL, 0, queueState.str(), queueStateDetails.str());
         }
     }
 
@@ -809,7 +818,7 @@ void CWsSMCEx::getServersAndWUs(IEspContext &context, IEspActivityRequest &req,
                             e->Release();
                         }
                     }
-                    addServerJobQueue(version, serverJobQueues, queueName, serverName, "DFUserver");
+                    addServerJobQueue(version, serverJobQueues, queueName, serverName, "DFUserver", NULL, 0);
                 }
             }
         } while (services->next());
@@ -1089,12 +1098,12 @@ void CWsSMCEx::getWUsNotOnTargetCluster(IEspContext &context, IPropertyTree* ser
         IPropertyTree& serverNode = it->query();
         const char* serverName = serverNode.queryProp("@name");
         const char* instance = serverNode.queryProp("@node");
-        if (!serverName || !*serverName || !instance || !*instance)
+        const char* queueName = serverNode.queryProp("@queue");
+        unsigned port = serverNode.getPropInt("@mpport", 0);
+        if (!serverName || !*serverName || !instance || !*instance || strieq(serverName, "DFUserver"))//DFUServer already handled separately
             continue;
 
-        bool hasWU = false;
-        StringBuffer queueName;
-        queueName.appendf("%s_on_%s", serverName, instance);
+        VStringBuffer instanceName("%s_on_%s:%d", serverName, instance, port);
         Owned<IPropertyTreeIterator> wuids(serverNode.getElements("WorkUnit"));
         ForEach(*wuids)
         {
@@ -1106,14 +1115,13 @@ void CWsSMCEx::getWUsNotOnTargetCluster(IEspContext &context, IPropertyTree* ser
                 continue;
 
             Owned<IEspActiveWorkunit> wu;
-            createActiveWorkUnit(wu, context, wuid, NULL, 0, serverName, queueName.str(), instance, NULL);
+            createActiveWorkUnit(wu, context, wuid, NULL, 0, serverName, queueName, instance, NULL);
             aws.append(*wu.getLink());
-            hasWU = true;
         }
-        if (hasWU && !uniqueServers.getValue(queueName))
+        if (!uniqueServers.getValue(instanceName))
         {
-            uniqueServers.setValue(queueName, true);
-            addServerJobQueue(version, serverJobQueues, queueName.str(), serverName, serverName);
+            uniqueServers.setValue(instanceName, true);
+            addServerJobQueue(version, serverJobQueues, queueName, instanceName, serverName, instance, port);
         }
     }
 
@@ -1177,7 +1185,7 @@ void CWsSMCEx::getDFUServersAndWUs(IEspContext &context, IPropertyTree* envRoot,
         {
             const char *queueName = queues.item(q);
             readDFUWUs(context, queueName, serverName, aws);
-            addServerJobQueue(version, serverJobQueues, queueName, serverName, "DFUserver");
+            addServerJobQueue(version, serverJobQueues, queueName, serverName, "DFUserver", NULL, 0);
         }
     }
 }
@@ -1346,9 +1354,12 @@ void CWsSMCEx::readTargetClusterInfo(IEspContext& context, IConstWUClusterInfo&
     targetCluster->statusServerName.set(statusServerName.str());
     targetCluster->queueName.set(jobQueue->queueName.str());
 
-    jobQueue->foundQueueInStatusServer = findQueueInStatusServer(context, serverStatusRoot, statusServerName.str(), targetCluster->queueName.get());
-    if (!jobQueue->foundQueueInStatusServer)
-        targetCluster->clusterStatusDetails.appendf("Cluster %s not attached; ", clusterName.str());
+    if (serverStatusRoot)
+    {
+        jobQueue->foundQueueInStatusServer = findQueueInStatusServer(context, serverStatusRoot, statusServerName.str(), targetCluster->queueName.get());
+        if (!jobQueue->foundQueueInStatusServer)
+            targetCluster->clusterStatusDetails.appendf("Cluster %s not attached; ", clusterName.str());
+    }
 
     return;
 }
@@ -1486,22 +1497,24 @@ void CWsSMCEx::readRunningWUsOnStatusServer(IEspContext& context, IPropertyTree*
 void CWsSMCEx::readWUsAndStateFromJobQueue(IEspContext& context, CIArrayOf<CWsSMCTargetCluster>& targetClusters, BoolHash& uniqueWUIDs, IArrayOf<IEspActiveWorkunit>& aws)
 {
     ForEachItemIn(i, targetClusters)
+        readWUsAndStateFromJobQueue(context, targetClusters.item(i), uniqueWUIDs, aws);
+}
+
+void CWsSMCEx::readWUsAndStateFromJobQueue(IEspContext& context, CWsSMCTargetCluster& targetCluster, BoolHash& uniqueWUIDs, IArrayOf<IEspActiveWorkunit>& aws)
+{
+    if (targetCluster.clusterType == ThorLCRCluster)
     {
-        CWsSMCTargetCluster& targetCluster = targetClusters.item(i);
-        if (targetCluster.clusterType == ThorLCRCluster)
-        {
-            readWUsAndStateFromJobQueue(context, targetCluster, targetCluster.clusterQueue, NULL, uniqueWUIDs, aws);
-            targetCluster.queueStatus.set(targetCluster.clusterQueue.queueState);
-        }
-        if (targetCluster.agentQueue.queueName.length())
-        {
-            readWUsAndStateFromJobQueue(context, targetCluster, targetCluster.agentQueue, targetCluster.agentQueue.queueName.str(), uniqueWUIDs, aws);
-            if (targetCluster.clusterType != ThorLCRCluster)
-                targetCluster.queueStatus.set(targetCluster.agentQueue.queueState);
-        }
-        if (targetCluster.serverQueue.queueName.length())
-            readWUsAndStateFromJobQueue(context, targetCluster, targetCluster.serverQueue, targetCluster.serverQueue.queueName.str(), uniqueWUIDs, aws);
+        readWUsAndStateFromJobQueue(context, targetCluster, targetCluster.clusterQueue, NULL, uniqueWUIDs, aws);
+        targetCluster.queueStatus.set(targetCluster.clusterQueue.queueState);
     }
+    if (targetCluster.agentQueue.queueName.length())
+    {
+        readWUsAndStateFromJobQueue(context, targetCluster, targetCluster.agentQueue, targetCluster.agentQueue.queueName.str(), uniqueWUIDs, aws);
+        if (targetCluster.clusterType != ThorLCRCluster)
+            targetCluster.queueStatus.set(targetCluster.agentQueue.queueState);
+    }
+    if (targetCluster.serverQueue.queueName.length())
+        readWUsAndStateFromJobQueue(context, targetCluster, targetCluster.serverQueue, targetCluster.serverQueue.queueName.str(), uniqueWUIDs, aws);
 }
 
 CWsSMCTargetCluster* CWsSMCEx::findTargetCluster(const char* clusterName, CIArrayOf<CWsSMCTargetCluster>& targetClusters)
@@ -1593,7 +1606,8 @@ void CWsSMCEx::setESPTargetClusters(IEspContext& context, CIArrayOf<CWsSMCTarget
     }
 }
 
-void CWsSMCEx::addServerJobQueue(double version, IArrayOf<IEspServerJobQueue>& jobQueues, const char* queueName, const char* serverName, const char* serverType)
+void CWsSMCEx::addServerJobQueue(double version, IArrayOf<IEspServerJobQueue>& jobQueues, const char* queueName, const char* serverName,
+    const char* serverType, const char* networkAddress, unsigned port)
 {
     if (!queueName || !*queueName || !serverName || !*serverName || !serverType || !*serverType)
         return;
@@ -1607,10 +1621,11 @@ void CWsSMCEx::addServerJobQueue(double version, IArrayOf<IEspServerJobQueue>& j
         queueState.set("paused");
     else
         queueState.set("running");
-    addServerJobQueue(version, jobQueues, queueName, serverName, serverType, queueState.str(), queueStateDetails.str());
+    addServerJobQueue(version, jobQueues, queueName, serverName, serverType, networkAddress, port, queueState.str(), queueStateDetails.str());
 }
 
-void CWsSMCEx::addServerJobQueue(double version, IArrayOf<IEspServerJobQueue>& jobQueues, const char* queueName, const char* serverName, const char* serverType, const char* queueState, const char* queueStateDetails)
+void CWsSMCEx::addServerJobQueue(double version, IArrayOf<IEspServerJobQueue>& jobQueues, const char* queueName, const char* serverName,
+    const char* serverType, const char* networkAddress, unsigned port, const char* queueState, const char* queueStateDetails)
 {
     if (!queueName || !*queueName || !serverName || !*serverName || !serverType || !*serverType)
         return;
@@ -1622,6 +1637,11 @@ void CWsSMCEx::addServerJobQueue(double version, IArrayOf<IEspServerJobQueue>& j
     jobQueue->setQueueName(queueName);
     jobQueue->setServerName(serverName);
     jobQueue->setServerType(serverType);
+    if ((version >= 1.19) && networkAddress && *networkAddress)
+    {
+        jobQueue->setNetworkAddress(networkAddress);
+        jobQueue->setPort(port);
+    }
     setServerJobQueueStatus(version, jobQueue, queueState, queueStateDetails);
 
     jobQueues.append(*jobQueue.getClear());
@@ -1922,6 +1942,10 @@ bool CWsSMCEx::onStopQueue(IEspContext &context, IEspSMCQueueRequest &req, IEspS
         Owned<IJobQueue> queue = createJobQueue(req.getQueueName());
         queue->stop(createQueueActionInfo(context, "stopped", req, info));
         AccessSuccess(context, "Stopped queue %s",req.getCluster());
+        double version = context.getClientVersion();
+        if (version >= 1.19)
+            getStatusServerInfo(context, req.getServerType(), req.getCluster(), req.getNetworkAddress(), req.getPort(), resp.updateStatusServerInfo());
+
         resp.setRedirectUrl("/WsSMC/");
     }
     catch(IException* e)
@@ -1941,6 +1965,10 @@ bool CWsSMCEx::onResumeQueue(IEspContext &context, IEspSMCQueueRequest &req, IEs
         Owned<IJobQueue> queue = createJobQueue(req.getQueueName());
         queue->resume(createQueueActionInfo(context, "resumed", req, info));
         AccessSuccess(context, "Resumed queue %s",req.getCluster());
+        double version = context.getClientVersion();
+        if (version >= 1.19)
+            getStatusServerInfo(context, req.getServerType(), req.getCluster(), req.getNetworkAddress(), req.getPort(), resp.updateStatusServerInfo());
+
         resp.setRedirectUrl("/WsSMC/");
     }
     catch(IException* e)
@@ -1977,6 +2005,10 @@ bool CWsSMCEx::onPauseQueue(IEspContext &context, IEspSMCQueueRequest &req, IEsp
         Owned<IJobQueue> queue = createJobQueue(req.getQueueName());
         queue->pause(createQueueActionInfo(context, "paused", req, info));
         AccessSuccess(context, "Paused queue %s",req.getCluster());
+        double version = context.getClientVersion();
+        if (version >= 1.19)
+            getStatusServerInfo(context, req.getServerType(), req.getCluster(), req.getNetworkAddress(), req.getPort(), resp.updateStatusServerInfo());
+
         resp.setRedirectUrl("/WsSMC/");
     }
     catch(IException* e)
@@ -1999,6 +2031,10 @@ bool CWsSMCEx::onClearQueue(IEspContext &context, IEspSMCQueueRequest &req, IEsp
             queue->clear();
         }
         AccessSuccess(context, "Cleared queue %s",req.getCluster());
+        double version = context.getClientVersion();
+        if (version >= 1.19)
+            getStatusServerInfo(context, req.getServerType(), req.getCluster(), req.getNetworkAddress(), req.getPort(), resp.updateStatusServerInfo());
+
         resp.setRedirectUrl("/WsSMC/");
     }
     catch(IException* e)
@@ -2480,3 +2516,265 @@ bool CWsSMCEx::onRoxieControlCmd(IEspContext &context, IEspRoxieControlCmdReques
     resp.setEndpoints(respEndpoints);
     return true;
 }
+
+bool CWsSMCEx::onGetStatusServerInfo(IEspContext &context, IEspGetStatusServerInfoRequest &req, IEspGetStatusServerInfoResponse &resp)
+{
+    getStatusServerInfo(context, req.getServerType(), req.getServerName(), req.getNetworkAddress(), req.getPort(), resp.updateStatusServerInfo());
+    return true;
+}
+
+void CWsSMCEx::getStatusServerInfo(IEspContext &context, const char *serverType, const char *server, const char *networkAddress, unsigned port,
+    IEspStatusServerInfo& statusServerInfo)
+{
+    if (!serverType || !*serverType)
+        throw MakeStringException(ECLWATCH_MISSING_PARAMS, "Server type not specified.");
+
+    if (strieq(serverType,STATUS_SERVER_THOR) || strieq(serverType,STATUS_SERVER_HTHOR) || strieq(serverType,STATUS_SERVER_ROXIE))
+    {
+        if (!server || !*server)
+            throw MakeStringException(ECLWATCH_MISSING_PARAMS, "cluster not specified.");
+        getStatusServerInfo(context, server, statusServerInfo);
+    }
+    else if (!strieq(serverType,STATUS_SERVER_DFUSERVER))
+    {
+        if (!networkAddress || !*networkAddress)
+            throw MakeStringException(ECLWATCH_MISSING_PARAMS, "Server network address not specified.");
+        getStatusServerInfo(context, serverType, networkAddress, port, statusServerInfo);
+    }
+    else
+    {
+        if (!server || !*server)
+            throw MakeStringException(ECLWATCH_MISSING_PARAMS, "Server not specified.");
+        getDFUServerInfo(context, server, statusServerInfo);
+    }
+    return;
+}
+
+void CWsSMCEx::getStatusServerInfo(IEspContext &context, const char* clusteName, IEspStatusServerInfo& statusServerInfo)
+{
+    double version = context.getClientVersion();
+
+    Owned<IConstWUClusterInfo> info = getTargetClusterInfo(clusteName);
+    if (!info)
+        throw MakeStringException(ECLWATCH_CANNOT_GET_ENV_INFO,"Failed to get target cluster information.");
+    CWsSMCTargetCluster targetCluster;
+    readTargetClusterInfo(context, *info, NULL, &targetCluster);
+
+    bool foundQueueInStatusServer = false;
+    Owned<IPropertyTree> statusServerTree = getStatusServerTree(info);
+    if (statusServerTree)
+    {
+        foundQueueInStatusServer = true;
+
+        BoolHash uniqueWUIDs;
+        IArrayOf<IEspActiveWorkunit> aws;
+        StringBuffer networkAddress;
+        statusServerTree->getProp("@node", networkAddress);
+        unsigned port = statusServerTree->getPropInt("@mpport");
+        readRunningWUsOnCluster(context, clusteName, networkAddress.str(), port, targetCluster, statusServerTree, uniqueWUIDs, aws);
+        readWUsAndStateFromJobQueue(context, targetCluster, uniqueWUIDs, aws);
+        statusServerInfo.setWorkunits(aws);
+    }
+
+    IEspTargetCluster& clusterInfo =  statusServerInfo.updateTargetClusterInfo();
+    clusterInfo.setClusterName(targetCluster.clusterName.get());
+    clusterInfo.setClusterSize(targetCluster.clusterSize);
+    clusterInfo.setClusterType(targetCluster.clusterType);
+    clusterInfo.setQueueName(targetCluster.queueName.get());
+    clusterInfo.setQueueStatus(targetCluster.queueStatus.get());
+    if (targetCluster.clusterType != ThorLCRCluster)
+        targetCluster.agentQueue.foundQueueInStatusServer = foundQueueInStatusServer;
+    else
+        targetCluster.clusterQueue.foundQueueInStatusServer = foundQueueInStatusServer;
+    setClusterStatus(context, targetCluster, &clusterInfo);
+}
+
+void CWsSMCEx::getStatusServerInfo(IEspContext &context, const char* type, const char *networkAddress, unsigned port, IEspStatusServerInfo& statusServerInfo)
+{
+    double version = context.getClientVersion();
+
+    Owned<IPropertyTree> statusServerTree = getStatusServerTree(networkAddress, port);
+    if (!statusServerTree)
+        throw MakeStringException(ECLWATCH_CANNOT_GET_STATUS_INFO, "Server not attached");
+
+    IEspServerJobQueue& serverInfo =  statusServerInfo.updateServerInfo();
+    serverInfo.setNetworkAddress(networkAddress);
+    serverInfo.setPort(port);
+
+    StringBuffer queueName, instance;
+    statusServerTree->getProp("@queue", queueName);
+    setServerJobQueue(version, type, NULL, queueName.str(), serverInfo);
+    instance.appendf("%s on %s:%d", type, networkAddress, port);
+
+    IArrayOf<IEspActiveWorkunit> aws;
+    Owned<IPropertyTreeIterator> wuids(statusServerTree->getElements("WorkUnit"));
+    ForEach(*wuids)
+    {
+        const char* wuid=wuids->query().queryProp(NULL);
+        if (!wuid || !*wuid)
+            continue;
+
+        Owned<IEspActiveWorkunit> wu;
+        createActiveWorkUnit(wu, context, wuid, NULL, 0, type, queueName.str(), instance, NULL);
+        aws.append(*wu.getLink());
+    }
+    statusServerInfo.setWorkunits(aws);
+}
+
+void CWsSMCEx::getDFUServerInfo(IEspContext &context, const char* serverName, IEspStatusServerInfo& statusServerInfo)
+{
+    double version = context.getClientVersion();
+
+    VStringBuffer xpath("/Environment/Software/%s[@name=\"%s\"]", eqDfu, serverName);
+    Owned<IRemoteConnection> connEnv = querySDS().connect(xpath.str(), myProcessSession(), RTM_LOCK_READ, SDS_LOCK_TIMEOUT);
+    if (!connEnv)
+        throw MakeStringException(ECLWATCH_CANNOT_GET_ENV_INFO,"Failed to get environment information.");
+    IPropertyTree* serviceTree = connEnv->queryRoot();
+    if (!serviceTree)
+        throw MakeStringException(ECLWATCH_CANNOT_GET_ENV_INFO,"Failed to get environment information.");
+    const char *queueName = serviceTree->queryProp("@queue");
+    if (!queueName || !*queueName)
+        throw MakeStringException(ECLWATCH_CANNOT_GET_ENV_INFO, "Server queue not found.");
+
+    setServerJobQueue(version, "DFUserver", serverName, queueName, statusServerInfo.updateServerInfo());
+
+    IArrayOf<IEspActiveWorkunit> aws;
+    readDFUWUs(context, queueName, serverName, aws);
+    statusServerInfo.setWorkunits(aws);
+}
+
+IPropertyTree* CWsSMCEx::getStatusServerTree(IConstWUClusterInfo* info)
+{
+    SCMStringBuffer str;
+    StringBuffer xpath;
+    if (info->getPlatform() != HThorCluster)
+    {
+        if (info->getPlatform() == ThorLCRCluster)
+            xpath.setf("/Status/Servers/Server[@name=\"%s\"][@cluster=\"%s\"]", getStatusServerTypeName(WsSMCSSTThorLCRCluster), info->getThorProcesses().item(0));
+        else
+            xpath.setf("/Status/Servers/Server[@name=\"%s\"][@cluster=\"%s\"]", getStatusServerTypeName(WsSMCSSTRoxieCluster), info->getRoxieProcess(str).str());
+        Owned<IRemoteConnection> connStatusServer = querySDS().connect(xpath.str(),myProcessSession(),RTM_LOCK_READ,SDS_LOCK_TIMEOUT);
+        if (!connStatusServer)
+            return NULL;
+
+        Owned<IPropertyTree> retServerTree = connStatusServer->queryRoot()->getBranch(NULL);
+        return retServerTree.getClear();
+    }
+    else
+    {
+        Owned<IRemoteConnection> connStatusServer = querySDS().connect("/Status/Servers",myProcessSession(),RTM_LOCK_READ,SDS_LOCK_TIMEOUT);
+        if (!connStatusServer)
+            throw MakeStringException(ECLWATCH_CANNOT_GET_STATUS_INFO, "Status servers not found");
+
+        info->getAgentQueue(str);
+        xpath.setf("Server[@name=\"%s\"]", getStatusServerTypeName(WsSMCSSTHThorCluster));
+        Owned<IPropertyTreeIterator> it(connStatusServer->queryRoot()->getElements(xpath));
+        ForEach(*it)
+        {
+            IPropertyTree &serverTree = it->query();
+            const char *queueNames = serverTree.queryProp("@queue");
+            if (!queueNames || !*queueNames)
+                continue;
+
+            StringArray qlist;
+            qlist.appendListUniq(queueNames, ",");
+            ForEachItemIn(q, qlist)
+            {
+                if (!strieq(qlist.item(q), str.str()))
+                    continue;
+
+                Owned<IPropertyTree> retServerTree = serverTree.getBranch(NULL);
+                return retServerTree.getClear();
+            }
+        }
+    }
+    return NULL;
+}
+
+IPropertyTree* CWsSMCEx::getStatusServerTree(const char *networkAddress, unsigned port)
+{
+    if (!networkAddress || !*networkAddress)
+        throw MakeStringException(ECLWATCH_INVALID_INPUT, "Network Address not specified");
+
+    VStringBuffer xpath("/Status/Servers/Server[@node=\"%s\"][@mpport=\"%d\"]", networkAddress, port);
+    Owned<IRemoteConnection> connStatusServer = querySDS().connect(xpath.str(),myProcessSession(),RTM_LOCK_READ,SDS_LOCK_TIMEOUT);
+    if (!connStatusServer)
+        return NULL;
+
+    Owned<IPropertyTree> retServerTree = connStatusServer->queryRoot()->getBranch(NULL);
+    return retServerTree.getClear();
+}
+
+void CWsSMCEx::readRunningWUsOnCluster(IEspContext& context, const char* serverName, const char* node, unsigned port,
+        CWsSMCTargetCluster& targetCluster, IPropertyTree* statusServerNode, BoolHash& uniqueWUIDs, IArrayOf<IEspActiveWorkunit>& aws)
+{
+    const char *cluster = statusServerNode->queryProp("Cluster");
+    StringBuffer queueName;
+    if (cluster) // backward compat check.
+        getClusterThorQueueName(queueName, cluster);
+    else
+        queueName.append(targetCluster.queueName.get());
+
+    CWsSMCQueue* jobQueue;
+    if (targetCluster.clusterType == ThorLCRCluster)
+        jobQueue = &targetCluster.clusterQueue;
+    else
+        jobQueue = &targetCluster.agentQueue;
+
+    StringBuffer instance;
+    if ((targetCluster.clusterType == ThorLCRCluster) || (targetCluster.clusterType == RoxieCluster))
+        statusServerNode->getProp("@cluster", instance);
+    else
+        instance.appendf("%s on %s:%d", serverName, node, port);
+
+    const char* targetClusterName = targetCluster.clusterName.get();
+    Owned<IPropertyTreeIterator> wuids(statusServerNode->getElements("WorkUnit"));
+    ForEach(*wuids)
+    {
+        const char* wuid=wuids->query().queryProp(NULL);
+        if (!wuid || !*wuid)
+            continue;
+
+        Owned<IEspActiveWorkunit> wu;
+        createActiveWorkUnit(wu, context, wuid, !strieq(targetClusterName, instance.str()) ? instance.str() : NULL, 0, serverName, queueName, instance.str(), targetClusterName);
+        if (wu->getStateID() == WUStateRunning) //'aborting' may be another possible status
+        {
+            StringBuffer durationStr, subgraphStr;
+            int sgDuration = statusServerNode->getPropInt("@sg_duration", -1);
+            int subgraph = statusServerNode->getPropInt("@subgraph", -1);
+            const char* graph = statusServerNode->queryProp("@graph");
+            durationStr.appendf("%d min", sgDuration);
+            subgraphStr.appendf("%d", subgraph);
+            if (subgraph > -1 && sgDuration > -1)
+            {
+                wu->setGraphName(graph);
+                wu->setDuration(durationStr.str());
+                wu->setGID(subgraphStr.str());
+            }
+
+            if (statusServerNode->getPropInt("@memoryBlocked ", 0) != 0)
+                wu->setMemoryBlocked(1);
+        }
+
+        aws.append(*wu.getLink());
+        jobQueue->countRunningJobs++;
+    }
+}
+
+void CWsSMCEx::setServerJobQueue(double version, const char* serverType, const char* serverName, const char* queueName, IEspServerJobQueue& serverInfo)
+{
+    StringBuffer queueState, queueStateDetails;
+    Owned<IJobQueue> queue = createJobQueue(queueName);
+    if (queue->stopped(queueStateDetails))
+        queueState.set("stopped");
+    else if (queue->paused(queueStateDetails))
+        queueState.set("paused");
+    else
+        queueState.set("running");
+
+    serverInfo.setQueueName(queueName);
+    serverInfo.setServerType(serverType);
+    if (serverName && *serverName)
+        serverInfo.setServerName(serverName);
+    setServerJobQueueStatus(version, &serverInfo, queueState, queueStateDetails);
+}

esp/services/ws_smc/ws_smcService.hpp

@@ -128,13 +128,16 @@ public:
 
     virtual bool onBrowseResources(IEspContext &context, IEspBrowseResourcesRequest & req, IEspBrowseResourcesResponse & resp);
     virtual bool onRoxieControlCmd(IEspContext &context, IEspRoxieControlCmdRequest &req, IEspRoxieControlCmdResponse &resp);
+    virtual bool onGetStatusServerInfo(IEspContext &context, IEspGetStatusServerInfoRequest &req, IEspGetStatusServerInfoResponse &resp);
 private:
     void addCapabilities( IPropertyTree* pFeatureNode, const char* access, 
                                  IArrayOf<IEspCapability>& capabilities);
     void addToThorClusterList(IArrayOf<IEspThorCluster>& clusters, IEspThorCluster* cluster, const char* sortBy, bool descending);
     void addToRoxieClusterList(IArrayOf<IEspRoxieCluster>& clusters, IEspRoxieCluster* cluster, const char* sortBy, bool descending);
-    void addServerJobQueue(double version, IArrayOf<IEspServerJobQueue>& jobQueues, const char* queueName, const char* serverName, const char* serverType);
-    void addServerJobQueue(double version, IArrayOf<IEspServerJobQueue>& jobQueues, const char* queueName, const char* serverName, const char* serverType, const char* queueState, const char* queueStateDetails);
+    void addServerJobQueue(double version, IArrayOf<IEspServerJobQueue>& jobQueues, const char* queueName, const char* serverName,
+        const char* serverType, const char* networkAddress, unsigned port);
+    void addServerJobQueue(double version, IArrayOf<IEspServerJobQueue>& jobQueues, const char* queueName, const char* serverName,
+        const char* serverType, const char* networkAddress, unsigned port, const char* queueState, const char* queueStateDetails);
     void getQueueState(int runningJobsInQueue, StringBuffer& queueState, BulletType& colorType);
     void readClusterTypeAndQueueName(CConstWUClusterInfoArray& clusters, const char* clusterName, StringBuffer& clusterType, SCMStringBuffer& clusterQueue);
     void addRunningWUs(IEspContext &context, IPropertyTree& node, CConstWUClusterInfoArray& clusters,
@@ -179,12 +182,23 @@ private:
     void readRunningWUsOnStatusServer(IEspContext& context, IPropertyTree* serverStatusRoot, WsSMCStatusServerType statusServerType,
             CIArrayOf<CWsSMCTargetCluster>& targetClusters, CIArrayOf<CWsSMCTargetCluster>& targetClusters1, CIArrayOf<CWsSMCTargetCluster>& targetClusters2,
             BoolHash& uniqueWUIDs, IArrayOf<IEspActiveWorkunit>& aws);
+    void readWUsAndStateFromJobQueue(IEspContext& context, CWsSMCTargetCluster& targetCluster, BoolHash& uniqueWUIDs, IArrayOf<IEspActiveWorkunit>& aws);
     void readWUsAndStateFromJobQueue(IEspContext& context, CIArrayOf<CWsSMCTargetCluster>& targetClusters, BoolHash& uniqueWUIDs, IArrayOf<IEspActiveWorkunit>& aws);
     void setESPTargetClusters(IEspContext& context, CIArrayOf<CWsSMCTargetCluster>& targetClusters, IArrayOf<IEspTargetCluster>& respTargetClusters);
     void updateActivityResponse(IEspContext &context,  IEspActivityRequest &req, IEspActivityResponse& resp,
             CIArrayOf<CWsSMCTargetCluster>& thorTargetClusters, CIArrayOf<CWsSMCTargetCluster>& roxieTargetClusters, CIArrayOf<CWsSMCTargetCluster>& hthorTargetClusters,
             IArrayOf<IEspActiveWorkunit>& aws, IArrayOf<IEspServerJobQueue>& serverJobQueues, IArrayOf<IEspDFUJob>& DFURecoveryJobs);
     const char *getStatusServerTypeName(WsSMCStatusServerType type);
+    void getStatusServerInfo(IEspContext &context, const char *serverType, const char *server, const char *networkAddress, unsigned port,
+        IEspStatusServerInfo& statusServerInfo);
+    void getStatusServerInfo(IEspContext &context, const char* name, IEspStatusServerInfo& statusServerInfo);
+    void getStatusServerInfo(IEspContext &context, const char* type, const char *node, unsigned port, IEspStatusServerInfo& statusServerInfo);
+    void getDFUServerInfo(IEspContext &context, const char* name, IEspStatusServerInfo& statusServerInfo);
+    void setServerJobQueue(double version, const char* type, const char* name, const char* queueName, IEspServerJobQueue& serverInfo);
+    IPropertyTree* getStatusServerTree(IConstWUClusterInfo* info);
+    IPropertyTree* getStatusServerTree(const char *networkAddress, unsigned port);
+    void readRunningWUsOnCluster(IEspContext& context, const char* serverName, const char* node, unsigned port,
+        CWsSMCTargetCluster& targetCluster, IPropertyTree* statusServerNode, BoolHash& uniqueWUIDs, IArrayOf<IEspActiveWorkunit>& aws);
 };
 
 

esp/services/ws_workunits/ws_workunitsQuerySets.cpp

@@ -339,7 +339,8 @@ void QueryFilesInUse::loadTarget(IPropertyTree *t, const char *target, unsigned
         if (pkgid && *pkgid)
             queryTree->setProp("@pkgid", pkgid);
 
-        Owned<IReferencedFileList> wufiles = createReferencedFileList(NULL, NULL, true);
+        IUserDescriptor **roxieUser = roxieUserMap.getValue(target);
+        Owned<IReferencedFileList> wufiles = createReferencedFileList(roxieUser ? *roxieUser : NULL, true);
         wufiles->addFilesFromQuery(cw, pm, queryid);
         if (aborting)
             return;
@@ -740,7 +741,7 @@ bool CWsWorkunitsEx::onWUPublishWorkunit(IEspContext &context, IEspWUPublishWork
     }
 
     if (!req.getDontCopyFiles())
-        copyQueryFilesToCluster(context, cw, daliIP, srcPrefix, target.str(), srcCluster, queryName.str(), false, req.getAllowForeignFiles());
+        copyQueryFilesToCluster(context, cw, daliIP, srcPrefix, target.str(), srcCluster, queryName.str(), req.getUpdateDfs(), req.getAllowForeignFiles());
 
     WorkunitUpdate wu(&cw->lock());
     if (req.getUpdateWorkUnitName() && notEmpty(req.getJobName()))
@@ -1188,6 +1189,8 @@ bool CWsWorkunitsEx::onWUListQueries(IEspContext &context, IEspWUListQueriesRequ
             sortOrder[0] = WUQSFwuid;
         else if (strieq(sortBy, "DLL"))
             sortOrder[0] = WUQSFdll;
+        else if (strieq(sortBy, "Activated"))
+            sortOrder[0] = WUQSFActivited;
         else if (strieq(sortBy, "MemoryLimit"))
             sortOrder[0] = (WUQuerySortField) (WUQSFmemoryLimit | WUQSFnumeric);
         else if (strieq(sortBy, "TimeLimit"))

esp/services/ws_workunits/ws_workunitsService.cpp

@@ -2189,7 +2189,7 @@ bool CWsWorkunitsEx::onWUQuery(IEspContext &context, IEspWUQueryRequest & req, I
     return true;
 }
 
-void appendResultSet(MemoryBuffer& mb, INewResultSet* result, const char *name, __int64 start, unsigned& count, __int64& total, bool bin, bool xsd, ESPSerializationFormat fmt)
+void appendResultSet(MemoryBuffer& mb, INewResultSet* result, const char *name, __int64 start, unsigned& count, __int64& total, bool bin, bool xsd, ESPSerializationFormat fmt, const IProperties *xmlns)
 {
     if (!result)
         return;
@@ -2217,7 +2217,7 @@ void appendResultSet(MemoryBuffer& mb, INewResultSet* result, const char *name,
         if (fmt==ESPSerializationJSON)
             count = getResultJSON(adaptor, result, name, (unsigned) start, count, (xsd) ? "myschema" : NULL);
         else
-            count = getResultXml(adaptor, result, name, (unsigned) start, count, (xsd) ? "myschema" : NULL);
+            count = getResultXml(adaptor, result, name, (unsigned) start, count, (xsd) ? "myschema" : NULL, xmlns);
     }
 }
 
@@ -2296,11 +2296,11 @@ void getWsWuResult(IEspContext &context, const char* wuid, const char *name, con
     else
         rs.setown(resultSetFactory->createNewResultSet(result, wuid));
     if (!filterBy || !filterBy->length())
-        appendResultSet(mb, rs, name, start, count, total, bin, xsd, context.getResponseFormat());
+        appendResultSet(mb, rs, name, start, count, total, bin, xsd, context.getResponseFormat(), result->queryXmlns());
     else
     {
         Owned<INewResultSet> filteredResult = createFilteredResultSet(rs, filterBy);
-        appendResultSet(mb, filteredResult, name, start, count, total, bin, xsd, context.getResponseFormat());
+        appendResultSet(mb, filteredResult, name, start, count, total, bin, xsd, context.getResponseFormat(), result->queryXmlns());
     }
 }
 
@@ -2673,11 +2673,11 @@ void getFileResults(IEspContext &context, const char* logicalName, const char* c
     Owned<IResultSetFactory> resultSetFactory = getSecResultSetFactory(context.querySecManager(), context.queryUser(), context.queryUserId(), context.queryPassword());
     Owned<INewResultSet> result(resultSetFactory->createNewFileResultSet(logicalName, cluster));
     if (!filterBy || !filterBy->length())
-        appendResultSet(buf, result, resname.str(), start, count, total, bin, xsd, context.getResponseFormat());
+        appendResultSet(buf, result, resname.str(), start, count, total, bin, xsd, context.getResponseFormat(), NULL);
     else
     {
         Owned<INewResultSet> filteredResult = createFilteredResultSet(result, filterBy);
-        appendResultSet(buf, filteredResult, resname.str(), start, count, total, bin, xsd, context.getResponseFormat());
+        appendResultSet(buf, filteredResult, resname.str(), start, count, total, bin, xsd, context.getResponseFormat(), NULL);
     }
 }
 

esp/services/ws_workunits/ws_workunitsService.hpp

@@ -33,6 +33,9 @@
 class QueryFilesInUse : public CInterface, implements ISDSSubscription
 {
     mutable CriticalSection crit;
+    MapStringTo<IUserDescriptor *> roxieUserMap;
+    IArrayOf<IUserDescriptor> roxieUsers;
+
     Owned<IPropertyTree> tree;
     SubscriptionId qsChange;
     SubscriptionId pmChange;
@@ -44,6 +47,23 @@ public:
     QueryFilesInUse() : aborting(false), qsChange(0), pmChange(0), psChange(0)
     {
         tree.setown(createPTree("QueryFilesInUse"));
+        updateUsers();
+    }
+
+    void updateUsers()
+    {
+        Owned<IStringIterator> clusters = getTargetClusters("RoxieCluster", NULL);
+        ForEach(*clusters)
+        {
+            SCMStringBuffer target;
+            clusters->str(target);
+
+            Owned<IConstWUClusterInfo> info = getTargetClusterInfo(target.str());
+            Owned<IUserDescriptor> user = createUserDescriptor();
+            user->set(info->getLdapUser(), info->getLdapPassword());
+            roxieUserMap.setValue(target.str(), user);
+            roxieUsers.append(*user.getClear());
+        }
     }
 
     const char *getPackageMap(const char *target)

esp/src/eclwatch/ActivityWidget.js

@@ -299,7 +299,15 @@ define([
                             return "<img src='" + img + "'/>&nbsp;<a href='#' class='" + context.id + "RowClick'>" + name + "</a>";
                         }
                     }),
-                    DisplaySize: { label: this.i18n.Size, width: 59, sortable: true },
+                    GID: {
+                        label: this.i18n.Graph, width: 90, sortable: true,
+                        formatter: function (_gid, row) {
+                            if (context.activity.isInstanceOfWorkunit(row)) {
+                                return "<a href='#' class='" + context.id + "GraphClick'>" + row.GraphName + "-" + row.GID + "</a>";
+                            }
+                            return "";
+                        }
+                    },
                     State: {
                         label: this.i18n.State,
                         sortable: false,
@@ -333,9 +341,21 @@ define([
             on(document, "." + this.id + "RowClick:click", function (evt) {
                 if (context._onRowDblClick) {
                     var row = retVal.row(evt).data;
-                    context._onRowDblClick(row);
+                    context._onRowDblClick(row, {
+                        OpenMode: "WU"
+                    });
+                }
+            });
+
+            on(document, "." + this.id + "GraphClick:click", function (evt) {
+                if (context._onRowDblClick) {
+                    var row = retVal.row(evt).data;
+                    context._onRowDblClick(row, {
+                        OpenMode: "Graph"
+                    });
                 }
             });
+
             return retVal;
         },
 
@@ -353,6 +373,21 @@ define([
                     }
                 });
             } else if (this.activity.isInstanceOfWorkunit(row)) {
+                if (lang.exists("OpenMode", params) && params.OpenMode === "Graph") {
+                    return new DelayLoadWidget({
+                        id: id,
+                        title: row.GraphName,
+                        closable: true,
+                        delayWidget: "GraphPageWidget",
+                        hpcc: {
+                            params: {
+                                Wuid: row.Wuid,
+                                GraphName: row.GraphName,
+                                SubGraphId: row.GID
+                            }
+                        }
+                    });
+                }
                 if (row.Server === "DFUserver") {
                     return new DelayLoadWidget({
                         id: id,

esp/src/eclwatch/ESPWorkunit.js

@@ -139,6 +139,10 @@ define([
             }
             this.set("timers", timers);
         },
+        _ResourceURLCountSetter: function (ResourceURLCount) {
+            //  All WU's have 1 resource URL, which we are not interested in  ---
+            this.set("resourceURLCount", ResourceURLCount - 1);
+        },
         _ResourceURLsSetter: function (resourceURLs) {
             var data = [];
             arrayUtil.forEach(resourceURLs.URL, function (url, idx) {
@@ -158,6 +162,7 @@ define([
                 }
             }, this);
             this.set("resourceURLs", data);
+            this.set("resourceURLCount", data.length);
         },
         _GraphsSetter: function (Graphs) {
             this.set("graphs", Graphs.ECLGraph);

esp/src/eclwatch/GridDetailsWidget.js

@@ -87,8 +87,8 @@ define([
             }
         },
 
-        _onRowDblClick: function (row) {
-            var tab = this.ensurePane(row);
+        _onRowDblClick: function (row, params) {
+            var tab = this.ensurePane(row, params);
             if (tab) {
                 this.selectChild(tab);
             }

esp/src/eclwatch/QuerySetQueryWidget.js

@@ -336,7 +336,6 @@ define([
                             node.innerHTML = dojoConfig.getImageHTML("active.png", context.i18n.Active);
                         },
                         width: 25,
-                        sortable: false,
                         formatter: function (activated) {
                             if (activated == true) {
                                 return dojoConfig.getImageHTML("active.png");
@@ -372,7 +371,8 @@ define([
                     },
                     PublishedBy: {
                         width: 100,
-                        label: this.i18n.PublishedBy
+                        label: this.i18n.PublishedBy,
+                        sortable: false
                     }
                 }
             }, this.id + "QuerySetGrid");

esp/src/eclwatch/WUDetailsWidget.js

@@ -415,7 +415,7 @@ define([
                 }
                 this.graphsWidget.set("tooltip", tooltip);
                 this.setDisabled(this.graphsWidget.id, false);
-            } else if (name === "ResourceURLCount" && newValue) {
+            } else if (name === "resourceURLCount" && newValue) {
                 this.widget._Resources.set("title", this.i18n.Resources + " (" + newValue + ")");
                 this.setDisabled(this.widget._Resources.id, false);
             } else if (name === "Archived") {

roxie/ccd/ccdcontext.cpp

@@ -2825,6 +2825,17 @@ public:
         return trim ? XWFtrim|XWFopt : XWFexpandempty;
     }
 
+    virtual const IProperties *queryXmlns(unsigned seqNo)
+    {
+        IConstWorkUnit *cw = serverQueryFactory->queryWorkUnit();
+        if (cw)
+        {
+            Owned<IConstWUResult> result = cw->getResultBySequence(seqNo);
+            if (result)
+                return result->queryXmlns();
+        }
+        return NULL;
+    }
     virtual unsigned getMemoryUsage()
     {
         return rowManager->getMemoryUsage();

roxie/ccd/ccdcontext.hpp

@@ -97,6 +97,7 @@ interface IRoxieServerContext : extends IInterface
     virtual bool outputResultsToSocket() const = 0;
 
     virtual IRoxieDaliHelper *checkDaliConnection() = 0;
+    virtual const IProperties *queryXmlns(unsigned seqNo) = 0;
 };
 
 interface IDeserializedResultStore : public IInterface

roxie/ccd/ccdquery.cpp

@@ -904,7 +904,7 @@ public:
         if (dll)
         {
             hashValue = rtlHash64VStr(dll->queryDll()->queryName(), hashValue);
-            if (!allFilesDynamic && !isDynamic)
+            if (!allFilesDynamic && !isDynamic && !package.isCompulsory())
             {
                 IConstWorkUnit *wu = dll->queryWorkUnit();
                 if (wu) // wu may be null in some unit test cases

roxie/ccd/ccdserver.cpp

@@ -19746,7 +19746,8 @@ public:
             response = serverContext->queryResult(sequence);
             if (response)
             {
-                response->startDataset("Dataset", helper.queryName(), sequence, (helper.getFlags() & POFextend) != 0);
+                const IProperties *xmlns = serverContext->queryXmlns(sequence);
+                response->startDataset("Dataset", helper.queryName(), sequence, (helper.getFlags() & POFextend) != 0, xmlns);
                 if (response->mlFmt==MarkupFmt_XML || response->mlFmt==MarkupFmt_JSON)
                 {
                     unsigned int writeFlags = serverContext->getXmlFlags();

roxie/ccd/ccdstate.cpp

@@ -109,7 +109,7 @@ static bool isSimpleIndexActivity(ThorActivityKind kind)
 const char *queryNodeFileName(const IPropertyTree &graphNode, ThorActivityKind kind)
 {
     if (isSimpleIndexActivity(kind))
-        return false;
+        return NULL;
     else
         return _queryNodeFileName(graphNode);
 }
@@ -519,10 +519,10 @@ protected:
     }
 
     // Use local package and its bases to resolve existing file into physical file info via all supported resolvers
-    IResolvedFile *lookupExpandedFileName(const char *fileName, bool useCache, bool cacheResult, bool writeAccess, bool alwaysCreate) const
+    IResolvedFile *lookupExpandedFileName(const char *fileName, bool useCache, bool cacheResult, bool writeAccess, bool alwaysCreate, bool checkCompulsory) const
     {
         IResolvedFile *result = lookupFile(fileName, useCache, cacheResult, writeAccess, alwaysCreate);
-        if (!result)
+        if (!result && (!checkCompulsory || !isCompulsory()))
             result = resolveLFNusingDaliOrLocal(fileName, useCache, cacheResult, writeAccess, alwaysCreate);
         return result;
     }
@@ -557,7 +557,7 @@ protected:
                     }
                     if (traceLevel > 9)
                         DBGLOG("Looking up subfile %s", subFileName.str());
-                    Owned<const IResolvedFile> subFileInfo = lookupExpandedFileName(subFileName, useCache, cacheResult, false, false);  // NOTE - overwriting a superfile does NOT require write access to subfiles
+                    Owned<const IResolvedFile> subFileInfo = lookupExpandedFileName(subFileName, useCache, cacheResult, false, false, false);  // NOTE - overwriting a superfile does NOT require write access to subfiles
                     if (subFileInfo)
                     {
                         if (!super)
@@ -623,13 +623,16 @@ public:
         if (traceLevel > 5)
             DBGLOG("lookupFileName %s", fileName.str());
 
-        const IResolvedFile *result = lookupExpandedFileName(fileName, useCache, cacheResult, false, false);
+        const IResolvedFile *result = lookupExpandedFileName(fileName, useCache, cacheResult, false, false, true);
         if (!result)
         {
+            StringBuffer compulsoryMsg;
+            if (isCompulsory())
+                    compulsoryMsg.append(" (Package is compulsory)");
             if (!opt)
-                throw MakeStringException(ROXIE_FILE_ERROR, "Could not resolve filename %s", fileName.str());
+                throw MakeStringException(ROXIE_FILE_ERROR, "Could not resolve filename %s%s", fileName.str(), compulsoryMsg.str());
             if (traceLevel > 4)
-                DBGLOG("Could not resolve OPT filename %s", fileName.str());
+                DBGLOG("Could not resolve OPT filename %s%s", fileName.str(), compulsoryMsg.str());
         }
         return result;
     }
@@ -679,6 +682,10 @@ public:
     {
         return CPackageNode::getEnableFieldTranslation();
     }
+    virtual bool isCompulsory() const
+    {
+        return CPackageNode::isCompulsory();
+    }
     virtual const IPropertyTree *queryTree() const
     {
         return CPackageNode::queryTree();

rtl/eclrtl/rtlxml.cpp

@@ -152,7 +152,7 @@ void outputXmlUtf8(unsigned len, const char *field, const char *fieldname, Strin
 {
     if (fieldname && *fieldname)
         out.append('<').append(fieldname).append('>');
-    encodeXML(field, out, 0, rtlUtf8Size(len, field), true); // output as UTF-8
+    encodeXML(field, out, 0, len, true); // output as UTF-8, encodeXml uses strlen and compensates for utf8 character size
     if (fieldname && *fieldname)
         out.append("</").append(fieldname).append('>');
 }

rtl/include/eclhelper.hpp

@@ -177,6 +177,7 @@ public:
     virtual void outputBeginArray(const char *fieldname) = 0;
     virtual void outputEndArray(const char *fieldname) = 0;
     virtual void outputInlineXml(const char *text) = 0; //for appending raw xml content
+    virtual void outputXmlns(const char *name, const char *uri) = 0;
     inline void outputCString(const char *field, const char *fieldname) { outputString((size32_t)strlen(field), field, fieldname); }
 };
 

testing/ecl/joinmulti.ecl

@@ -1,3 +1,4 @@
+//class=stress
 unsigned numgroups := 10000;
 unsigned largeGroupSize := 500;
 unsigned largeGroupFreq := 30;

testing/regress/ecl/key/xmlns.xml

@@ -0,0 +1,3 @@
+<Dataset name='Result 1' xmlns:star="urn:hpccsystems:ecl:unknown:star" xmlns:bk="urn:booklist:book" xmlns:bks="urn:booklist:books" xmlns="urn:noprefix:namespace">
+ <Row><bks:books><bk:book><bk:title>The Emperors New Mind</bk:title><bk:author>Sir Roger Penrose</bk:author><star:rating>*****</star:rating></bk:book><bk:book><bk:title>The Elegant Universe</bk:title><bk:author>Brian Greene</bk:author><star:rating>*****</star:rating></bk:book><bk:book><bk:title>A Brief History of Time</bk:title><bk:author>Stephen Hawking</bk:author><star:rating>*****</star:rating></bk:book></bks:books></Row>
+</Dataset>

testing/regress/ecl/key/xmlns2.xml

@@ -0,0 +1,3 @@
+<Dataset name='Result 1' xmlns:star="urn:hpccsystems:ecl:unknown:star" xmlns:bk="urn:booklist:电脑是新的:book" xmlns:bks="urn:booklist:电脑是新的:books" xmlns="urn:noprefix:namespace">
+ <Row><bks:books><bk:book><bk:title>The Emperors New Mind</bk:title><bk:author>Sir Roger Penrose</bk:author><star:rating>*****</star:rating></bk:book><bk:book><bk:title>The Elegant Universe</bk:title><bk:author>Brian Greene</bk:author><star:rating>*****</star:rating></bk:book><bk:book><bk:title>A Brief History of Time</bk:title><bk:author>Stephen Hawking</bk:author><star:rating>*****</star:rating></bk:book></bks:books></Row>
+</Dataset>

testing/regress/ecl/xmlns.ecl

@@ -0,0 +1,33 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2014 HPCC Systems.
+
+    This program is free software: you can redistribute it and/or modify
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+bookRec := RECORD
+string     title {xpath('bk:title')};
+string     author {xpath('bk:author')};
+string     rating {xpath('star:rating')};
+           END;
+
+collectionRec := RECORD
+  dataset(bookRec) books {xpath('bks:books/bk:book')};
+end;
+
+xmlinfo := u8'<Row><bks:books><bk:book><bk:author>Sir Roger Penrose</bk:author><bk:title>The Emperors New Mind</bk:title><star:rating>*****</star:rating></bk:book><bk:book><bk:author>Brian Greene</bk:author><bk:title>The Elegant Universe</bk:title><star:rating>*****</star:rating></bk:book><bk:book><bk:author>Stephen Hawking</bk:author><bk:title>A Brief History of Time</bk:title><star:rating>*****</star:rating></bk:book></bks:books></Row>';
+
+books := FROMXML(collectionRec, xmlinfo, TRIM);
+
+output(books, XMLNS(u8'bks',u8'urn:booklist:books'), XMLNS(U'bk',U'urn:booklist:book'), XMLNS('', 'urn:noprefix:namespace'));
+

testing/regress/ecl/xmlns2.ecl

@@ -0,0 +1,34 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2014 HPCC Systems.
+
+    This program is free software: you can redistribute it and/or modify
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+bookRec := RECORD
+string     title {xpath('bk:title')};
+string     author {xpath('bk:author')};
+string     rating {xpath('star:rating')};
+           END;
+
+collectionRec := RECORD
+  dataset(bookRec) books {xpath('bks:books/bk:book')};
+end;
+
+xmlinfo := u8'<Row><bks:books><bk:book><bk:author>Sir Roger Penrose</bk:author><bk:title>The Emperors New Mind</bk:title><star:rating>*****</star:rating></bk:book><bk:book><bk:author>Brian Greene</bk:author><bk:title>The Elegant Universe</bk:title><star:rating>*****</star:rating></bk:book><bk:book><bk:author>Stephen Hawking</bk:author><bk:title>A Brief History of Time</bk:title><star:rating>*****</star:rating></bk:book></bks:books></Row>';
+
+books := FROMXML(collectionRec, xmlinfo, TRIM);
+
+//testing extended unicode characters, but these are not valid URIs and viewing the result in a web browser may give an error
+output(books, XMLNS(u8'bks',u8'urn:booklist:电脑是新的:books'), XMLNS(U'bk',U'urn:booklist:电脑是新的:book'), XMLNS('', 'urn:noprefix:namespace'));
+