Merge remote-tracking branch 'origin/candidate-5.0.0' into closedown-5.0.x

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

common/environment/environment.cpp

@@ -888,6 +888,7 @@ IEnvironment& CLocalEnvironment::lock() const
 
 IStringVal & CLocalEnvironment::getName(IStringVal & str) const
 {
+    synchronized procedure(safeCache);
     str.set(p->queryProp("@name"));
     return str;
 }
@@ -895,13 +896,17 @@ IStringVal & CLocalEnvironment::getName(IStringVal & str) const
 IStringVal & CLocalEnvironment::getXML(IStringVal & str) const
 {
     StringBuffer xml;
-    toXML(p->queryBranch("."), xml);
+    {
+        synchronized procedure(safeCache);
+        toXML(p->queryBranch("."), xml);
+    }
     str.set(xml.str());
     return str;
 }
 
 IPropertyTree & CLocalEnvironment::getPTree() const
 {
+    synchronized procedure(safeCache);
     return *LINK(p);
 }
 
@@ -1059,6 +1064,7 @@ IConstInstanceInfo * CLocalEnvironment::getInstance(const char *type, const char
         xpath.append("[@version='").append(version).append("']");
     xpath.append("/Instance");
 
+    synchronized procedure(safeCache);
     Owned<IPropertyTreeIterator> _it = p->getElements(xpath);
     for (_it->first(); _it->isValid(); _it->next())
     {
@@ -1090,6 +1096,7 @@ CConstInstanceInfo * CLocalEnvironment::getInstanceByIP(const char *type, const
         xpath.append("[@version='").append(version).append("']");
     xpath.append("/Instance");
 
+    synchronized procedure(safeCache);
     assertex(p);
     Owned<IPropertyTreeIterator> _it = p->getElements(xpath);
     assertex(_it);
@@ -1120,6 +1127,7 @@ void CLocalEnvironment::unlockRemote()
 #else
    if (conn)
    {
+       synchronized procedure(safeCache);
        p.clear();
        conn.setown(querySDS().connect(xPath.str(), myProcessSession(), 0, SDS_LOCK_TIMEOUT));
        p.setown(conn->getRoot());
@@ -1129,12 +1137,14 @@ void CLocalEnvironment::unlockRemote()
 
 void CLocalEnvironment::preload()
 {
+    synchronized procedure(safeCache);
     p->queryBranch(".");
 }
 
 void CLocalEnvironment::setXML(const char *xml)
 {
     Owned<IPropertyTree> newRoot = createPTreeFromXMLString(xml);
+    synchronized procedure(safeCache);
     Owned<IPropertyTreeIterator> it = p->getElements("*");
     ForEach(*it)
     {

dali/base/dasds.cpp

@@ -1933,7 +1933,7 @@ public:
 
     void loadStore(const char *store=NULL, const bool *abort=NULL);
     void saveStore(const char *store=NULL, bool currentEdition=false);
-    bool unlock(__int64 treeId, ConnectionId connectionId);
+    bool unlock(__int64 treeId, ConnectionId connectionId, bool delayDelete=false);
     void unlockAll(__int64 treeId);
     void changeLockMode(CServerConnection &connection, unsigned newMode, unsigned timeout);
     void clearSDSLocks();
@@ -3666,7 +3666,7 @@ public:
         return true;
     }
 
-    bool unlock(ConnectionId id)
+    bool unlock(ConnectionId id, bool delayDelete=false)
     {
         bool ret = false;
         CPendingLockBlock b(*this); // carefully placed, removePending can destroy this, therefore must be destroyed last
@@ -3704,7 +3704,13 @@ public:
                 connectionInfo.remove(id);
                 if (parent && 0 == connectionInfo.count())
                 {
-                    clearLastRef();
+                    if (delayDelete)
+                    {
+                        parent.clear();
+                        child.clear();
+                    }
+                    else
+                        clearLastRef();
                     ret = true;
                 }
             }
@@ -4536,12 +4542,12 @@ void CSDSTransactionServer::processMessage(CMessageBuffer &mb)
                 if (queryTransactionLogging())
                 {
                     CServerConnection *conn = manager.queryConnection(connectionId);
-                    transactionLog.log("disconnect=%s, data=%s", disconnect?"true":"false", data?"true":"false");
+                    transactionLog.log("disconnect=%s, data=%s, deleteRoot=%s", disconnect?"true":"false", data?"true":"false", deleteRoot?"true":"false");
                 }
                 Owned<CLCLockBlock> lockBlock;
                 { 
                     CheckTime block1("DAMP_SDSCMD_DATA.1");
-                    if (data || disconnect)
+                    if (data || deleteRoot)
                         lockBlock.setown(new CLCWriteLockBlock(manager.dataRWLock, readWriteTimeout, __FILE__, __LINE__));
                     else
                         lockBlock.setown(new CLCReadLockBlock(manager.dataRWLock, readWriteTimeout, __FILE__, __LINE__));
@@ -4590,11 +4596,11 @@ void CSDSTransactionServer::processMessage(CMessageBuffer &mb)
                 catch (IException *)
                 {
                     if (disconnect)
-                        manager.disconnect(connectionId, deleteRoot, (data || disconnect)?NULL:&lockBlock);
+                        manager.disconnect(connectionId, deleteRoot, (data || deleteRoot)?NULL:&lockBlock);
                     throw;
                 }
                 if (disconnect)
-                    manager.disconnect(connectionId, deleteRoot, (data || disconnect)?NULL:&lockBlock);
+                    manager.disconnect(connectionId, deleteRoot, (data || deleteRoot)?NULL:&lockBlock);
 
                 break;
             }
@@ -7535,12 +7541,12 @@ bool CCovenSDSManager::unlock(__int64 connectionId, bool close, StringBuffer &co
     return true;
 }
 
-bool CCovenSDSManager::unlock(__int64 treeId, ConnectionId connectionId)
+bool CCovenSDSManager::unlock(__int64 treeId, ConnectionId connectionId, bool delayDelete)
 {
     CHECKEDCRITICALBLOCK(lockCrit, fakeCritTimeout);
     CLockInfo *lockInfo = queryLockInfo(treeId);
     if (lockInfo)
-        return lockInfo->unlock(connectionId);
+        return lockInfo->unlock(connectionId, delayDelete);
     return false;
 }
 
@@ -7900,7 +7906,6 @@ void CCovenSDSManager::disconnect(ConnectionId id, bool deleteRoot, Owned<CLCLoc
     unsigned index = (unsigned)-1;
     StringBuffer path;
     connection->queryPTreePath().getAbsolutePath(path);
-    bool noLockDelete = false;
     if (connection->queryParent())
     {
         if (deleteRoot || RTM_MODE(connection->queryMode(), RTM_DELETE_ON_DISCONNECT))
@@ -7913,12 +7918,7 @@ void CCovenSDSManager::disconnect(ConnectionId id, bool deleteRoot, Owned<CLCLoc
                 lockInfo->setDROLR((CServerRemoteTree *)connection->queryParent(), tree);
             }
             else
-                noLockDelete = deleteRoot = true;
-        }
-        if (lockBlock)
-        {
-            lockBlock->clear();
-            lockBlock->setown(new CLCWriteLockBlock(dataRWLock, readWriteTimeout, __FILE__, __LINE__));
+                deleteRoot = true;
         }
         if ((unsigned)-1 == index)
             index = connection->queryParent()->queryChildIndex(connection->queryRootUnvalidated());
@@ -7928,41 +7928,47 @@ void CCovenSDSManager::disconnect(ConnectionId id, bool deleteRoot, Owned<CLCLoc
 
     bool orphaned = ((CServerRemoteTree*)connection->queryRootUnvalidated())->isOrphaned();
     // Still want disconnection to be performed & recorded, if orphaned
-    if (noLockDelete)
-        connection->queryParent()->removeTree(tree);
-    else
-        deleteRoot |= unlock(tree->queryServerId(), id);
+    if (!deleteRoot && unlock(tree->queryServerId(), id, true)) // unlock returns true if last unlock and there was a setDROLR on it
+        deleteRoot = true;
     if (deleteRoot)
+    {
+        if (lockBlock)
+        {
+            lockBlock->clear();
+            lockBlock->setown(new CLCWriteLockBlock(dataRWLock, readWriteTimeout, __FILE__, __LINE__));
+        }
+        connection->queryParent()->removeTree(tree);
         writeTransactions++;
-    if (!orphaned && deleteRoot)
-    {
-        Owned<IPropertyTree> changeTree = createPTree(RESERVED_CHANGE_NODE);
-        IPropertyTree *d = changeTree->setPropTree(DELETE_TAG, createPTree());
-        d->setProp("@name", tree->queryName());
-        d->setPropInt("@pos", index+1);
-
-        Owned<CBranchChange> branchChange = new CBranchChange(*tree);
-        branchChange->noteChange(PDS_Deleted, PDS_Deleted);
-        CPTStack stack = connection->queryPTreePath();
-        stack.pop();
-        if (connection->queryRootUnvalidated() == SDSManager->queryRoot())
+        if (!orphaned)
+        {
+            Owned<IPropertyTree> changeTree = createPTree(RESERVED_CHANGE_NODE);
+            IPropertyTree *d = changeTree->setPropTree(DELETE_TAG, createPTree());
+            d->setProp("@name", tree->queryName());
+            d->setPropInt("@pos", index+1);
+
+            Owned<CBranchChange> branchChange = new CBranchChange(*tree);
+            branchChange->noteChange(PDS_Deleted, PDS_Deleted);
+            CPTStack stack = connection->queryPTreePath();
             stack.pop();
+            if (connection->queryRootUnvalidated() == SDSManager->queryRoot())
+                stack.pop();
 
-        if (!RTM_MODE(connection->queryMode(), RTM_INTERNAL))
-        {
-            connection->notify();
-            SDSManager->startNotification(*changeTree, stack, *branchChange);
-        }
+            if (!RTM_MODE(connection->queryMode(), RTM_INTERNAL))
+            {
+                connection->notify();
+                SDSManager->startNotification(*changeTree, stack, *branchChange);
+            }
 
-        StringBuffer head;
-        const char *tail = splitXPath(path.str(), head);
-        CHECKEDCRITICALBLOCK(blockedSaveCrit, fakeCritTimeout);
-        if (NotFound != index)
-            saveDelta(head.str(), *changeTree);
-        else
-        { // NB: don't believe this can happen, but last thing want to do is save duff delete delta.
-            WARNLOG("** CCovenSDSManager::disconnect - index position lost ** : noLockDelete=%d", noLockDelete);
-            PrintStackReport();
+            StringBuffer head;
+            const char *tail = splitXPath(path.str(), head);
+            CHECKEDCRITICALBLOCK(blockedSaveCrit, fakeCritTimeout);
+            if (NotFound != index)
+                saveDelta(head.str(), *changeTree);
+            else
+            { // NB: don't believe this can happen, but last thing want to do is save duff delete delta.
+                WARNLOG("** CCovenSDSManager::disconnect - index position lost **");
+                PrintStackReport();
+            }
         }
     }
     tree.clear();

dali/sasha/saarch.cpp

@@ -105,15 +105,18 @@ void WUiterate(ISashaCommand *cmd, const char *mask)
     StringBuffer after;
     StringBuffer tmppath;
     mkDateCompare(dfu,afterdt,after,'0');
-    mkDateCompare(dfu,beforedt,before,'9');     
+    mkDateCompare(dfu,beforedt,before,'9');
     bool haswusoutput = cmd->getAction()==SCA_WORKUNIT_SERVICES_GET;
     bool hasdtoutput = cmd->getAction()==SCA_LISTDT;
     MemoryBuffer WUSbuf;
     if (haswusoutput)
         cmd->setWUSresult(WUSbuf);  // swap in/out (in case ever do multiple)
-    if (cmd->getArchived()) {
 
-        Owned<IRemoteConnection> conn = querySDS().connect(dfu?"/DFU/WorkUnits":"/WorkUnits", myProcessSession(), 0, 5*60*1000);  
+    StringBuffer baseXPath = dfu ? "DFU/WorkUnits" : "WorkUnits";
+    Owned<IRemoteConnection> conn;
+    bool isWild = !mask || !*mask || isWildString(mask);
+    bool unfiltered = !mask || !*mask || (0==strcmp(mask,"*"));
+    if (cmd->getArchived()) {
         // used to check not online
         StringBuffer path;
         if (dfu)
@@ -122,7 +125,7 @@ void WUiterate(ISashaCommand *cmd, const char *mask)
             getLdsPath("Archive/WorkUnits",path);
         Owned<IFile> dir = createIFile(path.str());
         StringBuffer masktmp;
-        if (((mask==NULL)||(strcmp(mask,"*")==0))&&after.length()&&before.length()) {
+        if (unfiltered&&after.length()&&before.length()) {
             const char *lo = after.str();
             const char *hi = before.str();
             while (*lo&&(toupper(*lo)==toupper(*hi))) {
@@ -130,17 +133,20 @@ void WUiterate(ISashaCommand *cmd, const char *mask)
                 lo++;
                 hi++;
             }
-            if (*lo||*hi)
-                masktmp.append("*");
+            masktmp.append("*");
             mask = masktmp.str();
         }
-        StringBuffer head;
+        StringBuffer head, tmask;
         const char *hmask = NULL;
         if (mask&&*mask) {
             splitWUID(mask,head);
             if (head.length())
                 hmask = head.str();
+            tmask.clear().append(mask).toUpperCase();
         }
+        else
+            tmask.append("*");
+        tmask.append(".xml");
         Owned<IDirectoryIterator> di = dir->directoryFiles(hmask,false,true);
         StringBuffer name;
         unsigned index = 0;
@@ -150,10 +156,6 @@ void WUiterate(ISashaCommand *cmd, const char *mask)
             if (overflowed||(index>start+num))
                 break;
             if (di->isDir()) {
-                StringBuffer tmask("*");
-                if (mask)
-                    tmask.clear().append(mask).toUpperCase();
-                tmask.append(".xml");
                 Owned<IDirectoryIterator> di2 = di->query().directoryFiles(tmask.str(),false);
                 StringBuffer val;
                 ForEach(*di2) {
@@ -164,109 +166,118 @@ void WUiterate(ISashaCommand *cmd, const char *mask)
                         const char *wuid = name.str();
                         if ((name.length()>6)&&(stricmp(wuid+name.length()-6,"_HINTS")==0))
                             continue;
-                        if (!conn->queryRoot()->hasProp(wuid) &&
-                            (!mask||!*mask||WildMatch(wuid,mask,true)) &&
+                        if ((!mask||!*mask||WildMatch(wuid,mask,true)) &&
                             ((before.length()==0)||(stricmp(wuid,before.str())<=0)) &&
                             ((after.length()==0)||(stricmp(wuid,after.str())>=0))) {
-                            Owned<IPropertyTree> t;
-                            bool hasowner = owner&&*owner;
-                            bool hascluster = cluster&&*cluster;
-                            bool hasstate = state&&*state;
-                            bool hasjobname = jobname&&*jobname;
-                            bool hasoutput = outputformat&&*outputformat;
-                            bool inrange = (index>=start)&&(index<start+num);
-                            bool hascommand = cmdname&&*cmdname;
-                            bool haspriority = priority&&*priority;
-                            bool hasfileread = fileread&&*fileread;
-                            bool hasfilewritten = filewritten&&*filewritten;
-                            bool hasroxiecluster = roxiecluster&&*roxiecluster;
-                            bool haseclcontains = eclcontains&&*eclcontains;
-                            if ((cmd->getAction()==SCA_GET)||haswusoutput||hasowner||hasstate||hascluster||hasjobname||hascommand||(hasoutput&&inrange)||haspriority||hasfileread||hasfilewritten||hasroxiecluster||haseclcontains) {
-                                try {
-                                    t.setown(createPTree(di2->query()));
-                                    if (!t)
-                                        continue;
-                                    if (hasowner&&(!t->getProp("@submitID",val.clear())||!WildMatch(val.str(),owner,true))) 
-                                        continue;
-                                    if (hasstate&&(!t->getProp(dfu?"Progress/@state":"@state",val.clear())||!WildMatch(val.str(),state,true))) 
-                                        continue;
-                                    if (hascluster&&(!t->getProp("@clusterName",val.clear())||!WildMatch(val.str(),cluster,true))) 
-                                        continue;
-                                    if (hasjobname&&(!t->getProp("@jobName",val.clear())||!WildMatch(val.str(),jobname,true))) 
-                                        continue;
-                                    if (hascommand&&(!t->getProp("@command",val.clear())||!WildMatch(val.str(),cmdname,true))) 
-                                        continue;
-                                    if (haspriority&&(!t->getProp("@priorityClass",val.clear())||!WildMatch(val.str(),priority,true))) 
-                                        continue;
-                                    if (hasfileread&&!t->hasProp(tmppath.clear().appendf("FilesRead/File[@name=~?\"%s\"]",fileread).str()))
-                                        continue;
-                                    if (hasfilewritten&&!t->hasProp(tmppath.clear().appendf("Files/File[@name=~?\"%s\"]",filewritten).str()))
-                                        continue;
-                                    if (hasroxiecluster&&!t->hasProp(tmppath.clear().appendf("RoxieQueryInfo[@roxieClusterName=~?\"%s\"]",roxiecluster).str()))
-                                        continue;
-                                    if (haseclcontains&&!t->hasProp(tmppath.clear().appendf("Query[Text=~?\"*%s*\"]",eclcontains).str()))
+                            if (isWild) {
+                                if (!conn)
+                                    conn.setown(querySDS().connect(baseXPath.str(), myProcessSession(), 0, 5*60*1000)); // connection to all
+                            }
+                            else {
+                                VStringBuffer xpath("%s/%s", baseXPath.str(), mask);
+                                conn.setown(querySDS().connect(xpath.str(), myProcessSession(), 0, 5*60*1000));
+                            }
+                            if ((isWild && !conn->queryRoot()->hasProp(wuid)) || (!isWild && !conn)) { // check not online
+                                Owned<IPropertyTree> t;
+                                bool hasowner = owner&&*owner;
+                                bool hascluster = cluster&&*cluster;
+                                bool hasstate = state&&*state;
+                                bool hasjobname = jobname&&*jobname;
+                                bool hasoutput = outputformat&&*outputformat;
+                                bool inrange = (index>=start)&&(index<start+num);
+                                bool hascommand = cmdname&&*cmdname;
+                                bool haspriority = priority&&*priority;
+                                bool hasfileread = fileread&&*fileread;
+                                bool hasfilewritten = filewritten&&*filewritten;
+                                bool hasroxiecluster = roxiecluster&&*roxiecluster;
+                                bool haseclcontains = eclcontains&&*eclcontains;
+                                if ((cmd->getAction()==SCA_GET)||haswusoutput||hasowner||hasstate||hascluster||hasjobname||hascommand||(hasoutput&&inrange)||haspriority||hasfileread||hasfilewritten||hasroxiecluster||haseclcontains) {
+                                    try {
+                                        t.setown(createPTree(di2->query()));
+                                        if (!t)
+                                            continue;
+                                        if (hasowner&&(!t->getProp("@submitID",val.clear())||!WildMatch(val.str(),owner,true)))
+                                            continue;
+                                        if (hasstate&&(!t->getProp(dfu?"Progress/@state":"@state",val.clear())||!WildMatch(val.str(),state,true)))
+                                            continue;
+                                        if (hascluster&&(!t->getProp("@clusterName",val.clear())||!WildMatch(val.str(),cluster,true)))
+                                            continue;
+                                        if (hasjobname&&(!t->getProp("@jobName",val.clear())||!WildMatch(val.str(),jobname,true)))
+                                            continue;
+                                        if (hascommand&&(!t->getProp("@command",val.clear())||!WildMatch(val.str(),cmdname,true)))
+                                            continue;
+                                        if (haspriority&&(!t->getProp("@priorityClass",val.clear())||!WildMatch(val.str(),priority,true)))
+                                            continue;
+                                        if (hasfileread&&!t->hasProp(tmppath.clear().appendf("FilesRead/File[@name=~?\"%s\"]",fileread).str()))
+                                            continue;
+                                        if (hasfilewritten&&!t->hasProp(tmppath.clear().appendf("Files/File[@name=~?\"%s\"]",filewritten).str()))
+                                            continue;
+                                        if (hasroxiecluster&&!t->hasProp(tmppath.clear().appendf("RoxieQueryInfo[@roxieClusterName=~?\"%s\"]",roxiecluster).str()))
+                                            continue;
+                                        if (haseclcontains&&!t->hasProp(tmppath.clear().appendf("Query[Text=~?\"*%s*\"]",eclcontains).str()))
+                                            continue;
+                                    }
+                                    catch (IException *e) {
+                                        StringBuffer msg;
+                                        msg.appendf("WUiterate: Workunit %s failed to load", wuid);
+                                        EXCLOG(e,msg.str());
+                                        e->Release();
                                         continue;
+                                    }
                                 }
-                                catch (IException *e) {
-                                    StringBuffer msg;
-                                    msg.appendf("WUiterate: Workunit %s failed to load", wuid);
-                                    EXCLOG(e,msg.str());
-                                    e->Release();
+                                index++;
+                                if (!inrange)
                                     continue;
+                                if (hasoutput) {
+                                    char *saveptr;
+                                    char *parse = strdup(outputformat);
+                                    char *tok = strtok_r(parse, "|,",&saveptr);
+                                    while (tok) {
+                                        val.clear();
+                                        bool found = true;
+                                        if (stricmp(tok,"owner")==0)
+                                            t->getProp("@submitID",val);
+                                        else if (stricmp(tok,"cluster")==0)
+                                            t->getProp("@clusterName",val);
+                                        else if (stricmp(tok,"jobname")==0)
+                                            t->getProp("@jobName",val);
+                                        else if (stricmp(tok,"state")==0)
+                                            t->getProp(dfu?"Progress/@state":"@state",val);
+                                        else if (stricmp(tok,"command")==0)
+                                            t->getProp("@command",val);
+                                        else if (stricmp(tok,"wuid")==0)
+                                            t->getName(val);
+                                        else
+                                            found = false;
+                                        if (found) {
+                                            // remove commas TBD
+                                            name.append(',').append(val);
+                                        }
+                                        tok = strtok_r(NULL, "|,",&saveptr);
+                                    }
+                                    free(parse);
                                 }
-                            }
-                            index++;
-                            if (!inrange)
-                                continue;
-                            if (hasoutput) { 
-                                char *saveptr;
-                                char *parse = strdup(outputformat);
-                                char *tok = strtok_r(parse, "|,",&saveptr);
-                                while (tok) {
-                                    val.clear();
-                                    bool found = true;
-                                    if (stricmp(tok,"owner")==0)
-                                        t->getProp("@submitID",val);
-                                    else if (stricmp(tok,"cluster")==0)
-                                        t->getProp("@clusterName",val);
-                                    else if (stricmp(tok,"jobname")==0)
-                                        t->getProp("@jobName",val);
-                                    else if (stricmp(tok,"state")==0)
-                                        t->getProp(dfu?"Progress/@state":"@state",val);
-                                    else if (stricmp(tok,"command")==0)
-                                        t->getProp("@command",val);
-                                    else if (stricmp(tok,"wuid")==0)
-                                        t->getName(val);
-                                    else 
-                                        found = false;
-                                    if (found) {
-                                        // remove commas TBD
-                                        name.append(',').append(val);
+                                if (haswusoutput) {
+                                    if (!serializeWUSrow(*t,WUSbuf,false)) {
+                                        overflowed = true;
+                                        break;
                                     }
-                                    tok = strtok_r(NULL, "|,",&saveptr);
                                 }
-                                free(parse);
-                            }
-                            if (haswusoutput) { 
-                                if (!serializeWUSrow(*t,WUSbuf,false)) {
-                                    overflowed = true;
-                                    break; 
+                                else {
+                                    cmd->addId(name.str());
+                                    if (hasdtoutput) {
+                                        CDateTime dt;
+                                        di2->getModifiedTime(dt);
+                                        cmd->addDT(dt);
+                                    }
                                 }
-                            }
-                            else {
-                                cmd->addId(name.str());
-                                if (hasdtoutput) {
-                                    CDateTime dt;
-                                    di2->getModifiedTime(dt);
-                                    cmd->addDT(dt);
+                                if (cmd->getAction()==SCA_GET) {
+                                    StringBuffer xml;
+                                    toXML(t,xml);
+                                    if (!cmd->addResult(xml.str()))
+                                        break;
                                 }
                             }
-                            if (cmd->getAction()==SCA_GET) {
-                                StringBuffer xml;
-                                toXML(t,xml);
-                                if (!cmd->addResult(xml.str()))
-                                    break;
-                            }
                         }
                     }
                     if (index>start+num)
@@ -278,82 +289,91 @@ void WUiterate(ISashaCommand *cmd, const char *mask)
     if (cmd->getOnline()) {
         if (haswusoutput)
             throw MakeStringException(-1,"SCA_WORKUNIT_SERVICES_GET not implemented for online workunits!");
-        Owned<IRemoteConnection> conn = querySDS().connect("/", myProcessSession(), 0, 5*60*1000);  
-        Owned<IPropertyTreeIterator> iter = conn->queryRoot()->getElements(dfu?"DFU/WorkUnits/*":"WorkUnits/*");
-        unsigned index = 0;
-        StringBuffer val;
-        ForEach(*iter) {
-            IPropertyTree &pt=iter->query();
-            const char *wuid = pt.queryName();
-            if (index>start+num)
-                break;
-    //      PROGLOG("match before=%s after=%s wuid=%s",before.str(),after.str(),wuid);
-            if ((!mask||!*mask||WildMatch(wuid,mask,true)) &&
-                ((before.length()==0)||(stricmp(wuid,before)<0)) &&
-                ((after.length()==0)||(stricmp(wuid,after)>=0))) {
-    //          PROGLOG("matched before=%s after=%s wuid=%s",before.str(),after.str(),wuid);
-                bool hasowner = owner&&*owner;
-                bool hascluster = cluster&&*cluster;
-                bool hasstate = state&&*state;
-                bool hasjobname = jobname&&*jobname;
-                bool hasoutput = outputformat&&*outputformat;
-                bool inrange = (index>=start)&&(index<start+num);
-                if (hasowner||hasstate||hascluster||hasjobname||(hasoutput&&inrange)) {
-                    try {
-                        if (hasowner&&(!pt.getProp("@submitID",val.clear())||!WildMatch(val.str(),owner,true))) 
-                            continue;
-                        if (hasstate&&(!pt.getProp("@state",val.clear())||!WildMatch(val.str(),state,true))) 
-                            continue;
-                        if (hascluster&&(!pt.getProp("@clusterName",val.clear())||!WildMatch(val.str(),cluster,true))) 
-                            continue;
-                        if (hasjobname&&(!pt.getProp("@jobName",val.clear())||!WildMatch(val.str(),jobname,true))) 
+        StringBuffer xpath(baseXPath);
+        if (!conn)
+        {
+            if (!isWild)
+                xpath.append("/").append(mask);
+            conn.setown(querySDS().connect(xpath.str(), myProcessSession(), 0, 5*60*1000));
+        }
+        if (conn)
+        {
+            Owned<IPropertyTreeIterator> iter = conn->queryRoot()->getElements(isWild ? "*" : NULL);
+            unsigned index = 0;
+            StringBuffer val;
+            ForEach(*iter) {
+                IPropertyTree &pt=iter->query();
+                const char *wuid = pt.queryName();
+                if (index>start+num)
+                    break;
+        //      PROGLOG("match before=%s after=%s wuid=%s",before.str(),after.str(),wuid);
+                if ((!mask||!*mask||!isWild||WildMatch(wuid,mask,true)) &&
+                    ((before.length()==0)||(stricmp(wuid,before)<0)) &&
+                    ((after.length()==0)||(stricmp(wuid,after)>=0))) {
+        //          PROGLOG("matched before=%s after=%s wuid=%s",before.str(),after.str(),wuid);
+                    bool hasowner = owner&&*owner;
+                    bool hascluster = cluster&&*cluster;
+                    bool hasstate = state&&*state;
+                    bool hasjobname = jobname&&*jobname;
+                    bool hasoutput = outputformat&&*outputformat;
+                    bool inrange = (index>=start)&&(index<start+num);
+                    if (hasowner||hasstate||hascluster||hasjobname||(hasoutput&&inrange)) {
+                        try {
+                            if (hasowner&&(!pt.getProp("@submitID",val.clear())||!WildMatch(val.str(),owner,true)))
+                                continue;
+                            if (hasstate&&(!pt.getProp("@state",val.clear())||!WildMatch(val.str(),state,true)))
+                                continue;
+                            if (hascluster&&(!pt.getProp("@clusterName",val.clear())||!WildMatch(val.str(),cluster,true)))
+                                continue;
+                            if (hasjobname&&(!pt.getProp("@jobName",val.clear())||!WildMatch(val.str(),jobname,true)))
+                                continue;
+                        }
+                        catch (IException *e) {
+                            StringBuffer msg;
+                            msg.appendf("WUiterate: Workunit %s failed", wuid);
+                            EXCLOG(e,msg.str());
+                            e->Release();
                             continue;
+                        }
                     }
-                    catch (IException *e) {
-                        StringBuffer msg;
-                        msg.appendf("WUiterate: Workunit %s failed", wuid);
-                        EXCLOG(e,msg.str());
-                        e->Release();
+                    index++;
+                    if (!inrange)
                         continue;
+                    StringBuffer name(wuid);
+                    if (hasoutput) {
+                        char *saveptr;
+                        char *parse = strdup(outputformat);
+                        char *tok = strtok_r(parse, "|,",&saveptr);
+                        while (tok) {
+                            val.clear();
+                            bool found = true;
+                            if (stricmp(tok,"owner")==0)
+                                pt.getProp("@submitID",val);
+                            else if (stricmp(tok,"cluster")==0)
+                                pt.getProp("@clusterName",val);
+                            else if (stricmp(tok,"jobname")==0)
+                                pt.getProp("@jobName",val);
+                            else if (stricmp(tok,"state")==0)
+                                pt.getProp("@state",val);
+                            else
+                                found = false;
+                            if (found)
+                                name.append(',').append(val);
+                            tok = strtok_r(NULL, "|,",&saveptr);
+                        }
+                        free(parse);
                     }
-                }
-                index++;
-                if (!inrange)
-                    continue;
-                StringBuffer name(wuid);
-                if (hasoutput) {
-                    char *saveptr;
-                    char *parse = strdup(outputformat);
-                    char *tok = strtok_r(parse, "|,",&saveptr);
-                    while (tok) {
-                        val.clear();
-                        bool found = true;
-                        if (stricmp(tok,"owner")==0)
-                            pt.getProp("@submitID",val);
-                        else if (stricmp(tok,"cluster")==0)
-                            pt.getProp("@clusterName",val);
-                        else if (stricmp(tok,"jobname")==0)
-                            pt.getProp("@jobName",val);
-                        else if (stricmp(tok,"state")==0)
-                            pt.getProp("@state",val);
-                        else
-                            found = false;
-                        if (found)
-                            name.append(',').append(val);
-                        tok = strtok_r(NULL, "|,",&saveptr);
+                    cmd->addId(name.str());
+                    if (cmd->getAction()==SCA_GET) {
+                        StringBuffer xml;
+                        toXML(&pt,xml);
+                        if (!cmd->addResult(xml.str()))
+                            break;
                     }
-                    free(parse);
-                }
-                cmd->addId(name.str());
-                if (cmd->getAction()==SCA_GET) {
-                    StringBuffer xml;
-                    toXML(&pt,xml);
-                    if (!cmd->addResult(xml.str()))
-                        break;
                 }
+                if (index>start+num)
+                    break;
             }
-            if (index>start+num)
-                break;
         }
     }
     if (haswusoutput)

docs/ECLPlayground/ECLPlay-Mods/ECL_Playground.xml

@@ -69,9 +69,10 @@
             <listitem>
               <?dbfo keep-together="always"?>
 
-              <para>From ECL Watch page, click on the <emphasis
-              role="bold">ECL Playground </emphasis> link in the menu on the
-              left side.</para>
+              <para>From ECL Watch, click on the<emphasis
+              role="bold">ECL</emphasis> icon, then click the <emphasis
+              role="bold">Playground</emphasis> link from the navigation
+              sub-menu.</para>
 
               <para><figure>
                   <title>ECL Playground link</title>
@@ -104,7 +105,7 @@
 
             <mediaobject>
               <imageobject>
-                <imagedata fileref="../../images/ECLP_002d.jpg" />
+                <imagedata fileref="../../images/ECLPl002.jpg" />
               </imageobject>
             </mediaobject>
           </figure></para>
@@ -113,6 +114,7 @@
         contains the <emphasis>Editor</emphasis> area and the <emphasis>Graph
         Viewer</emphasis>. The Sample code drop list is at the top right. The
         bottom portion of the page displays the results.</para>
+        <!--***NEW INTERFACE DESCR***-->
 
         <para>The ECL Playground comes with a set of ready to run sample ECL
         code. The drop list contains code samples. Select any one of these
@@ -169,7 +171,7 @@
 
             <mediaobject>
               <imageobject>
-                <imagedata fileref="../../images/ECLP_004a.jpg" />
+                <imagedata fileref="../../images/ECLP_004.jpg" />
               </imageobject>
             </mediaobject>
           </figure>The status area displays the job status. If a job fails,
@@ -205,15 +207,16 @@
             <listitem>
               <?dbfo keep-together="always"?>
 
-              <para>Select <emphasis role="bold">Browse Workunits</emphasis>
-              from the ECL Watch menu.</para>
+              <para>Select <emphasis role="bold">Workunits</emphasis> from the
+              ECL Watch <inlinegraphic fileref="../../images/ECLWA001.jpg" />
+              menu.</para>
 
               <para><figure>
                   <title>Browse Workunits</title>
 
                   <mediaobject>
                     <imageobject>
-                      <imagedata fileref="../../images/ECLP_007.jpg"
+                      <imagedata fileref="../../images/ECLWA00A.jpg"
                                  vendor="eclwatchSS" />
                     </imageobject>
                   </mediaobject>
@@ -228,8 +231,8 @@
             <listitem>
               <?dbfo keep-together="always"?>
 
-              <para>Click on the <emphasis role="bold">ECL Playground
-              </emphasis>hyperlink to open it in the ECL Playground.<figure>
+              <para>Click on the<emphasis role="bold"> Playground
+              </emphasis>tab to open the workunit in the Playground.<figure>
                   <title>ECL Playground link</title>
 
                   <mediaobject>

docs/HPCCMonitoring/HPCCMonitoringAndReporting.xml

@@ -55,9 +55,16 @@
   <chapter id="GangliaIntroduction">
     <title>Introduction</title>
 
-    <para>The HPCC Systems platform supports a graphical monitoring and
-    reporting component. With the the graphical monitoring component you can:
-    <itemizedlist>
+    <para>The HPCC systems platform supports graphical monitoring and
+    reporting components.</para>
+
+    <para><emphasis role="bold">Ganglia:</emphasis></para>
+
+    <para>The HPCC monitoring component leverages Ganglia, an open source,
+    scalable, distributed monitoring system to display system information in a
+    graphical manner.</para>
+
+    <para>With the the graphical monitoring component you can: <itemizedlist>
         <listitem>
           <para>See system information at a glance</para>
         </listitem>
@@ -88,11 +95,55 @@
         </listitem>
       </itemizedlist></para>
 
-    <para>The HPCC monitoring component leverages Ganglia, an open-source,
-    scalable, distributed monitoring system to display system information in a
-    graphical manner.</para>
+    <para><emphasis role="bold">Nagios</emphasis></para>
+
+    <para>The HPCC reporting and alerting component leverages Nagios, a
+    powerful monitoring and notification system, which can help you identify
+    and resolve infrastructure problems before they affect critical
+    processes.</para>
+
+    <para>With the HPCC reporting and alerting component you can set up alerts
+    to inform of any changes to:</para>
+
+    <para><itemizedlist>
+        <listitem>
+          <para>SSH connectivity</para>
+        </listitem>
 
-    <!--***NOTE: At some point this next bit will need to get moved into the next chapter/section-->
+        <listitem>
+          <para>Users on system</para>
+        </listitem>
+
+        <listitem>
+          <para>System Load</para>
+        </listitem>
+
+        <listitem>
+          <para>Disk Usage</para>
+        </listitem>
+
+        <listitem>
+          <para>Roxie</para>
+        </listitem>
+
+        <listitem>
+          <para>Dali</para>
+        </listitem>
+
+        <listitem>
+          <para>Dafilesrv</para>
+        </listitem>
+
+        <listitem>
+          <para>Sasha</para>
+        </listitem>
+
+        <listitem>
+          <para>Bound services on each ESP</para>
+        </listitem>
+      </itemizedlist></para>
+
+    <!--***NOTE: At some point this next bit will need to get moved into the next chapter/section***-->
 
     <sect1 id="HPCC_Viewer">
       <title>The HPCC Ganglia Viewer</title>
@@ -119,7 +170,7 @@
   <chapter id="Ganglya_Overview">
     <title>Ganglia</title>
 
-    <para>The HPCC Monitoring component leverages Ganglia, an open-source,
+    <para>The HPCC Monitoring component leverages Ganglia, an open source,
     scalable, distributed monitoring system, to produce a graphical view of a
     Roxie cluster's servers. Ganglia leverages widely accepted technologies
     for data representation. It provides near real-time monitoring and
@@ -266,105 +317,11 @@
       <para>Evaluate the value of the content and decide what aspects of
       measurement are relevant to your needs.</para>
 
-      <sect2>
-        <title id="get_hpcc">Get the latest HPCC Virtual Image File</title>
-
-        <para>The complete details for installing and running HPCC in a
-        virtual machine are available in the document: <emphasis
-        role="bold">Running HPCC in a Virtual Machine</emphasis>, available
-        from <ulink
-        url="hpccsystems.com/download/docs">hpccsystems.com/download/docs</ulink>
-        .</para>
+      <!--INCLUDE-VM_STEPS-as-Sect2-->
 
-        <para>The following steps are a quick summary, assuming you have some
-        familiarity with running virtual machines.</para>
-
-        <para><orderedlist>
-            <listitem>
-              <para>Download the latest HPCC Virtual Machine image file
-              from:</para>
-
-              <para><ulink
-              url="http://HPCCsystems.com/download/hpcc-vm-image">http://hpccsystems.com/download/hpcc-vm-image</ulink></para>
-            </listitem>
-
-            <listitem>
-              <para>Save the file to a folder on your machine.</para>
-            </listitem>
-
-            <listitem>
-              <para>Open your virtualization software, import the virtual
-              machine and start it.</para>
-            </listitem>
-
-            <listitem>
-              <?dbfo keep-together="always"?>
-
-              <para>Once the VM initialization completes, you will see a
-              window similar to the following:</para>
-
-              <figure id="welcometovm">
-                <title xreflabel="welc">VM Welcome Screen</title>
-
-                <mediaobject>
-                  <imageobject>
-                    <imagedata fileref="images/GA-vm01.jpg"
-                               vendor="VM_welcome" />
-                  </imageobject>
-                </mediaobject>
-              </figure>
-
-              <para><informaltable colsep="1" frame="all" rowsep="1">
-                  <?dbfo keep-together="always"?>
-
-                  <tgroup cols="2">
-                    <colspec colwidth="49.50pt" />
-
-                    <colspec />
-
-                    <tbody>
-                      <row>
-                        <entry><inlinegraphic
-                        fileref="images/caution.png" /></entry>
-
-                        <entry>Your virtual IP address could be different from
-                        the ones provided in the example images. Please use
-                        the IP address provided by <emphasis
-                        role="bold">your</emphasis> installation.</entry>
-                      </row>
-                    </tbody>
-                  </tgroup>
-                </informaltable></para>
-
-              <para>Note the IP Address of your VM Instance.</para>
-            </listitem>
-
-            <listitem>
-              <para>In your browser, enter the URL displayed (circled in red
-              above) in the previous image (without the :8010) instead enter
-              the <emphasis>IP Address</emphasis>/ganglia.</para>
-
-              <para>For example,
-              <emphasis>http://nnn.nnn.nnn.nnn/ganglia</emphasis>, where
-              nnn.nnn.nnn.nnn is your Virtual Machine's IP address displayed
-              at the VM welcome screen.</para>
-            </listitem>
-          </orderedlist></para>
-
-        <para>We encourage experienced users to use SSH and log into the VM
-        and further examine the configuration of a 1-node monitoring
-        solution.</para>
-
-        <sect3 id="ViewTheMetrics">
-          <title>Viewing the Metrics</title>
-
-          <para>To view the metrics page, go to the following page in your
-          browser.<programlisting> <emphasis>http://nnn.nnn.nnn.nnn/ganglia</emphasis></programlisting></para>
-
-          <para>Where the <emphasis>nnn.nnn.nnn.nnn</emphasis> is your ESP
-          server running ECL Watch.</para>
-        </sect3>
-      </sect2>
+      <xi:include href="HPCCMonitoring/MonRep-Mods/MonRep-VM.xml"
+                  xpointer="get_hpcc"
+                  xmlns:xi="http://www.w3.org/2001/XInclude" />
     </sect1>
 
     <sect1 id="GangliaIntegration">
@@ -391,8 +348,8 @@
         </listitem>
 
         <listitem>
-          <para>Install the HPCC Systems monitoring component on every node.
-          </para>
+          <para>Install the HPCC Systems monitoring component on every
+          node.</para>
         </listitem>
 
         <listitem>
@@ -426,4 +383,283 @@
         </variablelist></para>
     </sect1>
   </chapter>
+
+  <chapter>
+    <title>Nagios</title>
+
+    <para>The HPCC Reporting component leverages Nagios, an open source,
+    system and network infrastructure monitoring application to monitor and
+    alert HPCC administrators. Nagios leverages established and accepted open
+    source technologies to alert users to changes or potential issues. It
+    provides near real-time system monitoring and reporting.</para>
+
+    <para>With the HPCC integration, you can generate Nagios configuration
+    files to monitor HPCC server health. Once the Nagios is configured, you
+    can monitor:<itemizedlist>
+        <listitem>
+          <para>SSH connectivity</para>
+        </listitem>
+
+        <listitem>
+          <para>Users on system</para>
+        </listitem>
+
+        <listitem>
+          <para>System Load</para>
+        </listitem>
+
+        <listitem>
+          <para>Disk Usage</para>
+        </listitem>
+
+        <listitem>
+          <para>Roxie</para>
+        </listitem>
+
+        <listitem>
+          <para>Dali</para>
+        </listitem>
+
+        <listitem>
+          <para>Dafilesrv</para>
+        </listitem>
+
+        <listitem>
+          <para>Sasha</para>
+        </listitem>
+
+        <listitem>
+          <para>Bound services on each ESP</para>
+        </listitem>
+      </itemizedlist></para>
+
+    <para>Nagios is a powerful monitoring and notification system, which can
+    be used with HPCC to help identify and resolve infrastructure problems
+    before they affect critical processes. Nagios hardware notifications can
+    help keep your system highly available and alerts can assist in
+    pre-emptive maintenance for processes which are down or behaving outside
+    expected parameters to ensure system stability, reliability, and uptime.
+    Scripts and tools are provided to extract HPCC Platform system metrics and
+    easily integrate that data into Nagios.</para>
+
+    <sect1 id="NagiosVM">
+      <title>Nagios in the Virtual Machine</title>
+
+      <para>An easy way to understand how the Nagios works and how to
+      implement it on a larger system, is to examine an established session in
+      action.</para>
+
+      <para>Nagios integration is built into the current HPCC Virtual Machine
+      images. Download and start up a virtual image and look at how the
+      monitoring component works.</para>
+
+      <para>The Nagios component for HPCC on the VM allows you:</para>
+
+      <itemizedlist>
+        <listitem>
+          <para>A preview of the alerts</para>
+        </listitem>
+
+        <listitem>
+          <para>A quickstart</para>
+        </listitem>
+
+        <listitem>
+          <para>A guide for set up</para>
+        </listitem>
+      </itemizedlist>
+
+      <para>Evaluate the value of the content and decide what aspects are
+      relevant to your needs.</para>
+
+      <!--INCLUDE-VM_STEPS-as-Sect2-->
+
+      <xi:include href="HPCCMonitoring/MonRep-Mods/MonRep-VM.xml"
+                  xpointer="get_hpcc"
+                  xmlns:xi="http://www.w3.org/2001/XInclude" />
+
+      <sect2>
+        <title>Nagios Interface</title>
+
+        <para>There are a number of Nagios configurations available. To get a
+        better understanding of Nagios configuration, look at the
+        configuration delivered with the VM. To login to the Nagios admin
+        page:</para>
+
+        <orderedlist>
+          <listitem>
+            <para>Go to
+            <emphasis>http://nnn.nnn.nnn.nnn/</emphasis>nagios3</para>
+
+            <para>Where the <emphasis>nnn.nnn.nnn.nnn</emphasis> is your ESP
+            server running ECL Watch.</para>
+          </listitem>
+
+          <listitem>
+            <para>Login with username : nagiosadmin</para>
+          </listitem>
+
+          <listitem>
+            <para>Enter the password : nagiosadmin</para>
+          </listitem>
+        </orderedlist>
+
+        <para>Once logged in the Nagios landing page displays. This page
+        displays information about Nagios and contains links to the various
+        components, items, and documentation.</para>
+
+        <para>To view the configuration, click on the <emphasis
+        role="bold">Host Groups</emphasis> link from the Nagios navigation
+        menu on the left side of the page.</para>
+
+        <para><figure>
+            <title>Nagios Host Groups</title>
+
+            <mediaobject>
+              <imageobject>
+                <imagedata fileref="images/NAG001.jpg" />
+              </imageobject>
+            </mediaobject>
+          </figure></para>
+
+        <para>This displays the Host Groups being monitored.</para>
+
+        <figure>
+          <title>Nagios Host Groups</title>
+
+          <mediaobject>
+            <imageobject>
+              <imagedata fileref="images/NAG002.jpg" />
+            </imageobject>
+          </mediaobject>
+        </figure>
+
+        <sect3>
+          <title>Nagios Services</title>
+
+          <para>Click on the <emphasis role="bold">Services</emphasis> link
+          from the Nagios navigation menu on the left side of the page.
+          <figure>
+              <title>Nagios Services</title>
+
+              <mediaobject>
+                <imageobject>
+                  <imagedata fileref="images/NAG003.jpg" />
+                </imageobject>
+              </mediaobject>
+            </figure></para>
+
+          <para>The services link displays the Service Status details for the
+          systems being monitored. <figure>
+              <title>Nagios Service status</title>
+
+              <mediaobject>
+                <imageobject>
+                  <imagedata fileref="images/NAG004.jpg" />
+                </imageobject>
+              </mediaobject>
+            </figure></para>
+
+          <para>You can see the service status for the systems being
+          monitored.</para>
+        </sect3>
+      </sect2>
+    </sect1>
+
+    <sect1>
+      <title>Installation of Nagios</title>
+
+      <para>The HPCC Nagios package provides tools and utilities for
+      generating Nagios configurations. These configurations check HPCC and
+      perform some of the HPCC specific checks. HPCC Nagios installation is
+      provided on the HPCC Systems portal. </para>
+
+      <sect2 id="HPCC_Nagios_Installation">
+        <title>HPCC Nagios Installation Package</title>
+
+        <para>To get the HPCC Nagios monitoring on your system you need the
+        Installation package. Download the installation package from the HPCC
+        Systems portal. </para>
+
+        <para>The HPCC Systems web portal is where you can find HPCC
+        resources, downloads, plug-ins, as well as helpful information.
+        </para>
+
+        <para><ulink
+        url="http://hpccsystems.com/download/free-community-edition/monitoring">http://hpccsystems.com/</ulink></para>
+
+        <para>You can find the HPCC Monitoring and Reporting Installation
+        packages at: </para>
+
+        <para><ulink
+        url="http://hpccsystems.com/download/free-community-edition/monitoring">http://hpccsystems.com/download/free-community-edition/monitoring</ulink></para>
+
+        <para>Download the appropriate installation package for your operating
+        system. </para>
+      </sect2>
+
+      <sect2>
+        <title>Install Nagios</title>
+
+        <para>To Install Nagios for HPCC, you must have HPCC System platform
+        installed and also have the open-source Nagios package installed.
+        </para>
+
+        <para><orderedlist>
+            <listitem>
+              <para>Install the <emphasis
+              role="bold">hpcc-nagios-monitoring</emphasis> on the node that
+              will be doing the monitoring. The node where you install the
+              Nagios monitoring must have network connectivity to all the
+              monitored nodes.</para>
+
+              <para>With the hpcc-nagios tools installed, you have HPCC check
+              utilities in:</para>
+
+              <para><programlisting> /usr/lib/nagios/plugins/ </programlisting></para>
+            </listitem>
+
+            <listitem>
+              <para>Generate Nagios configuration files.</para>
+            </listitem>
+          </orderedlist>Generate a host groups configuration for
+        Nagios.</para>
+
+        <programlisting> /opt/HPCCSystem/bin/hpcc-nagios-tools -env   \
+ /etc/HPCCSystems/environment.xml -h -out /etc/nagios3/config.d/hpcc_hostgroups.cfg 
+</programlisting>
+
+        <para>Generate a services configuration file.</para>
+
+        <programlisting> /opt/HPCCSystem/bin/hpcc-nagios-tools -env   \
+ /etc/HPCCSystems/environment.xml -g -out /etc/nagios3/config.d/hpcc_services.cfg
+</programlisting>
+
+        <para>You can use some or all of the configurations. You can use the
+        generated configurations, or you could merge them into any existing
+        Nagios configuration as needed. </para>
+
+        <para><orderedlist continuation="continues">
+            <listitem>
+              <para>Integrate the host and services configuration files into
+              the Nagios configuration folders.</para>
+
+              <para>You must restart Nagios for the new configuration to take
+              effect. </para>
+            </listitem>
+          </orderedlist> </para>
+
+        <sect3>
+          <title>Help</title>
+
+          <para>For help with HPCC Nagios enter:</para>
+
+          <programlisting> /opt/HPCCSystems/bin/hpcc-nagios-tools</programlisting>
+
+          <para>Entering the command without any parameters or options
+          specified displays all the available options.</para>
+        </sect3>
+      </sect2>
+    </sect1>
+  </chapter>
 </book>

docs/HPCCMonitoring/MonRep-Mods/MonRep-VM.xml

@@ -0,0 +1,114 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE section PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN"
+"http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd">
+<sect2 id="get_hpcc">
+  <title>Get the latest HPCC Virtual Image File</title>
+
+  <para>The complete details for installing and running HPCC in a virtual
+  machine are available in the document: <emphasis role="bold">Running HPCC in
+  a Virtual Machine</emphasis>, available from <ulink
+  url="hpccsystems.com/download/docs">hpccsystems.com/download/docs</ulink>
+  .</para>
+
+  <para>The following steps are a quick summary, assuming you have some
+  familiarity with running virtual machines.</para>
+
+  <para><orderedlist>
+      <listitem>
+        <para>Download the latest HPCC Virtual Machine image file from:</para>
+
+        <para><ulink
+        url="http://HPCCsystems.com/download/hpcc-vm-image">http://hpccsystems.com/download/hpcc-vm-image</ulink></para>
+      </listitem>
+
+      <listitem>
+        <para>Save the file to a folder on your machine.</para>
+      </listitem>
+
+      <listitem>
+        <para>Open your virtualization software, import the virtual machine
+        and start it.</para>
+      </listitem>
+
+      <listitem>
+        <?dbfo keep-together="always"?>
+
+        <para>Once the VM initialization completes, you will see a window
+        similar to the following:</para>
+
+        <figure id="welcometovm">
+          <title>VM Welcome Screen</title>
+
+          <mediaobject>
+            <imageobject>
+              <imagedata fileref="../../images/GA-vm01.jpg"
+                         vendor="VM_welcome" />
+            </imageobject>
+          </mediaobject>
+        </figure>
+
+        <para><informaltable colsep="1" frame="all" rowsep="1">
+            <?dbfo keep-together="always"?>
+
+            <tgroup cols="2">
+              <colspec colwidth="49.50pt" />
+
+              <colspec />
+
+              <tbody>
+                <row>
+                  <entry><inlinegraphic
+                  fileref="../../images/caution.png" /></entry>
+
+                  <entry>Your virtual IP address could be different from the
+                  ones provided in the example images. Please use the IP
+                  address provided by <emphasis role="bold">your</emphasis>
+                  installation.</entry>
+                </row>
+              </tbody>
+            </tgroup>
+          </informaltable></para>
+
+        <para>Note the IP Address of your VM Instance.</para>
+      </listitem>
+
+      <listitem>
+        <para>In your browser, enter the URL displayed (circled in red above)
+        in the previous image (without the :8010)</para>
+
+        <para>For Ganglia enter the <emphasis>IP Address</emphasis>/ganglia.
+        For Nagios enter the <emphasis>IP Address</emphasis>/nagios3.</para>
+
+        <para>For example,
+        <emphasis>http://nnn.nnn.nnn.nnn/nagios3</emphasis>, where
+        nnn.nnn.nnn.nnn is your Virtual Machine's IP address displayed at the
+        VM welcome screen.</para>
+      </listitem>
+    </orderedlist></para>
+
+  <para>We encourage experienced users to use SSH and log into the VM and
+  further examine the configuration of a 1-node monitoring solution.</para>
+
+  <sect3 id="ViewTheMetrics" role="brk">
+    <title>Viewing the Metrics</title>
+
+    <para>To view the metrics page, go to the following page(s) in your
+    browser.</para>
+
+    <para><emphasis role="bold">Ganglia:</emphasis></para>
+
+    <para><programlisting> 
+<emphasis>http://nnn.nnn.nnn.nnn/ganglia</emphasis></programlisting></para>
+
+    <para>Where the <emphasis>nnn.nnn.nnn.nnn</emphasis> is your ESP server
+    running ECL Watch.</para>
+
+    <para><emphasis role="bold">Nagios:</emphasis></para>
+
+    <para><programlisting> 
+<emphasis>http://nnn.nnn.nnn.nnn/nagios3</emphasis></programlisting></para>
+
+    <para>Where the <emphasis>nnn.nnn.nnn.nnn</emphasis> is your ESP server
+    running ECL Watch.</para>
+  </sect3>
+</sect2>

docs/HPCCSystemAdmin/HPCCSystemAdministratorsGuide.xml

@@ -743,8 +743,7 @@
         </mediaobject>
       </figure></para>
 
-   
-      <!--/*Including special SysAdmin Config Module -paras- */-->  
+    <!--/*Including special SysAdmin Config Module -paras- */-->
 
     <xi:include href="HPCCSystemAdmin/SA-Mods/SysAdminConfigMod.xml"
                 xpointer="cfgmgr_introP0"
@@ -762,12 +761,13 @@
                 xpointer="cfgmgr_introP2"
                 xmlns:xi="http://www.w3.org/2001/XInclude" />
 
-     <xi:include href="HPCCSystemAdmin/SA-Mods/SysAdminConfigMod.xml"
+    <xi:include href="HPCCSystemAdmin/SA-Mods/SysAdminConfigMod.xml"
                 xpointer="cfgmgr_introP3"
                 xmlns:xi="http://www.w3.org/2001/XInclude" />
 
-      <!--/*Including special SysAdmin Config Module -Sect1- */--> 
-     <xi:include href="HPCCSystemAdmin/SA-Mods/SysAdminConfigMod.xml"
+    <!--/*Including special SysAdmin Config Module -Sect1- */-->
+
+    <xi:include href="HPCCSystemAdmin/SA-Mods/SysAdminConfigMod.xml"
                 xpointer="configuring-a-multi-node-system"
                 xmlns:xi="http://www.w3.org/2001/XInclude" />
 
@@ -1129,7 +1129,7 @@ lock=/var/lock/HPCCSystems</programlisting>
 
         <para>Sasha should be run in an active/passive configuration.
         Active/passive meaning you would have two Sashas configured, one
-        primary (active), and the other standing by. </para>
+        primary (active), and the other standing by.</para>
       </sect2>
 
       <sect2>
@@ -1272,7 +1272,7 @@ lock=/var/lock/HPCCSystems</programlisting>
       </sect2>
 
       <sect2 id="HA_Middlewear">
-        <title>Middlewear</title>
+        <title>Middleware</title>
 
         <para>Replication of some components (ECL Agent, ESP/Eclwatch, DFU
         Server, etc.) are pretty straight forward as they really don’t have

ecl/hthor/hthorkey.cpp

@@ -1077,7 +1077,7 @@ extern HTHOR_API IHThorActivity *createIndexReadActivity(IAgentContext &_agent,
         StringBuffer buff;
         buff.append("Skipping OPT index read of nonexistent file ").append(lfn);
         WARNLOG("%s", buff.str());
-        _agent.addWuException(buff.str(), 0, ExceptionSeverityWarning, "hthor");
+        _agent.addWuException(buff.str(), 0, ExceptionSeverityInformation, "hthor");
         return new CHThorNullActivity(_agent, _activityId, _subgraphId, arg, _kind);
     }
     _agent.logFileAccess(dFile, "HThor", "READ");
@@ -1255,7 +1255,7 @@ extern HTHOR_API IHThorActivity *createIndexNormalizeActivity(IAgentContext &_ag
         StringBuffer buff;
         buff.append("Skipping OPT index normalize of nonexistent file ").append(lfn);
         WARNLOG("%s", buff.str());
-        _agent.addWuException(buff.str(), 0, ExceptionSeverityWarning, "hthor");
+        _agent.addWuException(buff.str(), 0, ExceptionSeverityInformation, "hthor");
         return new CHThorNullActivity(_agent, _activityId, _subgraphId, arg, _kind);
     }
     _agent.logFileAccess(dFile, "HThor", "READ");
@@ -1373,7 +1373,7 @@ extern HTHOR_API IHThorActivity *createIndexAggregateActivity(IAgentContext &_ag
         StringBuffer buff;
         buff.append("Skipping OPT index aggregate of nonexistent file ").append(lfn);
         WARNLOG("%s", buff.str());
-        _agent.addWuException(buff.str(), 0, ExceptionSeverityWarning, "hthor");
+        _agent.addWuException(buff.str(), 0, ExceptionSeverityInformation, "hthor");
         return new CHThorNullAggregateActivity(_agent, _activityId, _subgraphId, arg, arg, _kind);
     }
     _agent.logFileAccess(dFile, "HThor", "READ");
@@ -1477,7 +1477,7 @@ extern HTHOR_API IHThorActivity *createIndexCountActivity(IAgentContext &_agent,
         StringBuffer buff;
         buff.append("Skipping OPT index count of nonexistent file ").append(lfn);
         WARNLOG("%s", buff.str());
-        _agent.addWuException(buff.str(), 0, ExceptionSeverityWarning, "hthor");
+        _agent.addWuException(buff.str(), 0, ExceptionSeverityInformation, "hthor");
         return new CHThorNullCountActivity(_agent, _activityId, _subgraphId, arg, _kind);
     }
     _agent.logFileAccess(dFile, "HThor", "READ");
@@ -1587,7 +1587,7 @@ extern HTHOR_API IHThorActivity *createIndexGroupAggregateActivity(IAgentContext
         StringBuffer buff;
         buff.append("Skipping OPT index group aggregate of nonexistent file ").append(lfn);
         WARNLOG("%s", buff.str());
-        _agent.addWuException(buff.str(), 0, ExceptionSeverityWarning, "hthor");
+        _agent.addWuException(buff.str(), 0, ExceptionSeverityInformation, "hthor");
         return new CHThorNullActivity(_agent, _activityId, _subgraphId, arg, _kind);
     }
     _agent.logFileAccess(dFile, "HThor", "READ");
@@ -2206,7 +2206,7 @@ public:
                 StringBuffer buff;
                 buff.append("Skipping OPT fetch of nonexistent file ").append(lfn);
                 WARNLOG("%s", buff.str());
-                agent.addWuException(buff.str(), 0, ExceptionSeverityWarning, "hthor");
+                agent.addWuException(buff.str(), 0, ExceptionSeverityInformation, "hthor");
             }
         }
         inputThread.setown(new InputHandler(this));
@@ -2471,7 +2471,7 @@ public:
             StringBuffer buff;
             buff.append("Skipping OPT fetch of nonexistent file ").append(lfn);
             WARNLOG("%s", buff.str());
-            agent.addWuException(buff.str(), 0, ExceptionSeverityWarning, "hthor");
+            agent.addWuException(buff.str(), 0, ExceptionSeverityInformation, "hthor");
         }
             
         csvSplitter.init(_arg.getMaxColumns(), csvInfo, quotes, separators, terminators, escapes);
@@ -3918,7 +3918,7 @@ public:
             StringBuffer buff;
             buff.append("Skipping OPT keyed join against nonexistent file ").append(lfn);
             WARNLOG("%s", buff.str());
-            agent.addWuException(buff.str(), 0, ExceptionSeverityWarning, "hthor");
+            agent.addWuException(buff.str(), 0, ExceptionSeverityInformation, "hthor");
         }
         CHThorThreadedActivityBase::start();
     }

ecllibrary/std/Str.ecl

@@ -292,8 +292,8 @@ EXPORT SET OF STRING SplitWords(STRING src, STRING separator, BOOLEAN allow_blan
  * Returns the list of words extracted from the string.  Words are separated by one or more separator strings. No
  * spaces are stripped from either string before matching.
  *
- * @param words         The set of strings to be combined
- * @param separator     The string used to separate words
+ * @param words         The set of strings to be combined.
+ * @param separator     The string used to separate words.
  */
 
 EXPORT STRING CombineWords(SET OF STRING words, STRING separator) := lib_stringlib.StringLib.CombineWords(words, separator);
@@ -391,7 +391,7 @@ EXPORT STRING Repeat(STRING text, UNSIGNED4 n) := lib_stringlib.Stringlib.String
  * Converts the data value to a sequence of hex pairs.
  *
  * @param value         The data value that should be expanded as a sequence of hex pairs.
- * @return              A string containg a sequence of hex pairs.
+ * @return              A string containing a sequence of hex pairs.
  */
 
 EXPORT STRING ToHexPairs(DATA value) := lib_stringlib.StringLib.Data2String(value);

esp/eclwatch/ws_XSLT/wuidcommon.xslt

@@ -46,7 +46,7 @@
                    
                   <xsl:choose>
                     <xsl:when test="WUXMLSize &lt; 5000000">
-                      <a href="/esp/iframe?esp_iframe_title=ECL Workunit XML - {$wuid}&amp;inner=/WsWorkunits/WUFile%3fWuid%3d{$wuid}%26Type%3dXML%26Option%3d1" >XML</a><xsl:value-of select="WUXMLSize"/>
+                      <a href="/esp/iframe?esp_iframe_title=ECL Workunit XML - {$wuid}&amp;inner=/WsWorkunits/WUFile%3fWuid%3d{$wuid}%26Type%3dXML%26Option%3d0" >XML</a><xsl:value-of select="WUXMLSize"/>
                     </xsl:when>
                     <xsl:otherwise>
                       <a href="/esp/iframe?esp_iframe_title=Download ECL Workunit XML - {$wuid}&amp;inner=/WsWorkunits/WUFile%3fWuid%3d{$wuid}%26Type%3dXML%26Option%3d2" >Download XML</a>

esp/services/ws_workunits/ws_workunitsHelpers.cpp

@@ -1909,6 +1909,19 @@ void WsWuInfo::getWorkunitArchiveQuery(MemoryBuffer& buf)
     buf.append(queryText.length(), queryText.str());
 }
 
+void WsWuInfo::getWorkunitQueryShortText(MemoryBuffer& buf)
+{
+    Owned<IConstWUQuery> query = cw->getQuery();
+    if(!query)
+        throw MakeStringException(ECLWATCH_QUERY_NOT_FOUND_FOR_WU,"No query for workunit %s.",wuid.str());
+
+    SCMStringBuffer queryText;
+    query->getQueryShortText(queryText);
+    if (queryText.length() < 1)
+        throw MakeStringException(ECLWATCH_QUERY_NOT_FOUND_FOR_WU, "No query for workunit %s.",wuid.str());
+    buf.append(queryText.length(), queryText.str());
+}
+
 void WsWuInfo::getWorkunitDll(StringBuffer &dllname, MemoryBuffer& buf)
 {
     Owned<IConstWUQuery> query = cw->getQuery();

esp/services/ws_workunits/ws_workunitsHelpers.hpp

@@ -42,6 +42,7 @@ namespace ws_workunits {
 #define    File_XML "XML"
 #define    File_Res "res"
 #define    File_DLL "dll"
+#define    File_WUECL "WUECL"
 #define    File_ArchiveQuery "ArchiveQuery"
 
 #define    TOTALTHORTIME    "Total thor time"
@@ -178,6 +179,7 @@ public:
     void getWorkunitArchiveQuery(MemoryBuffer& buf);
     void getWorkunitDll(StringBuffer &name, MemoryBuffer& buf);
     void getWorkunitXml(const char* plainText, MemoryBuffer& buf);
+    void getWorkunitQueryShortText(MemoryBuffer& buf);
     void getWorkunitAssociatedXml(const char* name, const char* IPAddress, const char* plainText, const char* description, bool forDownload, MemoryBuffer& buf);
     void getWorkunitCpp(const char* cppname, const char* description, const char* ipAddress, MemoryBuffer& buf, bool forDownload);
     void getEventScheduleFlag(IEspECLWorkunit &info);

esp/services/ws_workunits/ws_workunitsService.cpp

@@ -2508,23 +2508,33 @@ bool CWsWorkunitsEx::onWUFile(IEspContext &context,IEspWULogFileRequest &req, IE
                 winfo.getWorkunitAssociatedXml(name, req.getIPAddress(), req.getPlainText(), req.getDescription(), opt > 0, mb);
                 openSaveFile(context, opt, ptr, HTTP_TYPE_APPLICATION_XML, mb, resp);
             }
-            else if (strieq(File_XML,req.getType()))
+            else if (strieq(File_XML,req.getType()) || strieq(File_WUECL,req.getType()))
             {
-                winfo.getWorkunitXml(req.getPlainText(), mb);
-                if (opt < 2)
+                StringBuffer mimeType, fileName;
+                if (strieq(File_WUECL,req.getType()))
                 {
-                    resp.setThefile(mb);
-                    const char* plainText = req.getPlainText();
-                    if (plainText && (!stricmp(plainText, "yes")))
-                        resp.setThefile_mimetype(HTTP_TYPE_TEXT_PLAIN);
-                    else
-                        resp.setThefile_mimetype(HTTP_TYPE_APPLICATION_XML);
+                    fileName.setf("%s.ecl", wuid.get());
+                    winfo.getWorkunitQueryShortText(mb);
+                    mimeType.set(HTTP_TYPE_TEXT_PLAIN);
                 }
                 else
                 {
-                    VStringBuffer xmlName("%s.xml", wuid.get());
-                    openSaveFile(context, 2, xmlName.str(), HTTP_TYPE_APPLICATION_XML, mb, resp);
+                    fileName.setf("%s.xml", wuid.get());
+                    winfo.getWorkunitXml(req.getPlainText(), mb);
+                    if (opt < 2)
+                    {
+                        const char* plainText = req.getPlainText();
+                        if (plainText && (!stricmp(plainText, "yes")))
+                            mimeType.set(HTTP_TYPE_TEXT_PLAIN);
+                        else
+                            mimeType.set(HTTP_TYPE_APPLICATION_XML);
+                    }
+                    else
+                    {
+                        mimeType.set(HTTP_TYPE_APPLICATION_XML);
+                    }
                 }
+                openSaveFile(context, opt, fileName.str(), mimeType.str(), mb, resp);
             }
         }
     }

esp/src/eclwatch/GetDFUWorkunitsWidget.js

@@ -341,7 +341,7 @@ define([
                             return "Unknown";
                         }
                     },
-                    Owner: { label: this.i18n.Owner, width: 90 },
+                    User: { label: this.i18n.Owner, width: 90 },
                     JobName: { label: this.i18n.JobName },
                     ClusterName: { label: this.i18n.Cluster, width: 126 },
                     StateMessage: { label: this.i18n.State, width: 72 },

esp/src/eclwatch/PermissionsWidget.js

@@ -20,6 +20,7 @@ define([
     "dojo/i18n!./nls/hpcc",
 
     "dijit/registry",
+    "dijit/form/CheckBox",
 
     "dgrid/tree",
     "dgrid/editor",
@@ -29,7 +30,7 @@ define([
     "hpcc/ESPUtil"
 
 ], function (declare, lang, i18n, nlsHPCC,
-                registry,
+                registry, CheckBox,
                 tree, editor,
                 GridDetailsWidget, WsAccess, ESPUtil) {
     return declare("PermissionsWidget", [GridDetailsWidget], {
@@ -67,39 +68,47 @@ define([
                     allow_access: editor({
                         width: 54,
                         editor: "checkbox",
+                        editorArgs: { value: true },
+                        className: "hpccCentered",
                         autoSave: true,
                         canEdit: function (object, value) { return object.__hpcc_type != "Permission"; },
                         renderHeaderCell: function (node) {
                             node.innerHTML = context.i18n.AllowAccess;
                         }
-                    }),
+                    }, CheckBox),
                     allow_read: editor({
                         width: 54,
                         editor: "checkbox",
+                        editorArgs: { value: true },
+                        className: "hpccCentered",
                         autoSave: true,
                         canEdit: function (object, value) { return object.__hpcc_type != "Permission"; },
                         renderHeaderCell: function (node) {
                             node.innerHTML = context.i18n.AllowRead;
                         }
-                    }),
+                    }, CheckBox),
                     allow_write: editor({
                         width: 54,
                         editor: "checkbox",
+                        editorArgs: { value: true },
+                        className: "hpccCentered",
                         autoSave: true,
                         canEdit: function (object, value) { return object.__hpcc_type != "Permission"; },
                         renderHeaderCell: function (node) {
                             node.innerHTML = context.i18n.AllowWrite;
                         }
-                    }),
+                    }, CheckBox),
                     allow_full: editor({
                         width: 54,
                         editor: "checkbox",
+                        editorArgs: { value: true },
+                        className: "hpccCentered",
                         autoSave: true,
                         canEdit: function (object, value) { return object.__hpcc_type != "Permission"; },
                         renderHeaderCell: function (node) {
                             node.innerHTML = context.i18n.AllowFull;
                         }
-                    }),
+                    }, CheckBox),
                     padding: {
                         width:20,
                         label: " "
@@ -107,39 +116,47 @@ define([
                     deny_access: editor({
                         width: 54,
                         editor: "checkbox",
+                        editorArgs: { value: true },
+                        className: "hpccCentered",
                         autoSave: true,
                         canEdit: function (object, value) { return object.__hpcc_type != "Permission"; },
                         renderHeaderCell: function (node) {
                             node.innerHTML = context.i18n.DenyAccess
                         }
-                    }),
+                    }, CheckBox),
                     deny_read: editor({
                         width: 54,
                         editor: "checkbox",
+                        editorArgs: { value: true },
+                        className: "hpccCentered",
                         autoSave: true,
                         canEdit: function (object, value) { return object.__hpcc_type != "Permission"; },
                         renderHeaderCell: function (node) {
                             node.innerHTML = context.i18n.DenyRead
                         }
-                    }),
+                    }, CheckBox),
                     deny_write: editor({
                         width: 54,
                         editor: "checkbox",
+                        editorArgs: { value: true },
+                        className: "hpccCentered",
                         autoSave: true,
                         canEdit: function (object, value) { return object.__hpcc_type != "Permission"; },
                         renderHeaderCell: function (node) {
                             node.innerHTML = context.i18n.DenyWrite
                         }
-                    }),
+                    }, CheckBox),
                     deny_full: editor({
                         width: 54,
                         editor: "checkbox",
+                        editorArgs: { value: true },
+                        className: "hpccCentered",
                         autoSave: true,
                         canEdit: function (object, value) { return object.__hpcc_type != "Permission"; },
                         renderHeaderCell: function (node) {
                             node.innerHTML = context.i18n.DenyFull
                         }
-                    })
+                    }, CheckBox)
                 }
             }, domID);
 

esp/src/eclwatch/css/hpcc.css

@@ -217,6 +217,10 @@ hr.dashedLine {
 
 .claro .dgrid-cell { text-overflow: ellipsis; white-space: nowrap; }
 
+.claro .hpccCentered { 
+    text-align: center; 
+}
+
 #stubTitlebar :focus {
     outline: none;
 }

initfiles/componentfiles/configxml/roxie.xsd.in

@@ -609,6 +609,13 @@
         </xs:appinfo>
       </xs:annotation>
     </xs:attribute>
+    <xs:attribute name="lockSuperFiles" type="xs:boolean" use="optional" default="false">
+      <xs:annotation>
+        <xs:appinfo>
+          <tooltip>If enabled, superfiles will be locked while queries that use them are loaded</tooltip>
+        </xs:appinfo>
+      </xs:annotation>
+    </xs:attribute>
     <xs:attribute name="lowTimeout" type="xs:nonNegativeInteger" use="optional" default="10000">
       <xs:annotation>
         <xs:appinfo>

initfiles/etc/DIR_NAME/environment.xml.in

@@ -846,6 +846,7 @@
                 linuxYield="false"
                 localFilesExpire="-1"
                 localSlave="true"
+                lockSuperFiles="false"
                 logFullQueries="false"
                 logQueueDrop="32"
                 logQueueLen="512"

initfiles/sbin/CMakeLists.txt

@@ -41,6 +41,7 @@ FOREACH( oFILES
     ${CMAKE_CURRENT_SOURCE_DIR}/alter_confs.sh
     ${CMAKE_CURRENT_SOURCE_DIR}/cluster_script.py
     ${CMAKE_CURRENT_SOURCE_DIR}/deploy-java-files.exp
+    ${CMAKE_CURRENT_SOURCE_DIR}/install-hpcc.exp
 )
     install ( PROGRAMS ${oFILES} DESTINATION sbin COMPONENT Runtime )
 ENDFOREACH ( oFILES )

initfiles/sbin/cluster_script.py

@@ -283,6 +283,12 @@ class ScriptExecution(object):
             self.usage()
             exit(0)
 
+        if not os.path.isfile(self.script_file):
+            print("\nFile " + self.script_file + " does not exist.\n")
+            exit(0)
+
+
+
 
     def log_input_parameters(self):
         self.logger.info("Current parameters:")

initfiles/sbin/hpcc-push.sh.in

@@ -15,54 +15,126 @@
 #    limitations under the License.
 ################################################################################
 #
-# Usage: hpcc-push.sh <from> <to>
+# Uusage: hpcc-push.sh -s <source> -t <target> -n  <number_concurrent> -x
 #
 # This is acomplished with a standard scp command with the use of the
 # runtime users id_rsa file.
 
 ###<REPLACE>###
 
-
-source  ${INSTALL_DIR}/etc/init.d/lock.sh
-source  ${INSTALL_DIR}/etc/init.d/pid.sh
 source  ${INSTALL_DIR}/etc/init.d/hpcc_common
 source  ${INSTALL_DIR}/etc/init.d/init-functions
 source  ${INSTALL_DIR}/etc/init.d/export-path
 
 
-getIPS(){
-	IPS=`${INSTALL_DIR}/sbin/configgen -env ${envfile} -machines | awk -F, '{print \$1}'  | sort | uniq`
+usage() {
+    echo ""
+    echo "usage: hpcc-push.sh -s <source> -t <target> -n  <concurrent> -x"
+    echo "   -n:  when specified, denotes the number of concurrent executions threads."
+    echo "        The default is 5."
+    echo "   -s:  source file or directory."
+    echo "   -t:  target file or directory."
+    echo "   -x:  when specified, this option excludes execution on the current host."
+    echo ""
+
+    exit 1
+
 }
+createScriptFile() {
+
+   cat > $SCRIPT_FILE <<SCRIPTFILE
+#~/bin/bash
+IP=\$1
+
+if ping -c 1 -w 5 -n \$IP > /dev/null 2>&1; then
+    echo "\$IP: Host is alive."
+    CAN_SSH="\`ssh -i $home/$user/.ssh/id_rsa -o BatchMode=yes -o StrictHostKeyChecking=no $user@\$IP exit > /dev/null 2>&1; echo \$?\`"
+    if [ "\$CAN_SSH" -eq 255 ]; then
+       echo "\$IP: Cannot SSH to host.";
+    else
+       echo "\$IP: Copying $source to $target on \$IP";
+       SCP=\$(scp -r -i $home/$user/.ssh/id_rsa $source $user@\$IP:$target; echo \$?)
+       if [ "\$SCP" -eq 0 ]; then
+          echo "\$IP: Success";
+       else
+          echo "\$IP: Failure";
+          exit 1
+       fi
+    fi
+else
+    echo "\$IP: Cannot Ping host? (Host Alive?)"
+    exit 1
+fi
+SCRIPTFILE
 
-copyFile(){
-	echo "$IP: Copying $1 to $2 on $IP";
-	SCP=`scp -i $home/$user/.ssh/id_rsa $1 $user@$IP:$2; echo $?`
-	if [ "$SCP" -eq 0 ]; then
-		echo "$IP: Success";
-	else
-		echo "$IP: Failure";
-	fi
+chmod +x ${SCRIPT_FILE}
 }
 
-if [ $# -ne 2 ]; then
-	echo "usage: hpcc-push.sh [user@]host1:]file1 [[user@]host2:]file2"
+
+############################################
+#
+# MAIN
+#
+############################################
+if [ "${USER}" != "root" ]; then
+   echo ""
+   echo "The script must run as root or sudo."
+   echo ""
+   exit 1
 fi
 
 set_environmentvars
-envfile=$configs/$environment
-
-getIPS
-
-for IP in $IPS; do
-	if ping -c 1 -w 5 -n $IP > /dev/null 2>&1; then
-		echo "$IP: Host is alive."
-		CAN_SSH="`ssh -i $home/$user/.ssh/id_rsa -o BatchMode=yes -o StrictHostKeyChecking=no $user@$IP exit > /dev/null 2>&1; echo $?`"
-		if [ "$CAN_SSH" -eq 255 ]; then
-			echo "$IP: Cannot SSH to host.";
-		else
-			copyFile $1 $2
-		fi
-	else
-        echo "$IP: Cannot Ping host? (Host Alive?)"
-	fi
+
+source=
+target=
+exclude=0
+OPTION="-e ${CONFIG_DIR}/${ENV_CONF_FILE} -s ${SECTION:-DEFAULT}"
+
+
+TEMP=`/usr/bin/getopt -o n:s:t:hx --long help,conrrent:,source:,target:,exclude -n 'hpcc-push' -- "$@"`
+if [ $? != 0 ] ; then echo "Failure to parse commandline." >&2 ; end 1 ; fi
+eval set -- "$TEMP"
+while true ; do
+    case "$1" in
+        -n|--concurrent) 
+            if [ -n "$2" ] && [[ $2 =~ ^[0-9]+$ ]]
+            then
+               [ $2 -gt 0 ] &&  $OPTION="${OPTION:+"$OPTION "}-n $2"
+            fi
+            shift 2 ;;
+        -s|--source) source="$2"
+            shift 2 ;;
+        -t|--target) target="$2"
+            shift 2 ;;
+        -x|--exclude) OPTION="${OPTION:+"$OPTION "}-x"
+            exclude=1
+            shift ;;
+        -h|--help) usage
+            shift ;;
+        --) shift ; break ;;
+        *) usage ;;
+    esac
 done
+
+
+if [ -z "$source"  ] || [ -z "$target" ]; then
+   usage
+fi
+
+
+SCRIPT_FILE=/tmp/hpcc-push_$$
+createScriptFile
+
+python_expected_version=2.6
+is_python_installed ${python_expected_version}
+if [ $? -eq 0 ]
+then
+   eval ${INSTALL_DIR}/sbin/cluster_script.py -f ${SCRIPT_FILE} $OPTION
+else
+   echo ""
+   echo "Cannot detect python version ${python_expected_version}+. Will run on the cluster hosts sequentially."
+   echo ""
+   run_cluster ${SCRIPT_FILE} ${exclude}
+fi
+
+rm -rf ${SCRIPT_FILE}

initfiles/sbin/install-cluster.sh.in

@@ -22,51 +22,70 @@
 # Flow:
 #
 # 1. SSH Keys Generated.
-# 2. Script copied to node.
-# 3. Install package copied to node.
-# 4. SSH Keys copied to node.
-# 5. Run CheckInstall to determine if  package is installed.
-# 6. Install/Upgrade if needed.
-# 7. Run CheckKeys to determine if sshkeys are installed.
-# 8. Install keys if different then installed or no keys installed.
-# 9. Return.
+# 2. Run install-hpcc.sh through cluster-script.py to perform HPCC install and 
+#        configuration on remote hosts
+# 3. Return.
 #
 
 ###<REPLACE>###
 
-source  ${INSTALL_DIR}/etc/init.d/lock.sh
-source  ${INSTALL_DIR}/etc/init.d/pid.sh
 source  ${INSTALL_DIR}/etc/init.d/hpcc_common
 source  ${INSTALL_DIR}/etc/init.d/init-functions
 source  ${INSTALL_DIR}/etc/init.d/export-path
 
-REMOTE_INSTALL="/tmp/remote_install"
-NEW="${REMOTE_INSTALL}/new_keys"
-CMDPREFIX="sudo"
+export REMOTE_INSTALL="/tmp/remote_install"
+export NEW="${REMOTE_INSTALL}/new_keys"
+export CMDPREFIX="sudo"
 
 print_usage(){
-    echo "usage: install-cluster.sh [-h|--help] [-k|--newkey] <Platform Package>"
+    echo ""
+    echo "usage: install-cluster.sh [-h|--help] [-k|--newkey] [-n|--concurrent <number>] <Platform Package>"
+    echo "   -k:  generate a new ssh key pair."
+    echo "   -n:  how many concurrent execution allowd. The default is 5"
+    echo "   <Platform Package>: HPCCSystems package file."
+    echo ""
     exit 1
 }
 
-getUser(){
-    read -p "Input admin username:" USER;
-    echo;
-    if [ "${USER}" == "root" ]; then
-        CMDPREFIX=""
-    fi
-}
-
-getPW(){
-    stty -echo;
-    read -p "Input admin password:" PASS;
-    stty echo;
-    echo;
-}
-
+getUserAndPasswd(){
+
+    trial=0
+    max_trial=3
+    while [ 1 ]
+    do
+       echo ""
+       read -p "Please enter admin username: " ADMIN_USER;
+       echo ""
+       echo "Please enter ssh/scp user password. If this is no password required (assume"
+       echo "the user has ssh private key: <user home>/.ssh/id_rsa) just press 'Enter':"
+       read -s PASS
+       echo ""
+
+       echo "You entered user $ADMIN_USER and a password ($(echo $PASS | sed 's/./\./g'))."
+       read -p  "Are these correct? [Y|n] " answer
+       if [ "$answer" = "Y" ] || [ "$answer" = "y" ]
+       then
+          break
+       fi
+
+       trial=$(expr $trial \+ 1)
+       if [ $trial -eq $max_trial ]
+       then
+          echo ""
+          echo "Exceeded maximum attempts. Giving up."
+          echo ""
+          exit 1
+       fi
+    done
+
+  if [ "$file_transfer_user" == "root" ]
+  then
+     CMDPREFIX=""
+  fi
+
+  export ADMIN_USER
+  export PASS
 
-getIPS(){
-    IPS=`${INSTALL_DIR}/sbin/configgen -env ${CONFIG_DIR}/${ENV_XML_FILE} -machines | awk -F, '{print $1}'  | sort | uniq`
 }
 
 
@@ -92,7 +111,10 @@ createPayload(){
     cp -r ${CONFIG_DIR}/${ENV_XML_FILE} ${REMOTE_INSTALL}
     cp -r ${CONFIG_DIR}/${ENV_CONF_FILE} ${REMOTE_INSTALL}
     cp -r ${INSTALL_DIR}/sbin/remote-install-engine.sh ${REMOTE_INSTALL}
+
+    echo "tar -zcvf /tmp/remote_install.tgz ${REMOTE_INSTALL}/*"
     tar -zcvf /tmp/remote_install.tgz ${REMOTE_INSTALL}/*
+    ls -l /tmp/remote_install.tgz 
     rm -rf ${REMOTE_INSTALL}
 }
 
@@ -100,89 +122,30 @@ removePayload(){
     rm /tmp/remote_install.tgz
 }
 
-copyPayload(){
-    expect -c "set timeout -1;
-        spawn scp /tmp/remote_install.tgz $USER@$1:~;
-        expect {
-            *?assword:* {
-                    send \"$PASS\r\";
-            } yes/no)? {
-                    send \"yes\r\";
-                    set timeout -1;
-            } timeout {
-                    exit;
-            } eof {
-                    exit;
-            }
-        }
-        interact;";
-}
-
-expandPayload(){
-    expect -c "set timeout -1;
-        spawn ssh $USER@$1 \"cd /; tar -zxf ~/remote_install.tgz\";
-        expect {
-            *?assword:* {
-                    send \"$PASS\r\";
-            } yes/no)? {
-                    send \"yes\r\";
-                    set timeout -1;
-            } timeout {
-                    exit;
-            } eof {
-                    exit;
-            }
-        }
-        interact;";
-}
-
-runPayload(){
-    basepkg=`basename ${PKG}`
-    expect -c "set timeout 3;
-        spawn ssh $USER@$1;
-        expect {
-            *?assword:* {
-                    send \"$PASS\r\";
-            } yes/no)? {
-                    send \"yes\r\";
-                    set timeout -1;
-            } timeout {
-                    exit;
-            } eof {
-                    exit;
-            } "${USER}@" { 
-                  # Allow non-password connection go through without changing following code.
-                  # This is will be replaced when HPCC-11156 is fixed.
-                  send \"hostname\r\" 
-            }
-        }
-        expect "${USER}@" {
-            send \"${CMDPREFIX} ${REMOTE_INSTALL}/remote-install-engine.sh ${REMOTE_INSTALL}/${basepkg}\r\";
-            expect *?assword* {
-                send \"${PASS}\r\";
-            }
-        }
-        expect "${USER}@" {
-            send \"exit\r\";
-        }
-        interact;"
-}
-
-
+if [ "${USER}" != "root" ]; then
+   echo ""
+   echo "The script must run as root or sudo."
+   echo ""
+   exit 1
+fi
 
-getUser
-getPW
-getIPS
 
 NEW_KEY=0
+OPTIONS="-l DEBUG -e ${CONFIG_DIR}/${ENV_CONF_FILE} -s ${SECTION:-DEFAULT}"
 
-TEMP=`/usr/bin/getopt -o kh --long help,newkey -n 'install-cluster' -- "$@"`
+TEMP=`/usr/bin/getopt -o n:s:kh --long help,newkey,concurrent: -n 'install-cluster' -- "$@"`
 if [ $? != 0 ] ; then echo "Failure to parse commandline." >&2 ; exit 1 ; fi
 eval set -- "$TEMP"
 while true ; do
     case "$1" in
         -k|--newkey) NEW_KEY=1
             shift ;;
+        -n|--concurrent) 
+            if [ -n "$2" ] && [[ $2 =~ ^[0-9]+$ ]]
+            then
+               [ $2 -gt 0 ] && OPTOINS="${OPTIONS:+"$OPTIONS "}-n $2"
+            fi
+            shift 2 ;;
         -h|--help) print_usage
                    shift ;;
         --) shift ; break ;;
@@ -191,6 +154,12 @@ while true ; do
 done
 for arg do arg=$arg; done
 PKG=${arg}
+[ -z "$PKG" ] && print_usage
+
+export PKG
+
+getUserAndPasswd
+
 
 pkgtype=`echo "${PKG}" | grep -i rpm`
 if [ -z $pkgtype ]; then
@@ -202,19 +171,22 @@ fi
 if [ ${NEW_KEY} -eq 1 ]; then
     generateKey
 fi
+export NEW_KEY
 
 createPayload;
 
-for IP in $IPS; do
-    if ping -c 1 -w 5 -n $IP > /dev/null 2>&1; then
-        echo "$IP: Host is alive."
-        copyPayload $IP;
-        expandPayload $IP;
-        runPayload $IP;
-        echo "$IP: Done.";
-    else
-        echo "$IP: Cannot Ping host? (Host Alive?)"
-    fi
-done
+expected_python_version=2.6
+is_python_installed ${expected_python_version}
+
+if [ $? -eq 0 ]
+then
+   chksum=$(md5sum ${INSTALL_DIR}/sbin/install-hpcc.exp | cut -d' ' -f1)
+   eval ${INSTALL_DIR}/sbin/cluster_script.py -f ${INSTALL_DIR}/sbin/install-hpcc.exp -c $chksum $OPTIONS
+else
+   echo ""
+   echo "Cannot detect python version ${expected_python_version}+. Will run on the cluster hosts sequentially."
+   echo ""
+   run_cluster ${INSTALL_DIR}/sbin/install-hpcc.exp 0
+fi
 
 removePayload;

initfiles/sbin/install-hpcc.exp

@@ -0,0 +1,218 @@
+#!/usr/bin/env expect 
+################################################################################
+#    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+################################################################################
+#
+# Usage: install-hpcc.exp
+#
+# This script is used as a remote engine for a cluster installation.
+#
+# Flow:
+#
+# 1. Copay HPCC package and script to node.
+# 2. call remote-install-engine.sh to install and configure HPCC.
+#
+
+
+
+proc print_usage {} {
+    puts ""
+    puts "Usage: [lindex [split $::argv0 '/' ] end]  <host ip>"
+    puts ""
+    exit 1
+}
+
+
+proc testWithPing {} {
+   global ip
+   spawn ping -c1 -W 5 ${ip}
+   expect eof {
+      catch wait result
+      if { [lindex $result 3] != 0 } {
+         echo "${ip}: Cannot Ping host? (Host Alive?)"
+         exit 1
+      }
+   }
+   sleep 1
+   puts "${ip}: Host is alive."
+}
+
+proc checkSSHConnection {} {
+   global ip user password prompt
+
+   set timeout 60
+   spawn ssh ${user}@${ip} 
+   expect {
+      *?assword:* {
+         send "${password}\r";
+         exp_continue
+      } "*?ermission denied*" {
+         puts "The user or password is wrong"
+         exit 1
+      } yes/no)? {
+         send "yes\r";
+         exp_continue
+      } timeout {
+         exit 1;
+      } eof {
+         exit 1;
+      }  -re "${prompt}" { 
+         puts "Connection is OK"
+         send "exit\r"
+      }
+   }
+   expect -re .*
+   interact;
+}
+
+proc copyPayload {} {
+   global ip user password prompt
+   
+   set timeout 300
+   spawn scp /tmp/remote_install.tgz ${user}@${ip}:~;
+   expect {
+      *?assword:* {
+         send "${password}\r";
+         exp_continue
+      } "*?ermission denied*" {
+         puts "The user or password is wrong"
+         exit 1
+      } yes/no)? {
+         send "yes\r";
+         exp_continue
+      } timeout {
+         exit 1;
+      } eof {
+         catch wait result
+         if { [lindex $result 3] != 0 } {
+            exit [lindex $result 3]
+         }
+      }
+   }
+   sleep 1
+}
+
+proc expandPayload {} {
+   global ip user password prompt
+
+   set timeout 180
+   spawn ssh ${user}@${ip} "cd /; tar -zxf ~/remote_install.tgz"
+   expect {
+      *?assword:* {
+         send "${password}\r"
+         exp_continue
+      } "*?ermission denied*" {
+         puts "The user or password is wrong"
+         exit 1
+      } yes/no)? {
+         send "yes\r";
+         exp_continue
+      } timeout {
+         exit 1;
+      } eof {
+         catch wait result
+         if { [lindex $result 3] != 0 } {
+            exit [lindex $result 3]
+         }
+      } 
+   }
+   sleep 1
+}
+
+proc runPayload {} {
+   global ip user password prompt pkg cmd_prefix remote_install
+
+   set basepkg  [file tail  ${pkg}]
+   set timeout 60
+   spawn ssh ${user}@${ip}
+   expect {
+      *?assword:* {
+         send "${password}\r"
+         exp_continue
+      } "*?ermission denied*" {
+         puts "The user or password is wrong"
+         exit 1
+      } yes/no)? {
+         send "yes\r"
+         exp_continue
+      } timeout {
+         exit 1;
+      } eof {
+         exit 1;
+      } -re "${prompt}" {}
+   }
+   send "${cmd_prefix} ${remote_install}/remote-install-engine.sh ${remote_install}/${basepkg}\r"
+   expect {
+      *?assword:* {
+         send "${password}\r"
+         exp_continue
+      } "*?ermission denied*" {
+         puts "The user or password is wrong"
+         exit 1
+      } timeout {
+         exit 1;
+      } eof {
+         exit 1;
+      }  -re "${prompt}" {}
+   }
+
+   expect  -re .*
+   send "echo \$?\r"
+   expect   -re "(\r\n| )(\[0-9]*)\r\n" {
+      if { [string compare $expect_out(2,string) "0" ] == 0 } {
+         puts "${ip}: Done."
+      } else {
+         puts "${ip}: Cannot Ping host? (Host Alive?)"
+         exit 1
+      }
+   }
+   send "exit\r"
+   interact
+}
+
+
+####################################################################
+#                                                                  #
+# Main                                                             #
+#                                                                  #
+####################################################################
+set ip  [lindex $argv 0]
+if { [string length $ip ] == 0 } {
+   puts "Missing hostname or ip"
+   print_usage
+   exit 1
+}
+
+# Check required environment variables
+if { [catch {
+        set user            $::env(ADMIN_USER)
+        set password        $::env(PASS)
+        set cmd_prefix      $::env(CMDPREFIX)
+        set remote_install  $::env(REMOTE_INSTALL)
+        set pkg             $::env(PKG)
+     } error] } {
+
+   puts stderr "Missing environment variable.\n$error"
+   exit 1
+}
+
+
+set prompt  "(${user}@|\\r\\n\\$ |\\r\\n# )"
+
+testWithPing
+checkSSHConnection
+copyPayload
+expandPayload
+runPayload

roxie/ccd/ccd.hpp

@@ -330,6 +330,7 @@ extern MapStringTo<int> *preferredClusters;
 extern StringArray allQuerySetNames;
 
 extern bool allFilesDynamic;
+extern bool lockSuperFiles;
 extern bool crcResources;
 extern bool logFullQueries;
 extern bool blindLogging;

roxie/ccd/ccddali.cpp

@@ -52,6 +52,8 @@ public:
     }
     ~CDaliPackageWatcher()
     {
+        if (change)
+            unsubscribe();
     }
     virtual void subscribe(bool exact)
     {

roxie/ccd/ccdfile.cpp

@@ -1744,7 +1744,7 @@ public:
                 }
                 // We have to clone the properties since we don't want to keep the superfile locked
                 properties.setown(createPTreeFromIPT(&dFile->queryAttributes()));
-                if (!isDynamic)
+                if (!isDynamic && !lockSuperFiles)
                 {
                     notifier.setown(daliHelper->getSuperFileSubscription(lfn, this));
                     dFile.clear();  // We don't lock superfiles, except dynamic ones

roxie/ccd/ccdmain.cpp

@@ -94,6 +94,7 @@ IPropertyTree* ccdChannels;
 StringArray allQuerySetNames;
 
 bool allFilesDynamic;
+bool lockSuperFiles;
 bool crcResources;
 bool useRemoteResources;
 bool checkFileDate;
@@ -669,6 +670,7 @@ int STARTQUERY_API start_query(int argc, const char *argv[])
         minIbytiDelay = topology->getPropInt("@minIbytiDelay", 2);
         initIbytiDelay = topology->getPropInt("@initIbytiDelay", 50);
         allFilesDynamic = topology->getPropBool("@allFilesDynamic", false);
+        lockSuperFiles = topology->getPropBool("@lockSuperFiles", false);
         crcResources = topology->getPropBool("@crcResources", false);
         ignoreOrphans = topology->getPropBool("@ignoreOrphans", true);
         chunkingHeap = topology->getPropBool("@chunkingHeap", true);

roxie/ccd/ccdquery.cpp

@@ -908,7 +908,7 @@ public:
             hashValue = rtlHash64VStr(dll->queryDll()->queryName(), hashValue);
             if (traceLevel > 8)
                 DBGLOG("getQueryHash: %s %"I64F"u from dll", id, hashValue);
-            if (!allFilesDynamic && !isDynamic && !package.isCompulsory())
+            if (!lockSuperFiles && !allFilesDynamic && !isDynamic && !package.isCompulsory())
             {
                 IConstWorkUnit *wu = dll->queryWorkUnit();
                 if (wu) // wu may be null in some unit test cases

roxie/ccd/ccdstate.cpp

@@ -486,7 +486,7 @@ protected:
                     }
                 }
             }
-            if (!result && resolveLocal)
+            if (!result && (resolveLocal || alwaysCreate))
             {
                 StringBuffer useName;
                 bool wasDFS = false;
@@ -2969,10 +2969,10 @@ void mergeQueries(IPropertyTree *dest, IPropertyTree *src)
         return;
     destQueries->setPropInt("@reporting", destQueries->getPropInt("@reporting") + srcQueries->getPropInt("@reporting"));
 
-    Owned<IPropertyTreeIterator> it = srcQueries->getElements("Query");
-    ForEach(*it)
+    Owned<IPropertyTreeIterator> queries = srcQueries->getElements("Query");
+    ForEach(*queries)
     {
-        IPropertyTree *srcQuery = &it->query();
+        IPropertyTree *srcQuery = &queries->query();
         const char *id = srcQuery->queryProp("@id");
         if (!id || !*id)
             continue;
@@ -2988,6 +2988,22 @@ void mergeQueries(IPropertyTree *dest, IPropertyTree *src)
         if (suspended)
             destQuery->setPropInt("@suspended", suspended);
     }
+    Owned<IPropertyTreeIterator> aliases = srcQueries->getElements("Alias");
+    ForEach(*aliases)
+    {
+        IPropertyTree *srcQuery = &aliases->query();
+        const char *id = srcQuery->queryProp("@id");
+        if (!id || !*id)
+            continue;
+        VStringBuffer xpath("Alias[@id='%s']", id);
+        IPropertyTree *destQuery = destQueries->queryPropTree(xpath);
+        if (!destQuery)
+        {
+            destQueries->addPropTree("Alias", LINK(srcQuery));
+            continue;
+        }
+        mergePTree(destQuery, srcQuery);
+    }
 }
 
 #ifdef _USE_CPPUNIT

testing/regress/README.rst

@@ -1,4 +1,4 @@
-Overview of Regression Suite usage (v:0.0.26)
+Overview of Regression Suite usage (v:0.0.28)
 ==============================================
 
 To use Regression Suite change directory to HPCC-Platform/testing/regress subdirectory.

testing/regress/ecl-test

@@ -31,7 +31,7 @@ from hpcc.util.ecl.file import ECLFile
 from hpcc.util.util import checkPqParam, getVersionNumbers, checkXParam, convertPath, getRealIPAddress
 from hpcc.common.error import Error
 
-prog_version = "0.0.26"
+prog_version = "0.0.28"
 
 # For coverage
 if ('coverage' in os.environ) and (os.environ['coverage'] == '1'):

testing/regress/ecl/key/mysqlembed.xml

@@ -15,23 +15,27 @@
  <Row><name>name2</name><value>2</value><boolval>false</boolval><r8>5.6</r8><r4>7.800000190734863</r4><d>3030</d><ddd>-1234567.89</ddd><u1>là</u1><u2>là      </u2></Row>
 </Dataset>
 <Dataset name='Result 6'>
- <Row><Result_6>2</Result_6></Row>
+ <Row><name>name1</name><value>1</value><boolval>true</boolval><r8>1.2</r8><r4>3.400000095367432</r4><d>6161353561613535</d><ddd>1234567.89</ddd><u1>Straße</u1><u2>Straße  </u2></Row>
+ <Row><name>name2</name><value>2</value><boolval>false</boolval><r8>5.6</r8><r4>7.800000190734863</r4><d>3030</d><ddd>-1234567.89</ddd><u1>là</u1><u2>là      </u2></Row>
 </Dataset>
 <Dataset name='Result 7'>
- <Row><Result_7>true</Result_7></Row>
+ <Row><Result_7>2</Result_7></Row>
 </Dataset>
 <Dataset name='Result 8'>
- <Row><Result_8>5.6</Result_8></Row>
+ <Row><Result_8>true</Result_8></Row>
 </Dataset>
 <Dataset name='Result 9'>
- <Row><Result_9>7.800000190734863</Result_9></Row>
+ <Row><Result_9>5.6</Result_9></Row>
 </Dataset>
 <Dataset name='Result 10'>
- <Row><Result_10>6161353561613535</Result_10></Row>
+ <Row><Result_10>7.800000190734863</Result_10></Row>
 </Dataset>
 <Dataset name='Result 11'>
- <Row><Result_11>Straße</Result_11></Row>
+ <Row><Result_11>6161353561613535</Result_11></Row>
 </Dataset>
 <Dataset name='Result 12'>
- <Row><Result_12>Straße  </Result_12></Row>
+ <Row><Result_12>Straße</Result_12></Row>
+</Dataset>
+<Dataset name='Result 13'>
+ <Row><Result_13>Straße  </Result_13></Row>
 </Dataset>

testing/regress/hpcc/regression/regress.py

@@ -415,17 +415,25 @@ class Regression:
         wuid = None
         if ECLCC().makeArchive(query):
             eclCmd = ECLcmd()
+            try:
+                if publish:
+                    res = eclCmd.runCmd("publish", cluster, query, report[0],
+                                      server=self.config.ip,
+                                      username=self.config.username,
+                                      password=self.config.password)
+                else:
+                    res = eclCmd.runCmd("run", cluster, query, report[0],
+                                      server=self.config.ip,
+                                      username=self.config.username,
+                                      password=self.config.password)
+            except Error as e:
+                res = False
+                wuid = 'Not found'
+                query.setWuid(wuid)
+                query.diff = query.getBaseEcl()+"\n\t"+str(e)
+                report[0].addResult(query)
+                pass
 
-            if publish:
-                res = eclCmd.runCmd("publish", cluster, query, report[0],
-                                  server=self.config.ip,
-                                  username=self.config.username,
-                                  password=self.config.password)
-            else:
-                res = eclCmd.runCmd("run", cluster, query, report[0],
-                                  server=self.config.ip,
-                                  username=self.config.username,
-                                  password=self.config.password)
             wuid = query.getWuid()
             logging.debug("CMD result: '%s', wuid:'%s'"  % ( res,  wuid),  extra={'taskId':cnt})
             if wuid == 'Not found':

testing/regress/hpcc/util/ecl/command.py

@@ -40,6 +40,10 @@ class ECLcmd(Shell):
         args.append('--target=' + cluster)
         args.append('--cluster=' + cluster)
 
+        server = kwargs.pop('server', False)
+        if server:
+            args.append('--server=' + server)
+
         username = kwargs.pop('username', False)
         if username:
                 args.append("--username=" + username)
@@ -62,9 +66,6 @@ class ECLcmd(Shell):
         else:
             args.append('--exception-level=warning')
             args.append('--noroot')
-            server = kwargs.pop('server', False)
-            if server:
-                args.append('--server=' + server)
 
             name = kwargs.pop('name', False)
             if not name:

testing/regress/hpcc/util/ecl/file.py

@@ -167,6 +167,9 @@ class ECLFile:
     def getWuid(self):
         return self.wuid
 
+    def setWuid(self,  wuid):
+        self.wuid = wuid
+
     def addResults(self, results, wuid):
         filename = self.getResults()
         self.wuid = wuid