Merge branch 'candidate-6.0.6'

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

common/environment/environment.cpp

@@ -1552,7 +1552,20 @@ void CLocalEnvironment::clearCache()
     if (conn)
     {
         p.clear();
-        conn->reload();
+        unsigned mode;
+        try
+        {
+            conn->reload();
+        }
+        catch (IException *e)
+        {
+            EXCLOG(e, "Failed to reload connection");
+            e->Release();
+            mode = conn->queryMode();
+            conn.clear();
+        }
+        if (!conn)
+            conn.setown(querySDS().connect(xPath, myProcessSession(), mode, SDS_LOCK_TIMEOUT));
         p.setown(conn->getRoot());
     }
     cache.kill();

common/remote/rmtssh.cpp

@@ -340,13 +340,6 @@ public:
  
     void exec(unsigned i,unsigned treefrom)
     {
-        {
-            CriticalBlock block1(sect);
-            if (!dryrun) {
-                if (slaves.ordinality()>1)
-                    PROGLOG("%d: starting %s (%d of %d finished)",i,slaves.item(i),done.ordinality(),slaves.ordinality());
-            }
-        }
         int retcode=-1;
         StringBuffer outbuf;
         try {
@@ -447,7 +440,7 @@ public:
                             firsterr = false;
                             if (outbuf.length())
                                 outbuf.append('\n');
-                            outbuf.append("ERR: ");
+                            outbuf.append("STDERR: ");
                         }
                         outbuf.append(read,(const char *)buf);
                     }
@@ -512,7 +505,6 @@ public:
         if (dryrun)
             return;
         if (slaves.ordinality()>1) {
-            PROGLOG("Results: (%d of %d finished)",done.ordinality(),slaves.ordinality());
             int errCode = 0;
             Owned<IMultiException> multiException = MakeMultiException();
             for (unsigned i=0;i<done.ordinality();i++) {
@@ -520,21 +512,9 @@ public:
                 StringBuffer res(replytext.item(n));
                 while (res.length()&&(res.charAt(res.length()-1)<=' '))
                     res.setLength(res.length()-1);
-                if (res.length()==0 && !reply.item(n))
-                    PROGLOG("%d: %s(%d): [OK]",n+1,slaves.item(n),reply.item(n));
-                else if (strchr(res.str(),'\n')==NULL) {
-                    PROGLOG("%d: %s(%d): %s",n+1,slaves.item(n),reply.item(n),res.str());
-                    if (reply.item(n)) {
-                        errCode = reply.item(n);
-                        multiException->append(*MakeStringExceptionDirect(reply.item(n),res.str()));
-                    }
-                }
-                else {
-                    PROGLOG("%d: %s(%d):\n---------------------------\n%s\n===========================",n+1,slaves.item(n),reply.item(n),res.str());
-                    if (reply.item(n)) {
-                        errCode = reply.item(n);
-                        multiException->append(*MakeStringExceptionDirect(reply.item(n),res.str()));
-                    }
+                if (reply.item(n)) {
+                    errCode = reply.item(n);
+                    multiException->append(*MakeStringExceptionDirect(errCode,res.str()));
                 }
             }
             if (errCode)
@@ -544,7 +524,6 @@ public:
             StringBuffer res(replytext.item(0));
             while (res.length()&&(res.charAt(res.length()-1)<=' '))
                 res.setLength(res.length()-1);
-            PROGLOG("%s result(%d):\n%s",useplink?"plink":"ssh",reply.item(0),res.str());
             if (reply.item(0))
                 throw MakeStringExceptionDirect(reply.item(0), res.str());
         }

dali/base/dadfs.cpp

@@ -1385,7 +1385,7 @@ public:
                 }
             }
             file.clear();
-            PROGLOG("CDelayedDelete: pausing");
+            PROGLOG("CDelayedDelete: pausing due to locked file = %s", logicalname);
             Sleep(SDS_TRANSACTION_RETRY/2+(getRandom()%SDS_TRANSACTION_RETRY));
         }
     }

dali/base/dautils.cpp

@@ -341,12 +341,30 @@ inline void normalizeScope(const char *name, const char *scope, unsigned len, St
     }
 }
 
+void normalizeNodeName(const char *node, unsigned len, SocketEndpoint &ep, bool strict)
+{
+    if (!strict)
+    {
+        while (isspace(*node))
+        {
+            node++;
+            len--;
+        }
+    }
+
+    StringBuffer nodename;
+    nodename.append(len, node);
+    if (!strict)
+        nodename.clip();
+    ep.set(nodename.str());
+}
+
 void CDfsLogicalFileName::normalizeName(const char *name, StringAttr &res, bool strict)
 {
     // NB: If !strict(default) allows spaces to exist either side of scopes (no idea why would want to permit that, but preserving for bwrd compat.)
     StringBuffer str;
     StringBuffer nametmp;
-    const char *ct = NULL;    
+    const char *ct = nullptr;
     bool wilddetected = false;
     if ('~' == *name) // allowed 1 leading ~
     {
@@ -361,7 +379,7 @@ void CDfsLogicalFileName::normalizeName(const char *name, StringAttr &res, bool
         switch (c)
         {
             case '@': ct = s; break;
-            case ':': ct = NULL; break;
+            case ':': ct = nullptr; break;
             case '?':
             case '*': wilddetected = true; break;
             case '~':
@@ -379,10 +397,9 @@ void CDfsLogicalFileName::normalizeName(const char *name, StringAttr &res, bool
         }
         c = *++s;
     }
-    bool isext = memicmp(name,EXTERNAL_SCOPE "::",sizeof(EXTERNAL_SCOPE "::")-1)==0;
-    if (!isext && !allowWild && wilddetected)
+    if (!allowWild && wilddetected)
         throw MakeStringException(-1, "Wildcards not allowed in filename (%s)", name);
-    if (!isext&&ct&&(ct-name>=1)) // trailing @
+    if (ct&&(ct-name>=1)) // trailing @
     {
         if ((ct[1]=='@')||(ct[1]=='^')) // escape
         {
@@ -410,40 +427,19 @@ void CDfsLogicalFileName::normalizeName(const char *name, StringAttr &res, bool
         {
             normalizeScope(name, name, s-name, str, strict);
             bool isForeign = 0 == stricmp(str.str(),FOREIGN_SCOPE);
-            if (isext || isForeign) // normalize node
+            if (isForeign) // normalize node
             {
                 const char *s1 = s+2;
                 const char *ns1 = strstr(s1,"::");
-                if (ns1) // TBD accept groupname here (in the case of isext)
+                if (ns1)
                 {
-                    if (!strict)
-                        skipSp(s1);
-                    StringBuffer nodename;
-                    nodename.append(ns1-s1,s1);
-                    if (!strict)
-                        nodename.clip();
-                    SocketEndpoint ep(nodename.str());
+                    SocketEndpoint ep;
+                    normalizeNodeName(s1, ns1-s1, ep, strict);
                     if (!ep.isNull())
                     {
                         ep.getUrlStr(str.append("::"));
                         s = ns1;
-                        if (isext)
-                        {
-                            external = true;
-                            if (s[2]=='>')
-                            {
-                                str.append("::");
-                                tailpos = str.length();
-                                str.append(s+2);
-                                res.set(str);
-                                return;
-                            }
-                        }
-                        else
-                        {
-                            dbgassertex(isForeign);
-                            localpos = str.length()+2;
-                        }
+                        localpos = str.length()+2;
                     }
                 }
             }
@@ -466,13 +462,62 @@ void CDfsLogicalFileName::normalizeName(const char *name, StringAttr &res, bool
     }
     str.append("::");
     tailpos = str.length();
-    if (strstr(s,"::")!=NULL)
+    if (strstr(s,"::")!=nullptr)
         ERRLOG("Tail contains '::'!");
     normalizeScope(name, s, strlen(name)-(s-name), str, strict);
     str.toLowerCase();
     res.set(str);
 }
 
+bool CDfsLogicalFileName::normalizeExternal(const char * name, StringAttr &res, bool strict)
+{
+    // TODO Should check the name is a valid OS filename
+    if ('~' == *name) // allowed 1 leading ~
+    {
+        name++;
+        if (!strict)
+            skipSp(name);
+    }
+    bool retVal = memicmp(name,EXTERNAL_SCOPE "::",sizeof(EXTERNAL_SCOPE "::")-1)==0;
+    if (retVal)
+    {
+        lfn.clear();
+        StringBuffer str;
+        const char *s=strstr(name,"::");
+        normalizeScope(name, name, s-name, str, strict);
+
+        const char *s1 = s+2;
+        const char *ns1 = strstr(s1,"::");
+        if (!ns1)
+            retVal = false;
+        else
+        {
+            SocketEndpoint ep;
+            normalizeNodeName(s1, ns1-s1, ep, strict);
+            if (ep.isNull())
+                retVal = false;
+            else
+            {
+                ep.getUrlStr(str.append("::"));
+                s = ns1;
+                if (s[2] == '>')
+                {
+                    str.append("::");
+                    tailpos = str.length();
+                    str.append(s+2);
+                }
+                else
+                {
+                    str.append(s);
+                    str.toLowerCase();
+                }
+                res.set(str);
+            }
+        }
+
+    }
+    return retVal;
+}
 
 void CDfsLogicalFileName::set(const char *name, bool removeForeign)
 {
@@ -480,7 +525,7 @@ void CDfsLogicalFileName::set(const char *name, bool removeForeign)
     if (!name)
         return;
     skipSp(name);
-    if (allowospath&&(isAbsolutePath(name)||(stdIoHandle(name)>=0)||(strstr(name,"::")==NULL)))
+    if (allowospath&&(isAbsolutePath(name)||(stdIoHandle(name)>=0)||(strstr(name,"::")==nullptr)))
     {
         RemoteFilename rfn;
         rfn.setRemotePath(name);
@@ -515,12 +560,18 @@ void CDfsLogicalFileName::set(const char *name, bool removeForeign)
         lfn.set(full);
         return;
     }
-    normalizeName(name, lfn, false);
-    if (removeForeign)
+
+    if (normalizeExternal(name, lfn, false))
+        external = true;
+    else
     {
-        StringAttr _lfn = get(true);
-        lfn.clear();
-        lfn.set(_lfn);
+        normalizeName(name, lfn, false);
+        if (removeForeign)
+        {
+            StringAttr _lfn = get(true);
+            lfn.clear();
+            lfn.set(_lfn);
+        }
     }
 }
 

dali/base/dautils.hpp

@@ -139,7 +139,10 @@ public:
     void setAllowWild(bool b=true) { allowWild = b; } // allow wildcards
     bool isExpanded() const;
     void expand(IUserDescriptor *user);
+
+protected:
     void normalizeName(const char * name, StringAttr &res, bool strict);
+    bool normalizeExternal(const char * name, StringAttr &res, bool strict);
 };
 
 // abstract class, define getCmdText to return tracing text of commands

ecl/eclcc/eclcc.cpp

@@ -1239,7 +1239,7 @@ void EclCC::processSingleQuery(EclCompileInstance & instance,
     if (syntaxChecking || optGenerateMeta || optEvaluateResult)
         return;
 
-    if (optSaveQueryArchive && instance.wu)
+    if (optSaveQueryArchive && instance.wu && instance.archive)
     {
         Owned<IWUQuery> q = instance.wu->updateQuery();
         StringBuffer buf;
@@ -1678,7 +1678,7 @@ void EclCC::processReference(EclCompileInstance & instance, const char * queryAt
     const char * outputFilename = instance.outputFilename;
 
     instance.wu.setown(createLocalWorkUnit(NULL));
-    if (optArchive || optGenerateDepend)
+    if (optArchive || optGenerateDepend || optSaveQueryArchive)
         instance.archive.setown(createAttributeArchive());
 
     EclRepositoryArray repositories;

esp/services/ws_workunits/ws_workunitsQuerySets.cpp

@@ -1267,7 +1267,7 @@ unsigned CWsWorkunitsEx::getGraphIdsByQueryId(const char *target, const char *qu
         IPropertyTree &graph = graphs->query();
         const char* graphId = graph.queryProp("@id");
         if (graphId && *graphId)
-            graphIds.append(graphId);
+            graphIds.appendUniq(graphId);
     }
     return graphIds.length();
 }

esp/src/eclwatch/ResultsWidget.js

@@ -89,6 +89,15 @@ define([
                 }
             }).placeAt(this.widget.Open.domNode, "after");
 
+            this.openLegacy = new Button({
+                label: this.i18n.OpenLegacyMode,
+                onClick: function (event) {
+                    context._onOpen(event, {
+                        legacyMode: true
+                    });
+                }
+            }).placeAt(this.widget.Open.domNode, "after");
+
             var retVal = new declare([ESPUtil.Grid(false, true)])({
                 store: this.store,
                 columns: {
@@ -159,8 +168,12 @@ define([
 
         getDetailID: function (row, params) {
             var retVal = "Detail" + row[this.idProperty];
-            if (params && params.vizMode) {
-                retVal += "Viz";
+            if (params) {
+                if (params.vizMode) {
+                    retVal += "Viz";
+                } else if (params.legacyMode) {
+                    retVal += "Legacy";
+                }
             }
             return retVal;
         },
@@ -180,6 +193,19 @@ define([
                         }
                     }
                 });
+            } else if (params && params.legacyMode) {
+                return new DelayLoadWidget({
+                    id: id,
+                    title: "[L] " + row.Name,
+                    closable: true,
+                    delayWidget: "IFrameWidget",
+                    hpcc: {
+                        type: "IFrameWidget",
+                        params: {
+                            src: "/WsWorkunits/WUResult?Wuid=" + row.Wuid + "&Sequence=" + row.Sequence
+                        }
+                    }
+                });
             } else if (row.FileName && params && params.logicalFile) {
                 return new DelayLoadWidget({
                     id: id,
@@ -242,6 +268,7 @@ define([
         refreshActionState: function (selection) {
             this.inherited(arguments);
 
+            this.openLegacy.set("disabled", !this.wu || !selection.length);
             this.openViz.set("disabled", !this.wu || !selection.length);
         }
 

esp/src/eclwatch/WUDetailsWidget.js

@@ -393,17 +393,17 @@ define([
                     context.slaveNumber.set("maxLength", context.maxSlaves);
                     dom.byId("SlavesMaxNumber").innerHTML = context.i18n.NumberofSlaves + " " + response.WUInfoResponse.Workunit.ThorLogList.ThorLogInfo[0].NumberSlaves;
                     context.logDate = response.WUInfoResponse.Workunit.ThorLogList.ThorLogInfo[0].LogDate;
-                    context.clusterGroup = response.WUInfoResponse.Workunit.ThorLogList.ThorLogInfo[0].ClusterGroup;
+                    context.clusterGroup = response.WUInfoResponse.Workunit.ThorLogList.ThorLogInfo[0].ProcessName;
                     context.slaveLogs.set("disabled", false);
                     context.includeSlaveLogsCheckbox.set("disabled", false);
                     var targetData = response.WUInfoResponse.Workunit.ThorLogList.ThorLogInfo;
                         for (var i = 0; i < targetData.length; ++i) {
                             context.thorProcess.options.push({
-                                label: targetData[i].ClusterGroup,
-                                value: targetData[i].ClusterGroup
+                                label: targetData[i].ProcessName,
+                                value: targetData[i].ProcessName
                             });
                         }
-                        context.thorProcess.set("value", targetData[0].ClusterGroup);
+                        context.thorProcess.set("value", targetData[0].ProcessName);
                 } else {
                    context.slaveLogs.set("disabled", true);
                    context.includeSlaveLogsCheckbox.set("disabled", true);

esp/src/eclwatch/templates/GraphTreeWidget.html

@@ -39,25 +39,25 @@
                 </select>
             </div>
             <div id="${id}OverviewTabContainer" data-dojo-props="region: 'center', tabPosition: 'bottom'" data-dojo-type="dijit.layout.StackContainer">
-                <div id="${id}TreeGridCP" title="${i18n.Tree}" style="padding: 0px; overflow: hidden" data-dojo-props="iconClass:'iconFolderTree', showTitle: false" data-dojo-type="dijit.layout.ContentPane">
+                <div id="${id}TreeGridCP" tooltip="${i18n.Tree}" style="padding: 0px; overflow: hidden" data-dojo-props="iconClass:'iconFolderTree', showTitle: false" data-dojo-type="dijit.layout.ContentPane">
                     <div id="${id}TreeGrid">
                     </div>
                 </div>
-                <div id="${id}SubgraphsGridCP" title="${i18n.Subgraphs}" style="padding: 0px; overflow: hidden" data-dojo-props="iconClass:'iconFolderList', showTitle: false" data-dojo-type="dijit.layout.ContentPane">
+                <div id="${id}SubgraphsGridCP" tooltip="${i18n.Subgraphs}" style="padding: 0px; overflow: hidden" data-dojo-props="iconClass:'iconFolderList', showTitle: false" data-dojo-type="dijit.layout.ContentPane">
                     <div id="${id}SubgraphsGrid">
                     </div>
                 </div>
-                <div id="${id}VerticesGridCP" title="${i18n.Activities}" style="padding: 0px; overflow: hidden" data-dojo-props="iconClass:'iconFileList', showTitle: false" data-dojo-type="dijit.layout.ContentPane">
+                <div id="${id}VerticesGridCP" tooltip="${i18n.Activities}" style="padding: 0px; overflow: hidden" data-dojo-props="iconClass:'iconFileList', showTitle: false" data-dojo-type="dijit.layout.ContentPane">
                     <div id="${id}VerticesGrid">
                     </div>
                 </div>
-                <div id="${id}EdgesGridCP" title="${i18n.Edges}" style="padding: 0px; overflow: hidden" data-dojo-props="iconClass:'iconEdgeList', showTitle: false" data-dojo-type="dijit.layout.ContentPane">
+                <div id="${id}EdgesGridCP" tooltip="${i18n.Edges}" style="padding: 0px; overflow: hidden" data-dojo-props="iconClass:'iconEdgeList', showTitle: false" data-dojo-type="dijit.layout.ContentPane">
                     <div id="${id}EdgesGrid">
                     </div>
                 </div>
-                <div id="${id}TimingsTreeMap" title="${i18n.TimingsMap}" data-dojo-props="iconClass:'iconTreeMap', showTitle: false" data-dojo-type="TimingTreeMapWidget">
+                <div id="${id}TimingsTreeMap" tooltip="${i18n.TimingsMap}" data-dojo-props="iconClass:'iconTreeMap', showTitle: false" data-dojo-type="TimingTreeMapWidget">
                 </div>
-                <div id="${id}ActivitiesTreeMap" title="${i18n.ActivityMap}" data-dojo-props="iconClass:'iconTreeMap', showTitle: false" data-dojo-type="TimingTreeMapWidget">
+                <div id="${id}ActivitiesTreeMap" tooltip="${i18n.ActivityMap}" data-dojo-props="iconClass:'iconTreeMap', showTitle: false" data-dojo-type="TimingTreeMapWidget">
                 </div>
             </div>
             <div id="${id}LocalTabContainer" style="height: 33%" data-dojo-props="region: 'bottom', splitter:true, minSize: 120, tabPosition: 'bottom'" data-dojo-type="dijit.layout.TabContainer">

initfiles/bin/init_thor.in

@@ -74,7 +74,7 @@ kill_slaves()
         # we want to kill only slaves that have already been started in run_thor
         if [[ -r $instancedir/uslaves ]]; then
             clusternodes=$(cat $instancedir/uslaves 2> /dev/null | wc -l)
-            $deploydir/frunssh $instancedir/slaves "/bin/sh -c '$deploydir/init_thorslave stop localhost $slavespernode $THORSLAVEPORT $slaveportinc $THORMASTER $THORMASTERPORT $LOG_DIR $instancedir $deploydir $THORNAME ${INSTALL_DIR}/sbin $logredirect'" -i:$SSHidentityfile -u:$SSHusername -pe:$SSHpassword -t:$SSHtimeout -a:$SSHretries -n:$clusternodes 2>&1
+            $deploydir/frunssh $instancedir/uslaves "/bin/sh -c '$deploydir/init_thorslave stop localhost $slavespernode $THORSLAVEPORT $slaveportinc $THORMASTER $THORMASTERPORT $LOG_DIR $instancedir $deploydir $THORNAME $PATH_PRE $logredirect'" -i:$SSHidentityfile -u:$SSHusername -pe:$SSHpassword -t:$SSHtimeout -a:$SSHretries -n:$clusternodes 2>&1
             FRUNSSH_RC=$?
             if [[ ${FRUNSSH_RC} -gt 0 ]]; then
                 log "Error ${FRUNSSH_RC} in frunssh"

plugins/javaembed/javaembed.cpp

@@ -1097,23 +1097,44 @@ protected:
 
 // A Java function that returns a dataset will return a JavaRowStream object that can be
 // interrogated to return each row of the result in turn
-// Note that we can't cache the JNIEnv here - calls may be made on different threads (though not at the same time).
 
 static JNIEnv *queryJNIEnv();
 
+class JavaLocalFrame
+{
+public:
+    JavaLocalFrame(JNIEnv *_JNIenv, unsigned size = 16) : JNIenv(_JNIenv)
+    {
+        JNIenv->PushLocalFrame(size);
+    }
+    ~JavaLocalFrame()
+    {
+        JNIenv->PopLocalFrame(NULL);
+    }
+private:
+    JNIEnv *JNIenv;
+};
+
 class JavaRowStream : public CInterfaceOf<IRowStream>
 {
 public:
     JavaRowStream(jobject _iterator, IEngineRowAllocator *_resultAllocator)
     : resultAllocator(_resultAllocator)
     {
-        iterator = queryJNIEnv()->NewGlobalRef(_iterator);
+        JNIEnv *JNIenv = queryJNIEnv();
+        iterator = JNIenv->NewGlobalRef(_iterator);
+        // Note that we can't cache the JNIEnv, iterClass, or methodIds here - calls may be made on different threads (though not at the same time).
+    }
+    ~JavaRowStream()
+    {
+        stop();
     }
     virtual const void *nextRow()
     {
         if (!iterator)
             return NULL;
         JNIEnv *JNIenv = queryJNIEnv();
+        JavaLocalFrame lf(JNIenv);
         // Java code would be
         // if (!iterator.hasNext)
         // {
@@ -1145,16 +1166,20 @@ public:
     virtual void stop()
     {
         resultAllocator.clear();
-        if (iterator)
+        JNIEnv *JNIenv = queryJNIEnv();
+        if (JNIenv)
         {
-            queryJNIEnv()->DeleteGlobalRef(iterator);
-            iterator = NULL;
+            if (iterator)
+            {
+                JNIenv->DeleteGlobalRef(iterator);
+                iterator = NULL;
+            }
         }
     }
 
 protected:
     Linked<IEngineRowAllocator> resultAllocator;
-    jobject iterator;;
+    jobject iterator;
 };
 
 const char *esdl2JavaSig(IEsdlDefinition &esdl, const char *esdlType)

roxie/ccd/ccdcontext.cpp

@@ -3000,8 +3000,21 @@ public:
 
     virtual char *getDaliServers()
     {
-        //MORE: Should this now be implemented using IRoxieDaliHelper?
-        throwUnexpected();
+        try
+        {
+            IRoxieDaliHelper *daliHelper = checkDaliConnection();
+            if (daliHelper)
+            {
+                StringBuffer ip;
+                daliHelper->getDaliIp(ip);
+                return ip.detach();
+            }
+        }
+        catch (IException *E)
+        {
+            E->Release();
+        }
+        return strdup("");
     }
     virtual IHpccProtocolResponse *queryProtocol()
     {

roxie/ccd/ccddali.cpp

@@ -644,6 +644,22 @@ public:
         }
     }
 
+    virtual StringBuffer &getDaliIp(StringBuffer &ret) const
+    {
+        IGroup &group = queryCoven().queryComm().queryGroup();
+        Owned<INodeIterator> coven = group.getIterator();
+        bool first = true;
+        ForEach(*coven)
+        {
+            if (first)
+                first = false;
+            else
+                ret.append(',');
+            coven->query().endpoint().getUrlStr(ret);
+        }
+        return ret;
+    }
+
     static IRoxieDaliHelper *connectToDali(unsigned waitToConnect)
     {
         CriticalBlock b(daliHelperCrit);

roxie/ccd/ccddali.hpp

@@ -55,6 +55,7 @@ interface IRoxieDaliHelper : extends IInterface
     virtual void disconnect() = 0;
     virtual void noteQueuesRunning(const char *queueNames) = 0;
     virtual void noteWorkunitRunning(const char *wu, bool running) = 0;
+    virtual StringBuffer &getDaliIp(StringBuffer &ip) const = 0;
 };
 
 

roxie/ccd/ccddebug.cpp

@@ -116,6 +116,14 @@ public:
         else
             return in->queryConcreteInput(idx);
     }
+    virtual IEngineRowStream *queryConcreteOutputStream(unsigned whichInput)
+    {
+        return this;
+    }
+    virtual IStrandJunction *queryConcreteOutputJunction(unsigned idx) const
+    {
+        return nullptr;
+    }
     virtual IRoxieServerActivity *queryActivity()
     {
         return in->queryActivity();

roxie/ccd/ccdlistener.cpp

@@ -1310,7 +1310,7 @@ class RoxieProtocolMsgContext : implements IHpccProtocolMsgContext, public CInte
 {
 public:
     StringAttr queryName;
-    StringAttr uid;
+    StringAttr uid = "-";
     Owned<CascadeManager> cascade;
     Owned<IDebuggerContext> debuggerContext;
     Owned<CDebugCommandHandler> debugCmdHandler;
@@ -1387,6 +1387,15 @@ public:
         return *cascade;
     }
 
+    virtual void setTransactionId(const char *id)
+    {
+        if (!id || !*id)
+            return;
+        uid.set(id);
+        ensureContextLogger();
+        StringBuffer s;
+        logctx->set(ep.getIpText(s).appendf(":%u{%s}", ep.port, uid.str()).str());
+    }
     inline IDebuggerContext &ensureDebuggerContext(const char *id)
     {
         if (!debuggerContext)

roxie/ccd/ccdprotocol.cpp

@@ -1745,7 +1745,9 @@ readAnother:
 
                 uid = NULL;
                 sanitizeQuery(queryPT, queryName, sanitizedText, httpHelper, uid, isRequest, isRequestArray, isBlind, isDebug);
-                if (!uid)
+                if (uid)
+                    msgctx->setTransactionId(uid);
+                else
                     uid = "-";
 
                 sink->checkAccess(peer, queryName, sanitizedText, isBlind);

roxie/ccd/ccdserver.cpp

@@ -1178,6 +1178,8 @@ public:
         return ctx;
     }
 
+    virtual IEngineRowStream *queryConcreteOutputStream(unsigned whichInput) { assertex(whichInput==0); return this; }
+    virtual IStrandJunction *queryConcreteOutputJunction(unsigned idx) const { assertex(idx==0); return junction; }
     virtual IRoxieServerActivity *queryActivity() { return this; }
     virtual IIndexReadActivityInfo *queryIndexReadActivity() { return NULL; }
 
@@ -2242,6 +2244,9 @@ public:
         return NULL;
     }
 
+    virtual IEngineRowStream *queryConcreteOutputStream(unsigned whichInput) { return this; }
+    virtual IStrandJunction *queryConcreteOutputJunction(unsigned idx) const { return nullptr; }
+
     virtual IIndexReadActivityInfo *queryIndexReadActivity() 
     {
         return puller.queryInput()->queryIndexReadActivity();
@@ -2541,7 +2546,28 @@ public:
             return NULL;
     }
 
-    virtual void reset()    
+    virtual IFinalRoxieInput * queryConcreteInput(unsigned idx)
+    {
+        return queryInput(idx);
+    }
+
+    virtual IEngineRowStream *queryConcreteOutputStream(unsigned whichInput)
+    {
+        if (whichInput < numInputs)
+            return streamArray[whichInput];
+        else
+            return NULL;
+    }
+
+    virtual IStrandJunction *queryConcreteOutputJunction(unsigned idx) const
+    {
+        if (idx < numInputs)
+            return junctionArray[idx];
+        else
+            return NULL;
+    }
+
+    virtual void reset()
     {
         for (unsigned i = 0; i < numInputs; i++)
             inputArray[i]->reset();
@@ -4040,6 +4066,9 @@ public:
         meta.set(newmeta);
     }
 
+    virtual IEngineRowStream *queryConcreteOutputStream(unsigned whichInput) { assertex(whichInput==0); return this; }
+    virtual IStrandJunction *queryConcreteOutputJunction(unsigned idx) const { assertex(idx==0); return nullptr; }
+
     virtual IRoxieServerActivity *queryActivity()
     {
         return &activity;
@@ -6072,6 +6101,8 @@ public:
     {
         return input->queryTotalCycles();
     }
+    virtual IEngineRowStream *queryConcreteOutputStream(unsigned whichInput) { assertex(whichInput==0); return this; }
+    virtual IStrandJunction *queryConcreteOutputJunction(unsigned idx) const { assertex(idx==0); return nullptr; }
     virtual IRoxieServerActivity *queryActivity()
     {
         return input->queryActivity();
@@ -6138,6 +6169,9 @@ public:
         return 0;
     }
 
+    virtual IEngineRowStream *queryConcreteOutputStream(unsigned whichInput) { assertex(whichInput==0); return this; }
+    virtual IStrandJunction *queryConcreteOutputJunction(unsigned idx) const { assertex(idx==0); return nullptr; }
+
     virtual IRoxieServerActivity *queryActivity()
     {
         throwUnexpected();
@@ -6221,6 +6255,16 @@ public:
         return input->queryConcreteInput(idx);
     }
 
+    virtual IEngineRowStream *queryConcreteOutputStream(unsigned whichInput)
+    {
+        return input->queryConcreteOutputStream(whichInput);
+    }
+
+    virtual IStrandJunction *queryConcreteOutputJunction(unsigned idx) const
+    {
+        return input->queryConcreteOutputJunction(idx);
+    }
+
     virtual IOutputMetaData * queryOutputMeta() const 
     { 
         return input->queryOutputMeta(); 
@@ -8607,6 +8651,9 @@ public:
             stopped = false;
         }
 
+        virtual IEngineRowStream *queryConcreteOutputStream(unsigned whichInput) { assertex(idx==0); return this; }
+        virtual IStrandJunction *queryConcreteOutputJunction(unsigned idx) const { assertex(idx==0); return nullptr; }
+
         virtual IRoxieServerActivity *queryActivity()
         {
             return parent;
@@ -8616,7 +8663,7 @@ public:
         {
             return parent->queryIndexReadActivity();
         }
-        
+
         virtual unsigned __int64 queryTotalCycles() const
         {
             return totalCycles;
@@ -16075,6 +16122,7 @@ class CRoxieServerNWayInputBaseActivity : public CRoxieServerMultiInputBaseActiv
 protected:
     PointerArrayOf<IFinalRoxieInput> selectedInputs;
     PointerArrayOf<IEngineRowStream> selectedStreams;
+    PointerArrayOf<IStrandJunction> selectedJunctions;
 
 public:
     CRoxieServerNWayInputBaseActivity(IRoxieSlaveContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager, unsigned _numInputs)
@@ -16116,6 +16164,7 @@ public:
             selectedInputs.item(i)->reset();
         selectedInputs.kill();
         selectedStreams.kill();
+        selectedJunctions.kill();
         CRoxieServerMultiInputBaseActivity::reset(); 
     }
 
@@ -16132,8 +16181,20 @@ public:
 
     virtual IFinalRoxieInput * queryConcreteInput(unsigned idx)
     {
-        if (selectedInputs.isItem(idx))
-            return selectedInputs.item(idx);
+        return queryInput(idx);
+    }
+
+    virtual IEngineRowStream *queryConcreteOutputStream(unsigned whichInput)
+    {
+        if (selectedStreams.isItem(whichInput))
+            return selectedStreams.item(whichInput);
+        return NULL;
+    }
+
+    virtual IStrandJunction *queryConcreteOutputJunction(unsigned idx) const
+    {
+        if (selectedJunctions.isItem(idx))
+            return selectedJunctions.item(idx);
         return NULL;
     }
 };
@@ -16157,6 +16218,8 @@ public:
         helper.getInputSelection(selectionIsAll, selectionLen, selection.refdata());
 
         selectedInputs.kill();
+        selectedStreams.kill();
+        selectedJunctions.kill();
         assertex(numInputs==numStreams); // Will need refactoring when that ceases to be true
         if (selectionIsAll)
         {
@@ -16164,6 +16227,7 @@ public:
             {
                 selectedInputs.append(inputArray[i]);
                 selectedStreams.append(streamArray[i]);    // Assumes 1:1 relationship - is that good?
+                selectedJunctions.append(junctionArray[i]);
             }
         }
         else
@@ -16184,13 +16248,17 @@ public:
 
                 selectedInputs.append(inputArray[nextIndex-1]);
                 selectedStreams.append(streamArray[nextIndex-1]);    // Assumes 1:1 relationship - is that good?
+                selectedJunctions.append(junctionArray[nextIndex-1]);
             }
         }
 
-        ForEachItemIn(i2, selectedInputs)
-            selectedInputs.item(i2)->start(parentExtractSize, parentExtract, paused);
+        // NB: Whatever pulls this nwayinput activity, starts and stops the selectedInputs and selectedJunctions
     }
 
+    virtual void stop()
+    {
+        // NB: Whatever pulls this nwayinput activity, starts and stops the selectedInputs
+    }
 };
 
 class CRoxieServerNWayInputActivityFactory : public CRoxieServerMultiInputFactory
@@ -16259,15 +16327,9 @@ public:
                 resultInput->onCreate(colocalParent);
                 resultInput->start(parentExtractSize, parentExtract, paused);
                 selectedStreams.append(connectSingleStream(ctx, resultInput, 0, resultJunctions[i], true));
+                selectedJunctions.append(resultJunctions[i]);
             }
         }
-        else
-        {
-            ForEachItemIn(i, selectedInputs)
-                selectedInputs.item(i)->start(parentExtractSize, parentExtract, paused);
-        }
-        ForEachItemIn(i, selectedStreams)
-            startJunction(resultJunctions[i]);
     }
 
     virtual void reset()    
@@ -16301,6 +16363,7 @@ public:
             IFinalRoxieInput *in = processor.connectIterationOutput(selections[i], probeManager, probes, this, i);
             selectedInputs.append(in);
             selectedStreams.append(connectSingleStream(ctx, in, 0, resultJunctions[i], true));
+            selectedJunctions.append(resultJunctions[i]);
         }
     }
 
@@ -16434,52 +16497,84 @@ protected:
 
 //=================================================================================
 
-class CRoxieServerNaryActivity : public CRoxieServerMultiInputActivity
+class CRoxieServerNWayBaseActivity : public CRoxieServerMultiInputBaseActivity
 {
 public:
-    CRoxieServerNaryActivity(IRoxieSlaveContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager, unsigned _numInputs)
-        : CRoxieServerMultiInputActivity(_ctx, _factory, _probeManager, _numInputs), expandedJunctions(nullptr)
+    CRoxieServerNWayBaseActivity(IRoxieSlaveContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager, unsigned _numInputs)
+        : CRoxieServerMultiInputBaseActivity(_ctx, _factory, _probeManager, _numInputs)
     {
     }
 
-    virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
+    void startExpandedInputs(unsigned parentExtractSize, const byte *parentExtract, bool paused)
     {
-        CRoxieServerMultiInputActivity::start(parentExtractSize, parentExtract, paused);
-        for (unsigned i=0; i < numInputs; i++)
-        {
-            IFinalRoxieInput * cur = inputArray[i];
-            unsigned numRealInputs = cur->numConcreteOutputs();
-            for (unsigned j = 0; j < numRealInputs; j++)
-            {
-                IFinalRoxieInput * curReal = cur->queryConcreteInput(j);
-                expandedInputs.append(curReal);
-            }
-        }
-        expandedJunctions = new Owned<IStrandJunction> [expandedInputs.length()];
+        ForEachItemIn(ei, expandedInputs)
+            expandedInputs.item(ei)->start(parentExtractSize, parentExtract, paused);
         ForEachItemIn(idx, expandedInputs)
-        {
-            expandedStreams.append(connectSingleStream(ctx, expandedInputs.item(idx), 0, expandedJunctions[idx], true));  // MORE - is the index 0 right?
-            startJunction(expandedJunctions[idx]);
-        }
+            startJunction(expandedJunctions.item(idx));
     }
 
-    virtual void reset()    
+    virtual void stop()
+    {
+        ForEachItemIn(i, expandedStreams)
+            expandedStreams.item(i)->stop();
+        CRoxieServerMultiInputBaseActivity::stop();
+    }
+
+    virtual void reset()
     {
         ForEachItemIn(idx, expandedInputs)
-            resetJunction(expandedJunctions[idx]);
+            resetJunction(expandedJunctions.item(idx));
         expandedInputs.kill();
         expandedStreams.kill();
-        delete [] expandedJunctions;
-        expandedJunctions = nullptr;
-        CRoxieServerMultiInputActivity::reset(); 
+        expandedJunctions.kill();
+        CRoxieServerMultiInputBaseActivity::reset();
     }
-
 protected:
     PointerArrayOf<IFinalRoxieInput> expandedInputs;
     PointerArrayOf<IEngineRowStream> expandedStreams;
-    Owned<IStrandJunction> *expandedJunctions;
+    PointerArrayOf<IStrandJunction> expandedJunctions;
 };
 
+//=================================================================================
+
+class CRoxieServerNaryActivity : public CRoxieServerNWayBaseActivity
+{
+public:
+    CRoxieServerNaryActivity(IRoxieSlaveContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager, unsigned _numInputs)
+        : CRoxieServerNWayBaseActivity(_ctx, _factory, _probeManager, _numInputs)
+    {
+    }
+
+    virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
+    {
+        CRoxieServerNWayBaseActivity::start(parentExtractSize, parentExtract, paused);
+
+        for (unsigned i=0; i < numInputs; i++)
+        {
+            IFinalRoxieInput * cur = inputArray[i];
+            CRoxieServerNWayInputBaseActivity *nWayInput = dynamic_cast<CRoxieServerNWayInputBaseActivity *>(cur);
+            if (nWayInput)
+            {
+                nWayInput->start(parentExtractSize, parentExtract, paused);
+                unsigned numRealInputs = cur->numConcreteOutputs();
+                for (unsigned j = 0; j < numRealInputs; j++)
+                {
+                    expandedInputs.append(cur->queryConcreteInput(j));
+                    expandedStreams.append(cur->queryConcreteOutputStream(j));
+                    expandedJunctions.append(cur->queryConcreteOutputJunction(j));
+                }
+            }
+            else
+            {
+                expandedInputs.append(cur);
+                // NB: this activities input streams + junction have been setup and held in CRoxieServerMultiInputActivity base
+                expandedStreams.append(queryConcreteOutputStream(i));
+                expandedJunctions.append(queryConcreteOutputJunction(i));
+            }
+        }
+        startExpandedInputs(parentExtractSize, parentExtract, paused);
+    }
+};
 
 //=================================================================================
 
@@ -16781,50 +16876,62 @@ IRoxieServerActivityFactory *createRoxieServerNWayMergeJoinActivityFactory(unsig
 
 //=================================================================================
 
-class CRoxieServerNWaySelectActivity : public CRoxieServerMultiInputActivity
+class CRoxieServerNWaySelectActivity : public CRoxieServerNWayBaseActivity
 {
     IHThorNWaySelectArg &helper;
 public:
     CRoxieServerNWaySelectActivity(IRoxieSlaveContext *_ctx, const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager, unsigned _numInputs)
-        : CRoxieServerMultiInputActivity(_ctx, _factory, _probeManager, _numInputs),
+        : CRoxieServerNWayBaseActivity(_ctx, _factory, _probeManager, _numInputs),
           helper((IHThorNWaySelectArg &)basehelper)
     {
-        selectedInput = NULL;
-        selectedStream = NULL;
     }
 
     virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
     {
-        CRoxieServerMultiInputActivity::start(parentExtractSize, parentExtract, paused);
+        CRoxieServerNWayBaseActivity::start(parentExtractSize, parentExtract, paused);
 
         unsigned whichInput = helper.getInputIndex();
-        selectedInput = NULL;
-        selectedStream = NULL;
+        selectedInput = nullptr;
+        selectedStream = nullptr;
         if (whichInput--)
         {
             for (unsigned i=0; i < numInputs; i++)
             {
                 IFinalRoxieInput * cur = inputArray[i];
-                unsigned numRealInputs = cur->numConcreteOutputs();
-                if (whichInput < numRealInputs)
+                CRoxieServerNWayInputBaseActivity *nWayInput = dynamic_cast<CRoxieServerNWayInputBaseActivity *>(cur);
+                if (nWayInput)
                 {
-                    selectedInput = cur->queryConcreteInput(whichInput);
-                    selectedStream = connectSingleStream(ctx, selectedInput, 0, selectedJunction, true);  // Should this be passing whichInput??
-                    break;
+                    nWayInput->start(parentExtractSize, parentExtract, paused);
+                    unsigned numRealInputs = cur->numConcreteOutputs();
+                    if (whichInput < numRealInputs)
+                    {
+                        expandedInputs.append(cur->queryConcreteInput(whichInput));
+                        expandedStreams.append(cur->queryConcreteOutputStream(whichInput));
+                        expandedJunctions.append(cur->queryConcreteOutputJunction(whichInput));
+                        break;
+                    }
+                    whichInput -= numRealInputs;
+                }
+                else
+                {
+                    if (whichInput == 0)
+                    {
+                        expandedInputs.append(cur);
+                        // NB: this activities input streams + junction have been setup and held in CRoxieServerMultiInputActivity base
+                        expandedStreams.append(queryConcreteOutputStream(i));
+                        expandedJunctions.append(queryConcreteOutputJunction(i));
+                        break;
+                    }
+                    whichInput -= 1;
                 }
-                whichInput -= numRealInputs;
             }
         }
-        startJunction(selectedJunction);
-    }
-
-    virtual void reset()    
-    {
-        selectedInput = NULL;
-        selectedStream = NULL;
-        resetJunction(selectedJunction);
-        selectedJunction.clear();
-        CRoxieServerMultiInputActivity::reset(); 
+        if (expandedInputs.ordinality())
+        {
+            startExpandedInputs(parentExtractSize, parentExtract, paused);
+            selectedInput = expandedInputs.item(0);
+            selectedStream = expandedStreams.item(0);
+        }
     }
 
     const void * nextRow()
@@ -16864,8 +16971,8 @@ public:
     }
 
 protected:
-    IFinalRoxieInput * selectedInput;
-    IEngineRowStream * selectedStream;
+    IFinalRoxieInput * selectedInput = nullptr;
+    IEngineRowStream * selectedStream = nullptr;
     Owned<IStrandJunction> selectedJunction;
 };
 
@@ -20060,6 +20167,9 @@ public:
         }
     }
 
+    virtual IEngineRowStream *queryConcreteOutputStream(unsigned whichInput) { return this; }
+    virtual IStrandJunction *queryConcreteOutputJunction(unsigned idx) const { return nullptr; }
+
     virtual IIndexReadActivityInfo *queryIndexReadActivity()
     {
         IFinalRoxieInput *in = cond ? inputTrue  : inputFalse;
@@ -27459,6 +27569,8 @@ public:
         streams.append(this);
         return NULL;
     }
+    virtual IEngineRowStream *queryConcreteOutputStream(unsigned whichInput) { assertex(whichInput==0); return this; }
+    virtual IStrandJunction *queryConcreteOutputJunction(unsigned idx) const { assertex(idx==0); return nullptr; }
     virtual IRoxieServerActivity *queryActivity()
     {
         throwUnexpected();

roxie/ccd/ccdserver.hpp

@@ -102,6 +102,8 @@ interface IFinalRoxieInput : extends IInputBase
     virtual bool gatherConjunctions(ISteppedConjunctionCollector & collector) { return false; }
     virtual unsigned numConcreteOutputs() const { return 1; }
     virtual IFinalRoxieInput * queryConcreteInput(unsigned idx) { assertex(idx==0); return this; }
+    virtual IEngineRowStream *queryConcreteOutputStream(unsigned whichInput) = 0;
+    virtual IStrandJunction *queryConcreteOutputJunction(unsigned idx) const = 0;
     virtual IRoxieServerActivity *queryActivity() = 0;
     virtual IIndexReadActivityInfo *queryIndexReadActivity() = 0;
 

roxie/ccd/hpccprotocol.hpp

@@ -40,6 +40,7 @@ interface IHpccProtocolMsgContext : extends IInterface
     virtual void setIntercept(bool val) = 0;
     virtual bool getIntercept() = 0;
     virtual void outputLogXML(IXmlStreamFlusher &out) = 0;
+    virtual void setTransactionId(const char *id) = 0;
 };
 
 interface IHpccProtocolResultsWriter : extends IInterface

roxie/udplib/udpsha.cpp

@@ -297,9 +297,9 @@ void setLinuxThreadPriority(int level)
         param.sched_priority = level;
     }
     if(( rc = pthread_setschedparam(self, policy, &param)) != 0) 
-        DBGLOG("pthread_setschedparam error: %d policy=%i pr=%i id=%" I64F "i PID=%i", rc, policy, param.sched_priority, (unsigned __int64) self, getpid());
+        DBGLOG("pthread_setschedparam error: %d policy=%i pr=%i id=%" I64F "i TID=%i", rc, policy, param.sched_priority, (unsigned __int64) self, threadLogID());
     else
-        DBGLOG("priority set id=%" I64F "i policy=%i pri=%i PID=%i", (unsigned __int64) self, policy, param.sched_priority, getpid());
+        DBGLOG("priority set id=%" I64F "i policy=%i pri=%i TID=%i", (unsigned __int64) self, policy, param.sched_priority, threadLogID());
 }
 #endif
 

services/runagent/frunssh.cpp

@@ -67,6 +67,17 @@ int main( int argc, char *argv[] )
         Owned<IFRunSSH> runssh = createFRunSSH();
         runssh->init(argc,argv);
         runssh->exec();
+        const StringArray & strArray = runssh->getReplyText();
+        const UnsignedArray & unsArray = runssh->getReply();
+        for(unsigned i = 0;i < unsArray.ordinality();i++) {
+            StringBuffer buf = strArray.item(i);
+            // strip newlines off end of string buf
+            if (buf.length() && (buf.charAt(buf.length()-1)) == '\n') {
+                buf.setLength(buf.length()-1);
+                buf.clip();
+            }
+            PROGLOG("%d: ssh(%d): %s",i+1,unsArray.item(i),buf.str());
+        }
     }
     catch(IException *e)
     {

testing/unittests/dalitests.cpp

@@ -2452,5 +2452,105 @@ public:
 CPPUNIT_TEST_SUITE_REGISTRATION( CDaliUtils );
 CPPUNIT_TEST_SUITE_NAMED_REGISTRATION( CDaliUtils, "DaliUtils" );
 
+class CFileNameNormalizeUnitTest : public CppUnit::TestFixture, CDfsLogicalFileName
+{
+    CPPUNIT_TEST_SUITE(CFileNameNormalizeUnitTest);
+      CPPUNIT_TEST(testFileNameNormalize);
+    CPPUNIT_TEST_SUITE_END();
+
+public:
+    void testFileNameNormalize()
+    {
+        //Columns
+        const int inFileName = 0;
+        const int normalizedFileName = 1;
+
+        const char *validExternalLfns[][2] = {
+                //     input file name                          expected normalized file name
+                {"~file::192.168.16.1::dir1::file1",            "file::192.168.16.1::dir1::file1"},
+                {"~  file::  192.168.16.1::dir1::file1",        "file::192.168.16.1::dir1::file1"},
+                {"~file::192.168.16.1::>some query or another", "file::192.168.16.1::>some query or another"},
+                {"~file::192.168.16.1::>Some Query or Another", "file::192.168.16.1::>Some Query or Another"},
+                {"~file::192.168.16.1::wild?card1",             "file::192.168.16.1::wild?card1"},
+                {"~file::192.168.16.1::wild*card2",             "file::192.168.16.1::wild*card2"},
+                {"~file::192.168.16.1::^C^a^S^e^d",             "file::192.168.16.1::^c^a^s^e^d"},
+                {nullptr,                                       nullptr }        // terminator
+                 };
+
+         const char *validInternalLfns[][2] = {
+                 //     input file name                    expected normalized file name
+                 {"~foreign::192.168.16.1::scope1::file1", "foreign::192.168.16.1::scope1::file1"},
+                 {".::scope1::file",                       ".::scope1::file"},
+                 {"~ scope1 :: scope2 :: file  ",          "scope1::scope2::file"},
+                 {". :: scope1 :: file nine",              ".::scope1::file nine"},
+                 {". :: scope1 :: file ten  ",             ".::scope1::file ten"},
+                 {". :: scope1 :: file",                   ".::scope1::file"},
+                 {"~scope1::file@cluster1",                "scope1::file"},
+                 {"~scope::^C^a^S^e^d",                    "scope::^c^a^s^e^d"},
+                 {"~scope::CaSed",                         "scope::cased"},
+                 {"~scope::^CaSed",                         "scope::^cased"},
+                 {nullptr,                                 nullptr}   // terminator
+                 };
+
+        // Check results
+        const bool externalFile = true;
+        const bool internalFile = false;
+        const bool fileNameMatch = true;
+
+        PROGLOG("Checking external filenames detection and normalization");
+        unsigned nlfn=0;
+        loop
+        {
+            const char *lfn = validExternalLfns[nlfn][inFileName];
+            if (nullptr == lfn)
+                break;
+            PROGLOG("lfn = '%s'", lfn);
+            StringAttr res;
+            try
+            {
+                ASSERT(externalFile == normalizeExternal(lfn, res, false));
+                PROGLOG("res = '%s'", res.str());
+                ASSERT(fileNameMatch == streq(res.str(), validExternalLfns[nlfn][normalizedFileName]))
+            }
+            catch (IException *e)
+            {
+                VStringBuffer err("External filename '%s' ('%s') failed.", lfn, res.str());
+                EXCLOG(e, err.str());
+                e->Release();
+                CPPUNIT_FAIL(err.str());
+            }
+            nlfn++;
+        }
+
+        PROGLOG("Checking valid internal filenames");
+        nlfn=0;
+        loop
+        {
+            const char *lfn = validInternalLfns[nlfn][inFileName];
+            if (nullptr == lfn)
+                break;
+            PROGLOG("lfn = '%s'", lfn);
+            StringAttr res;
+            try
+            {
+                ASSERT(internalFile == normalizeExternal(lfn, res, false));
+                normalizeName(lfn, res, false);
+                PROGLOG("res = '%s'", res.str());
+                ASSERT(fileNameMatch == streq(res.str(), validInternalLfns[nlfn][normalizedFileName]))
+            }
+            catch (IException *e)
+            {
+                VStringBuffer err("Internal filename '%s' ('%s') failed.", lfn, res.str());
+                EXCLOG(e, err.str());
+                e->Release();
+                CPPUNIT_FAIL(err.str());
+            }
+            nlfn++;
+        }
+    }
+};
+
+CPPUNIT_TEST_SUITE_REGISTRATION( CFileNameNormalizeUnitTest );
+CPPUNIT_TEST_SUITE_NAMED_REGISTRATION( CFileNameNormalizeUnitTest, "CFileNameNormalizeUnitTest" );
 
 #endif // _USE_CPPUNIT

thorlcr/activities/countproject/thcountprojectslave.cpp

@@ -167,12 +167,12 @@ public:
     }
     virtual void stop()
     {
-        PARENT::stop();
         if (first) // nextRow, therefore getPrevCount()/sendCount() never called
         {
             prevRecCount = count = getPrevCount();
             signalNext();
         }
+        PARENT::stop();
     }
     virtual void abort()
     {

thorlcr/activities/funnel/thfunnelslave.cpp

@@ -132,7 +132,11 @@ class CParallelFunnel : implements IRowStream, public CSimpleInterface
     void push(const void *row)
     {   
         CriticalBlock b2(fullCrit); // exclusivity for totSize / full
-        if (stopped) return;
+        if (stopped)
+        {
+            ReleaseThorRow(row);
+            return;
+        }
         rows.enqueue(row);
         totSize += thorRowMemoryFootprint(serializer, row);
         while (totSize > FUNNEL_MIN_BUFF_SIZE)
@@ -206,9 +210,9 @@ public:
         }
         {
             CriticalBlock b(fullCrit);
+            stopped = true; // ensure any pending push()'s don't enqueue and if big row potentially block again.
             if (full)
             {
-                stopped = true; // ensure pending push()'s don't enqueue and if big row potentially block again.
                 loop
                 {
                     OwnedConstThorRow row = rows.dequeueNow();
@@ -767,7 +771,7 @@ class CNWaySelectActivity : public CSlaveActivity, public CThorSteppable
     typedef CSlaveActivity PARENT;
 
     IHThorNWaySelectArg *helper;
-    IThorDataLink *selectedInputITDL = nullptr;
+    IThorDataLink *selectedInput = nullptr;
     IEngineRowStream *selectedStream = nullptr;
     IStrandJunction *selectedJunction = nullptr;
 public:
@@ -783,45 +787,45 @@ public:
         ActivityTimer s(totalCycles, timeActivities);
 
         unsigned whichInput = helper->getInputIndex();
-        selectedInputITDL = nullptr;
+        selectedInput = nullptr;
         selectedStream = nullptr;
         selectedJunction = nullptr;
         if (whichInput--)
         {
             ForEachItemIn(i, inputs)
             {
-                IThorDataLink *cur = queryInput(i);
-                IThorNWayInput *nWayInput = dynamic_cast<IThorNWayInput *>(cur);
+                IThorDataLink *curInput = queryInput(i);
+                IThorNWayInput *nWayInput = dynamic_cast<IThorNWayInput *>(curInput);
                 if (nWayInput)
                 {
-                    cur->start();
-                    unsigned numRealInputs = nWayInput->numConcreteOutputs();
-                    if (whichInput < numRealInputs)
+                    curInput->start();
+                    unsigned numOutputs = nWayInput->numConcreteOutputs();
+                    if (whichInput < numOutputs)
                     {
-                        selectedInputITDL = nWayInput->queryConcreteInput(whichInput);
-                        selectedStream = nWayInput->queryConcreteInputStream(whichInput);
-                        selectedJunction = nWayInput->queryConcreteInputJunction(whichInput);
+                        selectedInput = nWayInput->queryConcreteOutput(whichInput);
+                        selectedStream = nWayInput->queryConcreteOutputStream(whichInput);
+                        selectedJunction = nWayInput->queryConcreteOutputJunction(whichInput);
                         break;
                     }
-                    whichInput -= numRealInputs;
+                    whichInput -= numOutputs;
                 }
                 else
                 {
                     if (whichInput == 0)
                     {
-                        selectedInputITDL = cur;
+                        selectedInput = curInput;
                         selectedStream = queryInputStream(i);
                         selectedJunction = queryInputJunction(i);
                         break;
                     }
                     whichInput -= 1;
                 }
-                if (selectedInputITDL)
+                if (selectedInput)
                     break;
             }
         }
-        if (selectedInputITDL)
-            selectedInputITDL->start();
+        if (selectedInput)
+            selectedInput->start();
         startJunction(selectedJunction);
         dataLinkStart();
     }
@@ -845,7 +849,7 @@ public:
     { 
         if (!selectedStream)
             return false;
-        return selectedInputITDL->gatherConjunctions(collector);
+        return selectedInput->gatherConjunctions(collector);
     }
     virtual void resetEOF()
     { 
@@ -868,11 +872,11 @@ public:
     {
         initMetaInfo(info);
         if (selectedStream)
-            calcMetaInfoSize(info, selectedInputITDL);
+            calcMetaInfoSize(info, selectedInput);
         else if (!hasStarted())
             info.canStall = true; // unkwown if !started
     }
-    virtual bool isGrouped() const override { return selectedInputITDL ? selectedInputITDL->isGrouped() : false; }
+    virtual bool isGrouped() const override { return selectedInput ? selectedInput->isGrouped() : false; }
 // steppable
     virtual void setInputStream(unsigned index, CThorInput &input, bool consumerOrdered) override
     {
@@ -881,8 +885,8 @@ public:
     }
     virtual IInputSteppingMeta *querySteppingMeta()
     {
-        if (selectedInputITDL)
-            return selectedInputITDL->querySteppingMeta();
+        if (selectedInput)
+            return selectedInput->querySteppingMeta();
         return NULL;
     }
 };
@@ -915,6 +919,11 @@ public:
         selectedInputs.kill();
         selectedInputStreams.kill();
         selectedInputJunctions.kill();
+
+        /* NB: all input streams have been connected and because NWayInput does not support handling multiple streams.
+         * i.e. not a CThorStrandedActivity, will use base getOutputStreams implementation and ensure single streams are created.
+         * To allow NWay activities to handle stranding would need handle at getOutputStreams level and conditional produce junctions if mismatched # of output streams
+         */
         if (selectionIsAll)
         {
             ForEachItemIn(i, inputs)
@@ -945,7 +954,7 @@ public:
                 selectedInputJunctions.append(queryInputJunction(nextIndex-1));
             }
         }
-        // NB: Whatever pulls this IThorNWayInput, starts and stops the selectedInputs
+        // NB: Whatever pulls this IThorNWayInput, starts and stops the selectedInputs and selectedInputJunctions
     }
     virtual void stop() override
     {
@@ -966,19 +975,19 @@ public:
     {
         return selectedInputs.ordinality();
     }
-    virtual IThorDataLink *queryConcreteInput(unsigned idx) const
+    virtual IThorDataLink *queryConcreteOutput(unsigned idx) const
     {
         if (selectedInputs.isItem(idx))
             return selectedInputs.item(idx);
         return NULL;
     }
-    virtual IEngineRowStream *queryConcreteInputStream(unsigned idx) const
+    virtual IEngineRowStream *queryConcreteOutputStream(unsigned whichInput) const
     {
-        if (selectedInputStreams.isItem(idx))
-            return selectedInputStreams.item(idx);
+        if (selectedInputStreams.isItem(whichInput))
+            return selectedInputStreams.item(whichInput);
         return NULL;
     }
-    virtual IStrandJunction *queryConcreteInputJunction(unsigned idx) const
+    virtual IStrandJunction *queryConcreteOutputJunction(unsigned idx) const
     {
         if (selectedInputJunctions.isItem(idx))
             return selectedInputJunctions.item(idx);

thorlcr/activities/loop/thloopslave.cpp

@@ -1341,7 +1341,7 @@ public:
         return inputs.ordinality();
     }
 
-    virtual IHThorInput * queryConcreteInput(unsigned idx) const
+    virtual IHThorInput * queryConcreteOutput(unsigned idx) const
     {
         if (inputs.isItem(idx))
             return &inputs.item(idx);

thorlcr/activities/nsplitter/thnsplitterslave.cpp

@@ -82,7 +82,7 @@ class NSplitterSlaveActivity : public CSlaveActivity, implements ISharedSmartBuf
     bool spill = false;
     bool eofHit = false;
     bool writeBlocked = false, pagedOut = false;
-    CriticalSection startLock, writeAheadCrit;
+    CriticalSection connectLock, prepareInputLock, writeAheadCrit;
     PointerArrayOf<Semaphore> stalledWriters;
     unsigned stoppedOutputs = 0;
     unsigned activeOutputs = 0;
@@ -112,7 +112,7 @@ class NSplitterSlaveActivity : public CSlaveActivity, implements ISharedSmartBuf
         {
             Semaphore writeBlockSem;
             while (!stopped && !parent.eofHit)
-                current = parent.writeahead(current, stopped, writeBlockSem);
+                current = parent.writeahead(current, stopped, writeBlockSem, UINT_MAX);
         }
         void start()
         {
@@ -130,7 +130,7 @@ class NSplitterSlaveActivity : public CSlaveActivity, implements ISharedSmartBuf
     } writer;
     void connectInput(bool consumerOrdered)
     {
-        CriticalBlock block(startLock);
+        CriticalBlock block(connectLock);
         bool inputOrdered = isInputOrdered(consumerOrdered);
         if (!inputConnected)
         {
@@ -173,7 +173,8 @@ public:
     }
     bool prepareInput()
     {
-        CriticalBlock block(startLock);
+        // NB: called from writeahead by outputs
+        CriticalBlock block(prepareInputLock);
         if (!inputPrepared)
         {
             inputPrepared = true;
@@ -226,7 +227,7 @@ public:
             throw LINK(writeAheadException);
         return row.getClear();
     }
-    rowcount_t writeahead(rowcount_t current, const bool &stopped, Semaphore &writeBlockSem)
+    rowcount_t writeahead(rowcount_t current, const bool &stopped, Semaphore &writeBlockSem, unsigned outIdx)
     {
         // NB: readers call writeahead, which will block others
         CriticalBlock b(writeAheadCrit);
@@ -247,10 +248,11 @@ public:
                 break;
         }
         ActivityTimer t(totalCycles, queryTimeActivities());
-        if (!prepareInput()) // returns true, if
-        {
+
+        // NB: Avoid calling prepareInput() from writer thread, as a) already setup and b) if last output is stopping, it stops writer thread and would deadlock if in prepareInput() crit
+        if ((UINT_MAX != outIdx) && !prepareInput())
             return RCMAX; // signals to requester that you are the only output
-        }
+
         pagedOut = false;
         OwnedConstThorRow row;
         loop
@@ -285,7 +287,7 @@ public:
     }
     void inputStopped(unsigned outIdx)
     {
-        CriticalBlock block(startLock);
+        CriticalBlock block(prepareInputLock);
         if (smartBuf)
         {
             /* If no output has started reading (nextRow()), then it will not have been prepared
@@ -432,7 +434,7 @@ const void *CSplitterOutput::nextRow()
 {
     if (rec == max)
     {
-        max = activity.writeahead(max, activity.queryAbortSoon(), writeBlockSem);
+        max = activity.writeahead(max, activity.queryAbortSoon(), writeBlockSem, outIdx);
         // NB: if this is sole input that actually started, writeahead will have returned RCMAX and calls to activity.nextRow will go directly to splitter input
     }
     ActivityTimer t(totalCycles, activity.queryTimeActivities());

thorlcr/graph/thgraph.cpp

@@ -466,6 +466,7 @@ IThorGraphDependencyIterator *CGraphElementBase::getDependsIterator() const
 
 void CGraphElementBase::reset()
 {
+    alreadyUpdated = false;
     onStartCalled = false;
 //  prepared = false;
     if (activity)
@@ -552,6 +553,8 @@ void CGraphElementBase::serializeCreateContext(MemoryBuffer &mb)
     DelayedSizeMarker sizeMark(mb);
     queryHelper()->serializeCreateContext(mb);
     sizeMark.write();
+    if (isSink())
+        mb.append(alreadyUpdated);
 }
 
 void CGraphElementBase::serializeStartContext(MemoryBuffer &mb)
@@ -568,6 +571,8 @@ void CGraphElementBase::deserializeCreateContext(MemoryBuffer &mb)
     mb.read(createCtxLen);
     createCtxMb.clear().append(createCtxLen, mb.readDirect(createCtxLen));
     haveCreateCtx = true;
+    if (isSink())
+        mb.read(alreadyUpdated);
 }
 
 void CGraphElementBase::deserializeStartContext(MemoryBuffer &mb)
@@ -666,7 +671,6 @@ bool CGraphElementBase::prepareContext(size32_t parentExtractSz, const byte *par
                     return false;
             }
             whichBranch = (unsigned)-1;
-            alreadyUpdated = false;
             switch (getKind())
             {
                 case TAKindexwrite:

thorlcr/graph/thgraph.hpp

@@ -263,7 +263,7 @@ public:
 
     const void *queryFindParam() const { return &queryId(); } // for SimpleHashTableOf
 
-    bool alreadyUpdated;
+    bool alreadyUpdated = false;
     EclHelperFactory helperFactory;
 
     CIOConnectionArray inputs, outputs, connectedInputs, connectedOutputs;
@@ -300,7 +300,7 @@ public:
     virtual void deserializeStartContext(MemoryBuffer &mb);
     virtual void serializeCreateContext(MemoryBuffer &mb); // called after onCreate and create() (of activity)
     virtual void serializeStartContext(MemoryBuffer &mb);
-    virtual bool checkUpdate() { return false; }
+    virtual bool checkUpdate() { return alreadyUpdated; }
     virtual void reset();
     void onStart(size32_t parentExtractSz, const byte *parentExtract);
     void onCreate();

thorlcr/graph/thgraphslave.cpp

@@ -186,7 +186,7 @@ void CSlaveActivity::setInputStream(unsigned index, CThorInput &_input, bool con
         _input.junction.setown(junction.getClear());
         if (0 == index)
             inputStream = _inputStream;
-        _input.itdl->setOutputStream(_input.sourceIdx, LINK(_inputStream)); // used by debug request only at moment.
+        _input.itdl->setOutputStream(_input.sourceIdx, LINK(_inputStream)); // used by debug request only at moment. // JCSMORE - this should probably be the junction outputstream if there is one
     }
 }
 

thorlcr/slave/slave.ipp

@@ -82,9 +82,9 @@ public:
 interface IThorNWayInput
 {
     virtual unsigned numConcreteOutputs() const = 0;
-    virtual IThorDataLink *queryConcreteInput(unsigned idx) const = 0;
-    virtual IEngineRowStream *queryConcreteInputStream(unsigned idx) const = 0;
-    virtual IStrandJunction *queryConcreteInputJunction(unsigned idx) const = 0;
+    virtual IThorDataLink *queryConcreteOutput(unsigned idx) const = 0;
+    virtual IEngineRowStream *queryConcreteOutputStream(unsigned whichInput) const = 0;
+    virtual IStrandJunction *queryConcreteOutputJunction(unsigned whichInput) const = 0;
 };
 
 
@@ -105,18 +105,18 @@ public:
     {
         ForEachItemIn(i, inputs)
         {
-            IThorDataLink *cur = queryInput(i);
-            CActivityBase *activity = cur->queryFromActivity();
-            IThorNWayInput *nWayInput = dynamic_cast<IThorNWayInput *>(cur);
+            IThorDataLink *curInput = queryInput(i);
+            CActivityBase *activity = curInput->queryFromActivity();
+            IThorNWayInput *nWayInput = dynamic_cast<IThorNWayInput *>(curInput);
             if (nWayInput)
             {
-                cur->start();
-                unsigned numRealInputs = nWayInput->numConcreteOutputs();
-                for (unsigned i=0; i < numRealInputs; i++)
+                curInput->start();
+                unsigned numOutputs = nWayInput->numConcreteOutputs();
+                for (unsigned i=0; i < numOutputs; i++)
                 {
-                    IThorDataLink *curReal = nWayInput->queryConcreteInput(i);
-                    IEngineRowStream *curRealStream = nWayInput->queryConcreteInputStream(i);
-                    IStrandJunction *curRealJunction = nWayInput->queryConcreteInputJunction(i);
+                    IThorDataLink *curReal = nWayInput->queryConcreteOutput(i);
+                    IEngineRowStream *curRealStream = nWayInput->queryConcreteOutputStream(i);
+                    IStrandJunction *curRealJunction = nWayInput->queryConcreteOutputJunction(i);
                     expandedInputs.append(curReal);
                     expandedStreams.append(curRealStream);
                     expandedJunctions.append(curRealJunction);
@@ -124,7 +124,7 @@ public:
             }
             else
             {
-                expandedInputs.append(cur);
+                expandedInputs.append(curInput);
                 expandedStreams.append(queryInputStream(i));
                 expandedJunctions.append(queryInputJunction(i));
             }