Merge branch 'candidate-5.4.0'

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

.travis.yml

@@ -0,0 +1,21 @@
+language: cpp
+
+compiler:
+  - gcc
+
+before_install:
+  - echo $LANG
+  - echo $LC_ALL
+  - sudo apt-get update && sudo apt-get install -y  g++ gcc make cmake bison flex binutils-dev libldap2-dev libcppunit-dev libicu-dev libxslt1-dev zlib1g-dev libboost-regex-dev libssl-dev libarchive-dev python2.7-dev libv8-dev openjdk-6-jdk libapr1-dev libaprutil1-dev libhiredis-dev
+
+before_script:
+  - cd ..
+  - mkdir build
+  - cd build
+  - cmake ../HPCC-Platform
+
+script:
+  - cmake --build . --target package -- -j3
+  
+os:
+  - linux

common/thorhelper/roxiehelper.cpp

@@ -1427,6 +1427,17 @@ bool CSafeSocket::readBlock(MemoryBuffer &ret, unsigned timeout, unsigned maxBlo
     }
 }
 
+int readHttpHeaderLine(IBufferedSocket *linereader, char *headerline, unsigned maxlen)
+{
+    Owned<IMultiException> me = makeMultiException("roxie");
+    int bytesread = linereader->readline(headerline, maxlen, true, me);
+    if (me->ordinality())
+        throw me.getClear();
+    if(bytesread <= 0 || bytesread > maxlen)
+        throw MakeStringException(THORHELPER_DATA_ERROR, "HTTP-GET Bad Request");
+    return bytesread;
+}
+
 bool CSafeSocket::readBlock(StringBuffer &ret, unsigned timeout, HttpHelper *pHttpHelper, bool &continuationNeeded, bool &isStatus, unsigned maxBlockSize)
 {
     continuationNeeded = false;
@@ -1456,6 +1467,7 @@ bool CSafeSocket::readBlock(StringBuffer &ret, unsigned timeout, HttpHelper *pHt
         if (pHttpHelper != NULL && strncmp((char *)&len, "POST", 4) == 0)
         {
 #define MAX_HTTP_HEADERSIZE 8000
+            pHttpHelper->setIsHttp(true);
             char header[MAX_HTTP_HEADERSIZE + 1]; // allow room for \0
             sock->read(header, 1, MAX_HTTP_HEADERSIZE, bytesRead, timeout);
             header[bytesRead] = 0;
@@ -1492,10 +1504,46 @@ bool CSafeSocket::readBlock(StringBuffer &ret, unsigned timeout, HttpHelper *pHt
             else
                 left = len = 0;
 
-            pHttpHelper->setIsHttp(true);
             if (!len)
                 throw MakeStringException(THORHELPER_DATA_ERROR, "Badly formed HTTP header");
         }
+        else if (pHttpHelper != NULL && strncmp((char *)&len, "GET", 3) == 0)
+        {
+#define MAX_HTTP_GET_LINE 16000 //arbitrary per line limit, most web servers are lower, but urls for queries can be complex..
+                pHttpHelper->setIsHttp(true);
+                char headerline[MAX_HTTP_GET_LINE + 1];
+                Owned<IBufferedSocket> linereader = createBufferedSocket(sock);
+
+                int bytesread = readHttpHeaderLine(linereader, headerline, MAX_HTTP_GET_LINE);
+                pHttpHelper->parseHTTPRequestLine(headerline);
+
+                bytesread = readHttpHeaderLine(linereader, headerline, MAX_HTTP_GET_LINE);
+                while(bytesread >= 0 && *headerline && *headerline!='\r')
+                {
+                    // capture authentication token
+                    if (!strnicmp(headerline, "Authorization: Basic ", 21))
+                        pHttpHelper->setAuthToken(headerline+21);
+                    bytesread = readHttpHeaderLine(linereader, headerline, MAX_HTTP_GET_LINE);
+                }
+
+                StringBuffer queryName;
+                const char *target = pHttpHelper->queryTarget();
+                if (!target || !*target)
+                    throw MakeStringException(THORHELPER_DATA_ERROR, "HTTP-GET Target not specified");
+                else if (!pHttpHelper->validateTarget(target))
+                    throw MakeStringException(THORHELPER_DATA_ERROR, "HTTP-GET Target not found");
+                const char *query = pHttpHelper->queryQueryName();
+                if (!query || !*query)
+                    throw MakeStringException(THORHELPER_DATA_ERROR, "HTTP-GET Query not specified");
+
+                queryName.append(query);
+                Owned<IPropertyTree> req = createPTreeFromHttpParameters(queryName, pHttpHelper->queryUrlParameters(), true, pHttpHelper->queryContentFormat()==MarkupFmt_JSON);
+                if (pHttpHelper->queryContentFormat()==MarkupFmt_JSON)
+                    toJSON(req, ret);
+                else
+                    toXML(req, ret);
+                return true;
+        }
         else if (strnicmp((char *)&len, "STAT", 4) == 0)
             isStatus = true;
         else
@@ -1521,6 +1569,13 @@ bool CSafeSocket::readBlock(StringBuffer &ret, unsigned timeout, HttpHelper *pHt
 
         return len != 0;
     }
+    catch (IException *E)
+    {
+        if (pHttpHelper)
+            checkSendHttpException(*pHttpHelper, E, NULL);
+        heartbeat = false;
+        throw;
+    }
     catch (...)
     {
         heartbeat = false;
@@ -1528,11 +1583,11 @@ bool CSafeSocket::readBlock(StringBuffer &ret, unsigned timeout, HttpHelper *pHt
     }
 }
 
-void CSafeSocket::setHttpMode(const char *queryName, bool arrayMode, TextMarkupFormat _mlfmt)
+void CSafeSocket::setHttpMode(const char *queryName, bool arrayMode, HttpHelper &httphelper)
 {
     CriticalBlock c(crit); // Should not be needed
     httpMode = true;
-    mlFmt = _mlfmt;
+    mlFmt = httphelper.queryContentFormat();
     heartbeat = false;
     assertex(contentHead.length()==0 && contentTail.length()==0);
     if (mlFmt==MarkupFmt_JSON)
@@ -1543,19 +1598,91 @@ void CSafeSocket::setHttpMode(const char *queryName, bool arrayMode, TextMarkupF
     else
     {
         StringAttrBuilder headText(contentHead), tailText(contentTail);
-        headText.append(
-            "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
-            "<soap:Envelope xmlns:soap=\"http://schemas.xmlsoap.org/soap/envelope/\">"
-            "<soap:Body>");
+        if (httphelper.getUseEnvelope())
+            headText.append(
+                "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
+                "<soap:Envelope xmlns:soap=\"http://schemas.xmlsoap.org/soap/envelope/\">"
+                "<soap:Body>");
         if (arrayMode)
         {
             headText.append("<").append(queryName).append("ResponseArray>");
             tailText.append("</").append(queryName).append("ResponseArray>");
         }
-        tailText.append("</soap:Body></soap:Envelope>");
+        if (httphelper.getUseEnvelope())
+            tailText.append("</soap:Body></soap:Envelope>");
     }
 }
 
+void CSafeSocket::checkSendHttpException(HttpHelper &httphelper, IException *E, const char *queryName)
+{
+    if (!httphelper.isHttp())
+        return;
+    if (httphelper.queryContentFormat()==MarkupFmt_JSON)
+        sendJsonException(E, queryName);
+    else
+        sendSoapException(E, queryName);
+}
+
+void CSafeSocket::sendSoapException(IException *E, const char *queryName)
+{
+    try
+    {
+        if (!queryName)
+            queryName = "Unknown"; // Exceptions when parsing query XML can leave queryName unset/unknowable....
+
+        StringBuffer response;
+        response.append("<").append(queryName).append("Response");
+        response.append(" xmlns=\"urn:hpccsystems:ecl:").appendLower(strlen(queryName), queryName).append("\">");
+        response.appendf("<Results><Result><Exception><Source>Roxie</Source><Code>%d</Code>", E->errorCode());
+        response.append("<Message>");
+        StringBuffer s;
+        E->errorMessage(s);
+        encodeXML(s.str(), response);
+        response.append("</Message></Exception></Result></Results>");
+        response.append("</").append(queryName).append("Response>");
+        write(response.str(), response.length());
+    }
+    catch(IException *EE)
+    {
+        StringBuffer error("While reporting exception: ");
+        EE->errorMessage(error);
+        DBGLOG("%s", error.str());
+        EE->Release();
+    }
+#ifndef _DEBUG
+    catch(...) {}
+#endif
+}
+
+void CSafeSocket::sendJsonException(IException *E, const char *queryName)
+{
+    try
+    {
+        if (!queryName)
+            queryName = "Unknown"; // Exceptions when parsing query XML can leave queryName unset/unknowable....
+
+        StringBuffer response;
+        appendfJSONName(response, "%sResponse", queryName).append(" {");
+        appendJSONName(response, "Results").append(" {");
+        appendJSONName(response, "Exception").append(" [{");
+        appendJSONValue(response, "Source", "Roxie");
+        appendJSONValue(response, "Code", E->errorCode());
+        StringBuffer s;
+        appendJSONValue(response, "Message", E->errorMessage(s).str());
+        response.append("}]}}");
+        write(response.str(), response.length());
+    }
+    catch(IException *EE)
+    {
+        StringBuffer error("While reporting exception: ");
+        DBGLOG("%s", EE->errorMessage(error).str());
+        EE->Release();
+    }
+#ifndef _DEBUG
+    catch(...) {}
+#endif
+}
+
 void CSafeSocket::setHeartBeat()
 {
     CriticalBlock c(crit);

common/thorhelper/roxiehelper.hpp

@@ -31,10 +31,12 @@ class THORHELPER_API HttpHelper : public CInterface
 {
 private:
     bool _isHttp;
+    bool useEnvelope;
     StringAttr url;
     StringAttr authToken;
     StringAttr contentType;
     StringArray pathNodes;
+    StringArray *validTargets;
     Owned<IProperties> parameters;
 private:
     inline void setHttpHeaderValue(StringAttr &s, const char *v, bool ignoreExt)
@@ -51,12 +53,16 @@ private:
 
 public:
     IMPLEMENT_IINTERFACE;
-    HttpHelper() { _isHttp = false; parameters.setown(createProperties(true));}
+    HttpHelper(StringArray *_validTargets) : validTargets(_validTargets) { _isHttp = false; useEnvelope=true; parameters.setown(createProperties(true));}
     bool isHttp() { return _isHttp; }
+    bool getUseEnvelope(){return useEnvelope;}
+    void setUseEnvelope(bool _useEnvelope){useEnvelope=_useEnvelope;}
     bool getTrim() {return parameters->getPropBool(".trim", true); /*http currently defaults to true, maintain compatibility */}
     void setIsHttp(bool __isHttp) { _isHttp = __isHttp; }
     const char *queryAuthToken() { return authToken.str(); }
     const char *queryTarget() { return (pathNodes.length()) ? pathNodes.item(0) : NULL; }
+    const char *queryQueryName() { return (pathNodes.length()>1) ? pathNodes.item(1) : NULL; }
+
     inline void setAuthToken(const char *v)
     {
         setHttpHeaderValue(authToken, v, false);
@@ -75,8 +81,20 @@ public:
             parseURL();
         }
     }
-    TextMarkupFormat queryContentFormat(){return (strieq(queryContentType(), "application/json")) ? MarkupFmt_JSON : MarkupFmt_XML;}
+    TextMarkupFormat queryContentFormat()
+    {
+        if (!contentType.length())
+        {
+            if (pathNodes.length()>2 && strieq(pathNodes.item(2), "json"))
+                contentType.set("application/json");
+            else
+                contentType.set("text/xml");
+        }
+
+        return (strieq(queryContentType(), "application/json")) ? MarkupFmt_JSON : MarkupFmt_XML;
+    }
     IProperties *queryUrlParameters(){return parameters;}
+    bool validateTarget(const char *target){return (validTargets) ? validTargets->contains(target) : false;}
 };
 
 //==============================================================================================================
@@ -117,7 +135,10 @@ interface SafeSocket : extends IInterface
     virtual size32_t write(const void *buf, size32_t size, bool takeOwnership=false) = 0;
     virtual bool readBlock(MemoryBuffer &ret, unsigned maxBlockSize, unsigned timeout = (unsigned) WAIT_FOREVER) = 0;
     virtual bool readBlock(StringBuffer &ret, unsigned timeout, HttpHelper *pHttpHelper, bool &, bool &, unsigned maxBlockSize) = 0;
-    virtual void setHttpMode(const char *queryName, bool arrayMode, TextMarkupFormat txtfmt) = 0;
+    virtual void checkSendHttpException(HttpHelper &httphelper, IException *E, const char *queryName) = 0;
+    virtual void sendSoapException(IException *E, const char *queryName) = 0;
+    virtual void sendJsonException(IException *E, const char *queryName) = 0;
+    virtual void setHttpMode(const char *queryName, bool arrayMode, HttpHelper &httphelper) = 0;
     virtual void setHeartBeat() = 0;
     virtual bool sendHeartBeat(const IContextLogger &logctx) = 0;
     virtual void flush() = 0;
@@ -154,7 +175,11 @@ public:
     size32_t write(const void *buf, size32_t size, bool takeOwnership=false);
     bool readBlock(MemoryBuffer &ret, unsigned maxBlockSize, unsigned timeout = (unsigned) WAIT_FOREVER);
     bool readBlock(StringBuffer &ret, unsigned timeout, HttpHelper *pHttpHelper, bool &, bool &, unsigned maxBlockSize);
-    void setHttpMode(const char *queryName, bool arrayMode, TextMarkupFormat txtfmt);
+    void setHttpMode(const char *queryName, bool arrayMode, HttpHelper &httphelper);
+    void checkSendHttpException(HttpHelper &httphelper, IException *E, const char *queryName);
+    void sendSoapException(IException *E, const char *queryName);
+    void sendJsonException(IException *E, const char *queryName);
+
     void setHeartBeat();
     bool sendHeartBeat(const IContextLogger &logctx);
     void flush();

common/workunit/workunit.cpp

@@ -1429,8 +1429,8 @@ public:
             { return c->getDebugAgentListenerPort(); }
     virtual IStringVal & getDebugAgentListenerIP(IStringVal &ip) const
             { return c->getDebugAgentListenerIP(ip); }
-    virtual IStringVal & getXmlParams(IStringVal & params) const
-            { return c->getXmlParams(params); }
+    virtual IStringVal & getXmlParams(IStringVal & params, bool hidePasswords) const
+            { return c->getXmlParams(params, hidePasswords); }
     virtual const IPropertyTree *getXmlParams() const
             { return c->getXmlParams(); }
     virtual unsigned __int64 getHash() const
@@ -1747,7 +1747,7 @@ public:
     virtual IStringVal& getResultName(IStringVal &str) const;
     virtual int         getResultSequence() const;
     virtual bool        isResultScalar() const;
-    virtual IStringVal& getResultXml(IStringVal &str) const;
+    virtual IStringVal& getResultXml(IStringVal &str, bool hidePasswords) const;
     virtual unsigned    getResultFetchSize() const;
     virtual __int64     getResultTotalRowCount() const;
     virtual __int64     getResultRowCount() const;
@@ -1759,7 +1759,7 @@ public:
     virtual __int64     getResultInt() const;
     virtual bool        getResultBool() const;
     virtual double      getResultReal() const;
-    virtual IStringVal& getResultString(IStringVal & str) const;
+    virtual IStringVal& getResultString(IStringVal & str, bool hidePassword) const;
     virtual IDataVal&   getResultRaw(IDataVal & data, IXmlToRawTransformer * xmlTransformer, ICsvToRawTransformer * csvTransformer) const;
     virtual IDataVal&   getResultUnicode(IDataVal & data) const;
     virtual void        getResultDecimal(void * val, unsigned length, unsigned precision, bool isSigned) const;
@@ -3412,7 +3412,7 @@ bool CLocalWorkUnit::archiveWorkUnit(const char *base,bool del,bool ignoredllerr
         return false;
 
     StringBuffer buf;
-    exportWorkUnitToXML(this, buf, false, false);
+    exportWorkUnitToXML(this, buf, false, false, true);
 
     StringBuffer extraWorkUnitXML;
     StringBuffer xpath("/GraphProgress/");
@@ -3720,7 +3720,7 @@ void CLocalWorkUnit::serialize(MemoryBuffer &tgt)
 {
     CriticalBlock block(crit);
     StringBuffer x;
-    tgt.append(exportWorkUnitToXML(this, x, false, false).str());
+    tgt.append(exportWorkUnitToXML(this, x, false, false, false).str());
 }
 
 void CLocalWorkUnit::deserialize(MemoryBuffer &src)
@@ -6579,16 +6579,55 @@ unsigned CLocalWorkUnit::getApplicationValueCount() const
     
 }
 
-IStringVal &CLocalWorkUnit::getXmlParams(IStringVal &str) const
+StringBuffer &appendPTreeOpenTag(StringBuffer &s, IPropertyTree *tree, const char *name, unsigned indent)
+{
+    appendXMLOpenTag(s, name, NULL, false);
+    Owned<IAttributeIterator> attrs = tree->getAttributes(true);
+    if (attrs->first())
+    {
+        unsigned attributeindent = indent + (size32_t) strlen(name);
+        unsigned count = attrs->count();
+        bool doindent = false;
+        ForEach(*attrs)
+        {
+            if (doindent)
+                s.append('\n').appendN(attributeindent, ' ');
+            else if (count > 3)
+                doindent = true;
+            appendXMLAttr(s, attrs->queryName()+1, attrs->queryValue());
+        }
+    }
+    s.append('>');
+    return s;
+}
+
+IStringVal &CLocalWorkUnit::getXmlParams(IStringVal &str, bool hidePasswords) const
 {
     CriticalBlock block(crit);
     IPropertyTree *paramTree = p->queryPropTree("Parameters");
-    if (paramTree)
+    if (!paramTree)
+        return str;
+
+    StringBuffer xml;
+    if (!hidePasswords)
+        toXML(paramTree, xml);
+    else
     {
-        StringBuffer temp;
-        toXML(paramTree, temp);
-        str.set(temp.str());
+        appendPTreeOpenTag(xml.append(' '), paramTree, "Parameters", 0).append('\n');
+
+        Owned<IPropertyTreeIterator> elems = paramTree->getElements("*");
+        ForEach(*elems)
+        {
+            const char *paramname = elems->query().queryName();
+            VStringBuffer xpath("Variables/Variable[@name='%s']/Format/@password", paramname);
+            if (p->getPropBool(xpath))
+                appendXMLTag(xml.append("  "), paramname, "***").append('\n');
+            else
+                toXML(&elems->query(), xml, 2);
+        }
+        appendXMLCloseTag(xml.append(' '), "Parameters").append('\n');
     }
+    str.set(xml);
     return str;
 }
 
@@ -7740,7 +7779,7 @@ void readRow(StringBuffer &out, MemoryBuffer &in, TypeInfoArray &types, StringAt
     }
 }
 
-IStringVal& CLocalWUResult::getResultXml(IStringVal &str) const
+IStringVal& CLocalWUResult::getResultXml(IStringVal &str, bool hidePassword) const
 {
     TypeInfoArray types;
     StringAttrArray names;
@@ -7753,7 +7792,13 @@ IStringVal& CLocalWUResult::getResultXml(IStringVal &str) const
     else
         xml.append("<Dataset>\n");
 
-    if (p->hasProp("Value"))
+    if (hidePassword && p->getPropBool("Format/@password"))
+    {
+        xml.append(" <Row>");
+        appendXMLTag(xml, name, "****");
+        xml.append("</Row>\n");
+    }
+    else if (p->hasProp("Value"))
     {
         MemoryBuffer raw;
         p->getPropBin("Value", raw);
@@ -8113,8 +8158,13 @@ void CLocalWUResult::getResultDecimal(void * val, unsigned len, unsigned precisi
     }
 }
 
-IStringVal& CLocalWUResult::getResultString(IStringVal & str) const
+IStringVal& CLocalWUResult::getResultString(IStringVal & str, bool hidePassword) const
 {
+    if (hidePassword && p->getPropBool("@password"))
+    {
+        str.set("****");
+        return str;
+    }
     MemoryBuffer s;
     p->getPropBin("Value", s);
     if (s.length())
@@ -8763,7 +8813,84 @@ extern WORKUNIT_API ILocalWorkUnit * createLocalWorkUnit(const char *xml)
     return ret;
 }
 
-extern WORKUNIT_API StringBuffer &exportWorkUnitToXML(const IConstWorkUnit *wu, StringBuffer &str, bool unpack, bool includeProgress)
+void exportWorkUnitToXMLWithHiddenPasswords(IPropertyTree *p, IIOStream &out, unsigned extraXmlFlags)
+{
+    const char *name = p->queryName();
+    if (!name)
+        name = "__unnamed__";
+    StringBuffer temp;
+    writeStringToStream(out, appendPTreeOpenTag(temp, p, name, 1));
+
+    Owned<IPropertyTreeIterator> elems = p->getElements("*", iptiter_sort);
+    ForEach(*elems)
+    {
+        IPropertyTree &elem = elems->query();
+        if (streq(elem.queryName(), "Parameters"))
+        {
+            writeStringToStream(out, appendPTreeOpenTag(temp.clear().append(' '), &elem, "Parameters", 2).append('\n'));
+            Owned<IPropertyTreeIterator> params = elem.getElements("*", iptiter_sort);
+            ForEach(*params)
+            {
+                IPropertyTree &param = params->query();
+                const char *paramname = param.queryName();
+                VStringBuffer xpath("Variables/Variable[@name='%s']/Format/@password", paramname);
+                if (p->getPropBool(xpath))
+                    writeStringToStream(out, appendXMLTag(temp.clear().append("  "), paramname, "****").append('\n'));
+                else
+                {
+                    toXML(&param, out, 2, XML_Format|XML_SortTags|extraXmlFlags);
+                }
+            }
+            writeStringToStream(out, appendXMLCloseTag(temp.clear().append(' '), "Parameters").append('\n'));
+        }
+        else if (streq(elem.queryName(), "Variables"))
+        {
+            writeStringToStream(out, appendPTreeOpenTag(temp.clear().append(' '), &elem, "Variables", 2).append('\n'));
+            Owned<IPropertyTreeIterator> vars = elem.getElements("*", iptiter_sort);
+            ForEach(*vars)
+            {
+                Owned<IPropertyTree> var = LINK(&vars->query());
+                if (var->getPropBool("Format/@password"))
+                {
+                    var.setown(createPTreeFromIPT(var)); //copy and remove password values
+                    var->removeProp("Value");
+                    var->removeProp("xmlValue");
+                }
+                toXML(var, out, 2, XML_Format|XML_SortTags|extraXmlFlags);
+            }
+            writeStringToStream(out, appendXMLCloseTag(temp.clear().append(' '), "Variables").append('\n'));
+        }
+        else
+            toXML(&elem, out, 1, XML_Format|XML_SortTags|extraXmlFlags);
+    }
+    writeStringToStream(out, appendXMLCloseTag(temp.clear(), name));
+}
+
+StringBuffer &exportWorkUnitToXMLWithHiddenPasswords(IPropertyTree *p, StringBuffer &str)
+{
+    class CAdapter : public CInterface, implements IIOStream
+    {
+        StringBuffer &out;
+    public:
+        IMPLEMENT_IINTERFACE;
+        CAdapter(StringBuffer &_out) : out(_out) { }
+        virtual void flush() { }
+        virtual size32_t read(size32_t len, void * data) { UNIMPLEMENTED; return 0; }
+        virtual size32_t write(size32_t len, const void * data) { out.append(len, (const char *)data); return len; }
+    } adapter(str);
+    exportWorkUnitToXMLWithHiddenPasswords(p->queryBranch(NULL), adapter, 0);
+    return str;
+}
+
+void exportWorkUnitToXMLFileWithHiddenPasswords(IPropertyTree *p, const char *filename, unsigned extraXmlFlags)
+{
+    OwnedIFile ifile = createIFile(filename);
+    OwnedIFileIO ifileio = ifile->open(IFOcreate);
+    Owned<IIOStream> stream = createIOStream(ifileio);
+    exportWorkUnitToXMLWithHiddenPasswords(p->queryBranch(NULL), *stream, extraXmlFlags);
+}
+
+extern WORKUNIT_API StringBuffer &exportWorkUnitToXML(const IConstWorkUnit *wu, StringBuffer &str, bool unpack, bool includeProgress, bool hidePasswords)
 {
     const CLocalWorkUnit *w = QUERYINTERFACE(wu, const CLocalWorkUnit);
     if (!w)
@@ -8779,6 +8906,8 @@ extern WORKUNIT_API StringBuffer &exportWorkUnitToXML(const IConstWorkUnit *wu,
             p.setown(w->getUnpackedTree(includeProgress));
         else
             p.set(w->p);
+        if (hidePasswords && p->hasProp("Variables/Variable[Format/@password]"))
+            return exportWorkUnitToXMLWithHiddenPasswords(p, str);
         toXML(p, str, 0, XML_Format|XML_SortTags);
     }
     else
@@ -8786,14 +8915,14 @@ extern WORKUNIT_API StringBuffer &exportWorkUnitToXML(const IConstWorkUnit *wu,
     return str;
 }
 
-extern WORKUNIT_API IStringVal& exportWorkUnitToXML(const IConstWorkUnit *wu, IStringVal &str, bool unpack, bool includeProgress)
+extern WORKUNIT_API IStringVal& exportWorkUnitToXML(const IConstWorkUnit *wu, IStringVal &str, bool unpack, bool includeProgress, bool hidePasswords)
 {
     StringBuffer x;
-    str.set(exportWorkUnitToXML(wu,x,unpack, includeProgress).str());
+    str.set(exportWorkUnitToXML(wu,x,unpack, includeProgress, hidePasswords).str());
     return str;
 }
 
-extern WORKUNIT_API void exportWorkUnitToXMLFile(const IConstWorkUnit *wu, const char * filename, unsigned extraXmlFlags, bool unpack, bool includeProgress)
+extern WORKUNIT_API void exportWorkUnitToXMLFile(const IConstWorkUnit *wu, const char * filename, unsigned extraXmlFlags, bool unpack, bool includeProgress, bool hidePasswords)
 {
     const CLocalWorkUnit *w = QUERYINTERFACE(wu, const CLocalWorkUnit);
     if (!w)
@@ -8809,6 +8938,8 @@ extern WORKUNIT_API void exportWorkUnitToXMLFile(const IConstWorkUnit *wu, const
             p.setown(w->getUnpackedTree(includeProgress));
         else
             p.set(w->p);
+        if (hidePasswords && p->hasProp("Variables/Variable[Format/@password]"))
+            return exportWorkUnitToXMLFileWithHiddenPasswords(p, filename, extraXmlFlags);
         saveXML(filename, p, 0, XML_Format|XML_SortTags|extraXmlFlags);
     }
 }

common/workunit/workunit.hpp

@@ -275,7 +275,7 @@ interface IConstWUResult : extends IInterface
     virtual IStringVal & getResultName(IStringVal & str) const = 0;
     virtual int getResultSequence() const = 0;
     virtual bool isResultScalar() const = 0;
-    virtual IStringVal & getResultXml(IStringVal & str) const = 0;
+    virtual IStringVal & getResultXml(IStringVal & str, bool hidePasswords) const = 0;
     virtual unsigned getResultFetchSize() const = 0;
     virtual __int64 getResultTotalRowCount() const = 0;
     virtual __int64 getResultRowCount() const = 0;
@@ -286,7 +286,7 @@ interface IConstWUResult : extends IInterface
     virtual __int64 getResultInt() const = 0;
     virtual bool getResultBool() const = 0;
     virtual double getResultReal() const = 0;
-    virtual IStringVal & getResultString(IStringVal & str) const = 0;
+    virtual IStringVal & getResultString(IStringVal & str, bool hidePasswords) const = 0;
     virtual IDataVal & getResultRaw(IDataVal & data, IXmlToRawTransformer * xmlTransformer, ICsvToRawTransformer * csvTransformer) const = 0;
     virtual IDataVal & getResultUnicode(IDataVal & data) const = 0;
     virtual IStringVal & getResultEclSchema(IStringVal & str) const = 0;
@@ -1067,7 +1067,7 @@ interface IConstWorkUnit : extends IConstWorkUnitInfo
     virtual unsigned getApplicationValueCount() const = 0;
     virtual unsigned getDebugAgentListenerPort() const = 0;
     virtual IStringVal & getDebugAgentListenerIP(IStringVal & ip) const = 0;
-    virtual IStringVal & getXmlParams(IStringVal & params) const = 0;
+    virtual IStringVal & getXmlParams(IStringVal & params, bool hidePasswords) const = 0;
     virtual const IPropertyTree * getXmlParams() const = 0;
     virtual unsigned __int64 getHash() const = 0;
     virtual IStringIterator *getLogs(const char *type, const char *instance=NULL) const = 0;
@@ -1366,9 +1366,9 @@ extern WORKUNIT_API void setWorkUnitFactory(IWorkUnitFactory *_factory);
 extern WORKUNIT_API IWorkUnitFactory * getWorkUnitFactory();
 extern WORKUNIT_API IWorkUnitFactory * getWorkUnitFactory(ISecManager *secmgr, ISecUser *secuser);
 extern WORKUNIT_API ILocalWorkUnit* createLocalWorkUnit(const char *XML);
-extern WORKUNIT_API IStringVal& exportWorkUnitToXML(const IConstWorkUnit *wu, IStringVal &str, bool unpack, bool includeProgress);
-extern WORKUNIT_API StringBuffer &exportWorkUnitToXML(const IConstWorkUnit *wu, StringBuffer &str, bool unpack, bool includeProgress);
-extern WORKUNIT_API void exportWorkUnitToXMLFile(const IConstWorkUnit *wu, const char * filename, unsigned extraXmlFlags, bool unpack, bool includeProgress);
+extern WORKUNIT_API IStringVal& exportWorkUnitToXML(const IConstWorkUnit *wu, IStringVal &str, bool unpack, bool includeProgress, bool hidePasswords);
+extern WORKUNIT_API StringBuffer &exportWorkUnitToXML(const IConstWorkUnit *wu, StringBuffer &str, bool unpack, bool includeProgress, bool hidePasswords);
+extern WORKUNIT_API void exportWorkUnitToXMLFile(const IConstWorkUnit *wu, const char * filename, unsigned extraXmlFlags, bool unpack, bool includeProgress, bool hidePasswords);
 extern WORKUNIT_API void submitWorkUnit(const char *wuid, const char *username, const char *password);
 extern WORKUNIT_API void abortWorkUnit(const char *wuid);
 extern WORKUNIT_API void submitWorkUnit(const char *wuid, ISecManager *secmgr, ISecUser *secuser);

dali/dafilesrv/dafilesrv.cpp

@@ -492,6 +492,9 @@ int main(int argc,char **argv)
             SocketEndpoint listenep;
             bool useSSL;
             bool requireauthenticate;
+            unsigned parallelRequestLimit;
+            unsigned throttleDelayMs;
+            unsigned throttleCPULimit;
 
             
             class cpollthread: public Thread
@@ -514,8 +517,9 @@ int main(int argc,char **argv)
 
         public:
 
-            cserv(SocketEndpoint _listenep, bool _useSSL)
-                : listenep(_listenep),useSSL(_useSSL),pollthread(this)
+            cserv(SocketEndpoint _listenep, bool _useSSL, unsigned _parallelRequestLimit, unsigned _throttleDelayMs, unsigned _throttleCPULimit)
+                : listenep(_listenep),useSSL(_useSSL),pollthread(this),
+                  parallelRequestLimit(_parallelRequestLimit), throttleDelayMs(_throttleDelayMs), throttleCPULimit(_throttleCPULimit)
             {
                 stopped = false;
                 started = false;
@@ -592,7 +596,7 @@ int main(int argc,char **argv)
                 PROGLOG(DAFS_SERVICE_DISPLAY_NAME " Stopped");
                 stopped = true;
             }
-        } service(listenep, useSSL);
+        } service(listenep, useSSL, parallelRequestLimit, throttleDelayMs, throttleCPULimit);
         service.start();
         return 0;
 #else

dali/daliadmin/daliadmin.cpp

@@ -2417,7 +2417,7 @@ static void dumpWorkunit(const char *wuid, bool includeProgress)
 {
     Owned<IWorkUnitFactory> factory = getWorkUnitFactory();
     Owned<IConstWorkUnit> workunit = factory->openWorkUnit(wuid, false);
-    exportWorkUnitToXMLFile(workunit, "stdout:", 0, true, includeProgress);
+    exportWorkUnitToXMLFile(workunit, "stdout:", 0, true, includeProgress, true);
 }
 
 static void dumpProgress(const char *wuid, const char * graph)

docs/ECLLanguageReference/ECLR_mods/BltInFunc-FROMJSON.xml

@@ -0,0 +1,62 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE sect1 PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN"
+"http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd">
+<sect1 id="FROMJSON">
+  <title>FROMJSON</title>
+
+  <para><emphasis role="bold">FROMJSON<indexterm>
+      <primary>FROMJSON</primary>
+    </indexterm><indexterm>
+      <primary>FROMJSON function</primary>
+    </indexterm>(</emphasis><emphasis> record, jsonstring </emphasis><emphasis
+  role="bold">)</emphasis></para>
+
+  <para><informaltable colsep="1" frame="all" rowsep="1">
+      <tgroup cols="2">
+        <colspec colwidth="77.95pt" />
+
+        <tbody>
+          <row>
+            <entry><emphasis>record</emphasis></entry>
+
+            <entry>The RECORD structure to produce. Each field should specify
+            the XPATH to the data in the json<emphasis>string</emphasis> that
+            it should hold. If omitted, the lower-cased field names are
+            used.</entry>
+          </row>
+
+          <row>
+            <entry><emphasis>jsonstring</emphasis></entry>
+
+            <entry>A string containing the JSON to convert.</entry>
+          </row>
+
+          <row>
+            <entry>Return:</entry>
+
+            <entry>FROMJSON returns a single row (record).</entry>
+          </row>
+        </tbody>
+      </tgroup>
+    </informaltable></para>
+
+  <para>The <emphasis role="bold">FROMJSON </emphasis>function returns a
+  single row (record) in the <emphasis>record</emphasis> format from the
+  specified json<emphasis>string</emphasis>. This may be used anywhere a
+  single row can be used (similar to the ROW function).</para>
+
+  <para>Example:</para>
+
+  <programlisting>namesRec := RECORD  
+  UNSIGNED2 EmployeeID{xpath('EmpID')};  
+  STRING10 Firstname{xpath('FName')};  
+  STRING10 Lastname{xpath('LName')}; 
+END; 
+x := '{"FName": "George" , "LName": "Jetson", "EmpID": 42}'; 
+rec := FROMJSON(namesRec,x); 
+OUTPUT(rec);
+</programlisting>
+
+  <para>See Also: <link linkend="ROW">ROW</link>, <link
+  linkend="TOJSON">TOJSON</link></para>
+</sect1>

docs/ECLLanguageReference/ECLR_mods/BltInFunc-TOJSON.xml

@@ -0,0 +1,64 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE sect1 PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN"
+"http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd">
+<sect1 id="TOJSON">
+  <title>TOJSON</title>
+
+  <para><emphasis role="bold">TOJSON<indexterm>
+      <primary>TOJSON</primary>
+    </indexterm><indexterm>
+      <primary>TOJSON function</primary>
+    </indexterm>(</emphasis><emphasis> record </emphasis><emphasis
+  role="bold">)</emphasis></para>
+
+  <para><informaltable colsep="1" frame="all" rowsep="1">
+      <tgroup cols="2">
+        <colspec colwidth="77.95pt" />
+
+        <tbody>
+          <row>
+            <entry><emphasis>record</emphasis></entry>
+
+            <entry>The row (record) of data to convert to JSON format.</entry>
+          </row>
+
+          <row>
+            <entry>Return:</entry>
+
+            <entry>TOJSON returns a STRING.</entry>
+          </row>
+        </tbody>
+      </tgroup>
+    </informaltable></para>
+
+  <para>The <emphasis role="bold">TOJSON </emphasis>function returns a single
+  string with the data in the <emphasis>record</emphasis> re-formatted as
+  JSON. If the RECORD structure of the <emphasis>record</emphasis> has XPATHs
+  defined, then they will be used, otherwise the lower-cased field names are
+  used as the JSON tag names.</para>
+
+  <para>Example:</para>
+
+  <programlisting>namesRec1 := RECORD  
+ UNSIGNED2 EmployeeID{xpath('EmpID')};  
+ STRING10 Firstname{xpath('FName')};  
+ STRING10 Lastname{xpath('LName')}; 
+END; 
+str1 := TOJSON(ROW({42,'Fred','Flintstone'},namesRec1)); 
+OUTPUT(str1); 
+//returns this string: 
+//'"EmpID": 42, "FName": "Fred", "LName": "Flintstone"' 
+namesRec2 := RECORD  
+  UNSIGNED2 EmployeeID;  
+  STRING10 Firstname;  
+  STRING10 Lastname; 
+END; 
+str2 := TOJSON(ROW({42,'Fred','Flintstone'},namesRec2)); 
+OUTPUT(str2); 
+//returns this string: 
+//'"employeeid": 42, "firstname": "Fred", "lastname": "Flintstone"'
+</programlisting>
+
+  <para>See Also: <link linkend="ROW">ROW</link>, <link
+  linkend="FROMJSON">FROMJSON</link></para>
+</sect1>

ecl/eclagent/eclagent.cpp

@@ -335,7 +335,7 @@ public:
 
         Owned<IDebuggerContext> debuggerContext;
         unsigned slavesReplyLen = 0;
-        HttpHelper httpHelper;
+        HttpHelper httpHelper(NULL);
         try
         {
             client->querySocket()->getPeerAddress(peer);
@@ -748,7 +748,7 @@ void EclAgent::outputFormattedResult(const char * name, unsigned sequence, bool
     {
     case ofXML:
         {
-            res->getResultXml(buff);
+            res->getResultXml(buff, true);
             outputSerializer->fwrite(sequence, (const void*)buff.str(), 1, buff.length());
             break;
         }
@@ -902,7 +902,7 @@ char *EclAgent::getResultVarString(const char * stepname, unsigned sequence)
 {
     PROTECTED_GETRESULT(stepname, sequence, "VarString", "string",
         SCMStringBuffer result;
-        r->getResultString(result);
+        r->getResultString(result, false);
         return result.s.detach();
     );
 }
@@ -922,7 +922,7 @@ void EclAgent::getResultString(unsigned & tlen, char * & tgt, const char * stepn
 {
     PROTECTED_GETRESULT(stepname, sequence, "String", "string",
         SCMStringBuffer result;
-        r->getResultString(result);
+        r->getResultString(result, false);
         tlen = result.length();
         tgt = (char *)result.s.detach();
     );
@@ -933,7 +933,7 @@ void EclAgent::getResultStringF(unsigned tlen, char * tgt, const char * stepname
     PROTECTED_GETRESULT(stepname, sequence, "String", "string",
         //MORE: Could used a fixed size IStringVal implementation to save a memory allocation, but hardly worth it.
         SCMStringBuffer result;
-        r->getResultString(result);
+        r->getResultString(result, false);
         rtlStrToStr(tlen, tgt, result.length(), result.s.str());
     );
 }
@@ -942,7 +942,7 @@ void EclAgent::getResultData(unsigned & tlen, void * & tgt, const char * stepnam
 {
     PROTECTED_GETRESULT(stepname, sequence, "Data", "data",
         SCMStringBuffer result;
-        r->getResultString(result);
+        r->getResultString(result, false);
         tlen = result.length();
         tgt = (char *)result.s.detach();
     );

ecl/eclcc/eclcc.cpp

@@ -1659,7 +1659,7 @@ void EclCC::generateOutput(EclCompileInstance & instance)
         else
             xmlFilename.append(DEFAULT_OUTPUTNAME);
         xmlFilename.append(".xml");
-        exportWorkUnitToXMLFile(instance.wu, xmlFilename, 0, true, false);
+        exportWorkUnitToXMLFile(instance.wu, xmlFilename, 0, true, false, false);
     }
 }
 
@@ -2239,7 +2239,7 @@ void EclCC::processBatchedFile(IFile & file, bool multiThreaded)
             if (info.wu &&
                 (info.wu->getDebugValueBool("generatePartialOutputOnError", false) || info.queryErrorProcessor().errCount() == 0))
             {
-                exportWorkUnitToXMLFile(info.wu, xmlFilename, XML_NoBinaryEncode64, true, false);
+                exportWorkUnitToXMLFile(info.wu, xmlFilename, XML_NoBinaryEncode64, true, false, false);
                 Owned<IFile> xml = createIFile(xmlFilename);
                 info.stats.xmlSize = xml->size();
             }

ecl/hqlcpp/hqlecl.cpp

@@ -481,7 +481,7 @@ bool HqlDllGenerator::generateCode(HqlQueryContext & query)
 void HqlDllGenerator::addWorkUnitAsResource()
 {
     SCMStringBuffer wuXML;
-    exportWorkUnitToXML(wu, wuXML, false, false);
+    exportWorkUnitToXML(wu, wuXML, false, false, false);
     code->addCompressResource("WORKUNIT", wuXML.length(), wuXML.str(), NULL, 1000);
 }
 

ecl/hqlcpp/hqlsource.cpp

@@ -720,7 +720,7 @@ public:
     void buildLimits(BuildCtx & classctx, IHqlExpression * expr, unique_id_t id);
     void buildReadMembers( IHqlExpression * expr);
     void buildSteppingMeta(IHqlExpression * expr, MonitorExtractor * monitors);
-    void buildTransformBody(BuildCtx & transformCtx, IHqlExpression * expr, bool returnSize, bool ignoreFilters);
+    void buildTransformBody(BuildCtx & transformCtx, IHqlExpression * expr, bool returnSize, bool ignoreFilters, bool bindInputRow);
     void checkDependencies(BuildCtx & ctx, IHqlExpression * expr);
     bool containsStepping(IHqlExpression * expr);
     ABoundActivity * buildActivity(BuildCtx & ctx, IHqlExpression * expr, ThorActivityKind activityKind, const char *kind, ABoundActivity *input);
@@ -746,7 +746,7 @@ protected:
     void buildGroupAggregateHashHelper(ParentExtract * extractBuilder, IHqlExpression * dataset, IHqlExpression * fields);
     void buildGroupAggregateProcessHelper(ParentExtract * extractBuilder, IHqlExpression * aggregate, const char * name, bool doneAny);
     void buildGroupingMonitors(IHqlExpression * expr, MonitorExtractor & monitors);
-    void buildGroupAggregateTransformBody(BuildCtx & transformctx, IHqlExpression * expr, bool useExtract);
+    void buildGroupAggregateTransformBody(BuildCtx & transformctx, IHqlExpression * expr, bool useExtract, bool bindInputRow);
     void buildNormalizeHelpers(IHqlExpression * expr);
     void buildTargetCursor(Shared<BoundRow> & tempRow, Shared<BoundRow> & rowBuilder, BuildCtx & ctx, IHqlExpression * expr);
     void associateTargetCursor(BuildCtx & subctx, BuildCtx & ctx, BoundRow * tempRow, BoundRow * rowBuilder, IHqlExpression * expr);
@@ -1167,9 +1167,9 @@ void SourceBuilder::buildLimits(BuildCtx & classctx, IHqlExpression * expr, uniq
     }
 }
 
-void SourceBuilder::buildTransformBody(BuildCtx & transformCtx, IHqlExpression * expr, bool returnSize, bool ignoreFilters)
+void SourceBuilder::buildTransformBody(BuildCtx & transformCtx, IHqlExpression * expr, bool returnSize, bool ignoreFilters, bool bindInputRow)
 {
-    if (tableExpr)
+    if (tableExpr && bindInputRow)
     {
         IHqlExpression * mode = (tableExpr->getOperator() == no_table) ? tableExpr->queryChild(2) : NULL;
         if (mode && mode->getOperator() == no_csv)
@@ -2443,9 +2443,9 @@ void SourceBuilder::buildNormalizeHelpers(IHqlExpression * expr)
 }
 
 
-void SourceBuilder::buildGroupAggregateTransformBody(BuildCtx & transformCtx, IHqlExpression * expr, bool useExtract)
+void SourceBuilder::buildGroupAggregateTransformBody(BuildCtx & transformCtx, IHqlExpression * expr, bool useExtract, bool bindInputRow)
 {
-    buildTransformBody(transformCtx, expr, false, false);
+    buildTransformBody(transformCtx, expr, false, false, bindInputRow);
 
     IHqlExpression * aggregate = expr->queryChild(0);
     OwnedHqlExpr mappedAggregate = ensureAggregateGroupingAliased(aggregate);
@@ -2853,7 +2853,7 @@ void DiskReadBuilder::buildTransform(IHqlExpression * expr)
 
         //associateVirtualCallbacks(*this, funcctx, tableExpr);
 
-        buildTransformBody(funcctx, expr, true, false);
+        buildTransformBody(funcctx, expr, true, false, true);
         rootSelfRow = NULL;
 
         unsigned maxColumns = countTotalFields(tableExpr->queryRecord(), false);
@@ -2868,7 +2868,7 @@ void DiskReadBuilder::buildTransform(IHqlExpression * expr)
         transformCtx.addQuotedCompound("virtual size32_t transform(ARowBuilder & crSelf, const void * _left, IFilePositionProvider * fpp)");
     translator.ensureRowAllocated(transformCtx, "crSelf");
     transformCtx.addQuotedLiteral("unsigned char * left = (unsigned char *)_left;");
-    buildTransformBody(transformCtx, expr, true, false);
+    buildTransformBody(transformCtx, expr, true, false, true);
 }
 
 
@@ -2962,7 +2962,7 @@ void DiskNormalizeBuilder::buildTransform(IHqlExpression * expr)
     BuildCtx transformCtx(instance->startctx);
     transformCtx.addQuotedCompound("virtual size32_t transform(ARowBuilder & crSelf)");
     translator.ensureRowAllocated(transformCtx, "crSelf");
-    buildTransformBody(transformCtx, expr, true, false);
+    buildTransformBody(transformCtx, expr, true, false, false);
 }
 
 
@@ -3052,7 +3052,7 @@ void DiskAggregateBuilder::buildTransform(IHqlExpression * expr)
     BuildCtx transformCtx(instance->startctx);
     transformCtx.addQuotedCompound("void doProcessRow(ARowBuilder & crSelf, byte * left)");
     translator.ensureRowAllocated(transformCtx, "crSelf");
-    buildTransformBody(transformCtx, expr, false, false);
+    buildTransformBody(transformCtx, expr, false, false, true);
 }
 
 
@@ -3117,7 +3117,7 @@ void DiskCountBuilder::buildTransform(IHqlExpression * expr)
             cnt.setown(getSizetConstant(1));
 
         BuildCtx subctx(transformCtx);
-        buildTransformBody(subctx, expr, false, false);
+        buildTransformBody(subctx, expr, false, false, true);
         transformCtx.addReturn(cnt);
     }
 }
@@ -3197,7 +3197,7 @@ void DiskGroupAggregateBuilder::buildTransform(IHqlExpression * expr)
     BuildCtx transformCtx(instance->startctx);
     transformCtx.addQuotedCompound("void doProcessRow(byte * left, IHThorGroupAggregateCallback * callback)");
     bool accessesCallback = containsOperator(expr, no_filepos) || containsOperator(expr, no_file_logicalname); 
-    buildGroupAggregateTransformBody(transformCtx, expr, isNormalize || accessesCallback);
+    buildGroupAggregateTransformBody(transformCtx, expr, isNormalize || accessesCallback, true);
 }
 
 
@@ -3281,7 +3281,7 @@ void ChildNormalizeBuilder::buildTransform(IHqlExpression * expr)
     BuildCtx transformCtx(instance->startctx);
     transformCtx.addQuotedCompound("virtual size32_t transform(ARowBuilder & crSelf)");
     translator.ensureRowAllocated(transformCtx, "crSelf");
-    buildTransformBody(transformCtx, expr, true, false);
+    buildTransformBody(transformCtx, expr, true, false, false);
 }
 
 
@@ -3337,7 +3337,7 @@ void ChildAggregateBuilder::buildTransform(IHqlExpression * expr)
     BuildCtx transformCtx(instance->startctx);
     transformCtx.addQuotedCompound("virtual void processRows(ARowBuilder & crSelf)");
     translator.ensureRowAllocated(transformCtx, "crSelf");
-    buildTransformBody(transformCtx, expr, false, false);
+    buildTransformBody(transformCtx, expr, false, false, false);
 }
 
 
@@ -3400,7 +3400,7 @@ void ChildGroupAggregateBuilder::buildTransform(IHqlExpression * expr)
 {
     BuildCtx transformCtx(instance->startctx);
     transformCtx.addQuotedCompound("void processRows(IHThorGroupAggregateCallback * callback)");
-    buildGroupAggregateTransformBody(transformCtx, expr, true);
+    buildGroupAggregateTransformBody(transformCtx, expr, true, false);
 }
 
 
@@ -3461,7 +3461,7 @@ void ChildThroughNormalizeBuilder::buildTransform(IHqlExpression * expr)
     BuildCtx transformCtx(instance->startctx);
     transformCtx.addQuotedCompound("virtual size32_t transform(ARowBuilder & crSelf)");
     translator.ensureRowAllocated(transformCtx, "crSelf");
-    buildTransformBody(transformCtx, expr, true, false);
+    buildTransformBody(transformCtx, expr, true, false, false);
 }
 
 
@@ -6382,7 +6382,7 @@ void NewIndexReadBuilder::buildTransform(IHqlExpression * expr)
         translator.ensureRowAllocated(transformCtx, "crSelf");
         transformCtx.addQuotedLiteral("unsigned char * left = (unsigned char *)_left;");
         translator.associateBlobHelper(transformCtx, tableExpr, "fpp");
-        buildTransformBody(transformCtx, expr, true, false);
+        buildTransformBody(transformCtx, expr, true, false, true);
     }
 
     if (generateUnfilteredTransform)
@@ -6392,7 +6392,7 @@ void NewIndexReadBuilder::buildTransform(IHqlExpression * expr)
         translator.ensureRowAllocated(transformCtx, "crSelf");
         transformCtx.addQuotedLiteral("unsigned char * left = (unsigned char *)_left;");
         translator.associateBlobHelper(transformCtx, tableExpr, "fpp");
-        buildTransformBody(transformCtx, expr, true, true);
+        buildTransformBody(transformCtx, expr, true, true, true);
     }
 }
 
@@ -6469,7 +6469,7 @@ void IndexNormalizeBuilder::buildTransform(IHqlExpression * expr)
     OwnedHqlExpr simplified = removeMonitors(expr);
     lastTransformer.set(queryExpression(simplified->queryDataset()->queryTable()));
     useFilterMappings=false;
-    buildTransformBody(transformCtx, simplified, true, false);
+    buildTransformBody(transformCtx, simplified, true, false, false);
 }
 
 
@@ -6554,7 +6554,7 @@ void IndexAggregateBuilder::buildTransform(IHqlExpression * expr)
     transformCtx.addQuotedCompound("void doProcessRow(ARowBuilder & crSelf, byte * left)");
     translator.ensureRowAllocated(transformCtx, "crSelf");
     translator.associateBlobHelper(transformCtx, tableExpr, "fpp");
-    buildTransformBody(transformCtx, expr, false, false);
+    buildTransformBody(transformCtx, expr, false, false, true);
 }
 
 
@@ -6635,7 +6635,7 @@ void IndexCountBuilder::buildTransform(IHqlExpression * expr)
             cnt.setown(getSizetConstant(1));
 
         BuildCtx subctx(transformCtx);
-        buildTransformBody(subctx, expr, false, false);
+        buildTransformBody(subctx, expr, false, false, true);
         transformCtx.addReturn(cnt);
     }
 }
@@ -6799,7 +6799,7 @@ void IndexGroupAggregateBuilder::buildTransform(IHqlExpression * expr)
     BuildCtx transformCtx(instance->startctx);
     transformCtx.addQuotedCompound("void doProcessRow(byte * left, IHThorGroupAggregateCallback * callback)");
     translator.associateBlobHelper(transformCtx, tableExpr, "fpp");
-    buildGroupAggregateTransformBody(transformCtx, expr, isNormalize || transformAccessesCallback);
+    buildGroupAggregateTransformBody(transformCtx, expr, isNormalize || transformAccessesCallback, true);
 }
 
 
@@ -7186,7 +7186,7 @@ void FetchBuilder::buildTransform(IHqlExpression * expr)
     }
 
     translator.ensureRowAllocated(transformCtx, "crSelf");
-    buildTransformBody(transformCtx, expr, true, false);
+    buildTransformBody(transformCtx, expr, true, false, true);
 
     if (translator.xmlUsesContents)
         instance->classctx.addQuotedLiteral("virtual bool requiresContents() { return true; }");

ecl/wutest/wutest.cpp

@@ -56,7 +56,7 @@ bool dump(IConstWorkUnit &w, IProperties *globals)
         ForEach(*results)
         {
             SCMStringBuffer xml;
-            results->query().getResultXml(xml);
+            results->query().getResultXml(xml, true);
             printf("%s\n", xml.str());
             SCMStringBuffer schema;
             results->query().getResultEclSchema(schema);
@@ -66,7 +66,7 @@ bool dump(IConstWorkUnit &w, IProperties *globals)
     else if (stricmp(action, "dump")==0)
     {
         SCMStringBuffer xml;
-        exportWorkUnitToXML(&w, xml, true, false);
+        exportWorkUnitToXML(&w, xml, true, false, true);
         printf("%s\n", xml.str());
     }
     else if (stricmp(action, "temporaries")==0)

ecllibrary/teststd/uni/TestEditDistance.ecl

@@ -70,15 +70,15 @@ EXPORT TestEditDistance := MODULE
     EXPORT Test28a := ASSERT(Uni.EditDistance(U'BLVAREZ',U'ÁLVAREZ','en') = 1, CONST);
     // when character's encoding is from 0x00ffff - 0x10ffff range: 0x1D306 ; Description=TETRAGRAM FOR CENTER (Tai Xuan Jing Symbols)
     // UTF-16 representation is xD834,xDF06 (2 16-bit surrogates)
-    EXPORT Test27 := ASSERT(Uni.EditDistance(U'\uD834\uDF06XXX',U'XXXX') = 1, CONST);
-    EXPORT Test27a := ASSERT(Uni.EditDistance(U'\uD834\uDF06XXX',U'XXXX','en') = 1, CONST);
+    EXPORT Test29 := ASSERT(Uni.EditDistance(U'\uD834\uDF06XXX',U'XXXX') = 1, CONST);
+    EXPORT Test29a := ASSERT(Uni.EditDistance(U'\uD834\uDF06XXX',U'XXXX','en') = 1, CONST);
     // NFC (normalized form composed) for accented characters uses multiple 16-bit code units
     // for example: Ḍ̛ is encoded as 0x1E0C,0x031B, and Ḍ̛̇ as 0x1E0C,0x031B,0x0307
     // These are the cases where the fast function version (ToDo) does not work correctly, but this one does
-    EXPORT Test28 := ASSERT(Uni.EditDistance(U'\u1E0C\u031BDDD',U'DDDD') = 2, CONST);
-    EXPORT Test28a := ASSERT(Uni.EditDistance(U'\u1E0C\u031BDDD',U'DDDD','en') = 1, CONST);
+    EXPORT Test30 := ASSERT(Uni.EditDistance(U'\u1E0C\u031BDDD',U'DDDD') = 2, CONST);
+    EXPORT Test30a := ASSERT(Uni.EditDistance(U'\u1E0C\u031BDDD',U'DDDD','en') = 1, CONST);
     // Lithuanian 'i dot acute' is encoded as 0069 0307 0301
-    EXPORT Test29 := ASSERT(Uni.EditDistance(U'\u0069\u0307\u0301DDD',U'DDDD') = 3, CONST);
-    EXPORT Test29a := ASSERT(Uni.EditDistance(U'\u0069\u0307\u0301DDD',U'DDDD','lt') = 1, CONST);
+    EXPORT Test31 := ASSERT(Uni.EditDistance(U'\u0069\u0307\u0301DDD',U'DDDD') = 3, CONST);
+    EXPORT Test31a := ASSERT(Uni.EditDistance(U'\u0069\u0307\u0301DDD',U'DDDD','lt') = 1, CONST);
   END;
 END;

esp/services/common/jsonhelpers.hpp

@@ -31,97 +31,6 @@
 #define REQSF_ESCAPEFORMATTERS 0x0008
 #define REQSF_EXCLUSIVE (REQSF_SAMPLE_DATA | REQSF_TRIM)
 
-namespace HttpParamHelpers
-{
-    static const char * nextParameterTag(StringBuffer &tag, const char *path)
-    {
-        while (*path=='.')
-            path++;
-        const char *finger = strchr(path, '.');
-        if (finger)
-        {
-            tag.clear().append(finger - path, path);
-            finger++;
-        }
-        else
-            tag.set(path);
-        return finger;
-    }
-
-    static void ensureParameter(IPropertyTree *pt, StringBuffer &tag, const char *path, const char *value, const char *fullpath)
-    {
-        if (!tag.length())
-            return;
-
-        unsigned idx = 1;
-        if (path && isdigit(*path))
-        {
-            StringBuffer pos;
-            path = nextParameterTag(pos, path);
-            idx = (unsigned) atoi(pos.str())+1;
-            if (idx>25) //adf
-                throw MakeStringException(-1, "Array items above 25 not supported in HPCC WS HTTP parameters: %s", fullpath);
-        }
-
-        if (tag.charAt(tag.length()-1)=='$')
-        {
-            if (path && *path)
-                throw MakeStringException(-1, "'$' not allowed in parent node of parameter path: %s", fullpath);
-            tag.setLength(tag.length()-1);
-            StringArray values;
-            values.appendList(value, "\r");
-            ForEachItemIn(pos, values)
-            {
-                const char *itemValue = values.item(pos);
-                while (*itemValue=='\n')
-                    itemValue++;
-                pt->addProp(tag, itemValue);
-            }
-            return;
-        }
-        unsigned count = pt->getCount(tag);
-        while (count++ < idx)
-            pt->addPropTree(tag, createPTree(tag));
-        StringBuffer xpath(tag);
-        xpath.append('[').append(idx).append(']');
-        pt = pt->queryPropTree(xpath);
-
-        if (!path || !*path)
-        {
-            pt->setProp(NULL, value);
-            return;
-        }
-
-        StringBuffer nextTag;
-        path = HttpParamHelpers::nextParameterTag(nextTag, path);
-        ensureParameter(pt, nextTag, path, value, fullpath);
-    }
-
-    static void ensureParameter(IPropertyTree *pt, const char *path, const char *value)
-    {
-        const char *fullpath = path;
-        StringBuffer tag;
-        path = HttpParamHelpers::nextParameterTag(tag, path);
-        ensureParameter(pt, tag, path, value, fullpath);
-    }
-
-    static IPropertyTree *createPTreeFromHttpParameters(const char *name, IProperties *parameters)
-    {
-        Owned<IPropertyTree> pt = createPTree(name);
-        Owned<IPropertyIterator> props = parameters->getIterator();
-        ForEach(*props)
-        {
-            StringBuffer key = props->getPropKey();
-            if (!key.length() || key.charAt(key.length()-1)=='!')
-                continue;
-            const char *value = parameters->queryProp(key);
-            if (value && *value)
-                ensureParameter(pt, key, value);
-        }
-        return pt.getClear();
-    }
-};
-
 namespace JsonHelpers
 {
     static StringBuffer &appendJSONExceptionItem(StringBuffer &s, int code, const char *msg, const char *objname="Exceptions", const char *arrayName = "Exception")

esp/services/ws_ecl/ws_ecl_service.cpp

@@ -1166,6 +1166,8 @@ void CWsEclBinding::SOAPSectionToXsd(WsEclWuInfo &wuinfo, IPropertyTree *parmTre
                 unsigned cols = part.getPropInt("@width");
                 if (cols)
                     schema.appendf(" formCols='%u'", cols);
+                if (part.hasProp("@password"))
+                    schema.appendf(" password='%s'", part.queryProp("@password"));
                 schema.appendf("/></xsd:appinfo></xsd:annotation></xsd:element>");
             }
             else
@@ -1478,7 +1480,7 @@ void CWsEclBinding::getWsEcl2XmlRequest(StringBuffer& soapmsg, IEspContext &cont
         return;
     }
 
-    Owned<IPropertyTree> reqTree = HttpParamHelpers::createPTreeFromHttpParameters(wsinfo.queryname, parameters);
+    Owned<IPropertyTree> reqTree = createPTreeFromHttpParameters(wsinfo.queryname, parameters, true, false);
 
     if (!validate)
         toXML(reqTree, soapmsg, 0, 0);
@@ -1510,7 +1512,7 @@ void CWsEclBinding::getWsEclJsonRequest(StringBuffer& jsonmsg, IEspContext &cont
     try
     {
         IProperties *parameters = context.queryRequestParameters();
-        Owned<IPropertyTree> reqTree = HttpParamHelpers::createPTreeFromHttpParameters(wsinfo.queryname, parameters);
+        Owned<IPropertyTree> reqTree = createPTreeFromHttpParameters(wsinfo.queryname, parameters, true, false);
 
         if (!validate)
         {
@@ -2361,7 +2363,7 @@ int CWsEclBinding::onGet(CHttpRequest* request, CHttpResponse* response)
 
             if (!wsecl->connMap.getValue(target.str()))
                 throw MakeStringException(-1, "Target cluster not mapped to roxie process!");
-            Owned<IPropertyTree> pt = HttpParamHelpers::createPTreeFromHttpParameters(qid.str(), parms);
+            Owned<IPropertyTree> pt = createPTreeFromHttpParameters(qid.str(), parms, true, false);
             StringBuffer soapreq(
                 "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
                 "<soap:Envelope xmlns:soap=\"http://schemas.xmlsoap.org/soap/envelope/\""

esp/services/ws_ecl/ws_ecl_wuinfo.cpp

@@ -61,9 +61,10 @@ void appendVariableParmInfo(IArrayOf<IPropertyTree> &parts, IResultSetFactory *r
     SCMStringBuffer eclschema;
     var.getResultEclSchema(eclschema);
 
-    StringBuffer width, height, fieldSeq;
+    StringBuffer width, height, fieldSeq, isPassword;
     var.getResultFieldOpt("fieldwidth", StringBufferAdaptor(width));
     var.getResultFieldOpt("fieldheight", StringBufferAdaptor(height));
+    var.getResultFieldOpt("password", StringBufferAdaptor(isPassword));
     if (hashWebserviceSeq)
         fieldSeq.append(hashWebserviceSeq);
     else
@@ -125,6 +126,8 @@ void appendVariableParmInfo(IArrayOf<IPropertyTree> &parts, IResultSetFactory *r
             part->setProp("@height", height);
         if (fieldSeq.length())
             part->setProp("@sequence", fieldSeq);
+        if (isPassword.length())
+            part->setProp("@password", isPassword);
     }
     parts.append(*part.getClear());
 }

esp/services/ws_workunits/ws_workunitsHelpers.cpp

@@ -1025,7 +1025,7 @@ void WsWuInfo::getInfo(IEspECLWorkunit &info, unsigned flags)
     info.setActionEx(cw->getActionEx(s).str());
     info.setDescription(cw->getDebugValue("description", s).str());
     if (version > 1.21)
-        info.setXmlParams(cw->getXmlParams(s).str());
+        info.setXmlParams(cw->getXmlParams(s, true).str());
 
     info.setResultLimit(cw->getResultLimit());
     info.setArchived(false);
@@ -1480,7 +1480,7 @@ void WsWuInfo::getResult(IConstWUResult &r, IArrayOf<IEspECLResult>& results, un
         try
         {
             SCMStringBuffer xml;
-            r.getResultXml(xml);
+            r.getResultXml(xml, true);
 
             Owned<IPropertyTree> props = createPTreeFromXMLString(xml.str(), ipt_caseInsensitive);
             IPropertyTree *val = props->queryPropTree("Row/*");
@@ -2131,7 +2131,7 @@ void WsWuInfo::getWorkunitXml(const char* plainText, MemoryBuffer& buf)
         header = "<?xml version=\"1.0\" encoding=\"UTF-8\"?><?xml-stylesheet href=\"../esp/xslt/xmlformatter.xsl\" type=\"text/xsl\"?>";
 
     SCMStringBuffer xml;
-    exportWorkUnitToXML(cw, xml, true, false);
+    exportWorkUnitToXML(cw, xml, true, false, true);
 
     buf.append(strlen(header), header);
     buf.append(xml.length(), xml.str());

esp/services/ws_workunits/ws_workunitsService.cpp

@@ -3102,7 +3102,7 @@ bool CWsWorkunitsEx::onWUExport(IEspContext &context, IEspWUExportRequest &req,
         {
             Owned<IConstWorkUnit> cw = factory->openWorkUnit(it->c_str(), false);
             if (cw)
-                exportWorkUnitToXML(cw, xml, true, false);
+                exportWorkUnitToXML(cw, xml, true, false, true);
         }
         xml.append("</Workunits>");
 

esp/smc/SMCLib/WUXMLInfo.cpp

@@ -246,7 +246,7 @@ bool CWUXMLInfo::buildXmlResultList(IConstWorkUnit &wu,IPropertyTree& XMLStructu
                 else if (r.isResultScalar())
                 {
                     SCMStringBuffer x;
-                    r.getResultXml(x);
+                    r.getResultXml(x, true);
                     try
                     {
                         Owned<IPropertyTree> props = createPTreeFromXMLString(x.str(), ipt_caseInsensitive);

esp/xslt/wsecl3_form.xsl

@@ -1132,7 +1132,7 @@ function switchInputForm()
                     <xsl:otherwise>
                         <xsl:variable name="inputType">
                             <xsl:choose>
-                                <xsl:when test="$annot/@formType">password</xsl:when>
+                                <xsl:when test="$annot/@password">password</xsl:when>
                                 <xsl:otherwise>text</xsl:otherwise>
                             </xsl:choose>
                         </xsl:variable>

initfiles/bin/CMakeLists.txt

@@ -32,6 +32,7 @@ FOREACH( iFILES
     ${CMAKE_CURRENT_SOURCE_DIR}/init_roxie_cluster
     ${CMAKE_CURRENT_SOURCE_DIR}/init_sasha
     ${CMAKE_CURRENT_SOURCE_DIR}/init_thor
+    ${CMAKE_CURRENT_SOURCE_DIR}/init_thorslave
 )
     install ( PROGRAMS ${iFILES} DESTINATION ${EXEC_DIR} COMPONENT Runtime )
 ENDFOREACH ( iFILES )

initfiles/bin/init_thor

@@ -15,26 +15,197 @@
 #    limitations under the License.
 ################################################################################
 
-export deploydir=$(dirname $(type -path $0)) 
-PATH_PRE=`type -path hpcc_setenv` 
+deploydir=$(dirname $(type -path $0))
+PATH_PRE=$(type -path hpcc_setenv)
 source ${PATH_PRE} 
-PID_NAME="$PID/`basename $PWD`.pid"
 
-INIT_PID_NAME="$PID/init_`basename $PWD`.pid"
+INSTALL_DIR=$(dirname ${PATH_PRE})/..
+source  ${INSTALL_DIR}/etc/init.d/hpcc_common
+
+PID_NAME="$PID/$(basename $PWD).pid"
+INIT_PID_NAME="$PID/init_$(basename $PWD).pid"
 echo $$ > $INIT_PID_NAME
 
 export SENTINEL="thor.sentinel"
 rm -f ${SENTINEL}
 
-killed() {
-        echo "Stopping"
-        kill_process ${SENTINEL} ${PID_NAME} 3
-        exit 255
+if [[ -z "$deploydir" ]]; then
+    deploydir=$(pwd -P)
+fi
+
+compname=$(basename $PWD)
+instancedir=$(pwd -P)
+source $instancedir/setvars
+
+if [[ ! -z ${THORPRIMARY} ]]; then
+    groupName=${THORPRIMARY}
+else
+    groupName=${THORNAME}
+fi
+ln -s -f $deploydir/thormaster${LCR} thormaster_$THORNAME
+
+ENV_DIR=$(cat ${HPCC_CONFIG} | sed -n "/\[DEFAULT\]/,/\[/p" | grep "^configs=" | sed -e 's/^configs=//')
+logdir=$(updtdalienv $ENV_DIR/environment.xml -d log thor $THORNAME)
+logfile=$logdir/${THORNAME}.log
+
+exec >> $logfile
+
+contains()
+{
+    local n=$#
+    local value=${!n}
+    local i=0
+    for ((i=1;i < ${n};i++)); do
+        if [[ "${!i}" == "${value}" ]]; then
+            echo "${i}"
+            return 0
+        fi
+    done
+    echo "0"
+    return 1
+}
+
+makethorgroup()
+{
+    if [[ -z "$localthorportinc" ]]; then
+        localthorportinc=200
+    fi
+    rm -f thorgroup
+
+    declare -a ports_used
+    declare -a hosts
+    for slave in $(cat slaves); do
+        p=$(contains "${hosts[@]}" "${slave}")
+        if [[ 0 == ${p} ]]; then
+            echo "${slave}:${THORSLAVEPORT}" >> thorgroup
+            p=$((${#hosts[@]}+1))
+            ports[${p}]=${THORSLAVEPORT}
+            hosts=(${hosts[@]} $slave)
+        else
+            newport=$((${ports[${p}]}+${localthorportinc}))
+            echo "${slave}:${newport}" >> thorgroup
+            ports[${p}]=${newport}
+        fi
+    done
+}
+
+kill_slaves()
+{
+    if [[ "$localthor" = "true" ]]; then
+    $deploydir/init_thorslave stop localhost $THORMASTER $THORMASTERPORT $logdir $instancedir $deploydir $THORNAME $PATH_PRE $logredirect
+    else
+        # we want to kill only slaves that have already been started in run_thor
+        if [[ -r $instancedir/uslaves.start ]]; then
+            nslaves=$(cat $instancedir/uslaves.start 2> /dev/null | wc -l)
+            $deploydir/frunssh $instancedir/uslaves.start "/bin/sh -c '$deploydir/init_thorslave stop %a $THORMASTER $THORMASTERPORT $logdir $instancedir $deploydir $THORNAME $PATH_PRE $logredirect'" -i:$SSHidentityfile -u:$SSHusername -pe:$SSHpassword -t:$SSHtimeout -a:$SSHretries -n:$nslaves 2>&1 | egrep -v "no process killed"
+            echo slaves stopped
+        fi
+    fi
+
+    rm -f $instancedir/uslaves.start > /dev/null 2>&1
+}
+
+killed()
+{
+    echo "Stopping"
+    kill_process ${PID_NAME} thormaster_${compname} 30
+    kill_slaves
+    rm -f $INIT_PID_NAME $instancedir/uslaves.start > /dev/null 2>&1
+    exit 255
 }
 
-echo $$ > $PID_NAME
-$deploydir/start_thor $deploydir &
 trap "killed" SIGINT SIGTERM
-wait
-rm $PID_NAME
+# attempt to clean up any old slaves
+kill_slaves
+thorpid=0
+
+while [[ 1 ]]; do
+    # update slaves file in case state of environment has been altered since last run
+    daliadmin server=$DALISERVER dfsgroup ${groupName} slaves
+    errcode=$?
+    if [[ 0 != ${errcode} ]]; then
+    echo "failed to lookup dali group for $groupName"
+        exit 1
+    fi
+    makethorgroup
+    sort $instancedir/slaves | uniq > $instancedir/uslaves.start
+
+    echo "--------------------------"
+    echo "starting thorslaves ..."
+
+    logpthtail=$(date +%m_%d_%Y_%H_%M_%S)
+    logredirect="$logdir/init_thorslave_$logpthtail.log"
+    # Would be simpler, if there was simple way to test if ip is local and get rid of 'localthor' setting
+    if [[ "$localthor" = "true" ]]; then
+        slaveip=$(head -n 1 $instancedir/uslaves.start)
+        $deploydir/init_thorslave start $slaveip $THORMASTER $THORMASTERPORT $logdir $instancedir $deploydir $THORNAME $PATH_PRE $logredirect
+    else
+        nslaves=$(cat $instancedir/uslaves.start | wc -l)
+        $deploydir/frunssh $instancedir/uslaves.start "/bin/sh -c '$deploydir/init_thorslave start %a $THORMASTER $THORMASTERPORT $logdir $instancedir $deploydir $THORNAME $PATH_PRE $logredirect'" -i:$SSHidentityfile -u:$SSHusername -pe:$SSHpassword -t:$SSHtimeout -a:$SSHretries -n:$nslaves 2>&1
+        FRUNSSH_RC=$?
+        if [[ ${FRUNSSH_RC} -gt 0 ]]; then
+            echo "Error ${FRUNSSH_RC} in frunssh"
+            echo "Please check $(dirname ${logdir})/frunssh for more details"
+            # clean up any slaves it was able to reach
+            killed
+        fi
+    fi
+
+    echo thormaster cmd : $instancedir/thormaster_$THORNAME MASTER=$THORMASTER:$THORMASTERPORT
+    nohup $instancedir/thormaster_$THORNAME MASTER=$THORMASTER:$THORMASTERPORT 2> /dev/null 1>/dev/null &
+
+    thorpid=$!
+    if [[ "$thorpid" -ne "0" ]]; then
+        echo thormaster$LCR process started pid = $thorpid
+
+        echo $thorpid > $PID_NAME
+        wait $thorpid
+        errcode=$?
+        case $errcode in
+        # TEC_Clean
+        0)  echo "Thormaster ($thorpid) Exited cleanly"
+            rm -f $instancedir/uslaves.start $PID_NAME $INIT_PID_NAME > /dev/null 2>&1
+            exit 0
+            ;;
+        # TEC_CtrlC
+        1)  echo "Thormaster ($thorpid) Interrupted, Ctrl-C caught"
+            killed
+            ;;
+        # TEC_Idle, TEC_Watchdog, TEC_Swap, TEC_DaliDown
+        2|3|5|6)    [[ $errcode -eq 2 ]] && echo "Thormaster ($thorpid) Idle"
+                    [[ $errcode -eq 3 ]] && echo "Thormaster ($thorpid) Lost connection to slave(s)"
+                    [[ $errcode -eq 5 ]] && echo "Thormaster ($thorpid) Swap node required"
+                    [[ $errcode -eq 6 ]] && echo "Thormaster ($thorpid) Unable to connect to Dali"
+                    echo 'stopping thorslave(s) for restart'
+                    kill_slaves
+                    if [[ 0 != $autoSwapNode ]]; then
+                        echo "Running autoswap $THORNAME :: ($thorpid)"
+                        swapnode auto $DALISERVER $compname
+                        errcode=$?
+                        if [[ 0 != ${errcode} ]]; then
+                            echo "auto swap node failed, errcode=${errcode}"
+                            killed
+                        fi
+                    fi
+                # restarting thormaster
+                ;;
+        # TEC_SlaveInit
+        4)  echo "Thormaster ($thorpid) Slaves failed to initialize"
+            echo "Shutting down"
+            killed
+            ;;
+        *)  echo "Thormaster ($thorpid) Unknown error code.  Stopping"
+            killed
+            ;;
+        esac
+    else
+        echo failed to start thormaster$LCR, pausing for 30 seconds
+        sleep 30
+        kill_slaves
+    fi
+    if [[ ! -e $SENTINEL ]]; then
+        echo $SENTINEL 'has been removed or thormaster did not fully start - script stopping'
+        exit 0
+    fi
+done
 

initfiles/bin/init_thorslave

@@ -0,0 +1,120 @@
+#!/bin/bash
+################################################################################
+#    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+################################################################################
+
+cmd=$1
+ip=$2
+master=$3
+masterport=$4
+logpth=$5
+instancedir=$6
+deploydir=$7
+hpcc_compname=$8
+hpcc_setenv=$9
+logredirect=${10}
+
+source ${hpcc_setenv}
+
+slavename=thorslave_${hpcc_compname}
+
+stop_slaves()
+{
+    killall -0 $slavename > /dev/null 2>&1
+    if [[ $? -eq 0 ]];then
+        killall -9 $slavename > /dev/null 2>&1
+    fi
+    rm -f $PID/${slavename}_*.pid > /dev/null 2>&1
+}
+
+start_slaves()
+{
+    # insuring dafilesrv is running on the machine as it is a prerequisite
+    sudo /etc/init.d/dafilesrv status > /dev/null 2>&1
+    if [[ $? -ne 0 ]];then
+        sudo /etc/init.d/dafilesrv start > /dev/null 2>&1
+        if [[ $? -ne 0 ]];then
+            exit 1
+        fi
+    fi
+
+    # insuring parent directory structure is setup properly
+    mkdir -p $instancedir
+    mkdir -p $(dirname $logredirect)
+    exec >>$logredirect 2>&1
+
+    cd $instancedir
+
+    echo "$(date) Dependency dafilesrv is running"
+
+    ulimit -Sc hard > /dev/null 2>&1
+    [[ $? -ne 0 ]] && echo "$(date) Failed to set ulimit for core file size"
+    ulimit -Sn hard > /dev/null 2>&1
+    [[ $? -ne 0 ]] && echo "$(date) Failed to set ulimit for number of file descriptors open"
+
+    echo "$(date) slave(${ip}) init"
+    echo "$(date) slave(s) starting"
+
+    # create symlink for easier identification of slaves by compName
+    ln -s -f $deploydir/thorslave_lcr ${slavename}
+
+    # sync to current master thorgroup
+    echo "$(date) rsync -e ssh -o StrictHostKeyChecking=no ${master}:${instancedir}/thorgroup ${instancedir}/thorgroup.slave"
+    rsync -e "ssh -o StrictHostKeyChecking=no" $master:$instancedir/thorgroup $instancedir/thorgroup.slave
+
+    let "slavenum = 1";
+    for slave in $(cat $instancedir/thorgroup.slave); do
+        slaveip=${slave/:*/}
+        if [[ ${slaveip} = ${ip} ]]; then
+            slaveport=${slave/*:/}
+            if [[ "$slaveport" = "" ]]; then
+                slaveport=$THORSLAVEPORT
+            fi
+            echo "$(date) $slavename  master=$master:$masterport slave=.:$slaveport slavenum=$slavenum logDir=$logpth"
+            ./$slavename master=$master:$masterport slave=.:$slaveport slavenum=$slavenum logDir=$logpth 2>/dev/null 1>/dev/null &
+            slavepid=$!
+            if [[ "$slavepid" -eq "0" ]]; then
+                echo "$(date) failed to start"
+            else
+                echo "$(date) slave pid $slavepid started"
+                PID_NAME="$PID/${slavename}_${slavenum}.pid"
+                echo $slavepid > $PID_NAME
+            fi
+        fi
+        let "slavenum += 1";
+    done
+}
+
+print_usage()
+{
+  echo usage: cmd ip master masterport logdir workingdir deploydir hpcc_compname hpcc_setenv logredirect
+}
+
+##  Main
+if [[ $# -lt 10 ]]; then
+    print_usage
+    exit 1
+fi
+
+if [[ "start" = ${cmd} ]]; then
+    start_slaves
+elif [[ "stop" = ${cmd} ]]; then
+    stop_slaves
+else
+    print_usage
+    exit 1
+fi
+
+exit 0

initfiles/componentfiles/thor/CMakeLists.txt

@@ -5,18 +5,12 @@ ADD_CUSTOM_TARGET(ProcessFiles-initfiles-componentfiles-thor ALL DEPENDS ${outFi
 FOREACH( iFILES
     ${outFiles}
     ${CMAKE_CURRENT_SOURCE_DIR}/get_thor_options
-    ${CMAKE_CURRENT_SOURCE_DIR}/makethorgroup
     ${CMAKE_CURRENT_SOURCE_DIR}/multiscp
     ${CMAKE_CURRENT_SOURCE_DIR}/multissh
     ${CMAKE_CURRENT_SOURCE_DIR}/scpslaves
     ${CMAKE_CURRENT_SOURCE_DIR}/setup_nfs
     ${CMAKE_CURRENT_SOURCE_DIR}/setup_one_nfs
     ${CMAKE_CURRENT_SOURCE_DIR}/sshslaves
-    ${CMAKE_CURRENT_SOURCE_DIR}/start_slaves
-    ${CMAKE_CURRENT_SOURCE_DIR}/start_thor
-    ${CMAKE_CURRENT_SOURCE_DIR}/run_thor
-    ${CMAKE_CURRENT_SOURCE_DIR}/stop_thor
-    ${CMAKE_CURRENT_SOURCE_DIR}/stop_slaves
 )
     Install ( PROGRAMS ${iFILES} DESTINATION ${EXEC_DIR} )
 ENDFOREACH ( iFILES )

initfiles/componentfiles/thor/makethorgroup

@@ -1,51 +0,0 @@
-#!/bin/bash
-################################################################################
-#    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
-#
-#    Licensed under the Apache License, Version 2.0 (the "License");
-#    you may not use this file except in compliance with the License.
-#    You may obtain a copy of the License at
-#
-#       http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS,
-#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#    See the License for the specific language governing permissions and
-#    limitations under the License.
-################################################################################
-
-
-if [ -z "$localthorportinc" ]; then
-  export localthorportinc=200
-fi
-rm -f thorgroup
-
-function contains() {
-    local n=$#
-    local value=${!n}
-    for ((i=1;i < ${n};i++)) {
-        if [ "${!i}" == "${value}" ]; then
-            echo "${i}"
-            return 0
-        fi
-    }
-    echo "0"
-    return 1
-}
-
-declare -a ports_used
-declare -a hosts
-for slave in $(cat slaves); do
-    p=$(contains "${hosts[@]}" "${slave}")
-    if [ 0 == ${p} ]; then
-        echo "${slave}:${THORSLAVEPORT}" >> thorgroup
-        p=$((${#hosts[@]}+1))
-        ports[${p}]=${THORSLAVEPORT}
-        hosts=(${hosts[@]} $slave)
-    else
-        newport=$((${ports[${p}]}+${localthorportinc}))
-        echo "${slave}:${newport}" >> thorgroup
-        ports[${p}]=${newport}
-    fi
-done

initfiles/componentfiles/thor/run_thor

@@ -1,130 +0,0 @@
-#!/bin/bash
-################################################################################
-#    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
-#
-#    Licensed under the Apache License, Version 2.0 (the "License");
-#    you may not use this file except in compliance with the License.
-#    You may obtain a copy of the License at
-#
-#       http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS,
-#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#    See the License for the specific language governing permissions and
-#    limitations under the License.
-################################################################################
-
-thorpid=0
-
-
-
-
-RUN_THOR_PID_NAME="$PID/run_thor.`basename $PWD`.pid"
-# prevent two thors starting together
-if [[ -e $RUN_THOR_PID_NAME ]]; then
-    oldpid=$(cat $RUN_THOR_PID_NAME)
-    while ps h $oldpid ; do
-        echo "waiting for process $oldpid to finish"
-        sleep 5
-    done
-fi
-trap "rm -f $RUN_THOR_PID_NAME" exit
-echo $$ > $RUN_THOR_PID_NAME
-
-export SENTINEL="thor.sentinel"
-while [[ 1 ]]; do
-    # update slaves file in case state of environment has been altered since last run
-    daliadmin server=$DALISERVER dfsgroup ${groupName} slaves
-    errcode=$?
-    if [[ 0 != ${errcode} ]]; then
-    echo "failed to lookup dali group for $groupName"
-        exit 1
-    fi
-    $deploydir/makethorgroup
-    sort $instancedir/slaves | uniq > $instancedir/uslaves.start
-
-    echo "--------------------------"
-    echo "starting thorslaves ..."
-
-    logredirect="$logdir/start_slaves_$logpthtail.log"
-    # Would be simpler, if there was simple way to test if ip is local and get rid of 'localthor' setting
-    if [[ "$localthor" = "true" ]]; then
-        slaveip=$(head -n 1 $instancedir/uslaves.start)
-        $deploydir/start_slaves $slaveip $THORMASTER $THORMASTERPORT $logdir $instancedir $deploydir $THORNAME $PATH_PRE $logredirect
-    else
-        nslaves=$(cat $instancedir/uslaves.start | wc -l)
-        $deploydir/frunssh $instancedir/uslaves.start "/bin/sh -c '$deploydir/start_slaves %a $THORMASTER $THORMASTERPORT $logdir $instancedir $deploydir $THORNAME $PATH_PRE $logredirect'" -i:$SSHidentityfile -u:$SSHusername -pe:$SSHpassword -t:$SSHtimeout -a:$SSHretries -n:$nslaves 2>&1
-        FRUNSSH_RC=$?
-        if [[ ${FRUNSSH_RC} -gt 0 ]]; then
-            echo "Error ${FRUNSSH_RC} in frunssh"
-            echo "Please check `dirname ${logdir}`/frunssh for more details"
-            # clean up any slaves it was able to reach
-            $deploydir/stop_thor $deploydir
-            exit 0
-        fi
-    fi
-
-    echo thormaster cmd : $instancedir/thormaster_$THORNAME MASTER=$THORMASTER:$THORMASTERPORT
-    nohup $instancedir/thormaster_$THORNAME MASTER=$THORMASTER:$THORMASTERPORT 2> /dev/null 1>/dev/null &
-
-    thorpid=$!
-    if [[ "$thorpid" -ne "0" ]]; then
-        echo thormaster$LCR process started pid = $thorpid
-        
-        echo $thorpid > $PID_NAME
-        echo $thorpid > $PID_NAME_BASE
-        wait $thorpid
-        errcode=$?
-        case $errcode in
-        # TEC_Clean
-        0)  echo "Thormaster ($thorpid) Exited cleanly"
-            rm -rf $instancedir/uslaves.start $PID_NAME > /dev/null 2>&1
-            exit 0
-            ;;
-        # TEC_CtrlC
-        1)  echo "Thormaster ($thorpid) Interrupted, Ctrl-C caught"
-            $deploydir/stop_thor $deploydir
-            exit 1
-            ;;
-        # TEC_Idle, TEC_Watchdog, TEC_Swap, TEC_DaliDown
-        2|3|5|6)    [[ $errcode -eq 2 ]] && echo "Thormaster ($thorpid) Idle"
-                    [[ $errcode -eq 3 ]] && echo "Thormaster ($thorpid) Lost connection to slave(s)"
-                    [[ $errcode -eq 5 ]] && echo "Thormaster ($thorpid) Swap node required"
-                    [[ $errcode -eq 6 ]] && echo "Thormaster ($thorpid) Unable to connect to Dali"
-                    echo 'stopping thorslave(s) for restart'
-                    $deploydir/stop_thor $deploydir keep_sentinel
-                    if [[ 0 != $autoSwapNode ]]; then
-                        echo "Running autoswap $THORNAME :: ($thorpid)"
-                        compname=`basename $PWD`
-                        swapnode auto $DALISERVER $compname
-                        errcode=$?
-                        if [[ 0 != ${errcode} ]]; then
-                            echo "auto swap node failed, errcode=${errcode}"
-                            $deploydir/stop_thor $deploydir
-                            exit 1
-                        fi
-                    fi
-                # restarting thormaster
-                ;;
-        # TEC_SlaveInit
-        4)  echo "Thormaster ($thorpid) Slaves failed to initialize"
-            echo "Shutting down"
-            $deploydir/stop_thor $deploydir
-            exit 1
-            ;;
-        *)  echo "Thormaster ($thorpid) Unknown error code.  Stopping"
-            $deploydir/stop_thor $deploydir
-            exit 1
-            ;;
-        esac
-    else
-        echo failed to start thormaster$LCR, pausing for 30 seconds
-        $deploydir/stop_thor $deploydir
-        sleep 30
-    fi
-    if [[ ! -e $SENTINEL ]]; then
-        echo $SENTINEL 'has been removed or thormaster did not fully start - script stopping'
-        exit 0
-    fi
-done

initfiles/componentfiles/thor/start_slaves

@@ -1,105 +0,0 @@
-#!/bin/bash
-################################################################################
-#    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
-#
-#    Licensed under the Apache License, Version 2.0 (the "License");
-#    you may not use this file except in compliance with the License.
-#    You may obtain a copy of the License at
-#
-#       http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS,
-#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#    See the License for the specific language governing permissions and
-#    limitations under the License.
-################################################################################
-
-ip=$1
-master=$2
-masterport=$3
-logpth=$4
-instancedir=$5
-deploydir=$6
-hpcc_compname=$7
-hpcc_setenv=$8
-logredirect=$9
-
-source ${hpcc_setenv}
-
-if [ $# -lt 9 ]; then
-  echo usage: ip master masterport logdir workingdir deploydir hpcc_compname hpcc_setenv logredirect
-  exit 1
-fi
-
-# insuring parent directory structure is setup properly
-sudo /etc/init.d/dafilesrv status &>/dev/null
-if [ $? -ne 0 ];then
-  sudo /etc/init.d/dafilesrv start &>/dev/null
-  if [ $? -ne 0 ];then
-    exit 1
-  fi
-fi
-
-mkdir -p $instancedir
-mkdir -p `dirname $logredirect`
-exec >>$logredirect 2>&1
-
-cd $instancedir
-
-echo "`date` Dependency dafilesrv is running"
-echo "`date` slave(${ip}) init"
-
-lckfile="$PID/start_slaves_${hpcc_compname}_${ip}.pid"
-
-# prevent two of these scripts starting together
-while [ -e $lckfile ]; do
-  echo "`date` waiting on lckfile: ${lckfile}"
-  oldpid=`cat $lckfile`
-  if ps h $oldpid; then
-     echo "`date` killing pid ${oldpid} start_slaves"
-     kill -9 $oldpid
-     rm $lckfile                   # just in case
-     sleep 1
-  else
-     rm -f $lckfile
-  fi
-done
-trap "rm -f $lckfile" exit
-echo $$ > $lckfile
-
-ulimit -c unlimited > /dev/null 2>&1
-[[ $? -ne 0 ]] && echo "`date` Failed to set ulimit for core file size"
-ulimit -n 32768 > /dev/null 2>&1
-[[ $? -ne 0 ]] && echo "`date` Failed to set ulimit for number of file descriptors open"
-
-echo "`date` slave(s) starting"
-
-# create symlink for easier identification of slaves by compName
-ln -s -f $deploydir/thorslave_lcr thorslave_${hpcc_compname}
-
-# sync to current master thorgroup
-echo "`date` rsync -e ssh -o StrictHostKeyChecking=no ${master}:${instancedir}/thorgroup ${instancedir}/thorgroup.slave"
-rsync -e "ssh -o StrictHostKeyChecking=no" $master:$instancedir/thorgroup $instancedir/thorgroup.slave
-
-let "slavenum = 1";
-for slave in $(cat $instancedir/thorgroup.slave); do
-  slaveip=${slave/:*/}
-  if [ ${slaveip} = ${ip} ]; then
-    slaveport=${slave/*:/}
-    if [ "$slaveport" = "" ]; then
-      slaveport=$THORSLAVEPORT
-    fi
-    echo "`date` thorslave_$hpcc_compname  master=$master:$masterport slave=.:$slaveport slavenum=$slavenum logDir=$logpth"
-    ./thorslave_$hpcc_compname master=$master:$masterport slave=.:$slaveport slavenum=$slavenum logDir=$logpth 2>/dev/null 1>/dev/null &
-    slavepid=$!
-    PID_NAME="$PID/${hpcc_compname}_slave_${slavenum}.pid"
-    echo $slavepid > $PID_NAME
-    if [ "$slavepid" -eq "0" ]; then
-      echo "`date` failed to start"
-    else
-      echo "`date` slave pid $slavepid started"
-    fi
-  fi
-  let "slavenum += 1";
-done

initfiles/componentfiles/thor/start_thor

@@ -1,55 +0,0 @@
-#!/bin/bash
-################################################################################
-#    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
-#
-#    Licensed under the Apache License, Version 2.0 (the "License");
-#    you may not use this file except in compliance with the License.
-#    You may obtain a copy of the License at
-#
-#       http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS,
-#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#    See the License for the specific language governing permissions and
-#    limitations under the License.
-################################################################################
-
-if [ $# -lt 1 ]; then
-        echo usage: $0 thordeploydir 
-        exit 1
-fi
-
-shift
-
-export PATH_PRE=`type -path hpcc_setenv`
-source ${PATH_PRE} ""
-export PID_NAME="$PID/`basename $PWD`_master.pid" ## this needed when we use bash_init_system
-export PID_NAME_BASE="$PID/`basename $PWD`.pid"
-
-# deploydir = where binaries and scripts live
-if [ -z "$deploydir" ]; then
-export deploydir=$(pwd -P)
-fi
-# instancedir = [cwd] - on legacy system would be same as deploydir
-export instancedir=$(pwd -P)
-
-. $instancedir/setvars
-
-$deploydir/stop_thor $deploydir
-
-if [ ! -z ${THORPRIMARY} ]; then
-    export groupName=${THORPRIMARY}
-else
-    export groupName=${THORNAME}
-fi
-ln -s -f $deploydir/thormaster${LCR} thormaster_$THORNAME
-
-ENV_DIR=`cat ${HPCC_CONFIG} | sed -n "/\[DEFAULT\]/,/\[/p" | grep "^configs=" | sed -e 's/^configs=//'`
-export logdir=`updtdalienv $ENV_DIR/environment.xml -d log thor $THORNAME`
-if [ -z "$logdir" ]; then
-export logdir="./start_logs"
-fi
-mkdir -p $logdir
-export logpthtail="`date +%m_%d_%Y_%H_%M_%S`"
-$deploydir/run_thor >> $logdir/start_thor_$logpthtail.log 2>&1

initfiles/componentfiles/thor/stop_slaves

@@ -1,29 +0,0 @@
-#!/bin/bash
-################################################################################
-#    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
-#
-#    Licensed under the Apache License, Version 2.0 (the "License");
-#    you may not use this file except in compliance with the License.
-#    You may obtain a copy of the License at
-#
-#       http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS,
-#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#    See the License for the specific language governing permissions and
-#    limitations under the License.
-################################################################################
-
-compName=$1
-hpcc_setenv=$2
-
-source ${hpcc_setenv}
-
-slavename=thorslave_${compName}
-
-killall -0 $slavename &> /dev/null
-if [ $? -eq 0 ];then
-  killall -9 $slavename
-fi
-rm -f $PID/${compName}_slave_*.pid

initfiles/componentfiles/thor/stop_thor

@@ -1,70 +0,0 @@
-#!/bin/bash
-################################################################################
-#    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
-#
-#    Licensed under the Apache License, Version 2.0 (the "License");
-#    you may not use this file except in compliance with the License.
-#    You may obtain a copy of the License at
-#
-#       http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS,
-#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#    See the License for the specific language governing permissions and
-#    limitations under the License.
-################################################################################
-
-PATH_PRE=`type -path hpcc_setenv`
-INSTALL_DIR=`dirname ${PATH_PRE}`/..
-source  ${INSTALL_DIR}/etc/init.d/hpcc_common
-
-source ${PATH_PRE} ""
-comp_base=`basename $PWD`
-MASTER_PID_NAME="$PID/${comp_base}_master.pid" ## this needed when we use bash_init_system
-
-which_pidof
-
-if [ $# -lt 1 ]; then
-    echo usage: $0 thordeploydir 
-    exit 1
-fi
-
-export SENTINEL="thor.sentinel"
-
-# deploydir = where binaries and scripts live
-if [ -z "$deploydir" ]; then
-export deploydir=$(pwd -P)
-fi
-# instancedir = [cwd] - on legacy system would be same as deploydir
-export instancedir=$(pwd -P)
-
-. $instancedir/setvars
-
-if [ "$#" -lt "2" ] || [ "$2" != "keep_sentinel" ]; then
-    echo removing sentinel file
-    rm -f $instancedir/${SENTINEL}
-    sleep 1
-fi
-
-kill_process ${MASTER_PID_NAME} thormaster_${comp_base} 8
-if [ $? -eq 1 ]; then
-    echo "unable to kill ${THORNAME}, in zombie state"
-fi
-
-echo --------------------------
-echo stopping thor slaves 
-
-if [ "$localthor" = "true" ]; then
-  SLAVE_PID_NAME="$PID/${comp_base}_slave_*.pid"
-  slavepid=`cat ${SLAVE_PID_NAME} 2> /dev/null`
-  kill -9 $slavepid 2> /dev/null
-else
-  # we want to kill only slaves that have already been started in run_thor
-  if [ -r $instancedir/uslaves.start ]; then
-    nslaves=`cat $instancedir/uslaves.start | wc -l`
-    $deploydir/frunssh $instancedir/uslaves.start "/bin/sh -c '$deploydir/stop_slaves ${THORNAME} ${PATH_PRE}'" -i:$SSHidentityfile -u:$SSHusername -pe:$SSHpassword -t:$SSHtimeout -a:$SSHretries -n:$nslaves 2>&1 | egrep -v "no process killed"
-    echo slaves stopped
-    rm -rf $instancedir/uslaves.start
-  fi
-fi

roxie/ccd/ccdcontext.cpp

@@ -2590,7 +2590,7 @@ protected:
             client = NULL;
         updateSuppliedXmlParams(wu);
         SCMStringBuffer wuParams;
-        if (workUnit->getXmlParams(wuParams).length())
+        if (workUnit->getXmlParams(wuParams, false).length())
         {
             // Merge in params from WU. Ones on command line take precedence though...
             Owned<IPropertyTree> wuParamTree = createPTreeFromXMLString(wuParams.str(), ipt_caseInsensitive);

roxie/ccd/ccdlistener.cpp

@@ -58,76 +58,6 @@ static void controlException(StringBuffer &response, IException *E, const IRoxie
 
 //================================================================================================================
 
-static void sendSoapException(SafeSocket &client, IException *E, const char *queryName)
-{
-    try
-    {
-        if (!queryName)
-            queryName = "Unknown"; // Exceptions when parsing query XML can leave queryName unset/unknowable....
-
-        StringBuffer response;
-        response.append("<").append(queryName).append("Response");
-        response.append(" xmlns=\"urn:hpccsystems:ecl:").appendLower(strlen(queryName), queryName).append("\">");
-        response.appendf("<Results><Result><Exception><Source>Roxie</Source><Code>%d</Code>", E->errorCode());
-        response.append("<Message>");
-        StringBuffer s;
-        E->errorMessage(s);
-        encodeXML(s.str(), response);
-        response.append("</Message></Exception></Result></Results>");
-        response.append("</").append(queryName).append("Response>");
-        client.write(response.str(), response.length());
-    }
-    catch(IException *EE)
-    {
-        StringBuffer error("While reporting exception: ");
-        EE->errorMessage(error);
-        DBGLOG("%s", error.str());
-        EE->Release();
-    }
-#ifndef _DEBUG
-    catch(...) {}
-#endif
-}
-
-static void sendJsonException(SafeSocket &client, IException *E, const char *queryName)
-{
-    try
-    {
-        if (!queryName)
-            queryName = "Unknown"; // Exceptions when parsing query XML can leave queryName unset/unknowable....
-
-        StringBuffer response;
-        appendfJSONName(response, "%sResponse", queryName).append(" {");
-        appendJSONName(response, "Results").append(" {");
-        appendJSONName(response, "Exception").append(" [{");
-        appendJSONValue(response, "Source", "Roxie");
-        appendJSONValue(response, "Code", E->errorCode());
-        StringBuffer s;
-        appendJSONValue(response, "Message", E->errorMessage(s).str());
-        response.append("}]}}");
-        client.write(response.str(), response.length());
-    }
-    catch(IException *EE)
-    {
-        StringBuffer error("While reporting exception: ");
-        DBGLOG("%s", EE->errorMessage(error).str());
-        EE->Release();
-    }
-#ifndef _DEBUG
-    catch(...) {}
-#endif
-}
-
-static void sendHttpException(SafeSocket &client, TextMarkupFormat fmt, IException *E, const char *queryName)
-{
-    if (fmt==MarkupFmt_JSON)
-        sendJsonException(client, E, queryName);
-    else
-        sendSoapException(client, E, queryName);
-}
-
-//================================================================================================================
-
 class CHttpRequestAsyncFor : public CInterface, public CAsyncFor
 {
 private:
@@ -161,7 +91,7 @@ public:
         StringBuffer error("EXCEPTION: ");
         E->errorMessage(error);
         DBGLOG("%s", error.str());
-        sendHttpException(client, httpHelper.queryContentFormat(), E, queryName);
+        client.checkSendHttpException(httpHelper, E, queryName);
         E->Release();
     }
 
@@ -1467,9 +1397,14 @@ private:
                     else
                         throw MakeStringException(ROXIE_DATA_ERROR, "Malformed JSON request");
                 }
-                else if (strieq(queryName, "envelope"))
+                else
                 {
-                    queryXML.setown(queryXML->getPropTree("Body/*"));
+                    if (strieq(queryName, "envelope"))
+                        queryXML.setown(queryXML->getPropTree("Body/*"));
+                    else if (!strnicmp(httpHelper.queryContentType(), "application/soap", strlen("application/soap")))
+                        throw MakeStringException(ROXIE_DATA_ERROR, "Malformed SOAP request");
+                    else
+                        httpHelper.setUseEnvelope(false);
                     if (!queryXML)
                         throw MakeStringException(ROXIE_DATA_ERROR, "Malformed SOAP request (missing Body)");
                     String reqName(queryXML->queryName());
@@ -1494,8 +1429,6 @@ private:
 
                     queryXML->renameProp("/", queryName.get());  // reset the name of the tree
                 }
-                else
-                    throw MakeStringException(ROXIE_DATA_ERROR, "Malformed SOAP request");
             }
 
             // convert to XML with attribute values in single quotes - makes replaying queries easier
@@ -1528,7 +1461,7 @@ private:
 readAnother:
         Owned<IDebuggerContext> debuggerContext;
         unsigned slavesReplyLen = 0;
-        HttpHelper httpHelper;
+        HttpHelper httpHelper(&allQuerySetNames);
         try
         {
             if (client)
@@ -1764,7 +1697,7 @@ readAnother:
                         StringBuffer querySetName;
                         if (isHTTP)
                         {
-                            client->setHttpMode(queryName, isRequestArray, httpHelper.queryContentFormat());
+                            client->setHttpMode(queryName, isRequestArray, httpHelper);
                             querySetName.set(httpHelper.queryTarget());
                         }
                         queryFactory.setown(globalPackageSetManager->getQuery(queryName, &querySetName, NULL, logctx));
@@ -1917,7 +1850,7 @@ readAnother:
             if (client)
             {
                 if (isHTTP)
-                    sendHttpException(*client, httpHelper.queryContentFormat(), E, queryName);
+                    client->checkSendHttpException(httpHelper, E, queryName);
                 else
                     client->sendException("Roxie", code, error.str(), isBlocked, logctx);
             }
@@ -1942,7 +1875,7 @@ readAnother:
             if (client)
             {
                 if (isHTTP)
-                    sendHttpException(*client, httpHelper.queryContentFormat(), E, queryName);
+                    client->checkSendHttpException(httpHelper, E, queryName);
                 else
                     client->sendException("Roxie", code, error.str(), isBlocked, logctx);
             }
@@ -1962,7 +1895,7 @@ readAnother:
                 if (isHTTP)
                 {
                     Owned<IException> E = MakeStringException(ROXIE_INTERNAL_ERROR, "Unknown exception");
-                    sendHttpException(*client, httpHelper.queryContentFormat(), E, queryName);
+                    client->checkSendHttpException(httpHelper, E, queryName);
                 }
                 else
                     client->sendException("Roxie", ROXIE_INTERNAL_ERROR, "Unknown exception", isBlocked, logctx);

rtl/eclrtl/rtlfield.cpp

@@ -1460,6 +1460,11 @@ RtlFieldStrInfo::RtlFieldStrInfo(const char * _name, const char * _xpath, const
 }
 
 
+RtlFieldStrInfo::RtlFieldStrInfo(const char * _name, const char * _xpath, const RtlTypeInfo * _type)
+: RtlFieldInfo(rtlCreateFieldNameAtom(_name), _xpath, _type, NULL)
+{
+}
+
 
 
 /* 

rtl/eclrtl/rtlfield_imp.hpp

@@ -315,7 +315,8 @@ public:
 
 struct ECLRTL_API RtlFieldStrInfo : public RtlFieldInfo
 {
-    RtlFieldStrInfo(const char * _name, const char * _xpath, const RtlTypeInfo * _type, const char * _initializer = NULL);
+    RtlFieldStrInfo(const char * _name, const char * _xpath, const RtlTypeInfo * _type);
+    RtlFieldStrInfo(const char * _name, const char * _xpath, const RtlTypeInfo * _type, const char * _initializer);
 };
 
 

system/jlib/jptree.cpp

@@ -7126,3 +7126,97 @@ IPropertyTree *createPTreeFromJSONString(unsigned len, const char *json, byte fl
     reader->load();
     return LINK(iMaker->queryRoot());
 }
+
+
+static const char * nextHttpParameterTag(StringBuffer &tag, const char *path)
+{
+    while (*path=='.')
+        path++;
+    const char *finger = strchr(path, '.');
+    if (finger)
+    {
+        tag.clear().append(finger - path, path);
+        finger++;
+    }
+    else
+        tag.set(path);
+    return finger;
+}
+
+static void ensureHttpParameter(IPropertyTree *pt, StringBuffer &tag, const char *path, const char *value, const char *fullpath)
+{
+    if (!tag.length())
+        return;
+
+    unsigned idx = 1;
+    if (path && isdigit(*path))
+    {
+        StringBuffer pos;
+        path = nextHttpParameterTag(pos, path);
+        idx = (unsigned) atoi(pos.str())+1;
+    }
+
+    if (tag.charAt(tag.length()-1)=='$')
+    {
+        if (path && *path)
+            throw MakeStringException(-1, "'$' not allowed in parent node of parameter path: %s", fullpath);
+        tag.setLength(tag.length()-1);
+        StringArray values;
+        values.appendList(value, "\r");
+        ForEachItemIn(pos, values)
+        {
+            const char *itemValue = values.item(pos);
+            while (*itemValue=='\n')
+                itemValue++;
+            pt->addProp(tag, itemValue);
+        }
+        return;
+    }
+    unsigned count = pt->getCount(tag);
+    while (count++ < idx)
+        pt->addPropTree(tag, createPTree(tag));
+    StringBuffer xpath(tag);
+    xpath.append('[').append(idx).append(']');
+    pt = pt->queryPropTree(xpath);
+
+    if (!path || !*path)
+    {
+        pt->setProp(NULL, value);
+        return;
+    }
+
+    StringBuffer nextTag;
+    path = nextHttpParameterTag(nextTag, path);
+    ensureHttpParameter(pt, nextTag, path, value, fullpath);
+}
+
+static void ensureHttpParameter(IPropertyTree *pt, const char *path, const char *value)
+{
+    const char *fullpath = path;
+    StringBuffer tag;
+    path = nextHttpParameterTag(tag, path);
+    ensureHttpParameter(pt, tag, path, value, fullpath);
+}
+
+IPropertyTree *createPTreeFromHttpParameters(const char *name, IProperties *parameters, bool skipLeadingDotParameters, bool nestedRoot, ipt_flags flags)
+{
+    Owned<IPropertyTree> pt = createPTree(name, flags);
+
+    Owned<IPropertyIterator> props = parameters->getIterator();
+    ForEach(*props)
+    {
+        const char *key = props->getPropKey();
+        const char *value = parameters->queryProp(key);
+        if (skipLeadingDotParameters && key && *key=='.')
+            continue;
+        ensureHttpParameter(pt, key, value);
+    }
+    if (nestedRoot)
+    {
+        Owned<IPropertyTree> root = createPTree(flags);
+        root->setPropTree(name, pt.getClear());
+        return root.getClear();
+    }
+
+    return pt.getClear();
+}

system/jlib/jptree.hpp

@@ -22,6 +22,7 @@
 #include "jlib.hpp"
 #include "jexcept.hpp"
 #include "jiter.hpp"
+#include "jprop.hpp"
 
 enum TextMarkupFormat
 {
@@ -206,6 +207,8 @@ jlib_decl IPropertyTree *createPTreeFromIPT(const IPropertyTree *srcTree, ipt_fl
 
 jlib_decl IPropertyTree *createPTreeFromJSONString(const char *json, byte flags=ipt_none, PTreeReaderOptions readFlags=ptr_ignoreWhiteSpace, IPTreeMaker *iMaker=NULL);
 jlib_decl IPropertyTree *createPTreeFromJSONString(unsigned len, const char *json, byte flags=ipt_none, PTreeReaderOptions readFlags=ptr_ignoreWhiteSpace, IPTreeMaker *iMaker=NULL);
+jlib_decl IPropertyTree *createPTreeFromHttpParameters(const char *name, IProperties *parameters, bool skipLeadingDotParameters, bool nestedRoot, ipt_flags flags=ipt_none);
+
 
 #define XML_SortTags 0x01
 #define XML_Embed    0x02

testing/regress/ecl/formatstored.ecl

@@ -36,6 +36,8 @@ integer4 i4 := 0 : stored('i4', format(fieldwidth(4), sequence(4)));
 unsigned3 u3 := 0 : stored('u3', format(fieldwidth(3), sequence(13)));
 unsigned8 u8 := 0 : stored('u8', format(fieldwidth(8), sequence(18)));
 
+string pw := 'powow' : stored('pw', format(password, fieldwidth(40)));
+
 output (i1, named('i1'));
 output (i2, named('i2'));
 output (i3, named('i3'));
@@ -56,3 +58,4 @@ output (u8, named('u8'));
 
 output (s1, named('s1'));
 
+output (pw, named('showpw'));

testing/regress/ecl/key/formatstored.xml

@@ -49,3 +49,6 @@
 <Dataset name='s1'>
  <Row><s1>how now brown cow</s1></Row>
 </Dataset>
+<Dataset name='showpw'>
+ <Row><showpw>powow</showpw></Row>
+</Dataset>

thorlcr/graph/thgraphmaster.cpp

@@ -969,7 +969,7 @@ public:
     {
         PROTECTED_GETRESULT(stepname, sequence, "Data", "data",
             SCMStringBuffer result;
-            r->getResultString(result);
+            r->getResultString(result, false);
             tlen = result.length();
             tgt = (char *)result.s.detach();
         );
@@ -1017,7 +1017,7 @@ public:
     {
         PROTECTED_GETRESULT(stepname, sequence, "String", "string",
             SCMStringBuffer result;
-            r->getResultString(result);
+            r->getResultString(result, false);
             tlen = result.length();
             tgt = (char *)result.s.detach();
         );
@@ -1026,7 +1026,7 @@ public:
     {
         PROTECTED_GETRESULT(stepname, sequence, "String", "string",
             SCMStringBuffer result;
-            r->getResultString(result);
+            r->getResultString(result, false);
             rtlStrToStr(tlen, tgt, result.length(), result.s.str());
         );
     }
@@ -1044,7 +1044,7 @@ public:
     { 
         PROTECTED_GETRESULT(stepname, sequence, "VarString", "string",
             SCMStringBuffer result;
-            r->getResultString(result);
+            r->getResultString(result, false);
             return result.s.detach();
         );
     }

tools/wuget/wuget.cpp

@@ -57,7 +57,7 @@ int main(int argc, char **argv)
                 if (getWorkunitXMLFromFile(argv[i], xml))
                 {
                     Owned<ILocalWorkUnit> wu = createLocalWorkUnit(xml);
-                    exportWorkUnitToXML(wu, xml.clear(), true, false);
+                    exportWorkUnitToXML(wu, xml.clear(), true, false, true);
                     printf("%s\n", xml.str());
                 }
                 else