HPCC-13472 Add new Stored Format(password) option to hide passwords

Signed-off-by: Anthony Fishbeck <anthony.fishbeck@lexisnexis.com>

common/workunit/workunit.cpp

@@ -1084,8 +1084,8 @@ template <>  struct CachedTags<CLocalWUAppValue, IConstWUAppValue>
 
 class CLocalWorkUnit : public CInterface, implements IConstWorkUnit , implements ISDSSubscription, implements IExtendedWUInterface
 {
-    friend StringBuffer &exportWorkUnitToXML(const IConstWorkUnit *wu, StringBuffer &str, bool decodeGraphs, bool includeProgress);
-    friend void exportWorkUnitToXMLFile(const IConstWorkUnit *wu, const char * filename, unsigned extraXmlFlags, bool decodeGraphs, bool includeProgress);
+    friend StringBuffer &exportWorkUnitToXML(const IConstWorkUnit *wu, StringBuffer &str, bool decodeGraphs, bool includeProgress, bool hidePasswords);
+    friend void exportWorkUnitToXMLFile(const IConstWorkUnit *wu, const char * filename, unsigned extraXmlFlags, bool decodeGraphs, bool includeProgress, bool hidePasswords);
 
     // NOTE - order is important - we need to construct connection before p and (especially) destruct after p
     Owned<IRemoteConnection> connection;
@@ -1201,7 +1201,7 @@ public:
     virtual IConstWUStatistic * getStatistic(const char * creator, const char * scope, StatisticKind kind) const;
     virtual IConstWUWebServicesInfo * getWebServicesInfo() const;
     virtual IConstWURoxieQueryInfo * getRoxieQueryInfo() const;
-    virtual IStringVal & getXmlParams(IStringVal & params) const;
+    virtual IStringVal & getXmlParams(IStringVal & params, bool hidePasswords) const;
     virtual const IPropertyTree *getXmlParams() const;
     virtual unsigned __int64 getHash() const;
     virtual IStringIterator *getLogs(const char *type, const char *component) const;
@@ -1632,8 +1632,8 @@ public:
             { return c->getDebugAgentListenerPort(); }
     virtual IStringVal & getDebugAgentListenerIP(IStringVal &ip) const
             { return c->getDebugAgentListenerIP(ip); }
-    virtual IStringVal & getXmlParams(IStringVal & params) const
-            { return c->getXmlParams(params); }
+    virtual IStringVal & getXmlParams(IStringVal & params, bool hidePasswords) const
+            { return c->getXmlParams(params, hidePasswords); }
     virtual const IPropertyTree *getXmlParams() const
             { return c->getXmlParams(); }
     virtual unsigned __int64 getHash() const
@@ -2069,7 +2069,7 @@ public:
     virtual IStringVal& getResultName(IStringVal &str) const;
     virtual int         getResultSequence() const;
     virtual bool        isResultScalar() const;
-    virtual IStringVal& getResultXml(IStringVal &str) const;
+    virtual IStringVal& getResultXml(IStringVal &str, bool hidePasswords) const;
     virtual unsigned    getResultFetchSize() const;
     virtual __int64     getResultTotalRowCount() const;
     virtual __int64     getResultRowCount() const;
@@ -2081,7 +2081,7 @@ public:
     virtual __int64     getResultInt() const;
     virtual bool        getResultBool() const;
     virtual double      getResultReal() const;
-    virtual IStringVal& getResultString(IStringVal & str) const;
+    virtual IStringVal& getResultString(IStringVal & str, bool hidePassword) const;
     virtual IDataVal&   getResultRaw(IDataVal & data, IXmlToRawTransformer * xmlTransformer, ICsvToRawTransformer * csvTransformer) const;
     virtual IDataVal&   getResultUnicode(IDataVal & data) const;
     virtual void        getResultDecimal(void * val, unsigned length, unsigned precision, bool isSigned) const;
@@ -3666,7 +3666,7 @@ bool CLocalWorkUnit::archiveWorkUnit(const char *base,bool del,bool ignoredllerr
         return false;
 
     StringBuffer buf;
-    exportWorkUnitToXML(this, buf, false, false);
+    exportWorkUnitToXML(this, buf, false, false, false);
 
     StringBuffer extraWorkUnitXML;
     StringBuffer xpath("/GraphProgress/");
@@ -3974,7 +3974,7 @@ void CLocalWorkUnit::serialize(MemoryBuffer &tgt)
 {
     CriticalBlock block(crit);
     StringBuffer x;
-    tgt.append(exportWorkUnitToXML(this, x, false, false).str());
+    tgt.append(exportWorkUnitToXML(this, x, false, false, false).str());
 }
 
 void CLocalWorkUnit::deserialize(MemoryBuffer &src)
@@ -7028,16 +7028,55 @@ unsigned CLocalWorkUnit::getApplicationValueCount() const
     
 }
 
-IStringVal &CLocalWorkUnit::getXmlParams(IStringVal &str) const
+StringBuffer &appendPTreeOpenTag(StringBuffer &s, IPropertyTree *tree, const char *name, unsigned indent)
+{
+    appendXMLOpenTag(s, name, NULL, false);
+    Owned<IAttributeIterator> attrs = tree->getAttributes(true);
+    if (attrs->first())
+    {
+        unsigned attributeindent = indent + (size32_t) strlen(name);
+        unsigned count = attrs->count();
+        bool doindent = false;
+        ForEach(*attrs)
+        {
+            if (doindent)
+                s.append('\n').appendN(attributeindent, ' ');
+            else if (count > 3)
+                doindent = true;
+            appendXMLAttr(s, attrs->queryName()+1, attrs->queryValue());
+        }
+    }
+    s.append('>');
+    return s;
+}
+
+IStringVal &CLocalWorkUnit::getXmlParams(IStringVal &str, bool hidePasswords) const
 {
     CriticalBlock block(crit);
     IPropertyTree *paramTree = p->queryPropTree("Parameters");
-    if (paramTree)
+    if (!paramTree)
+        return str;
+
+    StringBuffer xml;
+    if (!hidePasswords)
+        toXML(paramTree, xml);
+    else
     {
-        StringBuffer temp;
-        toXML(paramTree, temp);
-        str.set(temp.str());
+        appendPTreeOpenTag(xml.append(' '), paramTree, "Parameters", 0).append('\n');
+
+        Owned<IPropertyTreeIterator> elems = paramTree->getElements("*");
+        ForEach(*elems)
+        {
+            const char *paramname = elems->query().queryName();
+            VStringBuffer xpath("Variables/Variable[@name='%s']/Format/@password", paramname);
+            if (p->getPropBool(xpath))
+                appendXMLTag(xml.append("  "), paramname, "***").append('\n');
+            else
+                toXML(&elems->query(), xml, 2);
+        }
+        appendXMLCloseTag(xml.append(' '), "Parameters").append('\n');
     }
+    str.set(xml);
     return str;
 }
 
@@ -8189,7 +8228,7 @@ void readRow(StringBuffer &out, MemoryBuffer &in, TypeInfoArray &types, StringAt
     }
 }
 
-IStringVal& CLocalWUResult::getResultXml(IStringVal &str) const
+IStringVal& CLocalWUResult::getResultXml(IStringVal &str, bool hidePassword) const
 {
     TypeInfoArray types;
     StringAttrArray names;
@@ -8202,7 +8241,13 @@ IStringVal& CLocalWUResult::getResultXml(IStringVal &str) const
     else
         xml.append("<Dataset>\n");
 
-    if (p->hasProp("Value"))
+    if (hidePassword && p->getPropBool("Format/@password"))
+    {
+        xml.append(" <Row>");
+        appendXMLTag(xml, name, "****");
+        xml.append("</Row>\n");
+    }
+    else if (p->hasProp("Value"))
     {
         MemoryBuffer raw;
         p->getPropBin("Value", raw);
@@ -8562,8 +8607,13 @@ void CLocalWUResult::getResultDecimal(void * val, unsigned len, unsigned precisi
     }
 }
 
-IStringVal& CLocalWUResult::getResultString(IStringVal & str) const
+IStringVal& CLocalWUResult::getResultString(IStringVal & str, bool hidePassword) const
 {
+    if (hidePassword && p->getPropBool("@password"))
+    {
+        str.set("****");
+        return str;
+    }
     MemoryBuffer s;
     p->getPropBin("Value", s);
     if (s.length())
@@ -9203,7 +9253,84 @@ extern WORKUNIT_API ILocalWorkUnit * createLocalWorkUnit()
     return ret;
 }
 
-extern WORKUNIT_API StringBuffer &exportWorkUnitToXML(const IConstWorkUnit *wu, StringBuffer &str, bool unpack, bool includeProgress)
+void exportWorkUnitToXMLWithHiddenPasswords(IPropertyTree *p, IIOStream &out, unsigned extraXmlFlags)
+{
+    const char *name = p->queryName();
+    if (!name)
+        name = "__unnamed__";
+    StringBuffer temp;
+    writeStringToStream(out, appendPTreeOpenTag(temp, p, name, 1));
+
+    Owned<IPropertyTreeIterator> elems = p->getElements("*", iptiter_sort);
+    ForEach(*elems)
+    {
+        IPropertyTree &elem = elems->query();
+        if (streq(elem.queryName(), "Parameters"))
+        {
+            writeStringToStream(out, appendPTreeOpenTag(temp.clear().append(' '), &elem, "Parameters", 2).append('\n'));
+            Owned<IPropertyTreeIterator> params = elem.getElements("*", iptiter_sort);
+            ForEach(*params)
+            {
+                IPropertyTree &param = params->query();
+                const char *paramname = param.queryName();
+                VStringBuffer xpath("Variables/Variable[@name='%s']/Format/@password", paramname);
+                if (p->getPropBool(xpath))
+                    writeStringToStream(out, appendXMLTag(temp.clear().append("  "), paramname, "****").append('\n'));
+                else
+                {
+                    toXML(&param, out, 2, XML_Format|XML_SortTags|extraXmlFlags);
+                }
+            }
+            writeStringToStream(out, appendXMLCloseTag(temp.clear().append(' '), "Parameters").append('\n'));
+        }
+        else if (streq(elem.queryName(), "Variables"))
+        {
+            writeStringToStream(out, appendPTreeOpenTag(temp.clear().append(' '), &elem, "Variables", 2).append('\n'));
+            Owned<IPropertyTreeIterator> vars = elem.getElements("*", iptiter_sort);
+            ForEach(*vars)
+            {
+                Owned<IPropertyTree> var = LINK(&vars->query());
+                if (var->getPropBool("Format/@password"))
+                {
+                    var.setown(createPTreeFromIPT(var)); //copy and remove password values
+                    var->removeProp("Value");
+                    var->removeProp("xmlValue");
+                }
+                toXML(var, out, 2, XML_Format|XML_SortTags|extraXmlFlags);
+            }
+            writeStringToStream(out, appendXMLCloseTag(temp.clear().append(' '), "Variables").append('\n'));
+        }
+        else
+            toXML(&elem, out, 1, XML_Format|XML_SortTags|extraXmlFlags);
+    }
+    writeStringToStream(out, appendXMLCloseTag(temp.clear(), name));
+}
+
+StringBuffer &exportWorkUnitToXMLWithHiddenPasswords(IPropertyTree *p, StringBuffer &str)
+{
+    class CAdapter : public CInterface, implements IIOStream
+    {
+        StringBuffer &out;
+    public:
+        IMPLEMENT_IINTERFACE;
+        CAdapter(StringBuffer &_out) : out(_out) { }
+        virtual void flush() { }
+        virtual size32_t read(size32_t len, void * data) { UNIMPLEMENTED; return 0; }
+        virtual size32_t write(size32_t len, const void * data) { out.append(len, (const char *)data); return len; }
+    } adapter(str);
+    exportWorkUnitToXMLWithHiddenPasswords(p->queryBranch(NULL), adapter, 0);
+    return str;
+}
+
+void exportWorkUnitToXMLFileWithHiddenPasswords(IPropertyTree *p, const char *filename, unsigned extraXmlFlags)
+{
+    OwnedIFile ifile = createIFile(filename);
+    OwnedIFileIO ifileio = ifile->open(IFOcreate);
+    Owned<IIOStream> stream = createIOStream(ifileio);
+    exportWorkUnitToXMLWithHiddenPasswords(p->queryBranch(NULL), *stream, extraXmlFlags);
+}
+
+extern WORKUNIT_API StringBuffer &exportWorkUnitToXML(const IConstWorkUnit *wu, StringBuffer &str, bool unpack, bool includeProgress, bool hidePasswords)
 {
     const CLocalWorkUnit *w = QUERYINTERFACE(wu, const CLocalWorkUnit);
     if (!w)
@@ -9219,6 +9346,8 @@ extern WORKUNIT_API StringBuffer &exportWorkUnitToXML(const IConstWorkUnit *wu,
             p.setown(w->getUnpackedTree(includeProgress));
         else
             p.set(w->p);
+        if (hidePasswords && p->hasProp("Variables/Variable[Format/@password]"))
+            return exportWorkUnitToXMLWithHiddenPasswords(p, str);
         toXML(p, str, 0, XML_Format|XML_SortTags);
     }
     else
@@ -9226,14 +9355,14 @@ extern WORKUNIT_API StringBuffer &exportWorkUnitToXML(const IConstWorkUnit *wu,
     return str;
 }
 
-extern WORKUNIT_API IStringVal& exportWorkUnitToXML(const IConstWorkUnit *wu, IStringVal &str, bool unpack, bool includeProgress)
+extern WORKUNIT_API IStringVal& exportWorkUnitToXML(const IConstWorkUnit *wu, IStringVal &str, bool unpack, bool includeProgress, bool hidePasswords)
 {
     StringBuffer x;
-    str.set(exportWorkUnitToXML(wu,x,unpack, includeProgress).str());
+    str.set(exportWorkUnitToXML(wu,x,unpack, includeProgress, hidePasswords).str());
     return str;
 }
 
-extern WORKUNIT_API void exportWorkUnitToXMLFile(const IConstWorkUnit *wu, const char * filename, unsigned extraXmlFlags, bool unpack, bool includeProgress)
+extern WORKUNIT_API void exportWorkUnitToXMLFile(const IConstWorkUnit *wu, const char * filename, unsigned extraXmlFlags, bool unpack, bool includeProgress, bool hidePasswords)
 {
     const CLocalWorkUnit *w = QUERYINTERFACE(wu, const CLocalWorkUnit);
     if (!w)
@@ -9249,6 +9378,8 @@ extern WORKUNIT_API void exportWorkUnitToXMLFile(const IConstWorkUnit *wu, const
             p.setown(w->getUnpackedTree(includeProgress));
         else
             p.set(w->p);
+        if (hidePasswords && p->hasProp("Variables/Variable[Format/@password]"))
+            return exportWorkUnitToXMLFileWithHiddenPasswords(p, filename, extraXmlFlags);
         saveXML(filename, p, 0, XML_Format|XML_SortTags|extraXmlFlags);
     }
 }

common/workunit/workunit.hpp

@@ -275,7 +275,7 @@ interface IConstWUResult : extends IInterface
     virtual IStringVal & getResultName(IStringVal & str) const = 0;
     virtual int getResultSequence() const = 0;
     virtual bool isResultScalar() const = 0;
-    virtual IStringVal & getResultXml(IStringVal & str) const = 0;
+    virtual IStringVal & getResultXml(IStringVal & str, bool hidePasswords) const = 0;
     virtual unsigned getResultFetchSize() const = 0;
     virtual __int64 getResultTotalRowCount() const = 0;
     virtual __int64 getResultRowCount() const = 0;
@@ -286,7 +286,7 @@ interface IConstWUResult : extends IInterface
     virtual __int64 getResultInt() const = 0;
     virtual bool getResultBool() const = 0;
     virtual double getResultReal() const = 0;
-    virtual IStringVal & getResultString(IStringVal & str) const = 0;
+    virtual IStringVal & getResultString(IStringVal & str, bool hidePasswords) const = 0;
     virtual IDataVal & getResultRaw(IDataVal & data, IXmlToRawTransformer * xmlTransformer, ICsvToRawTransformer * csvTransformer) const = 0;
     virtual IDataVal & getResultUnicode(IDataVal & data) const = 0;
     virtual IStringVal & getResultEclSchema(IStringVal & str) const = 0;
@@ -1068,7 +1068,7 @@ interface IConstWorkUnit : extends IInterface
     virtual unsigned getApplicationValueCount() const = 0;
     virtual unsigned getDebugAgentListenerPort() const = 0;
     virtual IStringVal & getDebugAgentListenerIP(IStringVal & ip) const = 0;
-    virtual IStringVal & getXmlParams(IStringVal & params) const = 0;
+    virtual IStringVal & getXmlParams(IStringVal & params, bool hidePasswords) const = 0;
     virtual const IPropertyTree * getXmlParams() const = 0;
     virtual unsigned __int64 getHash() const = 0;
     virtual IStringIterator *getLogs(const char *type, const char *instance=NULL) const = 0;
@@ -1362,9 +1362,9 @@ extern WORKUNIT_API IWorkUnitFactory * getWorkUnitFactory();
 extern WORKUNIT_API IWorkUnitFactory * getSecWorkUnitFactory(ISecManager &secmgr, ISecUser &secuser);
 extern WORKUNIT_API IWorkUnitFactory * getWorkUnitFactory(ISecManager *secmgr, ISecUser *secuser);
 extern WORKUNIT_API ILocalWorkUnit* createLocalWorkUnit();
-extern WORKUNIT_API IStringVal& exportWorkUnitToXML(const IConstWorkUnit *wu, IStringVal &str, bool unpack, bool includeProgress);
-extern WORKUNIT_API StringBuffer &exportWorkUnitToXML(const IConstWorkUnit *wu, StringBuffer &str, bool unpack, bool includeProgress);
-extern WORKUNIT_API void exportWorkUnitToXMLFile(const IConstWorkUnit *wu, const char * filename, unsigned extraXmlFlags, bool unpack, bool includeProgress);
+extern WORKUNIT_API IStringVal& exportWorkUnitToXML(const IConstWorkUnit *wu, IStringVal &str, bool unpack, bool includeProgress, bool hidePasswords);
+extern WORKUNIT_API StringBuffer &exportWorkUnitToXML(const IConstWorkUnit *wu, StringBuffer &str, bool unpack, bool includeProgress, bool hidePasswords);
+extern WORKUNIT_API void exportWorkUnitToXMLFile(const IConstWorkUnit *wu, const char * filename, unsigned extraXmlFlags, bool unpack, bool includeProgress, bool hidePasswords);
 extern WORKUNIT_API void submitWorkUnit(const char *wuid, const char *username, const char *password);
 extern WORKUNIT_API void abortWorkUnit(const char *wuid);
 extern WORKUNIT_API void submitWorkUnit(const char *wuid, ISecManager *secmgr, ISecUser *secuser);

dali/daliadmin/daliadmin.cpp

@@ -2417,7 +2417,7 @@ static void dumpWorkunit(const char *wuid, bool includeProgress)
 {
     Owned<IWorkUnitFactory> factory = getWorkUnitFactory();
     Owned<IConstWorkUnit> workunit = factory->openWorkUnit(wuid, false);
-    exportWorkUnitToXMLFile(workunit, "stdout:", 0, true, includeProgress);
+    exportWorkUnitToXMLFile(workunit, "stdout:", 0, true, includeProgress, false);  //should we hide passwords here?
 }
 
 static void dumpProgress(const char *wuid, const char * graph)

ecl/eclagent/eclagent.cpp

@@ -751,7 +751,7 @@ void EclAgent::outputFormattedResult(const char * name, unsigned sequence, bool
     {
     case ofXML:
         {
-            res->getResultXml(buff);
+            res->getResultXml(buff, false);
             outputSerializer->fwrite(sequence, (const void*)buff.str(), 1, buff.length());
             break;
         }
@@ -905,7 +905,7 @@ char *EclAgent::getResultVarString(const char * stepname, unsigned sequence)
 {
     PROTECTED_GETRESULT(stepname, sequence, "VarString", "string",
         SCMStringBuffer result;
-        r->getResultString(result);
+        r->getResultString(result, false);
         return result.s.detach();
     );
 }
@@ -925,7 +925,7 @@ void EclAgent::getResultString(unsigned & tlen, char * & tgt, const char * stepn
 {
     PROTECTED_GETRESULT(stepname, sequence, "String", "string",
         SCMStringBuffer result;
-        r->getResultString(result);
+        r->getResultString(result, false);
         tlen = result.length();
         tgt = (char *)result.s.detach();
     );
@@ -936,7 +936,7 @@ void EclAgent::getResultStringF(unsigned tlen, char * tgt, const char * stepname
     PROTECTED_GETRESULT(stepname, sequence, "String", "string",
         //MORE: Could used a fixed size IStringVal implementation to save a memory allocation, but hardly worth it.
         SCMStringBuffer result;
-        r->getResultString(result);
+        r->getResultString(result, false);
         rtlStrToStr(tlen, tgt, result.length(), result.s.str());
     );
 }
@@ -945,7 +945,7 @@ void EclAgent::getResultData(unsigned & tlen, void * & tgt, const char * stepnam
 {
     PROTECTED_GETRESULT(stepname, sequence, "Data", "data",
         SCMStringBuffer result;
-        r->getResultString(result);
+        r->getResultString(result, false);
         tlen = result.length();
         tgt = (char *)result.s.detach();
     );

ecl/eclcc/eclcc.cpp

@@ -1659,7 +1659,7 @@ void EclCC::generateOutput(EclCompileInstance & instance)
         else
             xmlFilename.append(DEFAULT_OUTPUTNAME);
         xmlFilename.append(".xml");
-        exportWorkUnitToXMLFile(instance.wu, xmlFilename, 0, true, false);
+        exportWorkUnitToXMLFile(instance.wu, xmlFilename, 0, true, false, false);
     }
 }
 
@@ -2239,7 +2239,7 @@ void EclCC::processBatchedFile(IFile & file, bool multiThreaded)
             if (info.wu &&
                 (info.wu->getDebugValueBool("generatePartialOutputOnError", false) || info.queryErrorProcessor().errCount() == 0))
             {
-                exportWorkUnitToXMLFile(info.wu, xmlFilename, XML_NoBinaryEncode64, true, false);
+                exportWorkUnitToXMLFile(info.wu, xmlFilename, XML_NoBinaryEncode64, true, false, false);
                 Owned<IFile> xml = createIFile(xmlFilename);
                 info.stats.xmlSize = xml->size();
             }

ecl/hqlcpp/hqlecl.cpp

@@ -481,7 +481,7 @@ bool HqlDllGenerator::generateCode(HqlQueryContext & query)
 void HqlDllGenerator::addWorkUnitAsResource()
 {
     SCMStringBuffer wuXML;
-    exportWorkUnitToXML(wu, wuXML, false, false);
+    exportWorkUnitToXML(wu, wuXML, false, false, false);
     code->addCompressResource("WORKUNIT", wuXML.length(), wuXML.str(), NULL, 1000);
 }
 

ecl/wutest/wutest.cpp

@@ -59,7 +59,7 @@ bool dump(IConstWorkUnit &w, IProperties *globals)
         ForEach(*results)
         {
             SCMStringBuffer xml;
-            results->query().getResultXml(xml);
+            results->query().getResultXml(xml, true);
             printf("%s\n", xml.str());
             SCMStringBuffer schema;
             results->query().getResultEclSchema(schema);
@@ -69,7 +69,7 @@ bool dump(IConstWorkUnit &w, IProperties *globals)
     else if (stricmp(action, "dump")==0)
     {
         SCMStringBuffer xml;
-        exportWorkUnitToXML(&w, xml, true, false);
+        exportWorkUnitToXML(&w, xml, true, false, false);
         printf("%s\n", xml.str());
     }
     else if (stricmp(action, "temporaries")==0)

esp/services/ws_ecl/ws_ecl_service.cpp

@@ -1166,6 +1166,8 @@ void CWsEclBinding::SOAPSectionToXsd(WsEclWuInfo &wuinfo, IPropertyTree *parmTre
                 unsigned cols = part.getPropInt("@width");
                 if (cols)
                     schema.appendf(" formCols='%u'", cols);
+                if (part.hasProp("@password"))
+                    schema.appendf(" password='%s'", part.queryProp("@password"));
                 schema.appendf("/></xsd:appinfo></xsd:annotation></xsd:element>");
             }
             else

esp/services/ws_ecl/ws_ecl_wuinfo.cpp

@@ -61,9 +61,10 @@ void appendVariableParmInfo(IArrayOf<IPropertyTree> &parts, IResultSetFactory *r
     SCMStringBuffer eclschema;
     var.getResultEclSchema(eclschema);
 
-    StringBuffer width, height, fieldSeq;
+    StringBuffer width, height, fieldSeq, isPassword;
     var.getResultFieldOpt("fieldwidth", StringBufferAdaptor(width));
     var.getResultFieldOpt("fieldheight", StringBufferAdaptor(height));
+    var.getResultFieldOpt("password", StringBufferAdaptor(isPassword));
     if (hashWebserviceSeq)
         fieldSeq.append(hashWebserviceSeq);
     else
@@ -125,6 +126,8 @@ void appendVariableParmInfo(IArrayOf<IPropertyTree> &parts, IResultSetFactory *r
             part->setProp("@height", height);
         if (fieldSeq.length())
             part->setProp("@sequence", fieldSeq);
+        if (isPassword.length())
+            part->setProp("@password", isPassword);
     }
     parts.append(*part.getClear());
 }

esp/services/ws_workunits/ws_workunitsHelpers.cpp

@@ -1023,7 +1023,7 @@ void WsWuInfo::getInfo(IEspECLWorkunit &info, unsigned flags)
     info.setActionEx(cw->getActionEx(s).str());
     info.setDescription(cw->getDebugValue("description", s).str());
     if (version > 1.21)
-        info.setXmlParams(cw->getXmlParams(s).str());
+        info.setXmlParams(cw->getXmlParams(s, true).str());
 
     info.setResultLimit(cw->getResultLimit());
     info.setArchived(false);
@@ -1494,7 +1494,7 @@ void WsWuInfo::getResult(IConstWUResult &r, IArrayOf<IEspECLResult>& results, un
         try
         {
             SCMStringBuffer xml;
-            r.getResultXml(xml);
+            r.getResultXml(xml, true);
 
             Owned<IPropertyTree> props = createPTreeFromXMLString(xml.str(), ipt_caseInsensitive);
             IPropertyTree *val = props->queryPropTree("Row/*");
@@ -2145,7 +2145,7 @@ void WsWuInfo::getWorkunitXml(const char* plainText, MemoryBuffer& buf)
         header = "<?xml version=\"1.0\" encoding=\"UTF-8\"?><?xml-stylesheet href=\"../esp/xslt/xmlformatter.xsl\" type=\"text/xsl\"?>";
 
     SCMStringBuffer xml;
-    exportWorkUnitToXML(cw, xml, true, false);
+    exportWorkUnitToXML(cw, xml, true, false, true);
 
     buf.append(strlen(header), header);
     buf.append(xml.length(), xml.str());

esp/services/ws_workunits/ws_workunitsService.cpp

@@ -3103,7 +3103,7 @@ bool CWsWorkunitsEx::onWUExport(IEspContext &context, IEspWUExportRequest &req,
         {
             Owned<IConstWorkUnit> cw = factory->openWorkUnit(it->c_str(), false);
             if (cw)
-                exportWorkUnitToXML(cw, xml, true, false);
+                exportWorkUnitToXML(cw, xml, true, false, true);
         }
         xml.append("</Workunits>");
 

esp/smc/SMCLib/WUXMLInfo.cpp

@@ -258,7 +258,7 @@ bool CWUXMLInfo::buildXmlResultList(IConstWorkUnit &wu,IPropertyTree& XMLStructu
                 else if (r.isResultScalar())
                 {
                     SCMStringBuffer x;
-                    r.getResultXml(x);
+                    r.getResultXml(x, true);
                     try
                     {
                         Owned<IPropertyTree> props = createPTreeFromXMLString(x.str(), ipt_caseInsensitive);

esp/xslt/wsecl3_form.xsl

@@ -1130,7 +1130,7 @@ function switchInputForm()
                     <xsl:otherwise>
                         <xsl:variable name="inputType">
                             <xsl:choose>
-                                <xsl:when test="$annot/@formType">password</xsl:when>
+                                <xsl:when test="$annot/@password">password</xsl:when>
                                 <xsl:otherwise>text</xsl:otherwise>
                             </xsl:choose>
                         </xsl:variable>

roxie/ccd/ccdcontext.cpp

@@ -2597,7 +2597,7 @@ protected:
             client = NULL;
         updateSuppliedXmlParams(wu);
         SCMStringBuffer wuParams;
-        if (workUnit->getXmlParams(wuParams).length())
+        if (workUnit->getXmlParams(wuParams, false).length())
         {
             // Merge in params from WU. Ones on command line take precedence though...
             Owned<IPropertyTree> wuParamTree = createPTreeFromXMLString(wuParams.str(), ipt_caseInsensitive);

testing/regress/ecl/formatstored.ecl

@@ -36,6 +36,8 @@ integer4 i4 := 0 : stored('i4', format(fieldwidth(4), sequence(4)));
 unsigned3 u3 := 0 : stored('u3', format(fieldwidth(3), sequence(13)));
 unsigned8 u8 := 0 : stored('u8', format(fieldwidth(8), sequence(18)));
 
+string pw := 'powow' : stored('pw', format(password, fieldwidth(40)));
+
 output (i1, named('i1'));
 output (i2, named('i2'));
 output (i3, named('i3'));
@@ -56,3 +58,4 @@ output (u8, named('u8'));
 
 output (s1, named('s1'));
 
+output (pw, named('showpw'));

testing/regress/ecl/key/formatstored.xml

@@ -49,3 +49,6 @@
 <Dataset name='s1'>
  <Row><s1>how now brown cow</s1></Row>
 </Dataset>
+<Dataset name='showpw'>
+ <Row><showpw>powow</showpw></Row>
+</Dataset>

thorlcr/graph/thgraphmaster.cpp

@@ -969,7 +969,7 @@ public:
     {
         PROTECTED_GETRESULT(stepname, sequence, "Data", "data",
             SCMStringBuffer result;
-            r->getResultString(result);
+            r->getResultString(result, false);
             tlen = result.length();
             tgt = (char *)result.s.detach();
         );
@@ -1017,7 +1017,7 @@ public:
     {
         PROTECTED_GETRESULT(stepname, sequence, "String", "string",
             SCMStringBuffer result;
-            r->getResultString(result);
+            r->getResultString(result, false);
             tlen = result.length();
             tgt = (char *)result.s.detach();
         );
@@ -1026,7 +1026,7 @@ public:
     {
         PROTECTED_GETRESULT(stepname, sequence, "String", "string",
             SCMStringBuffer result;
-            r->getResultString(result);
+            r->getResultString(result, false);
             rtlStrToStr(tlen, tgt, result.length(), result.s.str());
         );
     }
@@ -1044,7 +1044,7 @@ public:
     { 
         PROTECTED_GETRESULT(stepname, sequence, "VarString", "string",
             SCMStringBuffer result;
-            r->getResultString(result);
+            r->getResultString(result, false);
             return result.s.detach();
         );
     }

tools/wuget/wuget.cpp

@@ -58,7 +58,7 @@ int main(int argc, char **argv)
                 {
                     Owned<ILocalWorkUnit> wu = createLocalWorkUnit();
                     wu->loadXML(xml);
-                    exportWorkUnitToXML(wu, xml.clear(), true, false);
+                    exportWorkUnitToXML(wu, xml.clear(), true, false, false); //should we hide passwords here?
                     printf("%s\n", xml.str());
                 }
                 else