Big-Data-HPC-Platform/common/wuwebview/wuwebview.cpp

/*##############################################################################

    Copyright (C) 2011 HPCC Systems.

    All rights reserved. This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU Affero General Public License as
    published by the Free Software Foundation, either version 3 of the
    License, or (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Affero General Public License for more details.

    You should have received a copy of the GNU Affero General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.
############################################################################## */

#include "jlib.hpp"
#include "jexcept.hpp"
#include "jptree.hpp"
#include "workunit.hpp"
#include "dllserver.hpp"
#include "thorplugin.hpp"
#include "xslprocessor.hpp"
#include "fileview.hpp"

#include "wuwebview.hpp"
#include "wuweberror.hpp"

class WuExpandedResultBuffer : public CInterface, implements IPTreeNotifyEvent
{
public:
    IMPLEMENT_IINTERFACE;

    WuExpandedResultBuffer(const char *queryname, unsigned _flags=0) :
        name(queryname), datasetLevel(0), finalized(false), flags(_flags), hasXmlns(false)
    {
        if (flags & (WWV_INCL_NAMESPACES | WWV_INCL_GENERATED_NAMESPACES))
        {
            StringBuffer lower(name);
            ns.append("urn:hpccsystems:ecl:").append(lower.toLowerCase());
        }

        if (!(flags & WWV_OMIT_XML_DECLARATION))
            buffer.append("<?xml version=\"1.0\" encoding=\"UTF-8\"?>");
        if (flags & WWV_USE_DISPLAY_XSLT)
            buffer.append("<?xml-stylesheet type=\"text/xsl\" href=\"/esp/xslt/xmlformatter.xsl\"?>");
        if (flags & WWV_ADD_SOAP)
            buffer.append(
                "<soap:Envelope xmlns:soap=\"http://schemas.xmlsoap.org/soap/envelope\""
                  " xmlns:SOAP-ENC=\"http://schemas.xmlsoap.org/soap/encoding\">"
                    " <soap:Body>"
            );
        if (flags & WWV_ADD_RESPONSE_TAG)
        {
            buffer.append('<');
            if (queryname)
                buffer.append(queryname);
            buffer.append("Response");
            if (flags & WWV_INCL_NAMESPACES && ns.length())
                buffer.append(" xmlns=\"").append(ns.str()).append('\"');
            buffer.append('>');
        }
        if (flags & WWV_ADD_RESULTS_TAG)
            buffer.append("<Results>");
        if (!(flags & WWV_OMIT_RESULT_TAG))
            buffer.append("<Result>");
    }

    void appendResults(IConstWorkUnit *wu, const char *username, const char *pw)
    {
        StringBufferAdaptor resultXML(buffer);
        getFullWorkUnitResultsXML(username, pw, wu, resultXML, false, ExceptionSeverityError, true);
    }

    void appendSingleResult(IConstWorkUnit *wu, const char *resultname, const char *username, const char *pw)
    {
        SCMStringBuffer wuid;
        StringBufferAdaptor resultXML(buffer);
        Owned<IResultSetFactory> factory = getResultSetFactory(username, pw);
        Owned<INewResultSet> nr = factory->createNewResultSet(wu->getWuid(wuid).str(), 0, resultname);
        getResultXml(resultXML, nr.get(), resultname, 0, 0, NULL);
    }

    void appendDatasetsFromXML(const char *xml)
    {
        assertex(!finalized);
        Owned<IPullXMLReader> reader = createPullXMLStringReader(xml, *this);
        reader->load();
    }

    void append(const char *value)
    {
        assertex(!finalized);
        buffer.append(value);
    }

    void appendSchemaResource(IPropertyTree &res, ILoadedDllEntry &loadedDll)
    {
        if (flags & WWV_OMIT_SCHEMAS)
            return;
        if (res.getPropInt("@seq", -1)>=0 && res.hasProp("@id"))
        {
            int id = res.getPropInt("@id");
            size32_t len = 0;
            const void *data = NULL;
            if (loadedDll.getResource(len, data, "RESULT_XSD", (unsigned) id) && len>0)
            {
                buffer.append("<XmlSchema name=\"").append(res.queryProp("@name")).append("\">");
                if (res.getPropBool("@compressed"))
                {
                    StringBuffer decompressed;
                    decompressResource(len, data, decompressed);
                    if (flags & WWV_CDATA_SCHEMAS)
                        buffer.append("<![CDATA[");
                    buffer.append(decompressed.str());
                    if (flags & WWV_CDATA_SCHEMAS)
                        buffer.append("]]>");
                }
                else
                    buffer.append(len, (const char *)data);
                buffer.append("</XmlSchema>");
            }
        }
    }

    void appendManifestSchemas(IPropertyTree &manifest, ILoadedDllEntry &loadedDll)
    {
        assertex(!finalized);
        Owned<IPropertyTreeIterator> iter = manifest.getElements("Resource[@type='RESULT_XSD']");
        ForEach(*iter)
            appendSchemaResource(iter->query(), loadedDll);
    }

    void appendManifestResultSchema(IPropertyTree &manifest, const char *resultname, ILoadedDllEntry &loadedDll)
    {
        assertex(!finalized);
        VStringBuffer xpath("Resource[@name='%s'][@type='RESULT_XSD']", resultname);
        IPropertyTree *res=manifest.queryPropTree(xpath.str());
        if (res)
            appendSchemaResource(*res, loadedDll);
    }

    void appendXML(IPropertyTree *xml, const char *tag=NULL)
    {
        assertex(!finalized);
        if (tag)
            buffer.append('<').append(tag).append('>');
        if (xml)
            toXML(xml, buffer);
        if (tag)
            buffer.append("</").append(tag).append('>');
    }

    virtual void beginNode(const char *tag, offset_t startOffset)
    {
        if (streq("Dataset", tag) || streq("Exception", tag))
            datasetLevel++;
        if (datasetLevel)
            buffer.append('<').append(tag);
    }

    virtual void newAttribute(const char *name, const char *value)
    {
        if (datasetLevel)
        {
            if (streq(name, "@xmlns"))
            {
                if (!(flags & WWV_INCL_NAMESPACES))
                    return;
                if (datasetLevel==1)
                    hasXmlns=true;
            }
            if (datasetLevel==1 && streq(name, "@name"))
                dsname.set(value).toLowerCase().replace(' ', '_');;
            buffer.append(' ').append(name+1).append("=\"");
            encodeUtf8XML(value, buffer);
            buffer.append('\"');
        }
    }
    virtual void beginNodeContent(const char *tag)
    {
        if (datasetLevel==1 && streq("Dataset", tag))
        {
            if (!hasXmlns && dsname.length() && (flags & WWV_INCL_GENERATED_NAMESPACES))
            {
                StringBuffer s(ns);
                s.append(":result:").append(dsname.str());
                buffer.append(" xmlns=\"").append(s).append('\"');
            }
            dsname.clear();
            hasXmlns=false;
        }
        if (datasetLevel)
            buffer.append('>');
    }
    virtual void endNode(const char *tag, unsigned length, const void *value, bool binary, offset_t endOffset)
    {
        if (datasetLevel)
        {
            if (length)
            {
                if (binary)
                    JBASE64_Encode(value, length, buffer);
                else
                    encodeUtf8XML((const char *)value, buffer);
            }
            buffer.append("</").append(tag).append('>');
            if (streq("Dataset", tag) || streq("Exception", tag))
                datasetLevel--;
        }
    }
    StringBuffer &finalize()
    {
        if (!finalized)
        {
            if (!(flags & WWV_OMIT_RESULT_TAG))
                buffer.append("</Result>");
            if (flags & WWV_ADD_RESULTS_TAG)
                buffer.append("</Results>");
            if (flags & WWV_ADD_RESPONSE_TAG)
                buffer.appendf("</%sResponse>", name.sget());
            if (flags & WWV_ADD_SOAP)
                buffer.append("</soap:Body></soap:Envelope>");
            finalized=true;
        }
        return buffer;
    }

    const char *str()
    {
        finalize();
        return buffer.str();
    }

    size32_t length()
    {
        finalize();
        return buffer.length();
    }
public:
    StringBuffer buffer;
    StringAttr name;
    StringBuffer dsname;
    StringBuffer ns;
    bool hasXmlns;
    bool finalized;
    unsigned flags;
private:
    int datasetLevel;
};

class WuWebView : public CInterface,
    implements IWuWebView,
    implements IIncludeHandler
{
public:
    IMPLEMENT_IINTERFACE;

    WuWebView(IConstWorkUnit &wu, const char *queryname, const char *wdir, bool mapEspDir) :
        manifestIncludePathsSet(false), dir(wdir), mapEspDirectories(mapEspDir)
    {
        name.set(queryname);
        load(wu);
    }

    WuWebView(const char *wuid, const char *queryname, const char *wdir, bool mapEspDir) :
        manifestIncludePathsSet(false), dir(wdir), mapEspDirectories(mapEspDir)
    {
        name.set(queryname);
        load(wuid);
    }

    void load(IConstWorkUnit &wu);
    void load(const char *wuid);
    IPropertyTree *ensureManifest();

    virtual void getResultViewNames(StringArray &names);
    virtual void renderResults(const char *viewName, const char *xml, StringBuffer &html);
    virtual void renderResults(const char *viewName, StringBuffer &html);
    virtual void renderSingleResult(const char *viewName, const char *resultname, StringBuffer &html);
    virtual void expandResults(const char *xml, StringBuffer &out, unsigned flags);
    virtual void expandResults(StringBuffer &out, unsigned flags);
    virtual void applyResultsXSLT(const char *filename, const char *xml, StringBuffer &out);
    virtual void applyResultsXSLT(const char *filename, StringBuffer &out);
    virtual StringBuffer &aggregateResources(const char *type, StringBuffer &content);

    void renderExpandedResults(const char *viewName, WuExpandedResultBuffer &expanded, StringBuffer &out);

    void appendResultSchemas(WuExpandedResultBuffer &buffer);
    void getResultXSLT(const char *viewName, StringBuffer &xslt, StringBuffer &abspath);
    void getResource(IPropertyTree *res, StringBuffer &content);
    void getResource(const char *name, StringBuffer &content, StringBuffer &abspath, const char *type);

    void calculateResourceIncludePaths();
    virtual bool getInclude(const char *includename, MemoryBuffer &includebuf, bool &pathOnly);

protected:
    SCMStringBuffer dllname;
    Owned<IConstWorkUnit> wu;
    Owned<ILoadedDllEntry> loadedDll;
    Owned<IPropertyTree> manifest;
    SCMStringBuffer name;
    bool mapEspDirectories;
    bool manifestIncludePathsSet;
    StringAttr dir;
    StringAttr username;
    StringAttr pw;
};

IPropertyTree *WuWebView::ensureManifest()
{
    if (!manifest)
    {
        StringBuffer xml;
        manifest.setown((loadedDll && getEmbeddedManifestXML(loadedDll, xml)) ? createPTreeFromXMLString(xml.str()) : createPTree());
    }
    return manifest.get();
}

void WuWebView::calculateResourceIncludePaths()
{
    if (!manifestIncludePathsSet)
    {
        Owned<IPropertyTreeIterator> iter = ensureManifest()->getElements("Resource[@filename]");
        ForEach(*iter)
        {
            StringBuffer abspath;
            iter->query().setProp("@res_include_path", makeAbsolutePath(iter->query().queryProp("@filename"), dir.get(), abspath).str());
        }
        manifestIncludePathsSet=true;
    }
}

bool WuWebView::getInclude(const char *includename, MemoryBuffer &includebuf, bool &pathOnly)
{
    int len=strlen(includename);
    if (len<8)
        return false;
    //eliminate "file://"
    if (strncmp(includename, "file://", 7)==0)
        includename+=7;
    //eliminate extra '/' for windows absolute paths
    if (len>9 && includename[2]==':')
        includename++;
    StringBuffer relpath;
    if (mapEspDirectories && !strnicmp(includename, "/esp/", 5))
        relpath.append(includename+=5);
    else
        relpath.append(includename); //still correct for OS
    StringBuffer abspath;
    makeAbsolutePath(relpath.str(), dir.get(), abspath);

    IPropertyTree *res = NULL;
    if (manifest)
    {
        VStringBuffer xpath("Resource[@res_include_path='%s']", abspath.str());
        res = manifest->queryPropTree(xpath.str());
    }
    if (res)
    {
        StringBuffer xslt;
        getResource(res, xslt);
        includebuf.append(xslt.str());
    }
    else if (checkFileExists(abspath.str()))
    {
        Owned <IFile> f = createIFile(abspath.str());
        Owned <IFileIO> fio = f->open(IFOread);
        read(fio, 0, (size32_t) f->size(), includebuf);
    }
    //esp looks in two places for path starting with "xslt/"
    else if (mapEspDirectories && !strncmp(includename, "xslt/", 5))
    {
        StringBuffer relpath(dir.get());
        relpath.append("smc_").append(includename);;
        makeAbsolutePath(relpath.str(), abspath.clear());
        if (checkFileExists(abspath.str()))
        {
            Owned <IFile> f = createIFile(abspath.str());
            Owned <IFileIO> fio = f->open(IFOread);
            read(fio, 0, (size32_t) f->size(), includebuf);
        }
    }
    return true;
}

void WuWebView::getResultViewNames(StringArray &names)
{
    Owned<IPropertyTreeIterator> iter = ensureManifest()->getElements("Views/Results[@name]");
    ForEach(*iter)
    names.append(iter->query().queryProp("@name"));
}

void WuWebView::getResource(IPropertyTree *res, StringBuffer &content)
{
    if (!loadedDll)
        return;
    if (res->hasProp("@id"))
    {
        int id = res->getPropInt("@id");
        size32_t len = 0;
        const void *data = NULL;
        if (loadedDll->getResource(len, data, res->queryProp("@type"), (unsigned) id) && len>0)
        {
            if (res->getPropBool("@compressed"))
            {
                StringBuffer decompressed;
                decompressResource(len, data, content);
                content.append(decompressed.str());
            }
            else
                content.append(len, (const char *)data);
        }
    }
}

void WuWebView::getResource(const char *name, StringBuffer &content, StringBuffer &includepath, const char *type)
{
    VStringBuffer xpath("Resource[@name='%s']", name);
    if (type)
        xpath.append("[@type='").append(type).append("']");
    IPropertyTree *res = ensureManifest()->queryPropTree(xpath.str());
    calculateResourceIncludePaths();
    includepath.append(res->queryProp("@res_include_path"));
    if (res)
        getResource(res, content);
}

StringBuffer &WuWebView::aggregateResources(const char *type, StringBuffer &content)
{
    VStringBuffer xpath("Resource[@type='%s']", type);
    Owned<IPropertyTreeIterator> iter = ensureManifest()->getElements(xpath.str());
    ForEach(*iter)
        getResource(&iter->query(), content);
    return content;
}

void WuWebView::getResultXSLT(const char *viewName, StringBuffer &xslt, StringBuffer &abspath)
{
    VStringBuffer xpath("Views/Results[@name='%s']/@resource", viewName);
    const char *resource = ensureManifest()->queryProp(xpath.str());
    if (resource)
        getResource(resource, xslt, abspath, "XSLT");
}

void WuWebView::renderExpandedResults(const char *viewName, WuExpandedResultBuffer &expanded, StringBuffer &out)
{
    IPropertyTree *mf = ensureManifest();
    VStringBuffer xpath("Views/Results[@name='%s']", viewName);
    IPropertyTree *view = mf->queryPropTree(xpath.str());
    if (!view)
        throw MakeStringException(WUWEBERR_ViewResourceNotFound, "Result view %s not found", viewName);
    expanded.appendXML(view, "view");
    if (loadedDll)
        expanded.appendManifestSchemas(*mf, *loadedDll);
    expanded.finalize();
    const char *type=view->queryProp("@type");
    if (!type)
        throw MakeStringException(WUWEBERR_UnknownViewType, "No type defined for view %s", viewName);
    else if (strieq(type, "xml"))
    {
        out.swapWith(expanded.buffer);
        return;
    }
    else if (!strieq(type, "xslt"))
        throw MakeStringException(WUWEBERR_UnknownViewType, "View %s has an unknown type of %s", viewName, type);

    StringBuffer xslt;
    StringBuffer rootpath;
    getResultXSLT(viewName, xslt, rootpath);
    if (!xslt.length())
        throw MakeStringException(WUWEBERR_ViewResourceNotFound, "XSLT for %s view not found", viewName);
    Owned<IXslTransform> t = getXslProcessor()->createXslTransform();
    t->setIncludeHandler(this);
    StringBuffer cacheId(viewName);
    cacheId.append('@').append(dllname.str()); //using dllname, cloned workunits can share cache entry
    t->setXslSource(xslt.str(), xslt.length(), cacheId.str(), rootpath.str());
    t->setXmlSource(expanded.buffer.str(), expanded.buffer.length());
    t->transform(out);
}

void WuWebView::renderResults(const char *viewName, const char *xml, StringBuffer &out)
{
    WuExpandedResultBuffer buffer(name.str(), WWV_ADD_RESPONSE_TAG | WWV_ADD_RESULTS_TAG);
    buffer.appendDatasetsFromXML(xml);
    renderExpandedResults(viewName, buffer, out);
}

void WuWebView::renderResults(const char *viewName, StringBuffer &out)
{
    WuExpandedResultBuffer buffer(name.str(), WWV_ADD_RESPONSE_TAG | WWV_ADD_RESULTS_TAG);
    buffer.appendResults(wu, username.get(), pw.get());
    renderExpandedResults(viewName, buffer, out);
}

void WuWebView::renderSingleResult(const char *viewName, const char *resultname, StringBuffer &out)
{
    WuExpandedResultBuffer buffer(name.str(), WWV_ADD_RESPONSE_TAG | WWV_ADD_RESULTS_TAG);
    buffer.appendSingleResult(wu, resultname, username.get(), pw.get());
    renderExpandedResults(viewName, buffer, out);
}

void WuWebView::expandResults(const char *xml, StringBuffer &out, unsigned flags)
{
    WuExpandedResultBuffer expander(name.str(), flags);
    expander.appendDatasetsFromXML(xml);
    if (loadedDll)
        expander.appendManifestSchemas(*ensureManifest(), *loadedDll);
    expander.finalize();
    out.append(expander.buffer);
}

void WuWebView::expandResults(StringBuffer &out, unsigned flags)
{
    SCMStringBuffer xml;
    getFullWorkUnitResultsXML(username.get(), pw.get(), wu, xml, false, ExceptionSeverityInformation);
    expandResults(xml.str(), out, flags);
}

void WuWebView::applyResultsXSLT(const char *filename, const char *xml, StringBuffer &out)
{
    WuExpandedResultBuffer buffer(name.str(), WWV_ADD_RESPONSE_TAG | WWV_ADD_RESULTS_TAG);
    buffer.appendDatasetsFromXML(xml);
    if (loadedDll)
        buffer.appendManifestSchemas(*ensureManifest(), *loadedDll);

    Owned<IXslTransform> t = getXslProcessor()->createXslTransform();
    t->setIncludeHandler(this);
    //override default behavior using filename as cache identifier, there's a chance includes are
    //mapped to resources and need to be distinguished in cache
    StringBuffer cacheId(filename);
    cacheId.append('@').append(dllname.str()); //cloned workunits have same dll and resources
    t->loadXslFromFile(filename, cacheId.str());
    t->setXmlSource(buffer.str(), buffer.length());
    t->transform(out);
}

void WuWebView::applyResultsXSLT(const char *filename, StringBuffer &out)
{
    SCMStringBuffer xml;
    getFullWorkUnitResultsXML(username.get(), pw.get(), wu, xml, false, ExceptionSeverityInformation);
    applyResultsXSLT(filename, xml.str(), out);
}

void WuWebView::load(IConstWorkUnit &cwu)
{
    wu.set(&cwu);
    if (!name.length())
    {
        wu->getJobName(name);
        name.s.replace(' ','_');
    }
    Owned<IConstWUQuery> q = wu->getQuery();
    q->getQueryDllName(dllname);
    try
    {
        loadedDll.setown(queryDllServer().loadDll(dllname.str(), DllLocationAnywhere));
    }
    catch(...)
    {
        DBGLOG("Failed to load %s", dllname.str());
    }
}

void WuWebView::load(const char *wuid)
{
    Owned<IWorkUnitFactory> factory = getWorkUnitFactory();
    Owned<IConstWorkUnit> wu = factory->openWorkUnit(wuid, false);
    if (!wu)
        throw MakeStringException(WUWEBERR_WorkUnitNotFound, "Workunit not found %s", wuid);
    load(*wu);
}

extern WUWEBVIEW_API IWuWebView *createWuWebView(IConstWorkUnit &wu, const char *queryname, const char *dir, bool mapEspDirectories)
{
    try
    {
        return new WuWebView(wu, queryname, dir, mapEspDirectories);
    }
    catch (...)
    {
        SCMStringBuffer wuid;
        DBGLOG("ERROR loading workunit %s shared object.", wu.getWuid(wuid).str());
    }
    return NULL;
}

extern WUWEBVIEW_API IWuWebView *createWuWebView(const char *wuid, const char *queryname, const char *dir, bool mapEspDirectories)
{
    try
    {
        return new WuWebView(wuid, queryname, dir, mapEspDirectories);
    }
    catch (...)
    {
        DBGLOG("ERROR loading workunit %s shared object.", wuid);
    }
    return NULL;
}