HPCC-10203 Add manifest resource support for wildcards and relative URLS

Resources can be directly accessed in WsWorkunits using the following url format:

/WsWorkunits/res/W20131016-193456/resourcepath/resfile.ext

i.e. http://ip:8010/WsWorkunits/res/W20131016-193456/abc/xyz/tryit.html

A browser should automatically follow relative paths in html files.

The manifest is accessible at:

/WsWorkunits/manifest/W20131016-193456

Also add support for using wildcards in manifest resource filenames:




Signed-off-by: Anthony Fishbeck anthony.fishbeck@lexisnexis.com

common/dllserver/thorplugin.cpp

@@ -360,7 +360,7 @@ extern DLLSERVER_API ILoadedDllEntry * createExeDllEntry(const char *path)
     return result.getClear();
 }
 
-extern DLLSERVER_API bool decompressResource(size32_t len, const void *data, StringBuffer &result)
+extern DLLSERVER_API bool decompressResource(size32_t len, const void *data, MemoryBuffer &result)
 {
     bool hasVersion = len && (*(const byte *)data == 0x80);
     MemoryBuffer src;
@@ -372,16 +372,22 @@ extern DLLSERVER_API bool decompressResource(size32_t len, const void *data, Str
         src.read(version);
     }
 
-    MemoryBuffer tgt;
     switch (version)
     {
     case 1:
-        decompressToBuffer(tgt, src);
+        decompressToBuffer(result, src);
         break;
     default:
         throwUnexpected();
     }
 
+    return true;
+}
+
+extern DLLSERVER_API bool decompressResource(size32_t len, const void *data, StringBuffer &result)
+{
+    MemoryBuffer tgt;
+    decompressResource(len, data, tgt);
     tgt.append((char)0);
     unsigned expandedLen = tgt.length();
     result.setBuffer(expandedLen, reinterpret_cast<char *>(tgt.detach()), expandedLen-1);

common/dllserver/thorplugin.hpp

@@ -43,6 +43,7 @@ extern DLLSERVER_API bool getWorkunitXMLFromFile(const char *filename, StringBuf
 extern DLLSERVER_API bool getManifestXMLFromFile(const char *filename, StringBuffer &xml);
 
 extern DLLSERVER_API bool decompressResource(size32_t len, const void *data, StringBuffer &result);
+extern DLLSERVER_API bool decompressResource(size32_t len, const void *data, MemoryBuffer &result);
 extern DLLSERVER_API void compressResource(MemoryBuffer & compressed, size32_t len, const void *data);
 extern DLLSERVER_API void appendResource(MemoryBuffer & mb, size32_t len, const void *data, bool compress);
 

common/wuwebview/wuwebview.cpp

@@ -298,8 +298,12 @@ public:
 
     void appendResultSchemas(WuExpandedResultBuffer &buffer);
     void getResultXSLT(const char *viewName, StringBuffer &xslt, StringBuffer &abspath);
-    void getResource(IPropertyTree *res, StringBuffer &content);
+    bool getResource(IPropertyTree *res, StringBuffer &content);
+    bool getResource(IPropertyTree *res, MemoryBuffer &content);
+    bool getResource(IPropertyTree *res, size32_t & len, const void * & data);
     void getResource(const char *name, StringBuffer &content, StringBuffer &abspath, const char *type);
+    bool getResourceByPath(const char *path, MemoryBuffer &mb);
+    StringBuffer &getManifest(StringBuffer &mf){return toXML(ensureManifest(), mf);}
 
     void calculateResourceIncludePaths();
     virtual bool getInclude(const char *includename, MemoryBuffer &includebuf, bool &pathOnly);
@@ -313,6 +317,7 @@ public:
 
 protected:
     SCMStringBuffer dllname;
+    StringBuffer manifestDir;
     Owned<IConstWorkUnit> cw;
     Owned<ILoadedDllEntry> dll;
     bool delayedDll;
@@ -335,20 +340,26 @@ IPropertyTree *WuWebView::ensureManifest()
     return manifest.get();
 }
 
+StringBuffer &makeResourcePath(const char *path, const char *basedir, StringBuffer &respath)
+{
+    StringBuffer abspath;
+    makeAbsolutePath(path, basedir, abspath);
+
+    return makePathUniversal(abspath, respath);
+}
+
 void WuWebView::calculateResourceIncludePaths()
 {
     if (!manifestIncludePathsSet)
     {
-        Owned<IPropertyTreeIterator> iter = ensureManifest()->getElements("Resource[@filename]");
+        manifestDir.set(ensureManifest()->queryProp("@manifestDir"));
+        Owned<IPropertyTreeIterator> iter = manifest->getElements("Resource[@filename]");
         ForEach(*iter)
         {
             if (!iter->query().hasProp("@resourcePath")) //backward compatible
             {
-                StringBuffer abspath;
-                makeAbsolutePath(iter->query().queryProp("@filename"), dir.get(), abspath);
-
                 StringBuffer respath;
-                makePathUniversal(abspath.str(), respath);
+                makeResourcePath(iter->query().queryProp("@filename"), dir.get(), respath);
                 iter->query().setProp("@resourcePath", respath.str());
             }
         }
@@ -422,6 +433,26 @@ bool WuWebView::getInclude(const char *includename, MemoryBuffer &includebuf, bo
     return true;
 }
 
+bool WuWebView::getResourceByPath(const char *path, MemoryBuffer &mb)
+{
+    calculateResourceIncludePaths();
+
+    StringBuffer xpath;
+    if (!manifestDir.length())
+        xpath.setf("Resource[@filename='%s'][1]", path);
+    else
+    {
+        StringBuffer respath;
+        makeResourcePath(path, manifestDir.str(), respath);
+        xpath.setf("Resource[@resourcePath='%s'][1]", respath.str());
+    }
+
+    IPropertyTree *res = ensureManifest()->queryPropTree(xpath.str());
+    if (!res)
+        return false;
+    return getResource(res, mb);
+}
+
 void WuWebView::getResultViewNames(StringArray &names)
 {
     Owned<IPropertyTreeIterator> iter = ensureManifest()->getElements("Views/Results[@name]");
@@ -431,27 +462,46 @@ void WuWebView::getResultViewNames(StringArray &names)
         names.append("EmbeddedView");
 }
 
-void WuWebView::getResource(IPropertyTree *res, StringBuffer &content)
+bool WuWebView::getResource(IPropertyTree *res, size32_t & len, const void * & data)
 {
     if (!loadDll())
-        return;
+        return false;
     if (res->hasProp("@id") && (res->hasProp("@header")||res->hasProp("@compressed")))
     {
         int id = res->getPropInt("@id");
-        size32_t len = 0;
-        const void *data = NULL;
-        if (dll->getResource(len, data, res->queryProp("@type"), (unsigned) id) && len>0)
-        {
-            if (res->getPropBool("@compressed"))
-            {
-                StringBuffer decompressed;
-                decompressResource(len, data, content);
-                content.append(decompressed.str());
-            }
-            else
-                content.append(len, (const char *)data);
-        }
+        return (dll->getResource(len, data, res->queryProp("@type"), (unsigned) id) && len>0);
+    }
+    return false;
+}
+
+bool WuWebView::getResource(IPropertyTree *res, MemoryBuffer &content)
+{
+    size32_t len = 0;
+    const void *data = NULL;
+    if (getResource(res, len, data))
+    {
+        if (res->getPropBool("@compressed"))
+            decompressResource(len, data, content);
+        else
+            content.append(len, (const char *)data);
+        return true;
+    }
+    return false;
+}
+
+bool WuWebView::getResource(IPropertyTree *res, StringBuffer &content)
+{
+    size32_t len = 0;
+    const void *data = NULL;
+    if (getResource(res, len, data))
+    {
+        if (res->getPropBool("@compressed"))
+            decompressResource(len, data, content);
+        else
+            content.append(len, (const char *)data);
+        return true;
     }
+    return false;
 }
 
 void WuWebView::getResource(const char *name, StringBuffer &content, StringBuffer &includepath, const char *type)

common/wuwebview/wuwebview.hpp

@@ -53,9 +53,48 @@ interface IWuWebView : extends IInterface
     virtual void addInputsFromPTree(IPropertyTree *pt)=0;
     virtual void addInputsFromXml(const char *xml)=0;
     virtual void createWuidResponse(StringBuffer &out, unsigned flags)=0;
+    virtual bool getResourceByPath(const char *path, MemoryBuffer &mb)=0;
+    virtual StringBuffer &getManifest(StringBuffer &mf)=0;
+
 };
 
 extern WUWEBVIEW_API IWuWebView *createWuWebView(IConstWorkUnit &wu, const char *queryname, const char*dir, bool mapEspDir);
 extern WUWEBVIEW_API IWuWebView *createWuWebView(const char *wuid, const char *queryname, const char*dir, bool mapEspDir);
 
+static inline bool isPathSeparator(char sep)
+{
+    return (sep=='\\')||(sep=='/');
+}
+
+static inline const char *skipPathNodes(const char *&s, int skip)
+{
+    if (s) {
+        while (*s) {
+            if (isPathSeparator(*s++))
+                if (!skip--)
+                    return s;
+        }
+    }
+    return NULL;
+}
+
+static inline const char *nextPathNode(const char *&s, StringBuffer &node, int skip=0)
+{
+    if (skip)
+        skipPathNodes(s, skip);
+    if (s) while (*s) {
+        if (isPathSeparator(*s))
+            return s++;
+        node.append(*s++);
+    }
+    return NULL;
+}
+
+static inline const char *firstPathNode(const char *&s, StringBuffer &node)
+{
+    if (s && isPathSeparator(*s))
+        s++;
+    return nextPathNode(s, node);
+}
+
 #endif

ecl/hql/hqlmanifest.cpp

@@ -39,6 +39,9 @@ public:
 private:
     bool loadInclude(IPropertyTree &include, const char *dir);
     void expand();
+    void expandDirectory(IPropertyTree &res, IDirectoryIterator *it, const char*mask, bool recursive);
+    void expandDirectory(IPropertyTree &res, const char *path, const char*mask, bool recursive);
+
 public:
     Owned<IPropertyTree> manifest;
     StringBuffer absFilename;
@@ -87,14 +90,61 @@ bool ResourceManifest::loadInclude(IPropertyTree &include, const char *dir)
     return true;
 }
 
+void ResourceManifest::expandDirectory(IPropertyTree &res, IDirectoryIterator *it, const char*mask, bool recursive)
+{
+    if (!it)
+        return;
+    ForEach(*it)
+    {
+        if (it->isDir())
+        {
+            if (recursive)
+                expandDirectory(res, it->query().directoryFiles(mask, false, true), mask, recursive);
+            continue;
+        }
+        StringBuffer reldir;
+        Owned<IPropertyTree> newRes = createPTreeFromIPT(&res);
+        reldir.append(splitRelativePath(it->query().queryFilename(), dir, reldir));
+        VStringBuffer xpath("Resource[@filename='%s']", reldir.str());
+        if (manifest->hasProp(xpath))
+            continue;
+        newRes->setProp("@filename", reldir.str());
+        updateResourcePaths(*newRes, dir.str());
+        if (manifest->hasProp(xpath.setf("resource[@resourcePath='%s']", newRes->queryProp("@resourcePath"))))
+            continue;
+        manifest->addPropTree("Resource", newRes.getClear());
+    }
+}
+
+void ResourceManifest::expandDirectory(IPropertyTree &res, const char *path, const char*mask, bool recursive)
+{
+    Owned<IDirectoryIterator> it = createDirectoryIterator(path, mask);
+    expandDirectory(res, it, mask, recursive);
+}
+
+
 void ResourceManifest::expand()
 {
-    Owned<IPropertyTreeIterator> resources = manifest->getElements("Resource[@filename]");
+	manifest->setProp("@manifestDir", dir.str());
+	Owned<IPropertyTreeIterator> resources = manifest->getElements("Resource[@filename]");
     ForEach(*resources)
         updateResourcePaths(resources->query(), dir.str());
     Owned<IPropertyTreeIterator> includes = manifest->getElements("Include[@filename]");
     ForEach(*includes)
         loadInclude(includes->query(), dir.str());
+    resources.setown(manifest->getElements("Resource[@filename]"));
+    ForEach(*resources)
+    {
+        IPropertyTree &res = resources->query();
+        const char *name = res.queryProp("@originalFilename");
+        if (containsFileWildcard(name))
+        {
+            StringBuffer wildpath;
+            const char *tail = splitDirTail(name, wildpath);
+            expandDirectory(res, wildpath, tail, res.getPropBool("@recursive"));
+            manifest->removeTree(&res);
+        }
+    }
 }
 
 bool ResourceManifest::checkResourceFilesExist()

esp/bindings/bindutil.cpp

@@ -817,3 +817,37 @@ void xslTransformHelper(IXslProcessor *xslp, const char* xml, const char* xslFil
     }
 }
 
+const char *mimeTypeFromFileExt(const char *ext)
+{
+    if (!ext)
+        return "application/octet-stream";
+    if (*ext=='.')
+        ext++;
+    if (strieq(ext, "html") || strieq(ext, "htm"))
+        return "text/html";
+    if (strieq(ext, "xml") || strieq(ext, "xsl") || strieq(ext, "xslt"))
+       return "application/xml";
+    if (strieq(ext, "js"))
+       return "text/javascript";
+    if (strieq(ext, "css"))
+       return "text/css";
+    if (strieq(ext, "jpeg") || strieq(ext, "jpg"))
+       return "image/jpeg";
+    if (strieq(ext, "gif"))
+       return "image/gif";
+    if (strieq(ext, "png"))
+       return "image/png";
+    if (strieq(ext, "svg"))
+       return "image/svg+xml";
+    if (strieq(ext, "txt") || strieq(ext, "text"))
+       return "text/plain";
+    if (strieq(ext, "zip"))
+       return "application/zip";
+    if (strieq(ext, "pdf"))
+       return "application/pdf";
+    if (strieq(ext, "xpi"))
+       return "application/x-xpinstall";
+    if (strieq(ext, "exe") || strieq(ext, "class"))
+       return "application/octet-stream";
+    return "application/octet-stream";
+}

esp/bindings/bindutil.hpp

@@ -71,6 +71,8 @@ inline char* strupr(char* str)
 BINDUTIL_EXPORT IEspStringIntMap * createStringIntMap();
 
 BINDUTIL_EXPORT void buildArrayHint(StringBuffer& s, int indents, const char* tag, const char* cls);
+BINDUTIL_EXPORT const char *mimeTypeFromFileExt(const char *ext);
+
 
 void xslTransformHelper(IXslProcessor *xslp, const char* xml, const char* xslFile, StringBuffer& output, IProperties *params);
 

esp/bindings/http/platform/httptransport.cpp

@@ -86,44 +86,7 @@ bool httpContentFromFile(const char *filepath, StringBuffer &mimetype, MemoryBuf
         {
             size32_t filesize = (size32_t)io->size();
             io->read(0, filesize, fileContents.reserveTruncate(filesize));
-            mimetype.clear();
-
-            const char *ext = strrchr(filepath, '.');
-            if (ext)
-            {
-                ext++;
-                if (!stricmp(ext, "html") || !stricmp(ext, "htm"))
-                    mimetype.append("text/html");
-                else if (!stricmp(ext, "js"))
-                   mimetype.append("text/javascript");
-                else if (!stricmp(ext, "jpeg") || !stricmp(ext, "jpg"))
-                   mimetype.append("image/gif");
-                else if (!stricmp(ext, "gif"))
-                   mimetype.append("image/gif");
-                else if (!stricmp(ext, "png"))
-                   mimetype.append("image/png");
-                else if (!stricmp(ext, "xml") || !stricmp(ext, "xsl") || !stricmp(ext, "xslt"))
-                   mimetype.append("application/xml");
-                else if (!stricmp(ext, "txt") || !stricmp(ext, "text"))
-                   mimetype.append("text/plain");
-                else if (!stricmp(ext, "zip"))
-                   mimetype.append("application/zip");
-                else if (!stricmp(ext, "pdf"))
-                   mimetype.append("application/pdf");
-                else if (!stricmp(ext, "pdf"))
-                   mimetype.append("application/pdf");
-                else if (!stricmp(ext, "xpi"))
-                   mimetype.append("application/x-xpinstall");
-                else if (!stricmp(ext, "exe") || !stricmp(ext, "class"))
-                   mimetype.append("application/octet-stream");
-                else if (!stricmp(ext, "css"))
-                   mimetype.append("text/css");
-                else if (!stricmp(ext, "svg"))
-                   mimetype.append("image/svg+xml");
-            }
-            
-            if (!mimetype.length())
-               mimetype.append("application/octet-stream");
+            mimetype.set(mimeTypeFromFileExt(strrchr(filepath, '.')));
             return true;
         }
     }

esp/services/ws_ecl/ws_ecl_service.cpp

@@ -472,41 +472,6 @@ void CWsEclBinding::getDynNavData(IEspContext &context, IProperties *params, IPr
 }
 
 
-static inline bool isPathSeparator(char sep)
-{
-    return (sep=='\\')||(sep=='/');
-}
-
-static inline const char *skipPathNodes(const char *&s, int skip)
-{
-    if (s) {
-        while (*s) {
-            if (isPathSeparator(*s++))
-                if (!skip--)
-                    return s;
-        }
-    }
-    return NULL;
-}
-
-static inline const char *nextPathNode(const char *&s, StringBuffer &node, int skip=0)
-{
-    if (skip)
-        skipPathNodes(s, skip);
-    if (s) while (*s) {
-        if (isPathSeparator(*s))
-            return s++;
-        node.append(*s++);
-    }
-    return NULL;
-}
-
-static inline const char *firstPathNode(const char *&s, StringBuffer &node)
-{
-    if (s && isPathSeparator(*s))
-        s++;
-    return nextPathNode(s, node);
-}
 
 
 static void splitPathTailAndExt(const char *s, StringBuffer &path, StringBuffer &tail, StringBuffer *ext)

esp/services/ws_workunits/ws_workunitsAuditLogs.cpp

@@ -1379,11 +1379,49 @@ int CWsWorkunitsSoapBindingEx::onGet(CHttpRequest* request, CHttpResponse* respo
     IEspContext *ctx = request->queryContext();
     IProperties *params = request->queryParameters();
 
-    try
+    StringBuffer path;
+    request->getPath(path);
+
+    if(!strnicmp(path.str(), "/WsWorkunits/res/", strlen("/WsWorkunits/res/")))
+    {
+        const char *pos = path.str();
+        StringBuffer wuid;
+        nextPathNode(pos, wuid, 2);
+        Owned<IWuWebView> web = createWuWebView(wuid, wuid, getCFD(), true);
+        if (!web)
+            throw createEspHttpException(ECLWATCH_CANNOT_OPEN_WORKUNIT, "Cannot open workunit", HTTP_STATUS_OK);
+        MemoryBuffer mb;
+        StringAttr mimetype(mimeTypeFromFileExt(strrchr(pos, '.')));
+        if (!web->getResourceByPath(pos, mb))
+            throw createEspHttpException(ECLWATCH_RESOURCE_NOT_FOUND, "Cannot open resource", HTTP_STATUS_OK);
+
+        response->setContent(mb.length(), mb.toByteArray());
+        response->setContentType(mimetype.get());
+        response->setStatus(HTTP_STATUS_OK);
+        response->send();
+        return 0;
+    }
+    if(!strnicmp(path.str(), "/WsWorkunits/manifest/", strlen("/WsWorkunits/manifest/")))
     {
-         StringBuffer path;
-         request->getPath(path);
+        const char *pos = path.str();
+        StringBuffer wuid;
+        nextPathNode(pos, wuid, 2);
+        Owned<IWuWebView> web = createWuWebView(wuid, wuid, getCFD(), true);
+        if (!web)
+            throw createEspHttpException(ECLWATCH_CANNOT_OPEN_WORKUNIT, "Cannot open workunit", HTTP_STATUS_OK);
+        StringBuffer mf;
+        if (!web->getManifest(mf).length())
+            throw createEspHttpException(ECLWATCH_RESOURCE_NOT_FOUND, "Cannot open manifest", HTTP_STATUS_OK);
+
+        response->setContent(mf.str());
+        response->setContentType("text/xml");
+        response->setStatus(HTTP_STATUS_OK);
+        response->send();
+        return 0;
+    }
 
+    try
+    {
          if(!strnicmp(path.str(), "/WsWorkunits/JobList", 20))
          {
             const char *cluster = params->queryProp("Cluster");

esp/smc/SMCLib/exception_util.hpp

@@ -120,6 +120,7 @@
 #define ECLWATCH_CONTROL_QUERY_FAILED       ECLWATCH_ERROR_START+100
 #define ECLWATCH_QUERY_SUSPENDED            ECLWATCH_ERROR_START+101
 #define ECLWATCH_PACKAGEMAP_NOTRESOLVED     ECLWATCH_ERROR_START+102
+#define ECLWATCH_RESOURCE_NOT_FOUND         ECLWATCH_ERROR_START+103
 
 inline void FORWARDEXCEPTION(IEspContext &context, IException *e, unsigned codeNew)
 {