HPCC-21274 Allow jar files required by a workunit to be passed in manifest

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

common/dllserver/thorplugin.cpp

@@ -235,12 +235,65 @@ extern bool getResourceFromFile(const char *filename, MemoryBuffer &data, const
 
 //-------------------------------------------------------------------------------------------------------------------
 
+class ManifestFileList : public MappingBase
+{
+    StringArray filenames;
+    StringAttr type;
+    StringAttr dir;
+    void recursiveRemoveDirectory(const char *fullPath)
+    {
+        if (rmdir(fullPath) == 0 && !streq(fullPath, dir))
+        {
+            StringBuffer head;
+            splitFilename(fullPath, &head, &head, NULL, NULL);
+            if (head.length() > 1)
+            {
+                head.setLength(head.length()-1);
+                recursiveRemoveDirectory(head);
+            }
+        }
+    }
+    void removeFileAndEmptyParents(const char *fullFileName)
+    {
+        remove(fullFileName);
+        StringBuffer path;
+        splitFilename(fullFileName, &path, &path, NULL, NULL);
+        if (path.length() > 1)
+        {
+            path.setLength(path.length()-1);
+            recursiveRemoveDirectory(path.str());
+        }
+    }
+
+public:
+    ManifestFileList(const char *_type, const char *_dir) : type(_type), dir(_dir) {}
+    ~ManifestFileList()
+    {
+        ForEachItemIn(idx, filenames)
+        {
+            removeFileAndEmptyParents(filenames.item(idx));
+        }
+        rmdir(dir);  // If the specified temporary directory is now empty, remove it.
+    }
+    void append(const char *filename)
+    {
+        assertex(strncmp(filename, dir, strlen(dir))==0);
+        filenames.append(filename);
+    }
+    inline const StringArray &queryFileNames() { return filenames; }
+    virtual const void * getKey() const { return type; }
+};
+
 class HelperDll : implements ILoadedDllEntry, public CInterface
 {
     SharedObject so;
     StringAttr name;
     Linked<const IFileIO> dllFile;
     Owned<IMemoryMappedFile> mappedDll;
+    mutable std::atomic<IPropertyTree *> manifest {nullptr};
+    mutable CriticalSection manifestLock;
+    mutable StringMapOf<ManifestFileList> manifestFiles;
+
     bool logLoad;
 public:
     IMPLEMENT_IINTERFACE;
@@ -255,7 +308,8 @@ public:
     virtual const char * queryName() const;
     virtual const byte * getResource(unsigned id) const;
     virtual bool getResource(size32_t & len, const void * & data, const char * type, unsigned id, bool trace) const;
-
+    virtual IPropertyTree &queryManifest() const override;
+    virtual const StringArray &queryManifestFiles(const char *type, const char *wuid) const override;
     bool load(bool isGlobal, bool raiseOnError);
     bool loadCurrentExecutable();
     bool loadResources();
@@ -278,7 +332,7 @@ public:
 };
 
 HelperDll::HelperDll(const char *_name, const IFileIO *_dllFile)
-: name(_name), dllFile(_dllFile)
+: name(_name), dllFile(_dllFile), manifestFiles(false)
 {
     logLoad = false;
 }
@@ -312,6 +366,7 @@ HelperDll::~HelperDll()
 {
     if (logLoad)
         DBGLOG("Unloading dll %s", name.get());
+    ::Release(manifest.load(std::memory_order_relaxed));
 }
 
 HINSTANCE HelperDll::getInstance() const
@@ -434,6 +489,66 @@ bool HelperDll::getResource(size32_t & len, const void * & data, const char * ty
     }
 }
 
+IPropertyTree &HelperDll::queryManifest() const
+{
+    return *querySingleton(manifest, manifestLock, [this]{ return getEmbeddedManifestPTree(this); });
+}
+
+const StringArray &HelperDll::queryManifestFiles(const char *type, const char *wuid) const
+{
+    CriticalBlock b(manifestLock);
+    Linked<ManifestFileList> list = manifestFiles.find(type);
+    if (!list)
+    {
+        // The temporary path we unpack to is based on so file's current location and workunit
+        StringBuffer tempDir;
+        splitFilename(name, &tempDir, &tempDir, &tempDir, nullptr);
+        list.setown(new ManifestFileList(type, tempDir));
+        tempDir.append(PATHSEPCHAR).append(wuid);
+        VStringBuffer xpath("Resource[@type='%s']", type);
+        Owned<IPropertyTreeIterator> resourceFiles = queryManifest().getElements(xpath.str());
+        ForEach(*resourceFiles)
+        {
+            IPropertyTree &resourceFile = resourceFiles->query();
+            unsigned id = resourceFile.getPropInt("@id", 0);
+            size32_t len = 0;
+            const void *data = nullptr;
+            if (!getResource(len, data, type, id, false))
+                throwUnexpected();
+            MemoryBuffer decompressed;
+            if (resourceFile.getPropBool("@compressed"))
+            {
+                // MORE - would be better to try to spot files that are not worth recompressing (like jar files)?
+                decompressResource(len, data, decompressed);
+                data = decompressed.toByteArray();
+                len = decompressed.length();
+            }
+            else
+            {
+                // Data is preceded by the resource header
+                // MORE - does this depend on whether @header is set? is that what @header means?
+                data = ((const byte *) data) + resourceHeaderLength;
+                len -= resourceHeaderLength;
+            }
+            StringBuffer extractName(tempDir);
+            extractName.append(PATHSEPCHAR);
+            if (resourceFile.hasProp("@filename"))
+                resourceFile.getProp("@filename", extractName);
+            else
+                extractName.append(id).append('.').append(type);
+            recursiveCreateDirectoryForFile(extractName);
+            OwnedIFile f = createIFile(extractName);
+            OwnedIFileIO o = f->open(IFOcreaterw);
+            assertex(o.get() != nullptr);
+            o->write(0, len, data);
+            list->append(extractName);
+        }
+        manifestFiles.replaceOwn(*list.getLink());
+    }
+    return list->queryFileNames();
+}
+
+
 //-------------------------------------------------------------------------------------------------------------------
 
 bool PluginDll::init(IPluginContextEx * pluginCtx)
@@ -580,7 +695,7 @@ extern DLLSERVER_API bool getEmbeddedWorkUnitXML(ILoadedDllEntry *dll, StringBuf
     return decompressResource(len, data, xml);
 }
 
-extern DLLSERVER_API bool getEmbeddedManifestXML(ILoadedDllEntry *dll, StringBuffer &xml)
+extern DLLSERVER_API bool getEmbeddedManifestXML(const ILoadedDllEntry *dll, StringBuffer &xml)
 {
     size32_t len = 0;
     const void * data = NULL;
@@ -589,6 +704,12 @@ extern DLLSERVER_API bool getEmbeddedManifestXML(ILoadedDllEntry *dll, StringBuf
     return decompressResource(len, data, xml);
 }
 
+extern DLLSERVER_API IPropertyTree *getEmbeddedManifestPTree(const ILoadedDllEntry *dll)
+{
+    StringBuffer xml;
+    return getEmbeddedManifestXML(dll, xml) ? createPTreeFromXMLString(xml.str()) : createPTree();
+}
+
 extern DLLSERVER_API bool checkEmbeddedWorkUnitXML(ILoadedDllEntry *dll)
 {
     size32_t len = 0;

common/dllserver/thorplugin.hpp

@@ -31,12 +31,16 @@ interface ILoadedDllEntry : extends IInterface
     virtual const char * queryName() const = 0;
     virtual const byte * getResource(unsigned id) const = 0;
     virtual bool getResource(size32_t & len, const void * & data, const char * type, unsigned id, bool trace=true) const = 0;
+    virtual IPropertyTree &queryManifest() const = 0;
+    virtual const StringArray &queryManifestFiles(const char *type, const char *tempDir) const = 0;
 };
 
 extern DLLSERVER_API ILoadedDllEntry * createDllEntry(const char *name, bool isGlobal, const IFileIO *dllFile, bool resourcesOnly);
 extern DLLSERVER_API ILoadedDllEntry * createExeDllEntry(const char *name);
 extern DLLSERVER_API bool getEmbeddedWorkUnitXML(ILoadedDllEntry *dll, StringBuffer &xml);
-extern DLLSERVER_API bool getEmbeddedManifestXML(ILoadedDllEntry *dll, StringBuffer &xml);
+extern DLLSERVER_API bool getEmbeddedManifestXML(const ILoadedDllEntry *dll, StringBuffer &xml);
+extern DLLSERVER_API IPropertyTree *getEmbeddedManifestPTree(const ILoadedDllEntry *dll);
+
 extern DLLSERVER_API bool checkEmbeddedWorkUnitXML(ILoadedDllEntry *dll);
 extern DLLSERVER_API bool getResourceFromFile(const char *filename, MemoryBuffer &data, const char * type, unsigned id);
 extern DLLSERVER_API bool getResourceXMLFromFile(const char *filename, const char *type, unsigned id, StringBuffer &xml);

common/thorhelper/enginecontext.hpp

@@ -42,6 +42,7 @@ interface IEngineContext
     virtual bool allowDaliAccess() const = 0;
     virtual StringBuffer &getQueryId(StringBuffer &result, bool isShared) const = 0;
     virtual void onTermination(QueryTermCallback callback, const char *key, bool isShared) const = 0;
+    virtual const StringArray &queryManifestFiles(const char *type) const = 0;
 };
 
 #endif // ENGINECONTEXT_HPP

common/wuwebview/wuwebview.cpp

@@ -372,6 +372,8 @@ IPropertyTree *WuWebView::ensureManifest()
     if (!manifest)
     {
         StringBuffer xml;
+        // Is this threadsafe? Does it need to be? Should it use getEmbeddedManifestPTree ?
+        // It looks like WuWebView classes are created when needed, so not shared between threads.
         manifest.setown((loadDll() && getEmbeddedManifestXML(dll, xml)) ? createPTreeFromXMLString(xml.str()) : createPTree());
     }
     return manifest.get();

ecl/eclagent/eclagent.ipp

@@ -539,6 +539,10 @@ public:
         result.append("workunit"); // No distinction between global, workunit and query scopes for eclagent
         return result;
     }
+    virtual const StringArray &queryManifestFiles(const char *type) const override
+    {
+        return dll->queryManifestFiles(type, wuid);
+    }
 
     virtual void onTermination(QueryTermCallback callback, const char *key, bool isShared) const
     {

plugins/javaembed/javaembed.cpp

@@ -3003,14 +3003,22 @@ public:
     {
         if (instance)
             instance = JNIenv->NewGlobalRef(instance);
-        if (flags & EFthreadlocal)
-            sharedCtx->registerContext(this);
         argcount = 0;
         argsig = NULL;
         nonStatic = (instance != nullptr);
         javaClass = nullptr;
         StringArray opts;
         opts.appendList(options, ",");
+        engine = codeCtx->queryEngineContext();
+        StringBuffer lclassPath;
+        if (engine)
+        {
+            const StringArray &manifestJars = engine->queryManifestFiles("jar");
+            ForEachItemIn(idx, manifestJars)
+            {
+                lclassPath.append(';').append(manifestJars.item(idx));
+            }
+        }
         ForEachItemIn(idx, opts)
         {
             const char *opt = opts.item(idx);
@@ -3020,7 +3028,7 @@ public:
                 StringBuffer optName(val-opt, opt);
                 val++;
                 if (stricmp(optName, "classpath")==0)
-                    classpath.set(val);
+                    lclassPath.append(';').append(val);
                 else if (strieq(optName, "persist"))
                 {
                     if (persistMode != persistNone)
@@ -3030,7 +3038,6 @@ public:
                     {
                     case persistWorkunit:
                     case persistQuery:
-                        engine = codeCtx->queryEngineContext();
                         if (!engine)
                             throw MakeStringException(MSGAUD_user, 0, "javaembed: Persist mode '%s' not supported here", val);
                         break;
@@ -3040,6 +3047,10 @@ public:
                     throw MakeStringException(0, "javaembed: Unknown option %s", optName.str());
             }
         }
+        if (lclassPath.length()>1)
+            classpath.set(lclassPath.str()+1);
+        if (flags & EFthreadlocal)
+            sharedCtx->registerContext(this);  // Do at end - otherwise an exception thrown during construction will leave this reference dangling
     }
     ~JavaEmbedImportContext()
     {
@@ -3132,11 +3143,15 @@ public:
         else if (*returnType=='V' && strieq(methodName, "<init>"))
         {
             jobject thisObject = JNIenv->NewGlobalRef(result.l);
-            if (persistMode==persistThread)
-                UNIMPLEMENTED;
-            if (engine)
+            switch (persistMode)
             {
+            case persistThread:
+                UNIMPLEMENTED;  // MORE - think about what this means?
+                break;
+            case persistWorkunit:
+            case persistQuery:
                 // Register this object to be removed automatically at end of specified scope...
+                assertex(engine);
                 VStringBuffer scopeKey("O.%p", thisObject);
                 PersistedObjectCriticalBlock persistBlock;
                 persistBlock.enter(globalState->getGlobalObject(JNIenv, scopeKey));
@@ -4154,8 +4169,11 @@ protected:
                     instance = JNIenv->NewGlobalRef(JNIenv->NewObject(javaClass, constructor));
                     if (persistBlock.locked())
                     {
-                        if (engine)
+                        if (persistMode==persistQuery || persistMode==persistWorkunit)
+                        {
+                            assertex(engine);
                             engine->onTermination(JavaGlobalState::unregister, scopeKey.str(), persistMode==persistWorkunit);
+                        }
                         persistBlock.leave(JNIenv->NewGlobalRef(instance));
                     }
                 }

roxie/ccd/ccdcontext.cpp

@@ -3129,6 +3129,12 @@ public:
             logctx.getLogPrefix(result);
         return result;
     }
+    virtual const StringArray &queryManifestFiles(const char *type) const override
+    {
+        ILoadedDllEntry *dll = factory->queryDll();
+        StringBuffer id;
+        return dll->queryManifestFiles(type, getQueryId(id, true).str());
+    }
 
     mutable CIArrayOf<TerminationCallbackInfo> callbacks;
     mutable CriticalSection callbacksCrit;

thorlcr/graph/thgraphslave.cpp

@@ -1560,6 +1560,10 @@ public:
     {
         return result.append(jobChannel.queryJob().queryWuid());
     }
+    virtual const StringArray &queryManifestFiles(const char *type) const override
+    {
+        return querySo.queryManifestFiles(type, jobChannel.queryJob().queryWuid());
+    }
     virtual void onTermination(QueryTermCallback callback, const char *key, bool isShared) const
     {
         TerminationCallbackInfo *term(new TerminationCallbackInfo(callback, key));

tools/wuget/wuget.cpp

@@ -21,7 +21,11 @@
 
 void usage(const char *progname)
 {
-    fprintf(stderr, "%s - extract workunit information from query dll.\nUsage: %s filename\n", progname, progname);
+    printf("%s - extract workunit information from query dll.\nUsage: %s [-m] [-w] filename\n", progname, progname);
+    printf("Options:\n");
+    printf("  -m     Display manifest information\n");
+    printf("  -w     Display embedded workunit\n");
+    printf("/nIf no options specified, -w is assumed\n");
 }
 
 int main(int argc, char **argv)