HPCC-9919 Fix core accessing uncompressed workunit resources

The technique used to store the size and retrieve the size of stored
resources were not compatible.  As most resources are formatted
with headers designed to support compression this is usually not
noticed.  But resources stored without the headers would fail because
they relied on the size that was stored as an absolute data resource.

Headers will now always be stored with resources whether attempting to
compress them or not.  The size will be retrieved from the headers.

The old size information is still stored as it can be retrieved in
other ways besides the dlopen/dlsym technique used by dllserver.

Signed-off-by: Anthony Fishbeck <anthony.fishbeck@lexisnexis.com>

common/dllserver/thorplugin.cpp

@@ -178,6 +178,10 @@ const byte * HelperDll::getResource(unsigned id) const
 #endif
 }
 
+const byte resourceHeaderVersion=1;
+const size32_t resourceHeaderLength = sizeof(byte) + sizeof(byte) + sizeof(bool) + sizeof(size32_t);
+
+
 bool HelperDll::getResource(size32_t & len, const void * & data, const char * type, unsigned id) const
 {
 #ifdef _WIN32
@@ -189,15 +193,25 @@ bool HelperDll::getResource(size32_t & len, const void * & data, const char * ty
     data = (const byte *) LoadResource(dllHandle, hrsrc);
     return true;
 #else
-    StringBuffer baseName;
-    baseName.append(type).append("_").append(id);
-    StringBuffer symName, sizeName;
-    symName.append(baseName).append("_txt_start");
-    sizeName.append(baseName).append("_txt_size");
+    StringBuffer symName;
+    symName.append(type).append("_").append(id).append("_txt_start");
     data = (const void *) getEntry(symName.str());
     if (!data)
         return false;
-    len = (unsigned)(memsize_t)getEntry(sizeName.str());
+    byte bom;
+    byte version;
+    bool compressed;
+
+    MemoryBuffer mb;
+    mb.setBuffer(resourceHeaderLength, const_cast<void *>(data));
+    mb.read(bom);
+    if (bom!=0x80)
+        return false;
+    mb.read(version);
+    if (version>resourceHeaderVersion)
+        return false;
+    mb.read(compressed).read(len);
+    len+=resourceHeaderLength;
     return true;
 #endif
 }
@@ -374,11 +388,18 @@ extern DLLSERVER_API bool decompressResource(size32_t len, const void *data, Str
     return true;
 }
 
+extern DLLSERVER_API void appendResource(MemoryBuffer & mb, size32_t len, const void *data, bool compress)
+{
+    mb.append((byte)0x80).append(resourceHeaderVersion);
+    if (compress)
+        compressToBuffer(mb, len, data);
+    else
+        appendToBuffer(mb, len, data);
+}
+
 extern DLLSERVER_API void compressResource(MemoryBuffer & compressed, size32_t len, const void *data)
 {
-    byte version = 1;
-    compressed.append((byte)0x80).append(version);
-    compressToBuffer(compressed, len, data);
+    appendResource(compressed, len, data, true);
 }
 
 extern DLLSERVER_API bool getEmbeddedWorkUnitXML(ILoadedDllEntry *dll, StringBuffer &xml)

common/dllserver/thorplugin.hpp

@@ -44,6 +44,7 @@ extern DLLSERVER_API bool getManifestXMLFromFile(const char *filename, StringBuf
 
 extern DLLSERVER_API bool decompressResource(size32_t len, const void *data, StringBuffer &result);
 extern DLLSERVER_API void compressResource(MemoryBuffer & compressed, size32_t len, const void *data);
+extern DLLSERVER_API void appendResource(MemoryBuffer & mb, size32_t len, const void *data, bool compress);
 
 class DLLSERVER_API SimplePluginCtx : implements IPluginContextEx
 {

common/wuwebview/wuwebview.cpp

@@ -435,7 +435,7 @@ void WuWebView::getResource(IPropertyTree *res, StringBuffer &content)
 {
     if (!loadDll())
         return;
-    if (res->hasProp("@id"))
+    if (res->hasProp("@id") && (res->hasProp("@header")||res->hasProp("@compressed")))
     {
         int id = res->getPropInt("@id");
         size32_t len = 0;

ecl/hqlcpp/hqlres.cpp

@@ -74,28 +74,21 @@ void ResourceManager::addNamed(const char * type, unsigned len, const void * dat
         Owned<IPropertyTree> entry=createPTree("Resource");
         entry->setProp("@type", type);
         entry->setPropInt("@id", id);
-        if (compressed)
-            entry->setPropBool("@compressed", true);
+        entry->setPropBool("@compressed", compressed);
+        entry->setPropBool("@header", true);
         if (manifestEntry)
             mergePTree(entry, manifestEntry);
         ensureManifestInfo()->addPropTree("Resource", entry.getClear());
     }
-    resources.append(*new ResourceItem(type, id, len, data));
+    MemoryBuffer mb;
+    appendResource(mb, len, data, compressed);
+    resources.append(*new ResourceItem(type, id, mb.length(), mb.toByteArray()));
 }
 
 bool ResourceManager::addCompress(const char * type, unsigned len, const void * data, IPropertyTree *manifestEntry, unsigned id, bool addToManifest)
 {
-    bool isCompressed=false;
-    if (len>=32) //lzw assert if too small
-    {
-        isCompressed = true;
-        MemoryBuffer compressed;
-        compressResource(compressed, len, data);
-        addNamed(type, compressed.length(), compressed.toByteArray(), manifestEntry, id, addToManifest, isCompressed);
-    }
-    else
-        addNamed(type, len, data, manifestEntry, id, addToManifest, isCompressed);
-    return isCompressed;
+    addNamed(type, len, data, manifestEntry, id, addToManifest, true);
+    return true;
 }
 
 static void loadResource(const char *filepath, MemoryBuffer &content)

system/jlib/jlzw.cpp

@@ -685,6 +685,12 @@ Escape:
     return (size32_t)(in-(const byte *)src);
 }
 
+void appendToBuffer(MemoryBuffer & out, size32_t len, const void * src)
+{
+    out.append(false);
+    out.append(len);
+    out.append(len, src);
+}
 
 void compressToBuffer(MemoryBuffer & out, size32_t len, const void * src)
 {
@@ -696,7 +702,7 @@ void compressToBuffer(MemoryBuffer & out, size32_t len, const void * src)
     Owned<ICompressor> compressor = createLZWCompressor();
     void *newData = out.reserve(newSize);
     compressor->open(newData, newSize);
-    if (compressor->write(src, len)==len)
+    if (len>=32 && compressor->write(src, len)==len)
     {
         compressor->close();
         size32_t compressedLen = compressor->buflen();
@@ -707,9 +713,7 @@ void compressToBuffer(MemoryBuffer & out, size32_t len, const void * src)
     else // all or don't compress
     {
         out.setWritePos(originalLength);
-        out.append(false);
-        out.append(len);
-        out.append(len, src);
+        appendToBuffer(out, len, src);
     }
 }
 

system/jlib/jlzw.hpp

@@ -90,6 +90,8 @@ extern jlib_decl void decompressToBuffer(MemoryBuffer & out, const void * src);
 extern jlib_decl void decompressToBuffer(MemoryBuffer & out, MemoryBuffer & in);
 extern jlib_decl void decompressToAttr(MemoryAttr & out, const void * src);
 extern jlib_decl void decompressToBuffer(MemoryAttr & out, MemoryBuffer & in);
+extern jlib_decl void appendToBuffer(MemoryBuffer & out, size32_t len, const void * src); //format as failed compression
+
 
 #define COMPRESS_METHOD_ROWDIF 1
 #define COMPRESS_METHOD_LZW    2