ISSUE: #79 Support adding resources using a manifest file

Add eclcc command line support for adding arbitrary resources using a
manifest file.

Usage:

eclcc -manifest filename queryfile.ecl

The manifest can can contain both individual and composite resources.

The manifest file should contain xml of the following format:

<manifest>
  <resource type="TYPENAME" filepath="path/file.ext"/>
  <resource type="TYPENAME">
	--content--
  </resource>
  <othercontent/>
</manifest>

eclcc -E -manifest filename queryfile.ecl

Adds the manifest resources to the generated ECL Archive.

Compiling from an ECL Archive will look for Manifest resouces in the
Archive in the following format:

<Archive>
  <Query/>
  <AdditionalFiles>
    <Manifest>...manifest xml content...</Manifest>
    <Resource originalFilename="path">base64Content</Resource>
  </AddtionalFiles>
</Archive>

Signed-off-by: Anthony Fishbeck <Anthony.Fishbeck@lexisnexis.com>

ecl/eclcc/eclcc.cpp

@@ -31,7 +31,7 @@
 #include "hqlwuerr.hpp"
 #include "hqlfold.hpp"
 #include "hqlplugins.hpp"
-#include "hqlesp.hpp"
+#include "hqlmanifest.hpp"
 #include "hqlrepository.hpp"
 #include "hqlerror.hpp"
 
@@ -221,11 +221,11 @@ public:
     Linked<IPropertyTree> archive;
     Linked<IErrorReceiver> errs;
     Linked<IWorkUnit> wu;
-    Linked<IPropertyTree> webServiceInfo;
     StringAttr eclVersion;
     const char * outputFilename;
     FILE * errout;
     bool importAllModules;
+    Owned<IPropertyTree> srcArchive;
     bool fromArchive;
 };
 
@@ -262,7 +262,7 @@ protected:
     void applyDebugOptions(IWorkUnit * wu);
     bool checkWithinRepository(StringBuffer & attributePath, const char * sourcePathname);
     ICppCompiler * createCompiler(const char * coreName);
-    void instantECL(IWorkUnit *wu, IHqlExpression * query, IPropertyTree * webServiceInfo, const char * queryFullName, IErrorReceiver *errs, const char * outputFile);
+    void instantECL(EclCompileInstance & instance, IWorkUnit *wu, IHqlExpression * query, const char * queryFullName, IErrorReceiver *errs, const char * outputFile);
     void getComplexity(IWorkUnit *wu, IHqlExpression * query, IErrorReceiver *errs);
     void processSingleQuery(EclCompileInstance & instance, IEclRepository * dataServer, const char * sourceName,
                                const char * queryText,
@@ -291,6 +291,7 @@ protected:
     StringBuffer cclogFilename;
     StringAttr optLogfile;
     StringAttr optIniFilename;
+    StringAttr optManifestFilename;
     StringAttr optOutputDirectory;
     StringAttr optOutputFilename;
     FILE * batchLog;
@@ -530,7 +531,7 @@ void EclCC::reportCompileErrors(IErrorReceiver *errs, const char * processName)
 
 //=========================================================================================
 
-void EclCC::instantECL(IWorkUnit *wu, IHqlExpression * query, IPropertyTree * webServiceInfo, const char * queryFullName, IErrorReceiver *errs, const char * outputFile)
+void EclCC::instantECL(EclCompileInstance & instance, IWorkUnit *wu, IHqlExpression * query, const char * queryFullName, IErrorReceiver *errs, const char * outputFile)
 {
     OwnedHqlExpr qquery = LINK(query);
     StringBuffer processName(outputFile);
@@ -549,7 +550,10 @@ void EclCC::instantECL(IWorkUnit *wu, IHqlExpression * query, IPropertyTree * we
                 if (!optShared)
                     wu->setDebugValueInt("standAloneExe", 1, true);
                 EclGenerateTarget target = optWorkUnit ? EclGenerateNone : (optNoCompile ? EclGenerateCpp : optShared ? EclGenerateDll : EclGenerateExe);
-                generator->setWebServiceInfo(webServiceInfo);
+                if (optManifestFilename)
+                    generator->addManifest(optManifestFilename);
+                if (instance.srcArchive)
+                    generator->addManifestFromArchive(instance.srcArchive);
                 generator->setSaveGeneratedFiles(optSaveCpp);
 
                 bool generateOk = generator->processQuery(qquery, target);
@@ -923,8 +927,6 @@ void EclCC::processSingleQuery(EclCompileInstance & instance, IEclRepository * d
         if (instance.wu->getDebugValueBool("addTimingToWorkunit", true))
             instance.wu->setTimerInfo("EclServer: parse query", NULL, msTick()-startTime, 1, 0);
 
-        instance.webServiceInfo.setown(retrieveWebServicesInfo(queryText, ctx));
-
         if (qquery && !syntaxChecking)
             qquery.setown(convertAttributeToQuery(instance, qquery, ctx));
     }
@@ -966,7 +968,7 @@ void EclCC::processSingleQuery(EclCompileInstance & instance, IEclRepository * d
         targetFilename.append(".eclout");
 
     if (errs->errCount() == prevErrs)
-        instantECL(instance.wu, qquery, instance.webServiceInfo, scope->queryFullName(), errs, targetFilename);
+        instantECL(instance, instance.wu, qquery, scope->queryFullName(), errs, targetFilename);
     else 
     {
         if (stdIoHandle(targetFilename) == -1)
@@ -988,19 +990,19 @@ void EclCC::processSingleQuery(EclCompileInstance & instance, IEclRepository * d
 
 void EclCC::processXmlFile(EclCompileInstance & instance, const char *archiveXML)
 {
-    Owned<IPropertyTree> xml = createPTreeFromXMLString(archiveXML, ipt_caseInsensitive);
-    Owned<IEclRepository> dataServer = createXmlDataServer(xml, libraryRepository);
-    Owned<IPropertyTreeIterator> iter = xml->getElements("Option");
+    instance.srcArchive.setown(createPTreeFromXMLString(archiveXML, ipt_caseInsensitive));
+    Owned<IEclRepository> dataServer = createXmlDataServer(instance.srcArchive, libraryRepository);
+    Owned<IPropertyTreeIterator> iter = instance.srcArchive->getElements("Option");
     ForEach(*iter) 
     {
         IPropertyTree &item = iter->query();
         instance.wu->setDebugValue(item.queryProp("@name"), item.queryProp("@value"), true);
     }
 
-    const char * queryText = xml->queryProp("Query");
-    const char * queryAttributePath = xml->queryProp("Query/@attributePath");
-    const char * sourceFilename = xml->queryProp("Query/@originalFilename");
-    instance.eclVersion.set(xml->queryProp("@eclVersion"));
+    const char * queryText = instance.srcArchive->queryProp("Query");
+    const char * queryAttributePath = instance.srcArchive->queryProp("Query/@attributePath");
+    const char * sourceFilename = instance.srcArchive->queryProp("Query/@originalFilename");
+    instance.eclVersion.set(instance.srcArchive->queryProp("@eclVersion"));
     if (instance.eclVersion)
     {
         unsigned major, minor, subminor;
@@ -1031,9 +1033,9 @@ void EclCC::processXmlFile(EclCompileInstance & instance, const char *archiveXML
     else
     {
         //This is really only useful for regression testing
-        const char * queryText = xml->queryProp("SyntaxCheck");
-        const char * syntaxCheckModule = xml->queryProp("SyntaxCheck/@module");
-        const char * syntaxCheckAttribute = xml->queryProp("SyntaxCheck/@attribute");
+        const char * queryText = instance.srcArchive->queryProp("SyntaxCheck");
+        const char * syntaxCheckModule = instance.srcArchive->queryProp("SyntaxCheck/@module");
+        const char * syntaxCheckAttribute = instance.srcArchive->queryProp("SyntaxCheck/@attribute");
         if (queryText && syntaxCheckModule && syntaxCheckAttribute)
             processSingleQuery(instance, dataServer, syntaxCheckModule, queryText, NULL, syntaxCheckModule, syntaxCheckAttribute);
         else
@@ -1088,6 +1090,9 @@ void EclCC::processFile(EclCompileInstance & instance)
             instance.archive->setProp("Query/@originalFilename", fname.str());
         }
 
+        if (optManifestFilename)
+            addManifestResourcesToArchive(instance.archive, optManifestFilename);
+
         //Work around windows problem writing 64K to stdout if not redirected/piped
         StringBuffer archiveName;
         if (instance.outputFilename && !streq(instance.outputFilename, "-"))
@@ -1263,6 +1268,11 @@ bool EclCC::parseCommandLineOptions(int argc, const char* argv[])
                 return false;
             }
         }
+        else if (iter.matchOption(optManifestFilename, "-manifest"))
+        {
+            if (!isManifestFileValid(optManifestFilename))
+                return false;
+        }
         else if (iter.matchOption(tempArg, "-split"))
         {
             batchPart = atoi(tempArg)-1;
@@ -1361,6 +1371,7 @@ void EclCC::usage()
            "    -Ppath        Specify the path of the output files\n"
            "    -Wc,xx        Supply option for the c++ compiler\n"
            "    -save-temps   Do not delete intermediate files (implied if -g)\n"
+           "    -manifest     Specify path to manifest file listing resources to add\n"
            "\nECL options:\n"
            "    -E            Output preprocessed ECL in xml archive form\n"
            "    -S            Generate c++ output, but don't compile\n"

ecl/hql/CMakeLists.txt

@@ -36,6 +36,7 @@ set (   SRCS
         hqlexpr.cpp 
         hqlfold.cpp 
         hqlgram2.cpp 
+        hqlmanifest.cpp 
         hqlmeta.cpp 
         hqlopt.cpp 
         hqlparse.cpp 

ecl/hql/hqlesp.cpp

@@ -427,5 +427,3 @@ IPropertyTree * retrieveWebServicesInfo(const char * queryText, HqlLookupContext
     result->setPropInt("@crc", extractor.getVersion());
     return result.getClear();
 }
-
-

ecl/hql/hqlmanifest.cpp

@@ -0,0 +1,126 @@
+/*##############################################################################
+
+    Copyright (C) 2011 HPCC Systems.
+
+    All rights reserved. This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as
+    published by the Free Software Foundation, either version 3 of the
+    License, or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+############################################################################## */
+#include "jliball.hpp"
+#include "hql.hpp"
+#include "hqlmanifest.hpp"
+
+//-------------------------------------------------------------------------------------------------------------------
+// Process manifest resources.
+
+
+class ResourceManifest : public CInterface
+{
+public:
+    ResourceManifest(const char *filename)
+        : manifest(createPTreeFromXMLFile(filename)), origPath(filename)
+    {
+        makeAbsolutePath(filename, absFilename);
+        splitDirTail(absFilename, dir);
+    }
+
+    void addToArchive(IPropertyTree *archive);
+    void loadResource(const char *filepath, MemoryBuffer &content);
+    bool checkResourceFilesExist();
+public:
+    Owned<IPropertyTree> manifest;
+    StringAttr origPath;
+    StringBuffer absFilename;
+    StringBuffer dir;
+};
+
+bool ResourceManifest::checkResourceFilesExist()
+{
+    Owned<IPropertyTreeIterator> resources = manifest->getElements("Resource[@filename]");
+    ForEach(*resources)
+    {
+        const char *filename = resources->query().queryProp("@filename");
+        StringBuffer fullpath;
+        if (!isAbsolutePath(filename))
+            fullpath.append(dir);
+        fullpath.append(filename);
+        if (!checkFileExists(fullpath.str()))
+        {
+            StringBuffer absResPath;
+            makeAbsolutePath(fullpath.str(), absResPath);
+            ERRLOG("Error: RESOURCE file '%s' does not exist", absResPath.str());
+            return false;
+        }
+    }
+    return true;
+}
+
+void ResourceManifest::loadResource(const char *filepath, MemoryBuffer &content)
+{
+    Owned <IFile> f = createIFile(filepath);
+    Owned <IFileIO> fio = f->open(IFOread);
+    read(fio, 0, (size32_t) f->size(), content);
+}
+
+void ResourceManifest::addToArchive(IPropertyTree *archive)
+{
+    IPropertyTree *additionalFiles = ensurePTree(archive, "AdditionalFiles");
+
+    //xsi namespace required for proper representaion after PTree::setPropBin()
+    if (!additionalFiles->hasProp("@xmlns:xsi"))
+        additionalFiles->setProp("@xmlns:xsi", "http://www.w3.org/2001/XMLSchema-instance");
+
+    StringBuffer xml;
+    toXML(manifest, xml);
+    additionalFiles->setProp("Manifest", xml.str());
+    additionalFiles->setProp("Manifest/@originalFilename", origPath.sget());
+
+    Owned<IPropertyTreeIterator> resources = manifest->getElements("Resource[@filename]");
+    ForEach(*resources)
+    {
+        StringBuffer absResPath;
+        const char *filename = resources->query().queryProp("@filename");
+        if (!isAbsolutePath(filename))
+        {
+            StringBuffer relResPath(dir);
+            relResPath.append(filename);
+            makeAbsolutePath(relResPath.str(), absResPath);
+        }
+        else
+            absResPath.append(filename);
+
+        IPropertyTree *resTree = additionalFiles->addPropTree("Resource", createPTree("Resource"));
+        resTree->setProp("@originalFilename", filename);
+
+        MemoryBuffer content;
+        loadResource(absResPath.str(), content);
+        resTree->setPropBin(NULL, content.length(), content.toByteArray());
+    }
+}
+
+void addManifestResourcesToArchive(IPropertyTree *archive, const char *filename)
+{
+    ResourceManifest manifest(filename);
+    manifest.addToArchive(archive);
+}
+
+bool isManifestFileValid(const char *filename)
+{
+    if (!checkFileExists(filename))
+    {
+        ERRLOG("Error: MANIFEST file '%s' does not exist", filename);
+        return false;
+    }
+
+    ResourceManifest manifest(filename);
+    return manifest.checkResourceFilesExist();
+}

ecl/hql/hqlmanifest.hpp

@@ -0,0 +1,26 @@
+/*##############################################################################
+
+    Copyright (C) 2011 HPCC Systems.
+
+    All rights reserved. This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as
+    published by the Free Software Foundation, either version 3 of the
+    License, or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+############################################################################## */
+#ifndef HQLMANIFEST_HPP
+#define HQLMANIFEST_HPP
+
+#include "hql.hpp"
+
+extern HQL_API void addManifestResourcesToArchive(IPropertyTree *archive, const char *filename);
+extern HQL_API bool isManifestFileValid(const char *filename);
+
+#endif

ecl/hqlcpp/hqlcpp.cpp

@@ -1147,7 +1147,7 @@ void HqlCppInstance::addPluginsAsResource()
         return;
     StringBuffer pluginXML;
     toXML(plugins, pluginXML);
-    addResource("PLUGINS", 1, pluginXML.length(), pluginXML.str());
+    addResource("PLUGINS", pluginXML.length(), pluginXML.str(), NULL, 1);
 }
 
 
@@ -1241,12 +1241,12 @@ unsigned HqlCppInstance::addStringResource(unsigned len, const char * body)
     return resources.addString(len, body);
 }
 
-void HqlCppInstance::addResource(const char * type, unsigned id, unsigned len, const void * body)
+void HqlCppInstance::addResource(const char * type, unsigned len, const void * body, IPropertyTree *entryEx, unsigned id)
 {
-    resources.addNamed(type, id, len, body);
+    resources.addNamed(type, len, body, entryEx, id);
 }
 
-void HqlCppInstance::addCompressResource(const char * type, unsigned id, unsigned len, const void * data)
+void HqlCppInstance::addCompressResource(const char * type, unsigned len, const void * body, IPropertyTree *entryEx, unsigned id)
 {
 #ifdef ADD_RESOURCE_AS_CPP_COMMENT
     BuildCtx ctx(*this, includeAtom);
@@ -1255,9 +1255,7 @@ void HqlCppInstance::addCompressResource(const char * type, unsigned id, unsigne
     ctx.addQuoted(s);
 #endif
 
-    MemoryBuffer compressed;
-    compressResource(compressed, len, data);
-    addResource(type, id, compressed.length(), compressed.toByteArray());
+    resources.addCompress(type, len, body, entryEx, id);
 }
 
 void HqlCppInstance::flushHints()
@@ -1269,35 +1267,9 @@ void HqlCppInstance::flushHints()
     }
 }
 
-IPropertyTree * HqlCppInstance::ensureWebServiceInfo()
-{
-    if (!webServiceInfo)
-        webServiceInfo.setown(createPTree("WebServicesInfo"));
-    return webServiceInfo;
-}
-
-
-void HqlCppInstance::addWebServicesResource()
-{
-    if (webServiceInfo)
-    {
-        StringBuffer webXML;
-        toXML(webServiceInfo, webXML);
-        addCompressResource("SOAPINFO", 1000, webXML.length(), webXML.str());
-    }
-}
-
-
-void HqlCppInstance::addWebServices(IPropertyTree * info)
-{
-    if (info)
-        mergePTree(ensureWebServiceInfo(), info);
-}
-
 void HqlCppInstance::flushResources(const char *filename, ICodegenContextCallback * ctxCallback)
 {
     addPluginsAsResource();
-    addWebServicesResource();
     if (resources.count())
     {
         bool flushText = workunit->getDebugValueBool("flushResourceAsText", false);

ecl/hqlcpp/hqlcpp.hpp

@@ -124,9 +124,10 @@ public:
     virtual HqlStmts * ensureSection(_ATOM section) = 0;
     virtual const char * queryLibrary(unsigned idx) = 0;
     virtual HqlStmts * querySection(_ATOM section) = 0;
-    virtual void addResource(const char * type, unsigned id, unsigned len, const void * data) = 0;
-    virtual void addCompressResource(const char * type, unsigned id, unsigned len, const void * data) = 0;
-    virtual void addWebServices(IPropertyTree * info) = 0;
+    virtual void addResource(const char * type, unsigned len, const void * data, IPropertyTree *entryEx, unsigned id=(unsigned)-1) = 0;
+    virtual void addCompressResource(const char * type, unsigned len, const void * data, IPropertyTree *entryEx, unsigned id=(unsigned)-1) = 0;
+    virtual void addManifest(const char *filename) = 0;
+    virtual void addManifestFromArchive(IPropertyTree *archive) = 0;
     virtual void flushHints() = 0;
     virtual void flushResources(const char *filename, ICodegenContextCallback * ctxCallback) = 0;
 };

ecl/hqlcpp/hqlcpp.ipp

@@ -123,10 +123,10 @@ public:
     virtual HqlStmts * querySection(_ATOM section);
     virtual void flushHints();
     virtual void flushResources(const char *filename, ICodegenContextCallback * ctxCallback);
-    virtual void addResource(const char * type, unsigned id, unsigned len, const void * data);
-    virtual void addCompressResource(const char * type, unsigned id, unsigned len, const void * data);
-    virtual void addWebServices(IPropertyTree * info);
-    virtual IPropertyTree * ensureWebServiceInfo();
+    virtual void addResource(const char * type, unsigned len, const void * data, IPropertyTree *entryEx=NULL, unsigned id=(unsigned)-1);
+    virtual void addCompressResource(const char * type, unsigned len, const void * data, IPropertyTree *entryEx=NULL, unsigned id=(unsigned)-1);
+    virtual void addManifest(const char *filename){resources.addManifest(filename);}
+    virtual void addManifestFromArchive(IPropertyTree *archive){resources.addManifestFromArchive(archive);}
     
     bool useFunction(IHqlExpression * funcdef);
     void useInclude(const char * include);
@@ -139,7 +139,6 @@ public:
         
 private:
     void addPluginsAsResource();
-    void addWebServicesResource();
     void appendHintText(const char * xml);
 
 public:
@@ -153,7 +152,6 @@ public:
     StringAttr          wupathname;
     Owned<IPropertyTree> plugins;
     Owned<IFileIOStream> hintFile;
-    Owned<IPropertyTree> webServiceInfo;
 };
 
 //---------------------------------------------------------------------------
@@ -1934,7 +1932,7 @@ protected:
     void addSchemaField(IHqlExpression *field, MemoryBuffer &schema, IHqlExpression *selector);
     void addSchemaFields(IHqlExpression * record, MemoryBuffer &schema, IHqlExpression *selector);
     void addSchemaResource(int seq, const char * name, IHqlExpression * record);
-    void addSchemaResource(int seq, const char * name, const char * schemaXml);
+    void addSchemaResource(int seq, const char * name, unsigned len, const char * schemaXml);
     void doAddSchemaFields(IHqlExpression * record, MemoryBuffer &schema, IHqlExpression *selector);
     IWUResult * createDatasetResultSchema(IHqlExpression * sequenceExpr, IHqlExpression * name, IHqlExpression * record, bool createTransformer, bool isFile);
 

ecl/hqlcpp/hqlecl.cpp

@@ -74,7 +74,8 @@ public:
     virtual bool generateExe(ICppCompiler * compiler);
     virtual bool generatePackage(const char * packageName);
     virtual void setMaxCompileThreads(unsigned value) { defaultMaxCompileThreads = value; }
-    virtual void setWebServiceInfo(IPropertyTree * webServiceInfo) { if (webServiceInfo) code->addWebServices(webServiceInfo); }
+    virtual void addManifest(const char *filename) { code->addManifest(filename); }
+    virtual void addManifestFromArchive(IPropertyTree *archive) { code->addManifestFromArchive(archive); }
 
     virtual double getECLcomplexity(IHqlExpression * exprs);
     virtual void generateCppPrototypes(IHqlScope * scope);
@@ -313,7 +314,7 @@ void HqlDllGenerator::addWorkUnitAsResource()
 {
     SCMStringBuffer wuXML;
     exportWorkUnitToXML(wu, wuXML);
-    code->addCompressResource("WORKUNIT", 1000, wuXML.length(), wuXML.str());
+    code->addCompressResource("WORKUNIT", wuXML.length(), wuXML.str(), NULL, 1000);
 }
 
 

ecl/hqlcpp/hqlecl.hpp

@@ -46,7 +46,8 @@ public:
     virtual bool generatePackage(const char * packageName) = 0;
     virtual bool processQuery(IHqlExpression * expr, EclGenerateTarget generateTarget) = 0;
     virtual void setMaxCompileThreads(unsigned max) = 0;
-    virtual void setWebServiceInfo(IPropertyTree * webServiceInfo) = 0;
+    virtual void addManifest(const char *filename) = 0;
+    virtual void addManifestFromArchive(IPropertyTree *archive) = 0;
     virtual void setSaveGeneratedFiles(bool value) = 0;
 };
 

ecl/hqlcpp/hqlhtcpp.cpp

@@ -5120,7 +5120,7 @@ void HqlCppTranslator::buildSetResultInfo(BuildCtx & ctx, IHqlExpression * origi
                     xmlbuilder.addField(fieldName, *schemaType);
                     xmlbuilder.getXml(xml);
                 }
-                addSchemaResource(sequence, resultName.str(), xml.str());
+                addSchemaResource(sequence, resultName.str(), xml.length()+1, xml.str());
             }
         }
     }
@@ -10382,21 +10382,16 @@ void HqlCppTranslator::addSchemaResource(int seq, const char * name, IHqlExpress
 {
     StringBuffer xml;
     getRecordXmlSchema(xml, record, true);
-    addSchemaResource(seq, name, xml.str());
+    addSchemaResource(seq, name, xml.length()+1, xml.str());
 }
 
 
-void HqlCppTranslator::addSchemaResource(int seq, const char * name, const char * schemaXml)
+void HqlCppTranslator::addSchemaResource(int seq, const char * name, unsigned len, const char * schemaXml)
 {
-    Owned<IPropertyTree> resultTree = createPTree("Result");
-    resultTree->setPropInt("@sequence", seq);
-    resultTree->setProp("@name", name);
-    resultTree->addPropTree("xs:schema", createPTreeFromXMLString(schemaXml));
-    IPropertyTree * web = code->ensureWebServiceInfo();
-    IPropertyTree * schema = web->queryPropTree("SCHEMA");
-    if (!schema)
-        schema = web->addPropTree("SCHEMA", createPTree("SCHEMA"));
-    schema->addPropTree("Result", resultTree.getClear());
+    Owned<IPropertyTree> entryEx = createPTree("Resource");
+    entryEx->setProp("@name", name);
+    entryEx->setPropInt("@seq", seq);
+    code->addCompressResource("RESULT_XSD", len, schemaXml, entryEx);
 }
 
 

ecl/hqlcpp/hqlres.cpp

@@ -23,11 +23,13 @@
 #include "jmisc.hpp"
 #include "jexcept.hpp"
 #include "hqlcerrors.hpp"
+#include "thorplugin.hpp"
 #ifndef _WIN32
 #include "bfd.h"
 #endif
 
-#define RESOURCE_BASE 101
+#define BIGSTRING_BASE 101
+#define MANIFEST_BASE 1000
 
 class ResourceItem : public CInterface
 {
@@ -44,8 +46,10 @@ public:
 
 ResourceManager::ResourceManager()
 {
-    nextid = 0;
+    nextmfid = 1;
+    nextbsid = 0;
     totalbytes = 0;
+    finalized=false;
 }
 
 unsigned ResourceManager::count()
@@ -55,16 +59,128 @@ unsigned ResourceManager::count()
 
 unsigned ResourceManager::addString(unsigned len, const char *data)
 {
-    unsigned id = RESOURCE_BASE + nextid++;
+    unsigned id = BIGSTRING_BASE + nextbsid++;
     resources.append(*new ResourceItem("BIGSTRING", id, len, data));
     return id;
 }
 
-void ResourceManager::addNamed(const char * type, unsigned id, unsigned len, const void * data)
+void ResourceManager::addNamed(const char * type, unsigned len, const void * data, IPropertyTree *entryEx, unsigned id, bool addToManifest, bool compressed)
 {
+    if (id==(unsigned)-1)
+        id = MANIFEST_BASE + nextmfid++;
+    if (addToManifest && !finalized)
+    {
+        Owned<IPropertyTree> entry=createPTree("Resource");
+        entry->setProp("@type", type);
+        entry->setPropInt("@id", id);
+        if (compressed)
+            entry->setPropBool("compressed", true);
+        if (entryEx)
+            mergePTree(entry, entryEx);
+        ensureManifestInfo()->addPropTree("Resource", entry.getClear());
+    }
     resources.append(*new ResourceItem(type, id, len, data));
 }
 
+bool ResourceManager::addCompress(const char * type, unsigned len, const void * data, IPropertyTree *entryEx, unsigned id, bool addToManifest)
+{
+    bool isCompressed=false;
+    if (len>=32) //lzw assert if too small
+    {
+        isCompressed = true;
+        MemoryBuffer compressed;
+        compressResource(compressed, len, data);
+        addNamed(type, compressed.length(), compressed.toByteArray(), entryEx, id, addToManifest, isCompressed);
+    }
+    else
+        addNamed(type, len, data, entryEx, id, addToManifest, isCompressed);
+    return isCompressed;
+}
+
+static void loadResource(const char *filepath, MemoryBuffer &content)
+{
+    Owned <IFile> f = createIFile(filepath);
+    Owned <IFileIO> fio = f->open(IFOread);
+    read(fio, 0, (size32_t) f->size(), content);
+}
+
+void ResourceManager::addManifest(const char *filename)
+{
+    Owned<IPropertyTree> manifestSrc = createPTreeFromXMLFile(filename);
+
+    StringBuffer dir; 
+    splitDirTail(filename, dir);
+
+    Owned<IPropertyTreeIterator> itres = manifestSrc->getElements("Resource[@filename]");
+    ForEach(*itres)
+    {
+        unsigned id = MANIFEST_BASE + nextmfid++;
+        IPropertyTree &resource = itres->query();
+        resource.setPropInt("@id", id);
+        if (!resource.hasProp("@type"))
+            resource.setProp("@type", "UNKOWN");
+
+        StringBuffer fullpath;
+        const char *respath = resource.queryProp("@filename");
+        if (!isAbsolutePath(respath))
+            fullpath.append(dir);
+        fullpath.append(respath);
+
+        MemoryBuffer content;
+        loadResource(fullpath.str(), content);
+        if (addCompress(resource.queryProp("@type"), content.length(), content.toByteArray(), NULL, id, false))
+            resource.setPropBool("@compressed", true);
+    }
+    mergePTree(ensureManifestInfo(), manifestSrc);
+}
+
+void ResourceManager::addManifestFromArchive(IPropertyTree *archive)
+{
+    if (archive)
+    {
+        Owned<IPropertyTreeIterator> manifests = archive->getElements("AdditionalFiles/Manifest");
+        ForEach(*manifests)
+        {
+            const char *xml = manifests->query().queryProp(NULL);
+            Owned<IPropertyTree> manifestSrc = createPTreeFromXMLString(xml);
+
+            Owned<IPropertyTreeIterator> itres = manifestSrc->getElements("Resource[@filename]");
+            ForEach(*itres)
+            {
+                unsigned id = MANIFEST_BASE + nextmfid++;
+                IPropertyTree &resource = itres->query();
+                resource.setPropInt("@id", id);
+                if (!resource.hasProp("@type"))
+                    resource.setProp("@type", "UNKOWN");
+                const char *filename = resource.queryProp("@filename");
+                VStringBuffer xpath("AdditionalFiles/Resource[@originalFilename=\"%s\"]", filename);
+                MemoryBuffer content;
+                archive->getPropBin(xpath.str(), content);
+                if (content.length())
+                {
+                    if (addCompress(resource.queryProp("@type"), content.length(), content.toByteArray(), NULL, id, false))
+                        resource.setPropBool("@compressed", true);
+                }
+            }
+            mergePTree(ensureManifestInfo(), manifestSrc);
+        }
+    }
+}
+
+void ResourceManager::finalize()
+{
+    if (!finalized)
+    {
+        if (manifest)
+        {
+            StringBuffer content;
+            toXML(manifest, content);
+            addCompress("MANIFEST", content.length()+1, content.str(), NULL, MANIFEST_BASE, false);
+        }
+        finalized=true;
+    }
+}
+
 void ResourceManager::putbytes(int h, const void *b, unsigned len)
 {
     int written = _write(h, b, len);
@@ -74,6 +190,8 @@ void ResourceManager::putbytes(int h, const void *b, unsigned len)
 
 void ResourceManager::flushAsText(const char *filename)
 {
+    finalize();
+
     StringBuffer name;
     int len = strlen(filename);
     name.append(filename,0,len-4).append(".txt");
@@ -96,6 +214,8 @@ void ResourceManager::flushAsText(const char *filename)
 
 void ResourceManager::flush(const char *filename, bool flushText, bool target64bit)
 {
+    finalize();
+
     // Use "resources" for strings that are a bit large to generate in the c++ (some compilers had limits at 64k) 
     // or that we want to access without having to run the dll/so
     // In linux there is no .res concept but we can achieve the same effect by generating an object file with a specially-named section 

ecl/hqlcpp/hqlres.hpp

@@ -21,13 +21,22 @@
 
 class ResourceManager
 {
-    unsigned nextid;
+    unsigned nextmfid;
+    unsigned nextbsid;
     unsigned totalbytes;
     CIArray resources;
+    Owned<IPropertyTree> manifest;
+    bool finalized;
 public:
     ResourceManager();
     unsigned addString(unsigned len, const char *data);
-    void addNamed(const char * type, unsigned id, unsigned len, const void *data);
+    void addNamed(const char * type, unsigned len, const void *data, IPropertyTree *entry=NULL, unsigned id=(unsigned)-1, bool addToManifest=true, bool compressed=false);
+    bool addCompress(const char * type, unsigned len, const void *data, IPropertyTree *entry=NULL, unsigned id=(unsigned)-1, bool addToManifest=true);
+    void addManifest(const char *filename);
+    void addManifestFromArchive(IPropertyTree *archive);
+    IPropertyTree *ensureManifestInfo(){if (!manifest) manifest.setown(createPTree("Manifest")); return manifest;}
+    void finalize();
+
     unsigned count();
     void flush(const char *filename, bool flushText, bool target64bit);
     void flushAsText(const char *filename);