ISSUE: #79 Support adding resources using a manifest file

Add eclcc command line support for adding arbitrary resources using a
manifest file.

Usage:

eclcc -manifest filename queryfile.ecl

The manifest can can contain both individual and composite resources.

The manifest file should contain xml of the following format:

<manifest>
  <resource type="TYPENAME" filepath="path/file.ext"/>
  <resource type="TYPENAME">
	--content--
  </resource>
  <othercontent/>
</manifest>

eclcc -E -manifest filename queryfile.ecl

Adds the manifest resources to the generated ECL Archive.

Compiling from an ECL Archive will look for Manifest resouces in the
Archive in the following format:

<Archive>
  <Query/>
  <AdditionalFiles>
    <Manifest>...manifest xml content...</Manifest>
    <Resource originalFilename="path">base64Content</Resource>
  </AddtionalFiles>
</Archive>

Refactored: now merges manifest trees explicitly rather then calling mergePTree.
Also commons up duplicate resources.
Signed-off-by: Anthony Fishbeck <Anthony.Fishbeck@lexisnexis.com>

ecl/hql/hqlmanifest.cpp

@@ -89,21 +89,25 @@ void ResourceManifest::addToArchive(IPropertyTree *archive)
     {
         StringBuffer absResPath;
         const char *filename = resources->query().queryProp("@filename");
-        if (!isAbsolutePath(filename))
+        VStringBuffer xpath("Resource[@originalFilename='%s']", filename);
+        if (!additionalFiles->hasProp(xpath.str()))
         {
-            StringBuffer relResPath(dir);
-            relResPath.append(filename);
-            makeAbsolutePath(relResPath.str(), absResPath);
+            if (!isAbsolutePath(filename))
+            {
+                StringBuffer relResPath(dir);
+                relResPath.append(filename);
+                makeAbsolutePath(relResPath.str(), absResPath);
+            }
+            else
+                absResPath.append(filename);
+
+            IPropertyTree *resTree = additionalFiles->addPropTree("Resource", createPTree("Resource"));
+            resTree->setProp("@originalFilename", filename);
+
+            MemoryBuffer content;
+            loadResource(absResPath.str(), content);
+            resTree->setPropBin(NULL, content.length(), content.toByteArray());
         }
-        else
-            absResPath.append(filename);
-
-        IPropertyTree *resTree = additionalFiles->addPropTree("Resource", createPTree("Resource"));
-        resTree->setProp("@originalFilename", filename);
-
-        MemoryBuffer content;
-        loadResource(absResPath.str(), content);
-        resTree->setPropBin(NULL, content.length(), content.toByteArray());
     }
 }
 

ecl/hqlcpp/hqlcerrors.hpp

@@ -209,6 +209,7 @@
 #define HQLERR_OpArgDependsDataset              4180
 #define HQLERR_CouldNotDetermineMinSize         4181
 #define HQLERR_OutsideGroupAggregate            4182
+#define HQLERR_ResourceAddAfterFinalManifest    4183
 
 //Warnings....
 #define HQLWRN_PersistDataNotLikely             4500
@@ -491,6 +492,7 @@
 #define HQLERR_OpArgDependsDataset_Text         "%s: %s cannot be dependent on the dataset"
 #define HQLERR_CouldNotDetermineMinSize_Text    "Cannot determine the minimum size of the expression"
 #define HQLERR_OutsideGroupAggregate_Text       "%s used outside of a TABLE aggregation"
+#define HQLERR_ResourceAddAfterFinalManifest_Text "%s resource added after manifest was finalized"
 
 //Warnings.
 #define HQLWRN_CannotRecreateDistribution_Text  "Cannot recreate the distribution for a persistent dataset"

ecl/hqlcpp/hqlcpp.cpp

@@ -1241,12 +1241,12 @@ unsigned HqlCppInstance::addStringResource(unsigned len, const char * body)
     return resources.addString(len, body);
 }
 
-void HqlCppInstance::addResource(const char * type, unsigned len, const void * body, IPropertyTree *entryEx, unsigned id)
+void HqlCppInstance::addResource(const char * type, unsigned len, const void * body, IPropertyTree *manifestEntry, unsigned id)
 {
-    resources.addNamed(type, len, body, entryEx, id);
+    resources.addNamed(type, len, body, manifestEntry, id);
 }
 
-void HqlCppInstance::addCompressResource(const char * type, unsigned len, const void * body, IPropertyTree *entryEx, unsigned id)
+void HqlCppInstance::addCompressResource(const char * type, unsigned len, const void * body, IPropertyTree *manifestEntry, unsigned id)
 {
 #ifdef ADD_RESOURCE_AS_CPP_COMMENT
     BuildCtx ctx(*this, includeAtom);
@@ -1255,7 +1255,7 @@ void HqlCppInstance::addCompressResource(const char * type, unsigned len, const
     ctx.addQuoted(s);
 #endif
 
-    resources.addCompress(type, len, body, entryEx, id);
+    resources.addCompress(type, len, body, manifestEntry, id);
 }
 
 void HqlCppInstance::flushHints()

ecl/hqlcpp/hqlcpp.hpp

@@ -124,10 +124,11 @@ public:
     virtual HqlStmts * ensureSection(_ATOM section) = 0;
     virtual const char * queryLibrary(unsigned idx) = 0;
     virtual HqlStmts * querySection(_ATOM section) = 0;
-    virtual void addResource(const char * type, unsigned len, const void * data, IPropertyTree *entryEx, unsigned id=(unsigned)-1) = 0;
-    virtual void addCompressResource(const char * type, unsigned len, const void * data, IPropertyTree *entryEx, unsigned id=(unsigned)-1) = 0;
+    virtual void addResource(const char * type, unsigned len, const void * data, IPropertyTree *manifestEntry, unsigned id=(unsigned)-1) = 0;
+    virtual void addCompressResource(const char * type, unsigned len, const void * data, IPropertyTree *manifestEntry, unsigned id=(unsigned)-1) = 0;
     virtual void addManifest(const char *filename) = 0;
     virtual void addManifestFromArchive(IPropertyTree *archive) = 0;
+    virtual void addWebServiceInfo(IPropertyTree *wsinfo) = 0;
     virtual void flushHints() = 0;
     virtual void flushResources(const char *filename, ICodegenContextCallback * ctxCallback) = 0;
 };

ecl/hqlcpp/hqlcpp.ipp

@@ -123,10 +123,11 @@ public:
     virtual HqlStmts * querySection(_ATOM section);
     virtual void flushHints();
     virtual void flushResources(const char *filename, ICodegenContextCallback * ctxCallback);
-    virtual void addResource(const char * type, unsigned len, const void * data, IPropertyTree *entryEx=NULL, unsigned id=(unsigned)-1);
-    virtual void addCompressResource(const char * type, unsigned len, const void * data, IPropertyTree *entryEx=NULL, unsigned id=(unsigned)-1);
+    virtual void addResource(const char * type, unsigned len, const void * data, IPropertyTree *manifestEntry=NULL, unsigned id=(unsigned)-1);
+    virtual void addCompressResource(const char * type, unsigned len, const void * data, IPropertyTree *manifestEntry=NULL, unsigned id=(unsigned)-1);
     virtual void addManifest(const char *filename){resources.addManifest(filename);}
     virtual void addManifestFromArchive(IPropertyTree *archive){resources.addManifestFromArchive(archive);}
+    virtual void addWebServiceInfo(IPropertyTree *wsinfo){resources.addWebServiceInfo(wsinfo);}
     
     bool useFunction(IHqlExpression * funcdef);
     void useInclude(const char * include);

ecl/hqlcpp/hqlecl.cpp

@@ -76,6 +76,7 @@ public:
     virtual void setMaxCompileThreads(unsigned value) { defaultMaxCompileThreads = value; }
     virtual void addManifest(const char *filename) { code->addManifest(filename); }
     virtual void addManifestFromArchive(IPropertyTree *archive) { code->addManifestFromArchive(archive); }
+    virtual void addWebServiceInfo(IPropertyTree *wsinfo){ code->addWebServiceInfo(wsinfo); }
 
     virtual double getECLcomplexity(IHqlExpression * exprs);
     virtual void generateCppPrototypes(IHqlScope * scope);

ecl/hqlcpp/hqlecl.hpp

@@ -48,6 +48,7 @@ public:
     virtual void setMaxCompileThreads(unsigned max) = 0;
     virtual void addManifest(const char *filename) = 0;
     virtual void addManifestFromArchive(IPropertyTree *archive) = 0;
+    virtual void addWebServiceInfo(IPropertyTree *wsinfo) = 0;
     virtual void setSaveGeneratedFiles(bool value) = 0;
 };
 

ecl/hqlcpp/hqlhtcpp.cpp

@@ -10388,10 +10388,10 @@ void HqlCppTranslator::addSchemaResource(int seq, const char * name, IHqlExpress
 
 void HqlCppTranslator::addSchemaResource(int seq, const char * name, unsigned len, const char * schemaXml)
 {
-    Owned<IPropertyTree> entryEx = createPTree("Resource");
-    entryEx->setProp("@name", name);
-    entryEx->setPropInt("@seq", seq);
-    code->addCompressResource("RESULT_XSD", len, schemaXml, entryEx);
+    Owned<IPropertyTree> manifestEntry = createPTree("Resource");
+    manifestEntry->setProp("@name", name);
+    manifestEntry->setPropInt("@seq", seq);
+    code->addCompressResource("RESULT_XSD", len, schemaXml, manifestEntry);
 }
 
 

ecl/hqlcpp/hqlres.cpp

@@ -46,8 +46,8 @@ public:
 
 ResourceManager::ResourceManager()
 {
-    nextmfid = 1;
-    nextbsid = 0;
+    nextmfid = MANIFEST_BASE + 1;
+    nextbsid = BIGSTRING_BASE;
     totalbytes = 0;
     finalized=false;
 }
@@ -59,30 +59,32 @@ unsigned ResourceManager::count()
 
 unsigned ResourceManager::addString(unsigned len, const char *data)
 {
-    unsigned id = BIGSTRING_BASE + nextbsid++;
+    unsigned id = nextbsid++;
     resources.append(*new ResourceItem("BIGSTRING", id, len, data));
     return id;
 }
 
-void ResourceManager::addNamed(const char * type, unsigned len, const void * data, IPropertyTree *entryEx, unsigned id, bool addToManifest, bool compressed)
+void ResourceManager::addNamed(const char * type, unsigned len, const void * data, IPropertyTree *manifestEntry, unsigned id, bool addToManifest, bool compressed)
 {
     if (id==(unsigned)-1)
-        id = MANIFEST_BASE + nextmfid++;
-    if (addToManifest && !finalized)
+        id = nextmfid++;
+    if (addToManifest)
     {
+        if (finalized)
+            throwError1(HQLERR_ResourceAddAfterFinalManifest, type);
         Owned<IPropertyTree> entry=createPTree("Resource");
         entry->setProp("@type", type);
         entry->setPropInt("@id", id);
         if (compressed)
-            entry->setPropBool("compressed", true);
-        if (entryEx)
-            mergePTree(entry, entryEx);
+            entry->setPropBool("@compressed", true);
+        if (manifestEntry)
+            mergePTree(entry, manifestEntry);
         ensureManifestInfo()->addPropTree("Resource", entry.getClear());
     }
     resources.append(*new ResourceItem(type, id, len, data));
 }
 
-bool ResourceManager::addCompress(const char * type, unsigned len, const void * data, IPropertyTree *entryEx, unsigned id, bool addToManifest)
+bool ResourceManager::addCompress(const char * type, unsigned len, const void * data, IPropertyTree *manifestEntry, unsigned id, bool addToManifest)
 {
     bool isCompressed=false;
     if (len>=32) //lzw assert if too small
@@ -90,10 +92,10 @@ bool ResourceManager::addCompress(const char * type, unsigned len, const void *
         isCompressed = true;
         MemoryBuffer compressed;
         compressResource(compressed, len, data);
-        addNamed(type, compressed.length(), compressed.toByteArray(), entryEx, id, addToManifest, isCompressed);
+        addNamed(type, compressed.length(), compressed.toByteArray(), manifestEntry, id, addToManifest, isCompressed);
     }
     else
-        addNamed(type, len, data, entryEx, id, addToManifest, isCompressed);
+        addNamed(type, len, data, manifestEntry, id, addToManifest, isCompressed);
     return isCompressed;
 }
 
@@ -104,65 +106,172 @@ static void loadResource(const char *filepath, MemoryBuffer &content)
     read(fio, 0, (size32_t) f->size(), content);
 }
 
+bool ResourceManager::getDuplicateResourceId(const char *srctype, const char *filename, int &id)
+{
+    VStringBuffer xpath("Resource[@filename='%s']", filename);
+    Owned<IPropertyTreeIterator> iter = manifest->getElements(xpath.str());
+    ForEach (*iter)
+    {
+        IPropertyTree &item = iter->query();
+        if (item.hasProp("@id"))
+        {
+            const char *type = item.queryProp("@type");
+            if (type && strieq(type, srctype))
+            {
+                id=item.getPropInt("@id");
+                return true;
+            }
+        }
+    }
+    return false;
+}
+
 void ResourceManager::addManifest(const char *filename)
 {
+    if (finalized)
+        throwError1(HQLERR_ResourceAddAfterFinalManifest, "MANIFEST");
     Owned<IPropertyTree> manifestSrc = createPTreeFromXMLFile(filename);
 
     StringBuffer dir; 
     splitDirTail(filename, dir);
 
-    Owned<IPropertyTreeIterator> itres = manifestSrc->getElements("Resource[@filename]");
-    ForEach(*itres)
+    ensureManifestInfo();
+    Owned<IAttributeIterator> aiter = manifestSrc->getAttributes();
+    ForEach (*aiter)
+        manifest->setProp(aiter->queryName(), aiter->queryValue());
+    Owned<IPropertyTreeIterator> iter = manifestSrc->getElements("*");
+    ForEach(*iter)
     {
-        unsigned id = MANIFEST_BASE + nextmfid++;
-        IPropertyTree &resource = itres->query();
-        resource.setPropInt("@id", id);
-        if (!resource.hasProp("@type"))
-            resource.setProp("@type", "UNKOWN");
-
-        StringBuffer fullpath;
-        const char *respath = resource.queryProp("@filename");
-        if (!isAbsolutePath(respath))
-            fullpath.append(dir);
-        fullpath.append(respath);
-
-        MemoryBuffer content;
-        loadResource(fullpath.str(), content);
-        if (addCompress(resource.queryProp("@type"), content.length(), content.toByteArray(), NULL, id, false))
-            resource.setPropBool("@compressed", true);
+        IPropertyTree &item = iter->query();
+        if (streq(item.queryName(), "Resource") && item.hasProp("@filename"))
+        {
+            if (!item.hasProp("@type"))
+                item.setProp("@type", "UNKNOWN");
+            const char *filename = item.queryProp("@filename");
+            int id;
+            if (getDuplicateResourceId(item.queryProp("@type"), filename, id))
+            {
+                item.setPropInt("@id", id);
+                manifest->addPropTree("Resource", LINK(&item));
+            }
+            else
+            {
+                StringBuffer fullpath;
+                if (!isAbsolutePath(filename))
+                    fullpath.append(dir);
+                fullpath.append(filename);
+
+                MemoryBuffer content;
+                loadResource(fullpath.str(), content);
+                addCompress(item.queryProp("@type"), content.length(), content.toByteArray(), &item);
+            }
+        }
+        else
+            manifest->addPropTree(item.queryName(), LINK(&item));
     }
-    mergePTree(ensureManifestInfo(), manifestSrc);
 }
 
 void ResourceManager::addManifestFromArchive(IPropertyTree *archive)
 {
-    if (archive)
+    if (!archive)
+        return;
+    if (finalized)
+        throwError1(HQLERR_ResourceAddAfterFinalManifest, "MANIFEST");
+    ensureManifestInfo();
+    Owned<IPropertyTreeIterator> manifests = archive->getElements("AdditionalFiles/Manifest");
+    ForEach(*manifests)
     {
-        Owned<IPropertyTreeIterator> manifests = archive->getElements("AdditionalFiles/Manifest");
-        ForEach(*manifests)
+        const char *xml = manifests->query().queryProp(NULL);
+        Owned<IPropertyTree> manifestSrc = createPTreeFromXMLString(xml);
+        Owned<IAttributeIterator> aiter = manifestSrc->getAttributes();
+        ForEach (*aiter)
+            manifest->setProp(aiter->queryName(), aiter->queryValue());
+        Owned<IPropertyTreeIterator> iter = manifestSrc->getElements("*");
+        ForEach(*iter)
         {
-            const char *xml = manifests->query().queryProp(NULL);
-            Owned<IPropertyTree> manifestSrc = createPTreeFromXMLString(xml);
-
-            Owned<IPropertyTreeIterator> itres = manifestSrc->getElements("Resource[@filename]");
-            ForEach(*itres)
+            IPropertyTree &item = iter->query();
+            if (streq(item.queryName(), "Resource")&& item.hasProp("@filename"))
             {
-                unsigned id = MANIFEST_BASE + nextmfid++;
-                IPropertyTree &resource = itres->query();
-                resource.setPropInt("@id", id);
-                if (!resource.hasProp("@type"))
-                    resource.setProp("@type", "UNKOWN");
-                const char *filename = resource.queryProp("@filename");
-                VStringBuffer xpath("AdditionalFiles/Resource[@originalFilename=\"%s\"]", filename);
-                MemoryBuffer content;
-                archive->getPropBin(xpath.str(), content);
-                if (content.length())
+                if (!item.hasProp("@type"))
+                    item.setProp("@type", "UNKNOWN");
+                const char *filename = item.queryProp("@filename");
+                int id;
+                if (getDuplicateResourceId(item.queryProp("@type"), filename, id))
+                {
+                    item.setPropInt("@id", (int)id);
+                    manifest->addPropTree("Resource", LINK(&item));
+                }
+                else
                 {
-                    if (addCompress(resource.queryProp("@type"), content.length(), content.toByteArray(), NULL, id, false))
-                        resource.setPropBool("@compressed", true);
+                    VStringBuffer xpath("AdditionalFiles/Resource[@originalFilename=\"%s\"]", filename);
+                    MemoryBuffer content;
+                    archive->getPropBin(xpath.str(), content);
+                    addCompress(item.queryProp("@type"), content.length(), content.toByteArray(), &item);
                 }
             }
-            mergePTree(ensureManifestInfo(), manifestSrc);
+            else
+                manifest->addPropTree(item.queryName(), LINK(&item));
+        }
+    }
+}
+
+void ResourceManager::addWebServiceInfo(IPropertyTree *wsinfo)
+{
+    //convert legacy web service info to the new resource format
+    if (wsinfo)
+    {
+        if (wsinfo->hasProp("SOAP"))
+            ensureManifestInfo()->addProp("WS-PARAMS", wsinfo->queryProp("SOAP"));
+        if (wsinfo->hasProp("HELP"))
+        {
+            const char *content = wsinfo->queryProp("HELP");
+            addCompress("HELP", strlen(content)+1, content);
+        }
+        if (wsinfo->hasProp("INFO"))
+        {
+            const char *content = wsinfo->queryProp("INFO");
+            addCompress("INFO", strlen(content)+1, content);
+        }
+        if (wsinfo->hasProp("OTX"))
+        {
+            const char *content = wsinfo->queryProp("OTX");
+            addCompress("HYPER-LINK", strlen(content)+1, content);
+        }
+        if (wsinfo->hasProp("HTML"))
+        {
+            const char *content = wsinfo->queryProp("HTML");
+            Owned<IPropertyTree> manifestEntry = createPTree("Resource");
+            manifestEntry->setProp("@name", "Custom Form");
+            addCompress("XSLT", strlen(content)+1, content, manifestEntry);
+            IPropertyTree *view = ensurePTree(ensureManifestInfo(), "Views/XSLT/FORM");
+            view->setProp("@resource", "Custom Form");
+        }
+        if (wsinfo->hasProp("HTMLD"))
+        {
+            const char *content = wsinfo->queryProp("HTMLD");
+            Owned<IPropertyTree> manifestEntry = createPTree("Resource");
+            manifestEntry->setProp("@name", "Custom HTML");
+            addCompress("HTML", strlen(content)+1, content, manifestEntry);
+            IPropertyTree *view = ensurePTree(ensureManifestInfo(), "Views/HTML/FORM");
+            view->setProp("@resource", "Custom HTML");
+        }
+        if (wsinfo->hasProp("RESULT"))
+        {
+            const char *content = wsinfo->queryProp("RESULT");
+            Owned<IPropertyTree> manifestEntry = createPTree("Resource");
+            manifestEntry->setProp("@name", "Results");
+            addCompress("XSLT", strlen(content)+1, content, manifestEntry);
+            IPropertyTree *view = ensurePTree(ensureManifestInfo(), "Views/XSLT/RESULTS");
+            view->setProp("@resource", "Results");
+        }
+        if (wsinfo->hasProp("ERROR"))
+        {
+            const char *content = wsinfo->queryProp("ERROR");
+            Owned<IPropertyTree> manifestEntry = createPTree("Resource");
+            manifestEntry->setProp("@name", "Error");
+            addCompress("XSLT", strlen(content)+1, content, manifestEntry);
+            IPropertyTree *view = ensurePTree(ensureManifestInfo(), "Views/XSLT/ERROR");
+            view->setProp("@resource", "Error");
         }
     }
 }

ecl/hqlcpp/hqlres.hpp

@@ -34,7 +34,9 @@ public:
     bool addCompress(const char * type, unsigned len, const void *data, IPropertyTree *entry=NULL, unsigned id=(unsigned)-1, bool addToManifest=true);
     void addManifest(const char *filename);
     void addManifestFromArchive(IPropertyTree *archive);
+    void addWebServiceInfo(IPropertyTree *wsinfo);
     IPropertyTree *ensureManifestInfo(){if (!manifest) manifest.setown(createPTree("Manifest")); return manifest;}
+    bool getDuplicateResourceId(const char *srctype, const char *filename, int &id);
     void finalize();
 
     unsigned count();