Merge pull request #6911 from afishbeck/autoManifest4

HPCC-11482 Automatically add manifests named same as included ECL files

Reviewed-By: Gordon Smith <gordon.smith@lexisnexis.com>
Reviewed-By: Richard Chapman <rchapman@hpccsystems.com>

ecl/eclcc/eclcc.cpp

@@ -215,6 +215,7 @@ public:
     FILE * errout;
     Owned<IPropertyTree> srcArchive;
     Owned<IPropertyTree> generatedMeta;
+    Owned<IPropertyTree> globalDependTree;
     bool legacyImport;
     bool legacyWhen;
     bool fromArchive;
@@ -332,13 +333,15 @@ protected:
     StringBuffer cclogFilename;
     StringAttr optLogfile;
     StringAttr optIniFilename;
-    StringAttr optManifestFilename;
     StringAttr optOutputDirectory;
     StringAttr optOutputFilename;
     StringAttr optQueryRepositoryReference;
     StringAttr optComponentName;
     FILE * batchLog;
 
+    StringAttr optManifestFilename;
+    StringArray resourceManifestFiles;
+
     IFileArray inputFiles;
     StringArray inputFileNames;
     StringArray applicationOptions;
@@ -494,6 +497,8 @@ void EclCC::loadManifestOptions()
 {
     if (!optManifestFilename)
         return;
+    resourceManifestFiles.append(optManifestFilename);
+
     Owned<IPropertyTree> mf = createPTreeFromXMLFile(optManifestFilename);
     IPropertyTree *ecl = mf->queryPropTree("ecl");
     if (ecl)
@@ -714,6 +719,26 @@ void EclCC::reportCompileErrors(IErrorReceiver & errorProcessor, const char * pr
 
 //=========================================================================================
 
+void gatherResourceManifestFilenames(EclCompileInstance & instance, StringArray &filenames)
+{
+    IPropertyTree *tree = (instance.archive) ? instance.archive.get() : instance.globalDependTree.get();
+    if (!tree)
+        return;
+    Owned<IPropertyTreeIterator> iter = tree->getElements((instance.archive) ? "Module/Attribute" : "Attribute");
+    ForEach(*iter)
+    {
+        StringBuffer filename(iter->query().queryProp("@sourcePath"));
+        if (filename.length())
+        {
+            getFullFileName(filename, true).append(".manifest");
+            if (filenames.contains(filename))
+                continue;
+            if (checkFileExists(filename))
+                filenames.append(filename);
+        }
+    }
+}
+
 void EclCC::instantECL(EclCompileInstance & instance, IWorkUnit *wu, const char * queryFullName, IErrorReceiver & errorProcessor, const char * outputFile)
 {
     StringBuffer processName(outputFile);
@@ -732,8 +757,9 @@ void EclCC::instantECL(EclCompileInstance & instance, IWorkUnit *wu, const char
                 if (!optShared)
                     wu->setDebugValueInt("standAloneExe", 1, true);
                 EclGenerateTarget target = optWorkUnit ? EclGenerateNone : (optNoCompile ? EclGenerateCpp : optShared ? EclGenerateDll : EclGenerateExe);
-                if (optManifestFilename)
-                    generator->addManifest(optManifestFilename);
+                gatherResourceManifestFilenames(instance, resourceManifestFiles);
+                ForEachItemIn(i, resourceManifestFiles)
+                    generator->addManifest(resourceManifestFiles.item(i));
                 if (instance.srcArchive)
                 {
                     generator->addManifestFromArchive(instance.srcArchive);
@@ -1067,6 +1093,8 @@ void EclCC::processSingleQuery(EclCompileInstance & instance,
     {
         //Minimize the scope of the parse context to reduce lifetime of cached items.
         HqlParseContext parseCtx(instance.dataServer, instance.archive);
+        if (!instance.archive)
+            parseCtx.globalDependTree.setown(createPTree(ipt_none)); //to locate associated manifests, keep separate from user specified MetaOptions
         if (optGenerateMeta || optIncludeMeta)
         {
             HqlParseContext::MetaOptions options;
@@ -1172,6 +1200,9 @@ void EclCC::processSingleQuery(EclCompileInstance & instance,
             if (optIncludeMeta || optGenerateMeta)
                 instance.generatedMeta.setown(parseCtx.getMetaTree());
 
+            if (parseCtx.globalDependTree)
+                instance.globalDependTree.set(parseCtx.globalDependTree);
+
             if (optEvaluateResult && !errorProcessor.errCount() && instance.query)
                 evaluateResult(instance);
         }
@@ -1599,8 +1630,9 @@ void EclCC::generateOutput(EclCompileInstance & instance)
                 option->setProp("@value", valueStr.str());
                 instance.archive->addPropTree("Option", option.getClear());
             }
-            if (optManifestFilename)
-                addManifestResourcesToArchive(instance.archive, optManifestFilename);
+            gatherResourceManifestFilenames(instance, resourceManifestFiles);
+            ForEachItemIn(i, resourceManifestFiles)
+                addManifestResourcesToArchive(instance.archive, resourceManifestFiles.item(i));
 
             outputXmlToOutputFile(instance, instance.archive);
         }

ecl/hql/hqlexpr.cpp

@@ -711,10 +711,10 @@ void HqlParseContext::noteBeginAttribute(IHqlScope * scope, IFileContents * cont
         setDefinitionText(attr, "", contents);
     }
 
+    ISourcePath * sourcePath = contents->querySourcePath();
+
     if (checkBeginMeta())
     {
-        ISourcePath * sourcePath = contents->querySourcePath();
-
         IPropertyTree * attr = metaTree->addPropTree("Source", createPTree("Source"));
         setFullNameProp(attr, "@name", scope->queryFullName(), name->str());
         attr->setProp("@sourcePath", sourcePath->str());
@@ -726,6 +726,7 @@ void HqlParseContext::noteBeginAttribute(IHqlScope * scope, IFileContents * cont
         IPropertyTree * attr = globalDependTree->addPropTree("Attribute", createPTree("Attribute"));
         attr->setProp("@module", scope->queryFullName());
         attr->setProp("@name", name->str());
+        attr->setProp("@sourcePath", sourcePath->str());
         //attr->setPropInt("@flags", symbol->getObType());  MORE
     }
 }

ecl/hql/hqlmanifest.cpp

@@ -200,8 +200,10 @@ void ResourceManifest::addToArchive(IPropertyTree *archive)
 
     StringBuffer xml;
     toXML(manifest, xml);
-    additionalFiles->setProp("Manifest", xml.str());
-    additionalFiles->setProp("Manifest/@originalFilename", absFilename.str());
+
+    IPropertyTree *manifest = additionalFiles->addPropTree("Manifest", createPTree("Manifest", ipt_none));
+    manifest->setProp("@originalFilename", absFilename.str());
+    manifest->setProp(NULL, xml.str());
 }
 
 void addManifestResourcesToArchive(IPropertyTree *archive, const char *filename)

ecl/hqlcpp/hqlres.cpp

@@ -122,6 +122,50 @@ bool ResourceManager::getDuplicateResourceId(const char *srctype, const char *re
     return false;
 }
 
+void updateManifestResourcePaths(IPropertyTree &resource, const char *dir)
+{
+    StringBuffer filepath;
+    makeAbsolutePath(resource.queryProp("@filename"), dir, filepath);
+    resource.setProp("@originalFilename", filepath.str());
+
+    StringBuffer respath;
+    makePathUniversal(filepath.str(), respath);
+    resource.setProp("@resourcePath", respath.str());
+}
+
+void expandManifestDirectory(IPropertyTree *manifestSrc, IPropertyTree &res, StringBuffer &dir, IDirectoryIterator *it, const char*mask, bool recursive)
+{
+    if (!it)
+        return;
+    ForEach(*it)
+    {
+        if (it->isDir())
+        {
+            if (recursive)
+                expandManifestDirectory(manifestSrc, res, dir, it->query().directoryFiles(mask, false, true), mask, recursive);
+            continue;
+        }
+        StringBuffer reldir;
+        Owned<IPropertyTree> newRes = createPTreeFromIPT(&res);
+        reldir.append(splitRelativePath(it->query().queryFilename(), dir, reldir));
+        VStringBuffer xpath("Resource[@filename='%s']", reldir.str());
+        if (manifestSrc->hasProp(xpath))
+            continue;
+        newRes->setProp("@filename", reldir.str());
+        updateManifestResourcePaths(*newRes, dir.str());
+        if (manifestSrc->hasProp(xpath.setf("resource[@resourcePath='%s']", newRes->queryProp("@resourcePath"))))
+            continue;
+        manifestSrc->addPropTree("Resource", newRes.getClear());
+    }
+}
+
+void expandManifestDirectory(IPropertyTree *manifestSrc, IPropertyTree &res, StringBuffer &dir, const char *path, const char*mask, bool recursive)
+{
+    Owned<IDirectoryIterator> it = createDirectoryIterator(path, mask);
+    expandManifestDirectory(manifestSrc, res, dir, it, mask, recursive);
+}
+
+
 void ResourceManager::addManifestFile(const char *filename)
 {
     Owned<IPropertyTree> manifestSrc = createPTreeFromXMLFile(filename);
@@ -149,24 +193,39 @@ void ResourceManager::addManifestFile(const char *filename)
             item.setProp("@originalFilename", filepath.str());
             item.setProp("@resourcePath", respath.str());
 
-            if (!item.hasProp("@type"))
-                item.setProp("@type", "UNKNOWN");
-            int id;
-            if (getDuplicateResourceId(item.queryProp("@type"), respath.str(), NULL, id))
-            {
-                item.setPropInt("@id", id);
-                manifest->addPropTree("Resource", LINK(&item));
-            }
-            else
+            if (containsFileWildcard(filepath))
             {
-                MemoryBuffer content;
-                loadResource(filepath.str(), content);
-                addCompress(item.queryProp("@type"), content.length(), content.toByteArray(), &item);
+                StringBuffer wildpath;
+                const char *tail = splitDirTail(filepath, wildpath);
+                expandManifestDirectory(manifestSrc, item, dir, wildpath, tail, item.getPropBool("@recursive"));
+                manifestSrc->removeTree(&item);
             }
+
         }
         else
             manifest->addPropTree(item.queryName(), LINK(&item));
     }
+
+    Owned<IPropertyTreeIterator> resources = manifestSrc->getElements("Resource[@filename]");
+    ForEach(*resources)
+    {
+        IPropertyTree &item = resources->query();
+
+        if (!item.hasProp("@type"))
+            item.setProp("@type", "UNKNOWN");
+        int id;
+        if (getDuplicateResourceId(item.queryProp("@type"), item.queryProp("@resourcePath"), NULL, id))
+        {
+            item.setPropInt("@id", id);
+            manifest->addPropTree("Resource", LINK(&item));
+        }
+        else
+        {
+            MemoryBuffer content;
+            loadResource(item.queryProp("@originalFilename"), content);
+            addCompress(item.queryProp("@type"), content.length(), content.toByteArray(), &item);
+        }
+    }
 }
 
 void ResourceManager::addManifest(const char *filename)