HPCC-21377 Specify CPP sources via manifest file

Also fixes
HPCC-21371 Add option to disallow code in manifest
HPCC-21364 Manifest support for signed code

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

ecl/eclcc/eclcc.cpp

@@ -1919,7 +1919,7 @@ bool EclCC::generatePrecompiledHeader()
     }
     Owned<ICppCompiler> compiler = createCompiler("precompile", foundPath, NULL);
     compiler->setDebug(true);  // a precompiled header with debug can be used for no-debug, but not vice versa
-    compiler->addSourceFile("eclinclude4.hpp");
+    compiler->addSourceFile("eclinclude4.hpp", nullptr);
     compiler->setPrecompileHeader(true);
     if (compiler->compile())
     {

ecl/hql/hqlmanifest.cpp

@@ -16,6 +16,7 @@
 ############################################################################## */
 #include "jliball.hpp"
 #include "hql.hpp"
+#include "hqlutil.hpp"
 #include "hqlmanifest.hpp"
 
 //-------------------------------------------------------------------------------------------------------------------
@@ -29,12 +30,28 @@ public:
     {
         try
         {
-            manifest.setown(createPTreeFromXMLFile(filename));
+            fileContents.loadFile(filename, false);
+            const char *xml = fileContents.str();
+            StringBuffer body;
+            // Check for signature
+            if (startsWith(fileContents, "-----BEGIN PGP SIGNED MESSAGE-----"))
+            {
+                // Note - we do not check the signature here - we are creating an archive, and typically that means we
+                // are on the client machine, while the signature can only be checked on the server where the keys are installed.
+                stripSignature(body, fileContents);
+                xml = body.str();
+                isSigned = true;
+            }
+            manifest.setown(createPTreeFromXMLString(xml));
         }
-        catch (IException * e)
+        catch (IException * E)
         {
-            e->Release();
-            throw makeStringExceptionV(0, "Invalid manifest file '%s'", filename);
+            StringBuffer msg;
+            E->errorMessage(msg);
+            auto code = E->errorCode();
+            auto aud = E->errorAudience();
+            E->Release();
+            throw makeStringExceptionV(aud, code, "While loading manifest file %s: %s", filename, msg.str());
         }
         makeAbsolutePath(filename, absFilename);
         splitDirTail(absFilename, dir);
@@ -52,8 +69,10 @@ private:
 
 public:
     Owned<IPropertyTree> manifest;
+    StringBuffer fileContents;
     StringBuffer absFilename;
     StringBuffer dir;
+    bool isSigned = false;
 };
 
 void updateResourcePaths(IPropertyTree &resource, const char *dir)
@@ -181,7 +200,7 @@ void ResourceManifest::addToArchive(IPropertyTree *archive)
 {
     IPropertyTree *additionalFiles = ensurePTree(archive, "AdditionalFiles");
 
-    //xsi namespace required for proper representaion after PTree::setPropBin()
+    //xsi namespace required for proper representation after PTree::setPropBin()
     if (!additionalFiles->hasProp("@xmlns:xsi"))
         additionalFiles->setProp("@xmlns:xsi", "http://www.w3.org/2001/XMLSchema-instance");
 
@@ -189,29 +208,60 @@ void ResourceManifest::addToArchive(IPropertyTree *archive)
     ForEach(*resources)
     {
         IPropertyTree &item = resources->query();
-        const char *respath = item.queryProp("@resourcePath");
-
-        VStringBuffer xpath("Resource[@resourcePath='%s']", respath);
-        if (!additionalFiles->hasProp(xpath.str()))
+        const char *md5 = item.queryProp("@md5");
+        const char *filename = item.queryProp("@filename");
+        MemoryBuffer content;
+        if (isSigned)
         {
-            IPropertyTree *resTree = additionalFiles->addPropTree("Resource", createPTree("Resource"));
-
-            const char *filepath = item.queryProp("@originalFilename");
-            resTree->setProp("@originalFilename", filepath);
-            resTree->setProp("@resourcePath", respath);
-
-            MemoryBuffer content;
-            loadResource(filepath, content);
-            resTree->setPropBin(NULL, content.length(), content.toByteArray());
+            if (md5)
+            {
+                VStringBuffer xpath("Resource[@filename='%s'][@md5='%s']", filename, md5);
+                if (!additionalFiles->hasProp(xpath.str()))
+                {
+                    IPropertyTree *resTree = additionalFiles->addPropTree("Resource", createPTree("Resource"));
+                    resTree->setProp("@filename", filename);
+                    resTree->setProp("@md5", md5);
+                    loadResource(filename, content);
+                    resTree->setPropBin(NULL, content.length(), content.toByteArray());
+                }
+            }
+            else
+                throw makeStringExceptionV(0, "Signed manifest %s must provide MD5 values for referenced resource %s", absFilename.str(), filename);
+        }
+        else
+        {
+            const char *respath = item.queryProp("@resourcePath");
+            VStringBuffer xpath("Resource[@resourcePath='%s']", respath);
+            if (!additionalFiles->hasProp(xpath.str()))
+            {
+                IPropertyTree *resTree = additionalFiles->addPropTree("Resource", createPTree("Resource"));
+                const char *filepath = item.queryProp("@originalFilename");
+                resTree->setProp("@originalFilename", filepath);
+                resTree->setProp("@resourcePath", respath);
+                loadResource(filepath, content);
+                resTree->setPropBin(NULL, content.length(), content.toByteArray());
+            }
+        }
+        if (md5)
+        {
+            StringBuffer calculated;
+            md5_data(content, calculated);
+            if (!strieq(calculated, md5))
+                throw makeStringExceptionV(0, "MD5 mismatch on file %s in manifest %s", filename, absFilename.str());
         }
     }
 
-    StringBuffer xml;
-    toXML(manifest, xml);
-
-    IPropertyTree *manifest = additionalFiles->addPropTree("Manifest", createPTree("Manifest", ipt_none));
-    manifest->setProp("@originalFilename", absFilename.str());
-    manifest->setProp(NULL, xml.str());
+    IPropertyTree *manifestWrapper = additionalFiles->addPropTree("Manifest", createPTree("Manifest", ipt_none));
+    manifestWrapper->setProp("@originalFilename", absFilename.str());
+    manifestWrapper->setPropBool("@isSigned", isSigned);
+    if (isSigned)
+        manifestWrapper->setProp(NULL, fileContents.str());
+    else
+    {
+        StringBuffer xml;
+        toXML(manifest, xml);
+        manifestWrapper->setProp(NULL, xml.str());
+    }
 }
 
 void addManifestResourcesToArchive(IPropertyTree *archive, const char *filename)

ecl/hql/hqlparse.cpp

@@ -606,57 +606,16 @@ void HqlLex::checkSignature(const attribute & dummyToken)
 {
     if (yyParser->lookupCtx.queryParseContext().ignoreSignatures)
         return;
-    Owned<IPipeProcess> pipe = createPipeProcess();
-    if (!pipe->run("gpg", "gpg --verify -", ".", true, false, true, 0, false))
-    {
-        reportWarning(CategorySecurity, dummyToken, WRN_SECURITY_SIGNERROR, "Signature could not be checked because gpg was not found");
-        return;
-    }
     try
     {
-        pipe->write(text->length(), text->getText());
-        pipe->closeInput();
-        unsigned retcode = pipe->wait();
-
-        StringBuffer buf;
-        Owned<ISimpleReadStream> pipeReader = pipe->getErrorStream();
-        readSimpleStream(buf, *pipeReader);
-        DBGLOG("GPG %d %s", retcode, buf.str());
-        if (retcode)
-        {
-            StringArray allErrs;
-            allErrs.appendList(buf, "\n");
-            ForEachItemIn(idx, allErrs)
-            {
-                if (strlen(allErrs.item(idx)))
-                    reportWarning(CategorySecurity, dummyToken, WRN_SECURITY_SIGNERROR, "gpg error: %s", allErrs.item(idx));
-            }
-        }
-        else
-        {
-            yyParser->inSignedModule = true;
-            yyParser->gpgSignature.clear();
-            const char * sigprefix = "Good signature from \"";
-            const char * const s = buf.str();
-            const char * match = strstr(s, sigprefix);
-            if (match)
-            {
-                match += strlen(sigprefix);
-                const char * const end = strchr(match, '\"');
-                if (end)
-                {
-                    buf.setLength(end-s);
-                    const char * sig = buf.str() + (match-s);
-                    yyParser->gpgSignature.setown(createExprAttribute(_signed_Atom,createConstant(sig)) );
-                }
-            }
-        }
+        yyParser->gpgSignature.setown(::checkSignature(text->length(), text->getText()));
+        yyParser->inSignedModule = true;
     }
     catch (IException *e)
     {
         StringBuffer msg;
         e->errorMessage(msg);
-        reportWarning(CategorySecurity, dummyToken, WRN_SECURITY_SIGNERROR, "Failed to execute gpg: %d %s. Module will be treated as unsigned", e->errorCode(), msg.str());
+        reportWarning(CategorySecurity, dummyToken, WRN_SECURITY_SIGNERROR, "%s", msg.str());
         e->Release();
     }
 }

ecl/hql/hqlutil.cpp

@@ -10471,4 +10471,75 @@ const RtlTypeInfo *buildRtlType(IRtlFieldTypeDeserializer &deserializer, ITypeIn
     return deserializer.addType(info, type);
 }
 
+extern HQL_API IHqlExpression *checkSignature(unsigned fileSize, const char *fileContents)
+{
+    Owned<IPipeProcess> pipe = createPipeProcess();
+    if (!pipe->run("gpg", "gpg --verify -", ".", true, false, true, 0, false))
+        throw makeStringException(0, "Signature could not be checked because gpg was not found");
+    pipe->write(fileSize, fileContents);
+    pipe->closeInput();
+    unsigned retcode = pipe->wait();
+
+    StringBuffer buf;
+    Owned<ISimpleReadStream> pipeReader = pipe->getErrorStream();
+    readSimpleStream(buf, *pipeReader);
+    DBGLOG("GPG %d %s", retcode, buf.str());
+    if (retcode)
+    {
+        StringArray allErrs;
+        allErrs.appendList(buf, "\n");
+        ForEachItemInRev(idx, allErrs)
+        {
+            if (strlen(allErrs.item(idx)))
+                throw makeStringExceptionV(0, "gpg error: gpg returned %d: %s", retcode, allErrs.item(idx));
+        }
+        throw makeStringExceptionV(0, "gpg error: gpg returned %d", retcode);
+    }
+    else
+    {
+        const char * sigprefix = "Good signature from \"";
+        const char * const s = buf.str();
+        const char * match = strstr(s, sigprefix);
+        if (match)
+        {
+            match += strlen(sigprefix);
+            const char * const end = strchr(match, '\"');
+            if (end)
+            {
+                buf.setLength(end-s);
+                const char * sig = buf.str() + (match-s);
+                return createExprAttribute(_signed_Atom,createConstant(sig));
+            }
+        }
+        throw makeStringExceptionV(0, "gpg error: gpg response not recognised");
+    }
+}
+
+extern HQL_API StringBuffer &stripSignature(StringBuffer &out, const char *fileContents)
+{
+    assertex(startsWith(fileContents, "-----BEGIN PGP SIGNED MESSAGE-----"));
+    // Look for first blank line
+    const char *head = fileContents;
+    while ((head = strchr(head, '\n')) != nullptr)
+    {
+        head++;
+        if (*head=='\n')
+        {
+            head++;
+            break;
+        }
+        else if (*head=='\r' && head[1]=='\n')
+        {
+            head += 2;
+            break;
+        }
+    }
+    if (!head)
+        throw makeStringException(0, "End of PGP header not found");
+    // Now look for signature
+    const char *tail = strstr(head, "-----BEGIN PGP SIGNATURE-----");
+    if (!tail)
+        throw makeStringException(0, "PGP signature not found");
+    return out.append(tail-head, head);
+}
 

ecl/hql/hqlutil.hpp

@@ -249,6 +249,30 @@ extern HQL_API IHqlExpression * createTransformForField(IHqlExpression * field,
 extern HQL_API IHqlExpression * convertScalarToRow(IHqlExpression * value, ITypeInfo * fieldType);
 extern HQL_API bool splitResultValue(SharedHqlExpr & dataset, SharedHqlExpr & attribute, IHqlExpression * value);
 
+/**
+ * Check whether GPG signed file is valid.
+ *
+ * @param fileSize      Size of file contents
+ * @param fileContents  File contents
+ *
+ * @return             _signed_ attribute containing signature
+ *
+ * Will throw exception if signature cannot be verified.
+ */
+
+extern HQL_API IHqlExpression * checkSignature(unsigned fileSize, const char *fileContents);
+
+/**
+ * Strip GPG signature from file
+ *
+ * @param fileContents  File contents
+ * @param out           Buffer for returned value
+ *
+ * @return              Reference to out
+ */
+
+extern HQL_API StringBuffer &stripSignature(StringBuffer &out, const char *fileContents);
+
 //Is 'expr' really dependent on a parameter - expr->isFullyBound() can give false negatives.
 extern HQL_API bool isDependentOnParameter(IHqlExpression * expr);
 extern HQL_API bool isTimed(IHqlExpression * expr);

ecl/hqlcpp/hqlcpp.cpp

@@ -1114,7 +1114,7 @@ void insertUniqueString(StringAttrArray & array, const char * text)
     array.append(* new StringAttrItem(text));
 }
 
-HqlCppInstance::HqlCppInstance(IWorkUnit *_wu, const char * _wupathname)
+HqlCppInstance::HqlCppInstance(IWorkUnit *_wu, const char * _wupathname) : resources(*this)
 {
     workunit.set(_wu);
     wupathname.set(_wupathname);
@@ -1165,6 +1165,27 @@ const char * HqlCppInstance::querySourceFile(unsigned idx)
     return NULL;
 }
 
+const char * HqlCppInstance::querySourceFlags(unsigned idx)
+{
+    if (sourceFlags.isItem(idx))
+        return sourceFlags.item(idx).text;
+    return NULL;
+}
+
+bool HqlCppInstance::querySourceIsTemp(unsigned idx)
+{
+    if (sourceIsTemp.isItem(idx))
+        return sourceIsTemp.item(idx);
+    return false;
+}
+
+const char * HqlCppInstance::queryTempDirectory(unsigned idx)
+{
+    if (tempDirs.isItem(idx))
+        return tempDirs.item(idx).text;
+    return NULL;
+}
+
 HqlStmts * HqlCppInstance::querySection(IAtom * section)
 {
     ForEachItemIn(idx, sections)
@@ -1272,7 +1293,7 @@ bool HqlCppInstance::useFunction(IHqlExpression * func)
     if (include.length())
         useInclude(include.str());
     if (source.length())
-        useSourceFile(source);
+        useSourceFile(source, nullptr, false);
     return true;
 }
 
@@ -1292,9 +1313,26 @@ void HqlCppInstance::useObjectFile(const char * objname)
     insertUniqueString(objectFiles, objname);
 }
 
-void HqlCppInstance::useSourceFile(const char * srcname)
+void HqlCppInstance::useSourceFile(const char * srcname, const char *flags, bool isTemp)
+{
+    ForEachItemIn(idx, sourceFiles)
+    {
+        StringAttrItem & cur = sourceFiles.item(idx);
+        if (strcmp(cur.text, srcname) == 0)
+        {
+            assertex(!flags);
+            assertex(sourceIsTemp.item(idx)==isTemp);
+            return;
+        }
+    }
+    sourceFiles.append(* new StringAttrItem(srcname));
+    sourceFlags.append(* new StringAttrItem(flags));
+    sourceIsTemp.append(isTemp);
+}
+
+void HqlCppInstance::addTemporaryDir(const char * path)
 {
-    insertUniqueString(sourceFiles, srcname);
+    insertUniqueString(tempDirs, path);
 }
 
 void HqlCppInstance::addHint(const char * hintXml, ICodegenContextCallback * ctxCallback)
@@ -1381,7 +1419,7 @@ void HqlCppInstance::flushResources(const char *filename, ICodegenContextCallbac
         if (isObjectFile)
             useObjectFile(resname);
         else
-            useSourceFile(resname);
+            useSourceFile(resname, nullptr, true);
     }
 }
 
@@ -7685,7 +7723,7 @@ void HqlCppTranslator::processCppBodyDirectives(IHqlExpression * expr)
                 StringBuffer sourceName;
                 getStringValue(sourceName, cur->queryChild(0));
                 if (sourceName.length())
-                    code->useSourceFile(sourceName.str());
+                    code->useSourceFile(sourceName.str(), nullptr, false);
             }
         }
     }

ecl/hqlcpp/hqlcpp.hpp

@@ -109,15 +109,20 @@ public:
     virtual const char * queryLibrary(unsigned idx) = 0;
     virtual const char * queryObjectFile(unsigned idx) = 0;
     virtual const char * querySourceFile(unsigned idx) = 0;
+    virtual const char * querySourceFlags(unsigned idx) = 0;
+    virtual const char * queryTempDirectory(unsigned idx) = 0;
+    virtual bool querySourceIsTemp(unsigned idx) = 0;
     virtual HqlStmts * querySection(IAtom * section) = 0;
     virtual void addResource(const char * type, unsigned len, const void * data, IPropertyTree *manifestEntry, unsigned id=(unsigned)-1) = 0;
     virtual void addCompressResource(const char * type, unsigned len, const void * data, IPropertyTree *manifestEntry, unsigned id=(unsigned)-1) = 0;
-    virtual void addManifest(const char *filename) = 0;
-    virtual void addManifestsFromArchive(IPropertyTree *archive) = 0;
+    virtual void addManifest(const char *filename, ICodegenContextCallback *ctxCallback) = 0;
+    virtual void addManifestsFromArchive(IPropertyTree *archive, ICodegenContextCallback *ctxCallback) = 0;
     virtual void addWebServiceInfo(IPropertyTree *wsinfo) = 0;
     virtual void flushHints() = 0;
     virtual void flushResources(const char *filename, ICodegenContextCallback * ctxCallback) = 0;
     virtual void getActivityRange(unsigned cppIndex, unsigned & minActivityId, unsigned & maxActivityId) = 0;
+    virtual void useSourceFile(const char * srcname, const char *flags, bool isTemp) = 0;
+    virtual void addTemporaryDir(const char *path) = 0;
 };
 
 // A class used to hold the context about the expression being processed at this point in time.

ecl/hqlcpp/hqlcpp.ipp

@@ -143,21 +143,25 @@ public:
     virtual const char * queryLibrary(unsigned idx);
     virtual const char * queryObjectFile(unsigned idx);
     virtual const char * querySourceFile(unsigned idx);
+    virtual const char * querySourceFlags(unsigned idx);
+    virtual const char * queryTempDirectory(unsigned idx);
+    virtual bool querySourceIsTemp(unsigned idx);
     virtual HqlStmts * querySection(IAtom * section);
     virtual void flushHints();
     virtual void flushResources(const char *filename, ICodegenContextCallback * ctxCallback);
     virtual void addResource(const char * type, unsigned len, const void * data, IPropertyTree *manifestEntry=NULL, unsigned id=(unsigned)-1);
     virtual void addCompressResource(const char * type, unsigned len, const void * data, IPropertyTree *manifestEntry=NULL, unsigned id=(unsigned)-1);
-    virtual void addManifest(const char *filename){resources.addManifest(filename);}
-    virtual void addManifestsFromArchive(IPropertyTree *archive){resources.addManifestsFromArchive(archive);}
+    virtual void addManifest(const char *filename, ICodegenContextCallback *ctxCallback) { resources.addManifest(filename, ctxCallback); }
+    virtual void addManifestsFromArchive(IPropertyTree *archive, ICodegenContextCallback *ctxCallback) { resources.addManifestsFromArchive(archive, ctxCallback); }
     virtual void addWebServiceInfo(IPropertyTree *wsinfo){resources.addWebServiceInfo(wsinfo);}
     virtual void getActivityRange(unsigned cppIndex, unsigned & minActivityId, unsigned & maxActivityId);
+    virtual void useSourceFile(const char * srcname, const char *flags, bool isTemp);
+    virtual void addTemporaryDir(const char * path);
     
     bool useFunction(IHqlExpression * funcdef);
     void useInclude(const char * include);
     void useLibrary(const char * libname);
     void useObjectFile(const char * objname);
-    void useSourceFile(const char * srcname);
     unsigned addStringResource(unsigned len, const char * body);
     void addHint(const char * hintXml, ICodegenContextCallback * ctxCallback);
 
@@ -174,7 +178,10 @@ public:
     StringAttrArray     modules;
     StringAttrArray     objectFiles;
     StringAttrArray     sourceFiles;
+    StringAttrArray     sourceFlags;
+    StringAttrArray     tempDirs;
     StringAttrArray     includes;
+    BoolArray           sourceIsTemp;
     CIArray             extra;
     ResourceManager     resources;
     Owned<IWorkUnit>    workunit;

ecl/hqlcpp/hqlecl.cpp

@@ -91,8 +91,8 @@ public:
     virtual bool generateExe(ICppCompiler * compiler);
     virtual bool generatePackage(const char * packageName);
     virtual void setMaxCompileThreads(unsigned value) { defaultMaxCompileThreads = value; }
-    virtual void addManifest(const char *filename) { code->addManifest(filename); }
-    virtual void addManifestsFromArchive(IPropertyTree *archive) { code->addManifestsFromArchive(archive); }
+    virtual void addManifest(const char *filename) { code->addManifest(filename, ctxCallback); }
+    virtual void addManifestsFromArchive(IPropertyTree *archive) { code->addManifestsFromArchive(archive, ctxCallback); }
     virtual void addWebServiceInfo(IPropertyTree *wsinfo){ code->addWebServiceInfo(wsinfo); }
 
     virtual double getECLcomplexity(IHqlExpression * exprs);
@@ -125,7 +125,10 @@ protected:
     ClusterType targetClusterType;
     Linked<ICodegenContextCallback> ctxCallback;
     unsigned defaultMaxCompileThreads;
+    StringArray temporaryDirectories;
     StringArray sourceFiles;
+    StringArray sourceFlags;
+    BoolArray sourceIsTemp;
     StringArray libraries;
     StringArray objects;
     offset_t totalGeneratedSize;
@@ -186,6 +189,18 @@ void HqlDllGenerator::addLibrariesToCompiler()
             break;
         PrintLog("Adding source file: %s", src);
         sourceFiles.append(src);
+        sourceFlags.append(code->querySourceFlags(idx));
+        sourceIsTemp.append(code->querySourceIsTemp(idx));
+        idx++;
+    }
+    idx=0;
+    for (;;)
+    {
+        const char * dir = code->queryTempDirectory(idx);
+        if (!dir)
+            break;
+        PrintLog("Adding temporary directory: %s", dir);
+        temporaryDirectories.append(dir);
         idx++;
     }
 }
@@ -527,6 +542,8 @@ void HqlDllGenerator::doExpand(HqlCppTranslator & translator)
     StringBuffer fullname;
     expandCode(fullname, MAIN_MODULE_TEMPLATE, false, code, isMultiFile, 0, targetCompiler);
     sourceFiles.append(fullname);
+    sourceFlags.append(nullptr);
+    sourceIsTemp.append(true);
     if (isMultiFile)
     {
         expandCode(fullname, HEADER_TEMPLATE, true, code, true, 0, targetCompiler);
@@ -534,6 +551,8 @@ void HqlDllGenerator::doExpand(HqlCppTranslator & translator)
         {
             expandCode(fullname, CHILD_MODULE_TEMPLATE, false, code, true, i+1, targetCompiler);
             sourceFiles.append(fullname);
+            sourceFlags.append(nullptr);
+            sourceIsTemp.append(true);
         }
     }
 
@@ -550,7 +569,7 @@ bool HqlDllGenerator::doCompile(ICppCompiler * compiler)
     cycle_t startCycles = get_cycles_now();
     addTimeStamp(wu, SSTcompilestage, "compile:compile c++", StWhenStarted);
     ForEachItemIn(i, sourceFiles)
-        compiler->addSourceFile(sourceFiles.item(i));
+        compiler->addSourceFile(sourceFiles.item(i), sourceFlags.item(i));
 
     unsigned maxThreads = wu->getDebugValueInt("maxCompileThreads", defaultMaxCompileThreads);
     compiler->setMaxCompileThreads(maxThreads);
@@ -618,9 +637,16 @@ bool HqlDllGenerator::doCompile(ICppCompiler * compiler)
         removeFileTraceIfFail(temp.clear().append(wuname).append(".hpp").str());
         ForEachItemIn(i, sourceFiles)
         {
-            removeFileTraceIfFail(sourceFiles.item(i));
+            if (sourceIsTemp.item(i))
+                removeFileTraceIfFail(sourceFiles.item(i));
         }
     }
+    ForEachItemIn(j, temporaryDirectories)
+    {
+        Owned<IFile> tempDir = createIFile(temporaryDirectories.item(j));
+        if (tempDir)
+            recursiveRemoveDirectory(tempDir);
+    }
     return ok;
 }
 

ecl/hqlcpp/hqlres.cpp

@@ -19,6 +19,7 @@
 
 #include "jlib.hpp"
 #include "hqlres.hpp"
+#include "hqlcpp.ipp"
 #include "jmisc.hpp"
 #include "jexcept.hpp"
 #include "hqlcerrors.hpp"
@@ -40,7 +41,7 @@ public:
 };
 
 
-ResourceManager::ResourceManager()
+ResourceManager::ResourceManager(IHqlCppInstance &_cppInstance) : cppInstance(_cppInstance)
 {
     nextmfid = MANIFEST_BASE + 1;
     nextbsid = BIGSTRING_BASE;
@@ -165,9 +166,33 @@ void expandManifestDirectory(IPropertyTree *manifestSrc, IPropertyTree &res, Str
 }
 
 
-void ResourceManager::addManifestFile(const char *filename)
+void ResourceManager::addManifestFile(const char *filename, ICodegenContextCallback *ctxCallback)
 {
-    Owned<IPropertyTree> manifestSrc = createPTreeFromXMLFile(filename);
+    StringBuffer fileContents;
+    StringBuffer strippedFileContents;
+    fileContents.loadFile(filename, false);
+    bool isSigned = false;
+    const char *useContents = fileContents;
+    // Check for signature
+    if (startsWith(fileContents, "-----BEGIN PGP SIGNED MESSAGE-----"))
+    {
+        try
+        {
+            OwnedHqlExpr sig = checkSignature(fileContents.length(), fileContents.str());
+            useContents = stripSignature(strippedFileContents, fileContents).str();
+            isSigned = true;
+        }
+        catch (IException *E)
+        {
+            StringBuffer msg;
+            E->errorMessage(msg);
+            auto code = E->errorCode();
+            auto aud = E->errorAudience();
+            E->Release();
+            throw makeStringExceptionV(aud, code, "While loading manifest file %s: %s", filename, msg.str());
+        }
+    }
+    Owned<IPropertyTree> manifestSrc = createPTreeFromXMLString(useContents);
 
     StringBuffer dir; 
     splitDirTail(filename, dir);
@@ -181,7 +206,7 @@ void ResourceManager::addManifestFile(const char *filename)
     {
         IPropertyTree &item = iter->query();
         if (streq(item.queryName(), "Include") && item.hasProp("@filename"))
-            addManifestInclude(item, dir.str());
+            addManifestInclude(item, dir.str(), ctxCallback);
         else if (streq(item.queryName(), "Resource") && item.hasProp("@filename"))
         {
             StringBuffer filepath;
@@ -209,6 +234,17 @@ void ResourceManager::addManifestFile(const char *filename)
     ForEach(*resources)
     {
         IPropertyTree &item = resources->query();
+        const char *resourceFilename = item.queryProp("@originalFilename");
+        const char *md5 = item.queryProp("@md5");
+        if (md5)
+        {
+            StringBuffer calculated;
+            md5_filesum(resourceFilename, calculated);
+            if (!strieq(calculated, md5))
+                throw makeStringExceptionV(0, "MD5 mismatch on file %s in manifest %s", item.queryProp("@filename"), filename);
+        }
+        else if (isSigned)
+            throw makeStringExceptionV(0, "MD5 must be supplied for file %s in signed manifest %s", item.queryProp("@filename"), filename);
 
         if (!item.hasProp("@type"))
             item.setProp("@type", "UNKNOWN");
@@ -220,23 +256,35 @@ void ResourceManager::addManifestFile(const char *filename)
         }
         else
         {
-            MemoryBuffer content;
-            loadResource(item.queryProp("@originalFilename"), content);
-            addCompress(item.queryProp("@type"), content.length(), content.toByteArray(), &item);
+            const char *type = item.queryProp("@type");
+            if (strieq(type, "CPP") || strieq(type, "C"))
+            {
+                if (!ctxCallback->allowAccess("cpp", isSigned))
+                    throw makeStringExceptionV(0, "Embedded code via manifest file not allowed");
+                cppInstance.useSourceFile(resourceFilename, item.queryProp("@compileFlags"), false);
+            }
+            else
+            {
+                if (strieq(type, "jar") && !ctxCallback->allowAccess(type, isSigned))
+                    throw makeStringExceptionV(0, "Embedded %s files via manifest file not allowed", type);
+                MemoryBuffer content;
+                loadResource(resourceFilename, content);
+                addCompress(type, content.length(), content.toByteArray(), &item);
+            }
         }
     }
 }
 
-void ResourceManager::addManifest(const char *filename)
+void ResourceManager::addManifest(const char *filename, ICodegenContextCallback *ctxCallback)
 {
     StringBuffer path;
     Owned<IPropertyTree> t = createPTree();
     t->setProp("@originalFilename", makeAbsolutePath(filename, path).str());
     ensureManifestInfo()->addPropTree("Include", t.getClear());
-    addManifestFile(filename);
+    addManifestFile(filename, ctxCallback);
 }
 
-void ResourceManager::addManifestInclude(IPropertyTree &include, const char *dir)
+void ResourceManager::addManifestInclude(IPropertyTree &include, const char *dir, ICodegenContextCallback *ctxCallback)
 {
     StringBuffer includePath;
     makeAbsolutePath(include.queryProp("@filename"), dir, includePath);
@@ -245,10 +293,10 @@ void ResourceManager::addManifestInclude(IPropertyTree &include, const char *dir
         return;
     include.setProp("@originalFilename", includePath.str());
     manifest->addPropTree("Include", LINK(&include));
-    addManifestFile(includePath.str());
+    addManifestFile(includePath.str(), ctxCallback);
 }
 
-void ResourceManager::addManifestsFromArchive(IPropertyTree *archive)
+void ResourceManager::addManifestsFromArchive(IPropertyTree *archive, ICodegenContextCallback *ctxCallback)
 {
     if (!archive)
         return;
@@ -258,7 +306,31 @@ void ResourceManager::addManifestsFromArchive(IPropertyTree *archive)
     Owned<IPropertyTreeIterator> manifests = archive->getElements("AdditionalFiles/Manifest");
     ForEach(*manifests)
     {
-        const char *xml = manifests->query().queryProp(NULL);
+        StringBuffer tempDir;
+        StringBuffer manifestContents;
+        StringBuffer strippedManifestContents;
+        manifests->query().getProp(nullptr, manifestContents);
+        const char *xml = manifestContents;
+        bool isSigned = false;
+        // Check for signature
+        if (startsWith(xml, "-----BEGIN PGP SIGNED MESSAGE-----"))
+        {
+            try
+            {
+                OwnedHqlExpr sig = checkSignature(manifestContents.length(), manifestContents.str());
+                xml = stripSignature(strippedManifestContents, manifestContents).str();
+                isSigned = true;
+            }
+            catch (IException *E)
+            {
+                StringBuffer msg;
+                E->errorMessage(msg);
+                auto code = E->errorCode();
+                auto aud = E->errorAudience();
+                E->Release();
+                throw makeStringExceptionV(aud, code, "While loading manifest %s: %s", manifests->query().queryProp("@originalFileName"), msg.str());
+            }
+        }
         Owned<IPropertyTree> manifestSrc = createPTreeFromXMLString(xml);
         Owned<IAttributeIterator> aiter = manifestSrc->getAttributes();
         ForEach (*aiter)
@@ -288,10 +360,54 @@ void ResourceManager::addManifestsFromArchive(IPropertyTree *archive)
                 }
                 else
                 {
-                    VStringBuffer xpath("AdditionalFiles/Resource[@originalFilename=\"%s\"]", filename);
                     MemoryBuffer content;
+                    VStringBuffer xpath("AdditionalFiles/Resource[@originalFilename=\"%s\"]", filename);
+                    const char *md5=item.queryProp("@md5");
+                    if (!archive->hasProp(xpath.str()))
+                    {
+                        if (md5)
+                            xpath.clear().appendf("AdditionalFiles/Resource[@filename='%s'][@md5='%s']", filename, md5);
+                        else
+                            xpath.clear().appendf("AdditionalFiles/Resource[@originalFilename=\"%s\"]", filename);
+                        if (!archive->hasProp(xpath.str()))
+                            throw makeStringExceptionV(0, "Failed to locate resource for %s in archive", filename);
+                    }
                     archive->getPropBin(xpath.str(), content);
-                    addCompress(item.queryProp("@type"), content.length(), content.toByteArray(), &item);
+                    if (md5)
+                    {
+                        StringBuffer calculated;
+                        md5_data(content, calculated);
+                        if (!strieq(calculated, md5))
+                            throw makeStringExceptionV(0, "MD5 mismatch %s in archive", filename);
+                    }
+                    const char *type = item.queryProp("@type");
+                    if (strieq(type, "CPP") || strieq(type, "C"))
+                    {
+                        if (!ctxCallback->allowAccess("cpp", isSigned))
+                            throw makeStringExceptionV(0, "Embedded code via manifest file not allowed");
+                        if (!tempDir.length())
+                        {
+                            getTempFilePath(tempDir, "eclcc", nullptr);
+                            tempDir.append(PATHSEPCHAR).append("tmp.XXXXXX"); // Note - we share same temp dir for all from this manifest
+                            if (!mkdtemp((char *) tempDir.str()))
+                                throw makeStringExceptionV(0, "Failed to create temporary directory %s (error %d)", tempDir.str(), errno);
+                            cppInstance.addTemporaryDir(tempDir.str());
+                        }
+                        StringBuffer tempFileName;
+                        tempFileName.append(tempDir).append(PATHSEPCHAR).append(item.queryProp("@filename"));
+                        if (!recursiveCreateDirectoryForFile(tempFileName))
+                            throw makeStringExceptionV(0, "Failed to create temporary file %s (error %d)", tempFileName.str(), errno);
+                        FILE *source = fopen(tempFileName.str(), "wt");
+                        fwrite(content.toByteArray(), content.length(), 1, source);
+                        fclose(source);
+                        cppInstance.useSourceFile(tempFileName, item.queryProp("@compileFlags"), true);
+                    }
+                    else
+                    {
+                        if (strieq(type, "jar") && !ctxCallback->allowAccess(type, isSigned))
+                            throw makeStringExceptionV(0, "Embedded %s files via manifest file not allowed", type);
+                        addCompress(type, content.length(), content.toByteArray(), &item);
+                    }
                 }
             }
             else

ecl/hqlcpp/hqlres.hpp

@@ -18,6 +18,8 @@
 #define _HQLRES_INCL
 #include "jarray.hpp"
 
+interface IHqlCppInstance;
+
 class ResourceManager
 {
     unsigned nextmfid;
@@ -25,14 +27,15 @@ class ResourceManager
     unsigned totalbytes;
     CIArray resources;
     Owned<IPropertyTree> manifest;
+    IHqlCppInstance &cppInstance;
     bool finalized;
 public:
-    ResourceManager();
+    ResourceManager(IHqlCppInstance &);
     unsigned addString(unsigned len, const char *data);
     void addNamed(const char * type, unsigned len, const void *data, IPropertyTree *entry=NULL, unsigned id=(unsigned)-1, bool addToManifest=true, bool compressed=false);
     bool addCompress(const char * type, unsigned len, const void *data, IPropertyTree *entry=NULL, unsigned id=(unsigned)-1, bool addToManifest=true);
-    void addManifest(const char *filename);
-    void addManifestsFromArchive(IPropertyTree *archive);
+    void addManifest(const char *filename, ICodegenContextCallback *ctxCallback);
+    void addManifestsFromArchive(IPropertyTree *archive, ICodegenContextCallback *ctxCallback);
     void addWebServiceInfo(IPropertyTree *wsinfo);
     IPropertyTree *ensureManifestInfo(){if (!manifest) manifest.setown(createPTree("Manifest")); return manifest;}
     bool getDuplicateResourceId(const char *srctype, const char *respath, const char *filename, int &id);
@@ -44,8 +47,8 @@ public:
     bool queryWriteText(StringBuffer & resTextName, const char * filename);
 private:
     void putbytes(int h, const void *b, unsigned len);
-    void addManifestFile(const char *filename);
-    void addManifestInclude(IPropertyTree &include, const char *dir);
+    void addManifestFile(const char *filename, ICodegenContextCallback *ctxCallback);
+    void addManifestInclude(IPropertyTree &include, const char *dir, ICodegenContextCallback *ctxCallback);
 };
 
 #endif

system/jlib/jcomp.cpp

@@ -59,7 +59,8 @@
 #define BASE_ADDRESS "0x00480000"
 //#define BASE_ADDRESS        "0x10000000"
 
-static const char * CC_NAME[] =   { "\"#" PATHSEPSTR "bin" PATHSEPSTR "cl.bat\"",   "\"#" PATHSEPSTR "bin" PATHSEPSTR "g++\"" };
+static const char * CC_NAME_CPP[] =   { "\"#" PATHSEPSTR "bin" PATHSEPSTR "cl.bat\"",   "\"#" PATHSEPSTR "bin" PATHSEPSTR "g++\"" };
+static const char * CC_NAME_C[] =   { "\"#" PATHSEPSTR "bin" PATHSEPSTR "cl.bat\"",   "\"#" PATHSEPSTR "bin" PATHSEPSTR "gcc\"" };
 static const char * LINK_NAME[] = { "\"#" PATHSEPSTR "bin" PATHSEPSTR "link.bat\"", "\"#" PATHSEPSTR "bin" PATHSEPSTR "g++\"" };
 static const char * LIB_DIR[] = { "\"#\\lib\"", "\"#/lib\"" };
 static const char * LIB_OPTION_PREFIX[] = { "", "-Wl," };
@@ -80,12 +81,12 @@ static const char * LIBFLAG_DEBUG[] = { "/MDd", "" };
 static const char * LIBFLAG_RELEASE[] = { "/MD", "" };
 static const char * COMPILE_ONLY[] = { "/c", "-c" };
 
-static const char * CC_OPTION_CORE[] = { "", "-std=c++11 -fvisibility=hidden -DUSE_VISIBILITY=1 -Werror -Wno-tautological-compare" };
+static const char * CC_OPTION_CORE[] = { "", "-fvisibility=hidden -DUSE_VISIBILITY=1 -Werror -Wno-tautological-compare" };
 static const char * LINK_OPTION_CORE[] = { "/DLL /libpath:." , "" };
 static const char * CC_OPTION_DEBUG[] = { "/Zm500 /EHsc /GR /Zi /nologo /bigobj", "-g -fPIC  -O0" };
-
-
 static const char * CC_OPTION_RELEASE[] = { "/Zm500 /EHsc /GR /Oi /Ob1 /GF /nologo /bigobj", "-fPIC  -O0" };
+static const char * CC_OPTION_C[] = { "", "" };
+static const char * CC_OPTION_CPP[] = { "", "-std=c++11" };
 
 static const char * CC_OPTION_PRECOMPILEHEADER[] = { "", " -x c++-header" };
 
@@ -152,7 +153,7 @@ static void doSetCompilerPath(const char * path, const char * includes, const ch
     StringBuffer fname;
     if (path)
     {
-        const char *finger = CC_NAME[targetCompiler];
+        const char *finger = CC_NAME_CPP[targetCompiler];
         while (*finger)
         {
             if (*finger == '#')
@@ -179,7 +180,7 @@ static void doSetCompilerPath(const char * path, const char * includes, const ch
     }
     else
     {
-        fname.append(compilerRoot).append(CC_NAME[targetCompiler]);
+        fname.append(compilerRoot).append(CC_NAME_CPP[targetCompiler]);
         fname.replaceString("#",NULL);
     }
     if (verbose)
@@ -359,10 +360,11 @@ void CppCompiler::addLinkOption(const char * option)
         linkerOptions.append(' ').append(LIB_OPTION_PREFIX[targetCompiler]).append(option);
 }
 
-void CppCompiler::addSourceFile(const char * filename)
+void CppCompiler::addSourceFile(const char * filename, const char *flags)
 {
     DBGLOG("addSourceFile %s", filename);
     allSources.append(filename);
+    allFlags.append(flags);
 }
 
 void CppCompiler::addObjectFile(const char * filename)
@@ -401,7 +403,7 @@ bool CppCompiler::compile()
 
     ForEachItemIn(i0, allSources)
     {
-        ret = compileFile(pool, allSources.item(i0), finishedCompiling);
+        ret = compileFile(pool, allSources.item(i0), allFlags.item(i0), finishedCompiling);
         if (!ret)
             break;
         ++numSubmitted;
@@ -468,13 +470,15 @@ bool CppCompiler::compile()
     return ret;
 }
 
-bool CppCompiler::compileFile(IThreadPool * pool, const char * filename, Semaphore & finishedCompiling)
+bool CppCompiler::compileFile(IThreadPool * pool, const char * filename, const char *flags, Semaphore & finishedCompiling)
 {
     if (!filename || *filename == 0)
         return false;
 
     StringBuffer cmdline;
-    cmdline.append(CC_NAME[targetCompiler]);
+    const char *ext = pathExtension(filename);
+    bool isC = ext != nullptr && strieq(ext, ".c");
+    cmdline.append(isC ? CC_NAME_C[targetCompiler] : CC_NAME_CPP[targetCompiler]);
     if (precompileHeader)
         cmdline.append(CC_OPTION_PRECOMPILEHEADER[targetCompiler]);
     cmdline.append(" \"");
@@ -485,7 +489,7 @@ bool CppCompiler::compileFile(IThreadPool * pool, const char * filename, Semapho
     }
     cmdline.append(filename);
     cmdline.append("\" ");
-    expandCompileOptions(cmdline);
+    expandCompileOptions(cmdline, isC);
 
     if (useDebugLibrary)
         cmdline.append(" ").append(LIBFLAG_DEBUG[targetCompiler]);
@@ -512,6 +516,8 @@ bool CppCompiler::compileFile(IThreadPool * pool, const char * filename, Semapho
             getObjectName(cmdline, filename);
         cmdline.append("\"");
     }
+    if (flags)
+        cmdline.append(" ").append(flags);
     
     StringBuffer expanded;
     expandRootDirectory(expanded, cmdline);
@@ -647,7 +653,7 @@ void CppCompiler::extractErrors(IArrayOf<IError> & errors)
     }
 }
 
-void CppCompiler::expandCompileOptions(StringBuffer & target)
+void CppCompiler::expandCompileOptions(StringBuffer & target, bool isC)
 {
     target.append(" ").append(CC_OPTION_CORE[targetCompiler]).append(" ");
     if (targetDebug)
@@ -655,6 +661,10 @@ void CppCompiler::expandCompileOptions(StringBuffer & target)
     else
         target.append(CC_OPTION_RELEASE[targetCompiler]);
     target.append(compilerOptions).append(CC_EXTRA_OPTIONS);
+    if (isC)
+        target.append(" ").append(CC_OPTION_C[targetCompiler]);
+    else
+        target.append(" ").append(CC_OPTION_CPP[targetCompiler]);
 }
 
 bool CppCompiler::doLink()

system/jlib/jcomp.hpp

@@ -50,7 +50,7 @@ public:
     virtual void addLibraryPath(const char * libPath) = 0;
     virtual void addInclude(const char * includePath) = 0;
     virtual void addLinkOption(const char * option) = 0;
-    virtual void addSourceFile(const char * filename) = 0;
+    virtual void addSourceFile(const char * filename, const char *flags) = 0;
     virtual void addObjectFile(const char * filename) = 0;
     virtual bool compile() = 0;
     virtual void extractErrors(IArrayOf<IError> & errors) = 0;

system/jlib/jcomp.ipp

@@ -34,7 +34,7 @@ public:
     virtual void addLibraryPath(const char * libPath);
     virtual void addLinkOption(const char * option);
     virtual void addInclude(const char * includePath);
-    virtual void addSourceFile(const char * filename);
+    virtual void addSourceFile(const char * filename, const char *flags);
     virtual void addObjectFile(const char * filename);
     virtual bool compile();
     virtual void extractErrors(IArrayOf<IError> & errors);
@@ -53,11 +53,11 @@ public:
     virtual bool fireException(IException *e);
 
 protected:
-    void expandCompileOptions(StringBuffer & target);
+    void expandCompileOptions(StringBuffer & target, bool isC);
     void expandRootDirectory(StringBuffer & expanded, StringBuffer & in);
     StringBuffer & getObjectName(StringBuffer & out, const char * filename);
     void removeTemporaries();
-    bool compileFile(IThreadPool * pool, const char * filename, Semaphore & finishedCompiling);
+    bool compileFile(IThreadPool * pool, const char * filename, const char *flags, Semaphore & finishedCompiling);
     bool doLink();
     void writeLogFile(const char* filepath, StringBuffer& log);
 
@@ -71,6 +71,7 @@ protected:
     StringAttr      sourceDir;
     StringAttr      targetDir;
     StringArray     allSources;
+    StringArray     allFlags;
     StringArray     logFiles;
     StringAttr      ccLogPath;
     StringAttr      coreName;

system/jlib/jfile.cpp

@@ -4039,13 +4039,6 @@ void setDefaultUser(const char * username,const char *password)
 
 //---------------------------------------------------------------------------
 
-
-
-
-    
-
-
-
 bool recursiveCreateDirectory(const char * path)
 {
     Owned<IFile> file = createIFile(path);
@@ -4061,6 +4054,30 @@ bool recursiveCreateDirectoryForFile(const char *fullFileName)
     return recursiveCreateDirectory(path.str());
 }
 
+void recursiveRemoveDirectory(IFile *dir)
+{
+    Owned<IDirectoryIterator> files = dir->directoryFiles(NULL, false, true);
+    ForEach(*files)
+    {
+        IFile *thisFile = &files->query();
+        if (thisFile->isDirectory()==foundYes)
+            recursiveRemoveDirectory(thisFile);
+        else
+        {
+            thisFile->setReadOnly(false);
+            thisFile->remove();
+        }
+    }
+    dir->remove();
+}
+
+void recursiveRemoveDirectory(const char *dir)
+{
+    Owned<IFile> f = createIFile(dir);
+    if (f->isDirectory())
+        recursiveRemoveDirectory(f);
+}
+
 
 
 //---------------------------------------------------------------------------

system/jlib/jfile.hpp

@@ -252,6 +252,11 @@ extern jlib_decl void copyFile(const char *target, const char *source, size32_t
 extern jlib_decl void copyFile(IFile * target, IFile * source,size32_t buffersize=DEFAULT_COPY_BLKSIZE, ICopyFileProgress *progress=NULL,CFflags copyFlags=CFnone);
 extern jlib_decl bool recursiveCreateDirectory(const char * path);
 extern jlib_decl bool recursiveCreateDirectoryForFile(const char *filename);
+/**
+ * Recursively remove a directory and all its contents and subdirectories
+ */
+extern jlib_decl void recursiveRemoveDirectory(IFile *dir);
+extern jlib_decl void recursiveRemoveDirectory(const char *dir);
 
 extern jlib_decl void splitFilename(const char * filename, StringBuffer * drive, StringBuffer * path, StringBuffer * tail, StringBuffer * ext, bool longExt = false);
 extern jlib_decl bool splitUNCFilename(const char * filename, StringBuffer * machine, StringBuffer * path, StringBuffer * tail, StringBuffer * ext);

system/jlib/jmd5.cpp

@@ -439,6 +439,11 @@ void md5_string(StringBuffer& inpstring, StringBuffer& outstring)
     md5_string(inpstring, inpstring.length(), outstring);
 }
 
+void md5_data(const MemoryBuffer& in, StringBuffer& outstring)
+{
+    md5_string(in.toByteArray(), in.length(), outstring);
+}
+
 /* define chunk size to use with IFileIO read. */
 #define CHUNKSIZE 0x100000
 

system/jlib/jmd5.hpp

@@ -107,6 +107,8 @@ jlib_decl void md5_finish(md5_state_t *pms, md5_byte_t digest[16]);
 /* Take in a string and return it encoded */
 jlib_decl void md5_string (StringBuffer& inpstring, StringBuffer& outstring);
 jlib_decl void md5_string2(const char* inpstring, StringBuffer& outstring);
+/* Take in a string and return it encoded */
+jlib_decl void md5_data(const MemoryBuffer& in, StringBuffer& outstring);
 
 /* Takes in a filename and returns the md5 sum of the file */
 jlib_decl void md5_filesum(const char* filename, StringBuffer& outstring);

system/jlib/jutil.hpp

@@ -206,7 +206,7 @@ class StringPointerArrayMapper : public SimpleArrayMapper<const char *>
 public:
     static void construct(const char * & member, const char * newValue)
     {
-        member = strdup(newValue);
+        member = newValue ? strdup(newValue) : nullptr;
     }
     static void destruct(MEMBER & member)
     {