HPCC-14185 Support the option to sign code

Work in progress. Support separate permissions for PIPE, embedc++, and FOLD
depending on whether they are in signed code or not.

Actual signing bit remains TBD.

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

common/fileview2/fvtransform.cpp

@@ -298,7 +298,7 @@ void ViewTransformerRegistry::addPlugins(const char * name)
     dataServer.setown(createNewSourceFileEclRepository(errorReporter, name, ESFallowplugins, 0));
 
     HqlScopeArray scopes;
-    HqlParseContext parseCtx(dataServer, NULL);
+    HqlParseContext parseCtx(dataServer, NULL, NULL);
     HqlLookupContext ctx(parseCtx, errorReporter);
     getRootScopes(scopes, dataServer, ctx);
 

ecl/eclcc/eclcc.cpp

@@ -271,7 +271,8 @@ public:
         batchSplit = 1;
         batchLog = NULL;
         cclogFilename.append("cc.").append((unsigned)GetCurrentProcessId()).append(".log");
-        defaultAllowed = true;
+        defaultAllowed[false] = true;  // May want to change that?
+        defaultAllowed[true] = true;
     }
 
     bool printKeywordsToXml();
@@ -282,7 +283,7 @@ public:
     void processBatchedFile(IFile & file, bool multiThreaded);
 
     virtual void noteCluster(const char *clusterName);
-    virtual bool allowAccess(const char * category);
+    virtual bool allowAccess(const char * category, bool isSigned);
 
 protected:
     void addFilenameDependency(StringBuffer & target, EclCompileInstance & instance, const char * filename);
@@ -354,8 +355,9 @@ protected:
     StringArray libraryPaths;
 
     StringArray allowedPermissions;
+    StringArray allowSignedPermissions;
     StringArray deniedPermissions;
-    bool defaultAllowed;
+    bool defaultAllowed[2];
 
     ClusterType optTargetClusterType;
     CompilerType optTargetCompiler;
@@ -1100,7 +1102,7 @@ void EclCC::processSingleQuery(EclCompileInstance & instance,
 
     {
         //Minimize the scope of the parse context to reduce lifetime of cached items.
-        HqlParseContext parseCtx(instance.dataServer, instance.archive);
+        HqlParseContext parseCtx(instance.dataServer, this, instance.archive);
         if (!instance.archive)
             parseCtx.globalDependTree.setown(createPTree(ipt_none)); //to locate associated manifests, keep separate from user specified MetaOptions
         if (optGenerateMeta || optIncludeMeta)
@@ -1320,7 +1322,7 @@ void EclCC::processDefinitions(EclRepositoryArray & repositories)
         }
 
         //Create a repository with just that attribute.
-        Owned<IFileContents> contents = createFileContentsFromText(value, NULL);
+        Owned<IFileContents> contents = createFileContentsFromText(value, NULL, false);
         repositories.append(*createSingleDefinitionEclRepository(module, attr, contents));
     }
 }
@@ -1397,7 +1399,7 @@ void EclCC::processXmlFile(EclCompileInstance & instance, const char *archiveXML
     {
         const char * sourceFilename = archiveTree->queryProp("Query/@originalFilename");
         Owned<ISourcePath> sourcePath = createSourcePath(sourceFilename);
-        contents.setown(createFileContentsFromText(queryText, sourcePath));
+        contents.setown(createFileContentsFromText(queryText, sourcePath, false));
         if (queryAttributePath && queryText && *queryText)
         {
             Owned<IEclSourceCollection> inputFileCollection = createSingleDefinitionEclCollection(queryAttributePath, contents);
@@ -1418,7 +1420,7 @@ void EclCC::processXmlFile(EclCompileInstance & instance, const char *archiveXML
         queryAttributePath = fullPath.str();
 
         //Create a repository with just that attribute, and place it before the archive in the resolution order.
-        Owned<IFileContents> contents = createFileContentsFromText(queryText, NULL);
+        Owned<IFileContents> contents = createFileContentsFromText(queryText, NULL, false);
         repositories.append(*createSingleDefinitionEclRepository(syntaxCheckModule, syntaxCheckAttribute, contents));
     }
 
@@ -1456,7 +1458,7 @@ void EclCC::processFile(EclCompileInstance & instance)
 
     //On a system with userECL not allowed, all compilations must be from checked-in code that has been
     //deployed to the eclcc machine via other means (typically via a version-control system)
-    if (!allowAccess("userECL") && (!optQueryRepositoryReference || queryText->length()))
+    if (!allowAccess("userECL", false) && (!optQueryRepositoryReference || queryText->length()))
     {
         instance.queryErrorProcessor().reportError(HQLERR_UserCodeNotAllowed, HQLERR_UserCodeNotAllowed_Text, NULL, 1, 0, 0);
     }
@@ -1859,19 +1861,24 @@ bool EclCompileInstance::reportErrorSummary()
 void EclCC::noteCluster(const char *clusterName)
 {
 }
-bool EclCC::allowAccess(const char * category)
+bool EclCC::allowAccess(const char * category, bool isSigned)
 {
     ForEachItemIn(idx1, deniedPermissions)
     {
         if (stricmp(deniedPermissions.item(idx1), category)==0)
             return false;
     }
-    ForEachItemIn(idx2, allowedPermissions)
+    ForEachItemIn(idx2, allowSignedPermissions)
     {
-        if (stricmp(allowedPermissions.item(idx2), category)==0)
+        if (stricmp(allowSignedPermissions.item(idx2), category)==0)
+            return isSigned;
+    }
+    ForEachItemIn(idx3, allowedPermissions)
+    {
+        if (stricmp(allowedPermissions.item(idx3), category)==0)
             return true;
     }
-    return defaultAllowed;
+    return defaultAllowed[isSigned];
 }
 
 //=========================================================================================
@@ -1898,6 +1905,13 @@ bool EclCC::parseCommandLineOptions(int argc, const char* argv[])
         {
             allowedPermissions.append(tempArg);
         }
+        else if (iter.matchOption(tempArg, "--allowsigned"))
+        {
+            if (stricmp(tempArg, "all")==0)
+                defaultAllowed[true] = true;
+            else
+                allowSignedPermissions.append(tempArg);
+        }
         else if (iter.matchFlag(optBatchMode, "-b"))
         {
         }
@@ -1920,7 +1934,10 @@ bool EclCC::parseCommandLineOptions(int argc, const char* argv[])
         else if (iter.matchOption(tempArg, "--deny"))
         {
             if (stricmp(tempArg, "all")==0)
-                defaultAllowed = false;
+            {
+                defaultAllowed[false] = false;
+                defaultAllowed[true] = false;
+            }
             else
                 deniedPermissions.append(tempArg);
         }

ecl/eclccserver/eclccserver.cpp

@@ -280,7 +280,7 @@ class EclccCompileThread : public CInterface, implements IPooledThread, implemen
                 eclccCmd.appendf(" -I%s", value);
             else if (stricmp(optName, "libraryPath") == 0)
                 eclccCmd.appendf(" -L%s", value);
-            else if (stricmp(optName, "-allow")==0)
+            else if (strnicmp(optName, "-allow", 6)==0)
             {
                 if (isLocal)
                     throw MakeStringException(0, "eclcc-allow option can not be set per-workunit");  // for security reasons

ecl/hql/hql.hpp

@@ -188,6 +188,12 @@ interface IEclRepositoryCallback : public IEclRepository
     virtual IHqlExpression * loadSymbol(IHqlRemoteScope *scope, IIdAtom * searchName) = 0;
 };
 
+interface ICodegenContextCallback : public IInterface
+{
+    virtual void noteCluster(const char *clusterName) = 0;
+    virtual bool allowAccess(const char * category, bool isSigned) = 0;
+};
+
 
 #if defined(_DEBUG) && defined(_WIN32) && !defined(USING_MPATROL)
  #undef new

ecl/hql/hqlatoms.cpp

@@ -133,6 +133,7 @@ IAtom * definitionAtom;
 IAtom * deprecatedAtom;
 IAtom * descAtom;
 IAtom * diskAtom;
+IAtom *_disallowed_Atom;
 IAtom * distributedAtom;
 IAtom * _distributed_Atom;
 IAtom * _dot_Atom;
@@ -570,6 +571,7 @@ MODULE_INIT(INIT_PRIORITY_HQLATOM)
     MAKEATOM(definition);
     MAKEATOM(deprecated);
     MAKEATOM(desc);
+    MAKESYSATOM(disallowed);
     MAKEATOM(disk);
     MAKEATOM(distributed);
     MAKESYSATOM(distributed);

ecl/hql/hqlatoms.hpp

@@ -135,6 +135,7 @@ extern HQL_API IAtom * definitionAtom;
 extern HQL_API IAtom * deprecatedAtom;
 extern HQL_API IAtom * descAtom;
 extern HQL_API IAtom * diskAtom;
+extern HQL_API IAtom * _disallowed_Atom;
 extern HQL_API IAtom * distributedAtom;
 extern HQL_API IAtom * _distributed_Atom;
 extern HQL_API IAtom * _dot_Atom;

ecl/hql/hqlcollect.cpp

@@ -289,7 +289,7 @@ bool FileSystemFile::checkValid()
                         version.set(pb.version);
 
                         Owned<ISourcePath> pluginPath = createSourcePath(pb.moduleName);
-                        fileContents.setown(createFileContentsFromText(pb.ECL, pluginPath));
+                        fileContents.setown(createFileContentsFromText(pb.ECL, pluginPath, false)); // MORE - isSigned
 
                         //if (traceMask & PLUGIN_DLL_MODULE)
                         DBGLOG("Loading plugin %s[%s] version = %s", filename, pb.moduleName, version.get());
@@ -665,7 +665,7 @@ IFileContents * CXmlEclElement::queryFileContents()
                 getFullName(defaultName);
                 sourcePath.setown(createSourcePath(defaultName));
             }
-            fileContents.setown(createFileContentsFromText(text, sourcePath));
+            fileContents.setown(createFileContentsFromText(text, sourcePath, elemTree->getPropBool("@isSigned", false)));
         }
     }
     return fileContents;
@@ -799,9 +799,9 @@ IEclSourceCollection * createSingleDefinitionEclCollection(const char * moduleNa
     if (filename)
         attr->setProp("@sourcePath", filename);
 
-    StringBuffer temp;
-    temp.append(contents->length(), contents->getText());
-    attr->setProp("", temp.str());
+    attr->setProp("", contents->getText());
+    if (contents->isSignedModule())
+        attr->setPropBool("@isSigned", true);  // MORE - this is wrong!
     return createArchiveEclCollection(archive);
 }
 

ecl/hql/hqlerror.cpp

@@ -81,6 +81,8 @@ WarnErrorCategory getCategory(const char * category)
         return CategoryUnexpected;
     if (strieq(category, "cpp"))
         return CategoryCpp;
+    if (strieq(category, "security"))
+        return CategorySecurity;
     return CategoryUnknown;
 }
 

ecl/hql/hqlerrors.hpp

@@ -68,6 +68,8 @@
 #define WRN_SILLY_EXISTS            1051
 #define WRN_INT_OR_RANGE_EXPECTED   1052 /* Integer or integer range (i.e. 2..3) expected when real detected */
 #define WRN_UNRESOLVED_SYMBOL       1053
+#define WRN_REQUIRES_SIGNED         1054
+#define WRN_DISALLOWED              1055
 
 //Do not define any warnings > 1099 - use the range below instead
 
@@ -278,7 +280,7 @@
 #define ERR_SVC_NOSCOPEMODIFIER     2234 /* Function in service can not specify EXPORT or SHARED */
 #define ERR_SVC_NOENTRYPOINT        2235 /* Entrypoint is not defined, default to XXX */
 #define ERR_SVC_INVALIDINCLUDE      2236 /* Invalid include entry */
-#define ERR_SVC_NOYPENEEDED         2237 /* Serive does need a type */
+#define ERR_SVC_NOYPENEEDED         2237 /* Service does need a type */
 #define ERR_SVC_ATTRCONFLICTS       2238 /* Conflicted attributes defined */
 #define ERR_SVC_INVALIDINITFUNC     2239 /* Invalid initfunction: must be valid C identifier */
 

ecl/hql/hqlexpr.cpp

@@ -7538,6 +7538,7 @@ IHqlExpression * createJavadocAnnotation(IHqlExpression * _ownedBody, IPropertyT
 CFileContents::CFileContents(IFile * _file, ISourcePath * _sourcePath) : file(_file), sourcePath(_sourcePath)
 {
     delayedRead = false;
+    isSigned = false;
     if (!preloadFromFile())
         file.clear();
 }
@@ -7609,14 +7610,19 @@ void CFileContents::ensureLoaded()
 
     if (sizeRead != sizeToRead)
         throw MakeStringException(1, "File %s only read %u of %u bytes", file->queryFilename(), sizeRead, sizeToRead);
+    if (strstr(file->queryFilename(), "pipe"))  // TEmporary hack for testing!
+    {
+        DBGLOG("setting signed to true, %p", this);
+        isSigned = true;
+    }
 }
 
-CFileContents::CFileContents(const char *query, ISourcePath * _sourcePath) 
+CFileContents::CFileContents(const char *query, ISourcePath * _sourcePath, bool _isSigned)
 : sourcePath(_sourcePath)
 {
     if (query)
         setContents(strlen(query), query);
-
+    isSigned = _isSigned;
     delayedRead = false;
 }
 
@@ -7625,6 +7631,7 @@ CFileContents::CFileContents(unsigned len, const char *query, ISourcePath * _sou
 {
     setContents(len, query);
     delayedRead = false;
+    isSigned = false;
 }
 
 
@@ -7648,10 +7655,10 @@ IFileContents * createFileContentsFromText(unsigned len, const char * text, ISou
     return new CFileContents(len, text, sourcePath);
 }
 
-IFileContents * createFileContentsFromText(const char * text, ISourcePath * sourcePath)
+IFileContents * createFileContentsFromText(const char * text, ISourcePath * sourcePath, bool isSigned)
 {
     //MORE: Treatment of nulls?
-    return new CFileContents(text, sourcePath);
+    return new CFileContents(text, sourcePath, isSigned);
 }
 
 IFileContents * createFileContentsFromFile(const char * filename, ISourcePath * sourcePath)
@@ -7677,6 +7684,7 @@ public:
     virtual ISourcePath * querySourcePath() { return contents->querySourcePath(); }
     virtual const char *getText() { return contents->getText() + offset; }
     virtual size32_t length() { return len; }
+    virtual bool isSignedModule() { return contents->isSignedModule(); }
 
 protected:
     Linked<IFileContents> contents;
@@ -15975,7 +15983,7 @@ static void gatherAttributeDependencies(HqlLookupContext & ctx, const char * ite
 
 extern HQL_API IPropertyTree * gatherAttributeDependencies(IEclRepository * dataServer, const char * items)
 {
-    HqlParseContext parseCtx(dataServer, NULL);
+    HqlParseContext parseCtx(dataServer, NULL, NULL);
     parseCtx.nestedDependTree.setown(createPTree("Dependencies"));
 
     Owned<IErrorReceiver> errorHandler = createNullErrorReceiver();

ecl/hql/hqlexpr.hpp

@@ -836,6 +836,7 @@ public:
     virtual ISourcePath * querySourcePath() = 0;
     virtual const char * getText() = 0;
     virtual size32_t length() = 0;
+    virtual bool isSignedModule() = 0;
 };
 
 //This class ensures that the pointer to the owner is cleared before both links are released, which allows
@@ -876,8 +877,8 @@ public:
         bool includeJavadoc;
     };
 
-    HqlParseContext(IEclRepository * _eclRepository, IPropertyTree * _archive)
-    : archive(_archive), eclRepository(_eclRepository)
+    HqlParseContext(IEclRepository * _eclRepository, ICodegenContextCallback *_codegenCtx, IPropertyTree * _archive)
+    : archive(_archive), eclRepository(_eclRepository), codegenCtx(_codegenCtx)
     {
         expandCallsWhenBound = DEFAULT_EXPAND_CALL;
         ignoreUnknownImport = false;
@@ -916,6 +917,7 @@ public:
     bool expandCallsWhenBound;
     bool ignoreUnknownImport;
     bool aborting;
+    Linked<ICodegenContextCallback> codegenCtx;
 
 private:
     bool checkBeginMeta();
@@ -933,7 +935,7 @@ private:
 class HqlDummyParseContext : public HqlParseContext
 {
 public:
-    HqlDummyParseContext() : HqlParseContext(NULL, NULL) {}
+    HqlDummyParseContext() : HqlParseContext(NULL, NULL, NULL) {}
 };
 
 
@@ -1839,7 +1841,7 @@ extern HQL_API void extendAdd(SharedHqlExpr & value, IHqlExpression * expr);
 inline bool isOmitted(IHqlExpression * actual) { return !actual || actual->getOperator() == no_omitted; }
 
 extern HQL_API IFileContents * createFileContentsFromText(unsigned len, const char * text, ISourcePath * sourcePath);
-extern HQL_API IFileContents * createFileContentsFromText(const char * text, ISourcePath * sourcePath);
+extern HQL_API IFileContents * createFileContentsFromText(const char * text, ISourcePath * sourcePath, bool isSigned);
 extern HQL_API IFileContents * createFileContentsFromFile(const char * filename, ISourcePath * sourcePath);
 extern HQL_API IFileContents * createFileContentsSubset(IFileContents * contents, size32_t offset, size32_t len);
 extern HQL_API IFileContents * createFileContents(IFile * file, ISourcePath * sourcePath);

ecl/hql/hqlexpr.ipp

@@ -461,10 +461,11 @@ private:
     MemoryAttr fileContents;
     Linked<ISourcePath> sourcePath;
     bool delayedRead;
+    bool isSigned;
 
 public:
     CFileContents(IFile * _file, ISourcePath * _sourcePath);
-    CFileContents(const char *query, ISourcePath * _sourcePath);
+    CFileContents(const char *query, ISourcePath * _sourcePath, bool isSigned);
     CFileContents(unsigned len, const char *query, ISourcePath * _sourcePath);
 
     virtual IFile * queryFile() { return file; }
@@ -480,7 +481,11 @@ public:
         ensureLoaded();
         return fileContents.length()-1;
     }
-
+    virtual bool isSignedModule()
+    {
+        ensureLoaded();
+        return isSigned;
+    }
 private:
     bool preloadFromFile();
     void ensureLoaded();

ecl/hql/hqlfold.cpp

@@ -622,6 +622,12 @@ bool checkExternFoldable(IHqlExpression* expr, unsigned foldOptions, StringBuffe
             throw MakeStringException(ERR_TMPLT_NOFOLDFUNC, "%s does not have FOLD specified, can't constant fold it", str(expr->queryName()));
         return false;
     }
+    if (body->hasAttribute(_disallowed_Atom))
+    {
+        if (foldOptions & HFOthrowerror)
+            throw MakeStringException(ERR_TMPLT_NOFOLDFUNC, "You do not have permission to constant-fold %s", str(expr->queryName()));
+        return false;
+    }
     if(!body->hasAttribute(pureAtom) && !body->hasAttribute(templateAtom) && !(foldOptions & (HFOfoldimpure|HFOforcefold)))
     {
         if (foldOptions & HFOthrowerror)

ecl/hql/hqlgram.hpp

@@ -359,7 +359,10 @@ typedef CIArrayOf<HqlExprArrayItem> HqlExprArrayArray;
 class HqlGramCtx : public CInterface
 {
 public:
-    HqlGramCtx(HqlLookupContext & _lookupCtx) : lookupCtx(_lookupCtx) {}
+    HqlGramCtx(HqlLookupContext & _lookupCtx, bool _inSignedModule)
+      : lookupCtx(_lookupCtx), inSignedModule(_inSignedModule)
+    {
+    }
     bool hasAnyActiveParameters();
 public:
     CIArrayOf<ActiveScopeInfo> defineScopes;
@@ -368,6 +371,7 @@ public:
     Linked<ISourcePath> sourcePath;
     HqlLookupContext lookupCtx;
     HqlExprArray imports;
+    bool inSignedModule;
 };
 
 typedef const IAtom * const * AtomList;
@@ -470,6 +474,7 @@ public:
     void checkAggregateRecords(IHqlExpression * expr, IHqlExpression * record, attribute & errpos);
     void checkExportedModule(const attribute & errpos, IHqlExpression * scopeExpr);
     bool checkCompatibleSymbol(const attribute & errpos, IHqlExpression * prevValue, IHqlExpression * newValue);
+    bool checkAllowed(const attribute & errpos, const char *category, const char *description);
     IHqlExpression * createAveList(const attribute & errpos, IHqlExpression * list);
     IHqlExpression * createIff(attribute & condAttr, attribute & leftAttr, attribute & rightAttr);
     IHqlExpression * createListFromExpressionList(attribute & attr);
@@ -847,6 +852,7 @@ protected:
     bool parseConstantText;
     bool expandingMacroPosition;
     unsigned m_maxErrorsAllowed;
+    bool inSignedModule;
 
     IErrorArray pendingWarnings;
     Linked<ISourcePath> sourcePath;

ecl/hql/hqlgram.y

@@ -8791,9 +8791,12 @@ simpleDataSet
                         }
     | PIPE '(' expression ',' recordDef optPipeOptions ')'
                         {
+                            OwnedHqlExpr attrs = $6.getExpr();
+                            if (!parser->checkAllowed($1, "pipe", "PIPE"))
+                                attrs.setown(createComma(attrs.getClear(), createAttribute(_disallowed_Atom)));
                             parser->normalizeExpression($3, type_string, false);
                             parser->checkValidPipeRecord($5, $5.queryExpr(), $6.queryExpr(), NULL);
-                            $$.setExpr(createNewDataset(createConstant(""), $5.getExpr(), createValue(no_pipe, makeNullType(), $3.getExpr()), NULL, NULL, $6.getExpr()));
+                            $$.setExpr(createNewDataset(createConstant(""), $5.getExpr(), createValue(no_pipe, makeNullType(), $3.getExpr()), NULL, NULL, LINK(attrs)));
                             $$.setPosition($1);
                         }
     | PIPE '(' startTopFilter ',' expression optPipeOptions endTopFilter ')'
@@ -8801,6 +8804,8 @@ simpleDataSet
                             parser->normalizeExpression($5, type_string, false);
 
                             OwnedHqlExpr attrs = $6.getExpr();
+                            if (!parser->checkAllowed($1, "pipe", "PIPE"))
+                                attrs.setown(createComma(attrs.getClear(), createAttribute(_disallowed_Atom)));
                             parser->checkValidPipeRecord($3, $3.queryExpr()->queryRecord(), attrs, NULL);
                             parser->checkValidPipeRecord($3, $3.queryExpr()->queryRecord(), NULL, queryAttributeInList(outputAtom, attrs));
 
@@ -8812,6 +8817,8 @@ simpleDataSet
                             parser->normalizeExpression($5, type_string, false);
 
                             OwnedHqlExpr attrs = $8.getExpr();
+                            if (!parser->checkAllowed($1, "pipe", "PIPE"))
+                                attrs.setown(createComma(attrs.getClear(), createAttribute(_disallowed_Atom)));
                             parser->checkValidPipeRecord($3, $3.queryExpr()->queryRecord(), NULL, queryAttributeInList(outputAtom, attrs));
                             parser->checkValidPipeRecord($7, $7.queryExpr()->queryRecord(), attrs, NULL);
 
@@ -10104,7 +10111,10 @@ pipe
     : PIPE '(' expression optPipeOptions ')'    
                         {   
                             parser->normalizeExpression($3, type_string, false);
-                            $$.setExpr(createComma(createValue(no_pipe, makeNullType(), $3.getExpr()), $4.getExpr()));
+                            OwnedHqlExpr attrs = $4.getExpr();
+                            if (!parser->checkAllowed($1, "pipe", "PIPE"))
+                                attrs.setown(createComma(attrs.getClear(), createAttribute(_disallowed_Atom)));
+                            $$.setExpr(createComma(createValue(no_pipe, makeNullType(), $3.getExpr()), LINK(attrs)));
                         }
     ;
 

ecl/hql/hqlgram2.cpp

@@ -342,6 +342,7 @@ HqlGram::HqlGram(IHqlScope * _globalScope, IHqlScope * _containerScope, IFileCon
     moduleName = _containerScope->queryId();
     forceResult = false;
     parsingTemplateAttribute = false;
+    inSignedModule = _text->isSignedModule();
 
     lexObject = new HqlLex(this, _text, xmlScope, NULL);
 
@@ -367,6 +368,7 @@ HqlGram::HqlGram(HqlGramCtx & parent, IHqlScope * _containerScope, IFileContents
     for (unsigned i=0;i<parent.defaultScopes.length();i++)
         defaultScopes.append(*LINK(&parent.defaultScopes.item(i)));
     sourcePath.set(parent.sourcePath);
+    inSignedModule = parent.inSignedModule;
     errorHandler = lookupCtx.errs;
     moduleName = containerScope->queryId();
 
@@ -430,6 +432,7 @@ void HqlGram::init(IHqlScope * _globalScope, IHqlScope * _containerScope)
     moduleName = NULL;
     resolveSymbols = true;
     lastpos = 0;
+    inSignedModule = false;
     
     containerScope = _containerScope;
     globalScope = _globalScope;
@@ -921,6 +924,8 @@ IHqlExpression * HqlGram::processEmbedBody(const attribute & errpos, IHqlExpress
         }
         args.append(*createExprAttribute(languageAtom, getEmbedContextFunc.getClear()));
     }
+    if (!checkAllowed(errpos, "cpp", "Embedded code"))
+        args.append(*createExprAttribute(_disallowed_Atom));
     if (attribs)
         attribs->unwindList(args, no_comma);
     Linked<ITypeInfo> type = current_type;
@@ -2885,7 +2890,7 @@ void HqlGram::processForwardModuleDefinition(const attribute & errpos)
         return;
     }
 
-    HqlGramCtx * parentCtx = new HqlGramCtx(lookupCtx);
+    HqlGramCtx * parentCtx = new HqlGramCtx(lookupCtx, inSignedModule);
     saveContext(*parentCtx, true);
     Owned<IHqlScope> newScope = createForwardScope(queryGlobalScope(), parentCtx, lookupCtx.queryParseContext());
     IHqlExpression * newScopeExpr = queryExpression(newScope);
@@ -3605,6 +3610,8 @@ IHqlExpression* HqlGram::checkServiceDef(IHqlScope* serviceScope,IIdAtom * name,
         attrs->unwindList(attrArray,no_comma);
     
     bool hasEntrypoint = false;
+    bool foldSeen = false;
+    bool nofoldSeen = false;
     unsigned count = attrArray.length();
     if (count>0)
     {
@@ -3711,7 +3718,7 @@ IHqlExpression* HqlGram::checkServiceDef(IHqlScope* serviceScope,IIdAtom * name,
                 bcdApi = true;
                 checkSvcAttrNoValue(attr, errpos);
             }
-            else if (name == pureAtom || name == templateAtom || name == volatileAtom || name == onceAtom || name == actionAtom || name == foldAtom || name == nofoldAtom)
+            else if (name == pureAtom || name == templateAtom || name == volatileAtom || name == onceAtom || name == actionAtom)
             {
                 checkSvcAttrNoValue(attr, errpos);
             }
@@ -3727,10 +3734,13 @@ IHqlExpression* HqlGram::checkServiceDef(IHqlScope* serviceScope,IIdAtom * name,
             {
                 //backward compatibility
             }
+            else if (name == foldAtom)
+                foldSeen = true;
+            else if (name == nofoldAtom)
+                nofoldSeen = true;
             else // unsupported
                 reportWarning(CategorySyntax,WRN_SVC_UNSUPPORTED_ATTR, errpos.pos, "Unsupported service attribute: '%s'; ignored", str(name));
         }
-
         // check attribute conflicts
         int apiAttrs = 0;
         if (rtlApi) apiAttrs++;
@@ -3739,6 +3749,12 @@ IHqlExpression* HqlGram::checkServiceDef(IHqlScope* serviceScope,IIdAtom * name,
         if (apiAttrs>1)
             reportWarning(CategorySyntax, ERR_SVC_ATTRCONFLICTS, errpos.pos, "Attributes eclrtl, bcd, c are conflict: only 1 can be used at a time");
     }
+    if (foldSeen && !nofoldSeen)
+    {
+        // Check that we are allowed to fold...
+        if (!checkAllowed(errpos, "foldextern", "FOLD attribute"))
+            attrs = createComma(attrs, createAttribute(_disallowed_Atom));
+    }
 
     if (!hasEntrypoint)
     {
@@ -9018,6 +9034,20 @@ void HqlGram::checkDerivedCompatible(IIdAtom * name, IHqlExpression * scope, IHq
     }
 }
 
+bool HqlGram::checkAllowed(const attribute & errpos, const char *category, const char *description)
+{
+    if (lookupCtx.queryParseContext().codegenCtx && !lookupCtx.queryParseContext().codegenCtx->allowAccess(category, inSignedModule))
+    {
+        if (!inSignedModule && lookupCtx.queryParseContext().codegenCtx->allowAccess(category, true))
+            reportWarning(CategorySecurity, WRN_REQUIRES_SIGNED, errpos.pos, "%s is only permitted in a signed module", description);
+        else
+            reportWarning(CategorySecurity, WRN_DISALLOWED, errpos.pos, "%s is not permitted by your security settings", description);
+        return false;
+    }
+    return true;
+}
+
+
 bool HqlGram::okToAddSideEffects(IHqlExpression * expr)
 {
     switch (expr->getOperator())
@@ -11566,7 +11596,7 @@ IHqlExpression * reparseTemplateFunction(IHqlExpression * funcdef, IHqlScope *sc
     text.append("=>").append(contents->length(), contents->getText());
 
     //Could use a merge string implementation of IFileContents instead of expanding...
-    Owned<IFileContents> parseContents = createFileContentsFromText(text.str(), contents->querySourcePath());
+    Owned<IFileContents> parseContents = createFileContentsFromText(text.str(), contents->querySourcePath(), false);
     HqlGram parser(scope, scope, parseContents, ctx, NULL, hasFieldMap, true);
     unsigned startLine = funcdef->getStartLine();
 
@@ -11690,7 +11720,7 @@ extern HQL_API IHqlExpression * parseQuery(const char * text, IErrorReceiver * e
 {
     Owned<IHqlScope> scope = createScope();
     HqlDummyLookupContext ctx(errs);
-    Owned<IFileContents> contents = createFileContentsFromText(text, NULL);
+    Owned<IFileContents> contents = createFileContentsFromText(text, NULL, false);
     return parseQuery(scope, contents, ctx, NULL, NULL, true);
 }
 

ecl/hql/hqlparse.cpp

@@ -316,7 +316,7 @@ void HqlLex::pushText(IFileContents * text, int startLineNo, int startColumn)
 
 void HqlLex::pushText(const char *s, int startLineNo, int startColumn)
 {
-    Owned<IFileContents> macroContents = createFileContentsFromText(s, sourcePath);
+    Owned<IFileContents> macroContents = createFileContentsFromText(s, sourcePath, false); // MORE - may be wrong
     pushText(macroContents, startLineNo, startColumn);
 }
 
@@ -326,7 +326,7 @@ void HqlLex::pushText(const char *s)
 #ifdef TIMING_DEBUG
     MTIME_SECTION(timer, "HqlLex::pushText");
 #endif
-    Owned<IFileContents> macroContents = createFileContentsFromText(s, sourcePath);
+    Owned<IFileContents> macroContents = createFileContentsFromText(s, sourcePath, false);  // MORE - may be wrong
     inmacro = new HqlLex(yyParser, macroContents, NULL, NULL);
     inmacro->set_yyLineNo(yyLineNo);
     inmacro->set_yyColumn(yyColumn);
@@ -612,7 +612,7 @@ void HqlLex::processEncrypted()
     decryptEclAttribute(decrypted, encoded64.str());
     decrypted.append(0);    // add a null terminator to the string...
     Owned<ISourcePath> sourcePath = createSourcePath("<encrypted>");
-    Owned<IFileContents> decryptedContents = createFileContentsFromText((const char *)decrypted.toByteArray(), sourcePath);
+    Owned<IFileContents> decryptedContents = createFileContentsFromText((const char *)decrypted.toByteArray(), sourcePath, yyParser->inSignedModule);
     inmacro = new HqlLex(yyParser, decryptedContents, NULL, NULL);
     inmacro->setParentLex(this);
     inmacro->encrypted = true;
@@ -1030,7 +1030,7 @@ void HqlLex::doExport(YYSTYPE & returnToken, bool toXml)
         try
         {
             HqlLookupContext ctx(yyParser->lookupCtx);
-            Owned<IFileContents> exportContents = createFileContentsFromText(curParam.str(), sourcePath);
+            Owned<IFileContents> exportContents = createFileContentsFromText(curParam.str(), sourcePath, yyParser->inSignedModule);
             expr.setown(parseQuery(scope, exportContents, ctx, xmlScope, NULL, true));
 
             if (expr && (expr->getOperator() == no_sizeof))
@@ -1172,8 +1172,8 @@ void HqlLex::doFor(YYSTYPE & returnToken, bool doAll)
 
     forLoop = getSubScopes(returnToken, str(name), doAll);
     if (forFilterText.length())
-        forFilter.setown(createFileContentsFromText(forFilterText, sourcePath));
-    forBody.setown(createFileContentsFromText(forBodyText, sourcePath));
+        forFilter.setown(createFileContentsFromText(forFilterText, sourcePath, yyParser->inSignedModule));
+    forBody.setown(createFileContentsFromText(forBodyText, sourcePath, yyParser->inSignedModule));
 
     loopTimes = 0;
     if (forLoop && forLoop->first()) // more - check filter
@@ -1222,7 +1222,7 @@ void HqlLex::doLoop(YYSTYPE & returnToken)
     ::Release(forLoop);
     forLoop = new CDummyScopeIterator(ensureTopXmlScope());
     forFilter.clear();
-    forBody.setown(createFileContentsFromText(forBodyText, sourcePath));
+    forBody.setown(createFileContentsFromText(forBodyText, sourcePath, yyParser->inSignedModule));
     loopTimes = 0;
     if (forLoop->first()) // more - check filter
         checkNextLoop(returnToken, true,startLine,startCol);
@@ -1463,7 +1463,7 @@ void HqlLex::doIsValid(YYSTYPE & returnToken)
     {
         HqlLookupContext ctx(yyParser->lookupCtx);
         ctx.errs.clear();   //Deliberately ignore any errors
-        Owned<IFileContents> contents = createFileContentsFromText(curParam.str(), sourcePath);
+        Owned<IFileContents> contents = createFileContentsFromText(curParam.str(), sourcePath, yyParser->inSignedModule);
         expr = parseQuery(scope, contents, ctx, xmlScope, NULL, true);
 
         if(expr)
@@ -1730,7 +1730,7 @@ IHqlExpression *HqlLex::parseECL(IFileContents * contents, IXmlScope *xmlScope,
     //  Use an ECL reserved word as the scope name to avoid name conflicts with these defined localscope.
     Owned<IHqlScope> scope = new CHqlMultiParentScope(sharedId,yyParser->queryPrimaryScope(false),yyParser->queryPrimaryScope(true),yyParser->parseScope.get(),NULL);
 
-    HqlGramCtx parentContext(yyParser->lookupCtx);
+    HqlGramCtx parentContext(yyParser->lookupCtx, yyParser->inSignedModule);
     yyParser->saveContext(parentContext, false);
     HqlGram parser(parentContext, scope, contents, xmlScope, true);
     parser.getLexer()->set_yyLineNo(startLine);
@@ -1741,7 +1741,7 @@ IHqlExpression *HqlLex::parseECL(IFileContents * contents, IXmlScope *xmlScope,
 
 IHqlExpression *HqlLex::parseECL(const char * text, IXmlScope *xmlScope, int startLine, int startCol)
 {
-    Owned<IFileContents> contents = createFileContentsFromText(text, querySourcePath());
+    Owned<IFileContents> contents = createFileContentsFromText(text, querySourcePath(), yyParser->inSignedModule);
     return parseECL(contents, xmlScope, startLine, startCol);
 }
 

ecl/hql/hqlplugininfo.cpp

@@ -40,7 +40,7 @@ IEclRepository * loadPlugins(const char * pluginPath)
 
 IPropertyTree * createPluginPropertyTree(IEclRepository * plugins, bool includeModuleText)
 {
-    HqlParseContext parseCtx(plugins, NULL);
+    HqlParseContext parseCtx(plugins, NULL, NULL);
     HqlLookupContext ctx(parseCtx, NULL);
     HqlScopeArray scopes;
     getRootScopes(scopes, plugins, ctx);
@@ -99,7 +99,7 @@ IPropertyTree * getPlugin(IPropertyTree * p, IEclRepository * plugins, const cha
 
     if(load && !plugin->getPropInt("@loaded",0))
     {
-        HqlParseContext parseCtx(plugins, NULL);
+        HqlParseContext parseCtx(plugins, NULL, NULL);
         HqlLookupContext GHMOREctx(parseCtx, NULL);
         Owned<IHqlScope> resolved = getResolveDottedScope(modname, LSFpublic, GHMOREctx);
         if (resolved)

ecl/hqlcpp/hqlcpp.cpp

@@ -1367,7 +1367,7 @@ IHqlCppInstance * createCppInstance(IWorkUnit *wu, const char * wupathname)
 
 HqlCppTranslator::HqlCppTranslator(IErrorReceiver * _errors, const char * _soName, IHqlCppInstance * _code, ClusterType _targetClusterType, ICodegenContextCallback *_ctxCallback) : ctxCallback(_ctxCallback)
 {
-    //Insert a couple of warning mapping layers - one for global #onwarnigns, and another for local : onwarning
+    //Insert a couple of warning mapping layers - one for global #onwarnings, and another for local : onwarning
     globalOnWarnings.setown(new ErrorSeverityMapper(*_errors));
     localOnWarnings.setown(new ErrorSeverityMapper((IErrorReceiver &)*globalOnWarnings)); // horrible: cast required, otherwise copy constructor is called!
 
@@ -1390,7 +1390,7 @@ HqlCppTranslator::HqlCppTranslator(IErrorReceiver * _errors, const char * _soNam
             HqlDummyLookupContext ctx(&errs);
             cppSystemScope = createScope();
             Owned<ISourcePath> sysPath = createSourcePath("<system-definitions>");
-            Owned<IFileContents> systemContents = createFileContentsFromText(systemText.str(), sysPath);
+            Owned<IFileContents> systemContents = createFileContentsFromText(systemText.str(), sysPath, true);
             OwnedHqlExpr query = parseQuery(cppSystemScope, systemContents, ctx, NULL, NULL, false);
             if (errs.errCount())
             {
@@ -1425,9 +1425,6 @@ HqlCppTranslator::HqlCppTranslator(IErrorReceiver * _errors, const char * _soNam
     graphSeqNumber = 0;
     nlpParse = NULL;
     outputLibrary = NULL;
-    checkedEmbeddedCpp = false;
-    cachedAllowEmbeddedCpp = true;
-    checkedPipeAllowed = false;
     activitiesThisCpp = 0;
     curCppFile = 0;
     timeReporter.setown(createStdTimeReporter());
@@ -1968,26 +1965,6 @@ IHqlExpression *HqlCppTranslator::addStringLiteral(const char *lit)
         return createConstant(createStringValue(lit, litLen));
 }
 
-bool HqlCppTranslator::allowEmbeddedCpp()
-{
-    if (!checkedEmbeddedCpp)
-    {
-        cachedAllowEmbeddedCpp = ctxCallback->allowAccess("cpp");
-        checkedEmbeddedCpp = true;
-    }
-    return cachedAllowEmbeddedCpp;
-}
-
-void HqlCppTranslator::checkPipeAllowed()
-{
-    if (!checkedPipeAllowed)
-    {
-        if (!ctxCallback->allowAccess("pipe"))
-            throwError(HQLERR_PipeNotAllowed);
-        checkedPipeAllowed = true;
-    }
-}
-
 IHqlExpression * HqlCppTranslator::bindFunctionCall(IIdAtom * name, HqlExprArray & args)
 {
     OwnedHqlExpr function = needFunction(name);
@@ -7518,7 +7495,7 @@ void HqlCppTranslator::processCppBodyDirectives(IHqlExpression * expr)
 
 void HqlCppTranslator::doBuildExprEmbedBody(BuildCtx & ctx, IHqlExpression * expr, CHqlBoundExpr * tgt)
 {
-    if (!allowEmbeddedCpp())
+    if (expr->hasAttribute(_disallowed_Atom))
         throwError(HQLERR_EmbeddedCppNotAllowed);
 
     processCppBodyDirectives(expr);
@@ -11933,7 +11910,7 @@ void HqlCppTranslator::buildFunctionDefinition(IHqlExpression * funcdef)
 
     if (bodyCode->getOperator() == no_embedbody)
     {
-        if (!allowEmbeddedCpp())
+        if (bodyCode->hasAttribute(_disallowed_Atom))
             throwError(HQLERR_EmbeddedCppNotAllowed);
 
         IHqlExpression *languageAttr = bodyCode->queryAttribute(languageAtom);

ecl/hqlcpp/hqlcpp.hpp

@@ -84,12 +84,6 @@ The following debug options are currently supported by the code generator:
 
 #define CREATE_DEAULT_ROW_IF_NULL
 
-interface ICodegenContextCallback : public IInterface
-{
-    virtual void noteCluster(const char *clusterName) = 0;
-    virtual bool allowAccess(const char * category) = 0;
-};
-
 //interface that represents a implementation of a query
 //ready for passing to the thing that executes it.
 interface HQLCPP_API IHqlQueryInstance : public IInterface

ecl/hqlcpp/hqlcpp.ipp

@@ -1119,8 +1119,6 @@ public:
     void getRecordECL(IHqlExpression * record, StringBuffer & eclText);
     void ensureHasAddress(BuildCtx & ctx, CHqlBoundExpr & tgt);
     void normalizeBoundExpr(BuildCtx & ctx, CHqlBoundExpr & bound);
-    bool allowEmbeddedCpp();
-    void checkPipeAllowed();
 
     IWorkUnit * wu()           { return code->workunit; }
     void useInclude(const char * name)                      { code->useInclude(name); }
@@ -1941,9 +1939,6 @@ protected:
     Linked<IErrorReceiver> errorProcessor;
     HqlCppOptions       options;
     HqlCppDerived       derived;
-    bool                checkedEmbeddedCpp;
-    bool                cachedAllowEmbeddedCpp;
-    bool                checkedPipeAllowed;
     unsigned            activitiesThisCpp;
     unsigned            curCppFile;
     Linked<ICodegenContextCallback> ctxCallback;

ecl/hqlcpp/hqlecl.cpp

@@ -53,7 +53,7 @@ class NullContextCallback : public CInterface, implements ICodegenContextCallbac
     IMPLEMENT_IINTERFACE
 
     virtual void noteCluster(const char *clusterName) {}
-    virtual bool allowAccess(const char * category) { return true; }
+    virtual bool allowAccess(const char * category, bool isSigned) { return true; }
 };
 
 class HqlDllGenerator : public CInterface, implements IHqlExprDllGenerator, implements IAbortRequestCallback

ecl/hqlcpp/hqlhtcpp.cpp

@@ -10394,8 +10394,8 @@ ABoundActivity * HqlCppTranslator::doBuildActivityOutput(BuildCtx & ctx, IHqlExp
     else if (filename->getOperator()==no_pipe)
         pipe = filename->queryChild(0);
 
-    if (pipe)
-        checkPipeAllowed();
+    if (pipe && expr->hasAttribute(_disallowed_Atom))
+        throwError(HQLERR_PipeNotAllowed);
 
     Owned<ABoundActivity> boundDataset = buildCachedActivity(ctx, dataset);
     ThorActivityKind kind = TAKdiskwrite;
@@ -11329,8 +11329,8 @@ ABoundActivity * HqlCppTranslator::doBuildActivityDictionaryWorkunitWrite(BuildC
 
 ABoundActivity * HqlCppTranslator::doBuildActivityPipeThrough(BuildCtx & ctx, IHqlExpression * expr)
 {
-    checkPipeAllowed();
-
+    if (expr->hasAttribute(_disallowed_Atom))
+        throwError(HQLERR_PipeNotAllowed);
     IHqlExpression * dataset = expr->queryChild(0);
     IHqlExpression * pipe = expr->queryChild(1);
     IHqlExpression * output = expr->queryAttribute(outputAtom);

ecl/hqlcpp/hqlsource.cpp

@@ -2790,7 +2790,8 @@ void DiskReadBuilder::buildMembers(IHqlExpression * expr)
     //---- virtual const char * getPipeProgram() { return "grep"; } ----
     if (modeOp==no_pipe)
     {
-        translator.checkPipeAllowed();
+        if (expr->hasAttribute(_disallowed_Atom))
+            throwError(HQLERR_PipeNotAllowed);
         BuildCtx pipeCtx(instance->startctx);
         pipeCtx.addQuotedCompound("virtual const char * getPipeProgram()");
         translator.buildReturn(pipeCtx, mode->queryChild(0), unknownVarStringType);

system/jlib/jexcept.hpp

@@ -197,6 +197,7 @@ enum WarnErrorCategory
     CategoryUnusual,    // Not strictly speaking an error, but highly unusual and likely to be a mistake
     CategoryUnexpected, // Code that could be correct, but has the potential for unexpected behaviour
     CategoryCpp,        // Warning passed through from C++ compiler
+    CategorySecurity,   // Security warnings - operations that will be refused at codegen time unless unused.
 
     CategoryError,      // Typically severity fatal
     CategoryAll,