HPCC-14280 Access rights checks should include SERVICE calls

Code loaded from plugins directory (and the internal system definitions) are
considered implicitly signed.

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

ecl/eclcc/eclcc.cpp

@@ -1333,7 +1333,7 @@ void EclCC::processDefinitions(EclRepositoryArray & repositories)
         }
 
         //Create a repository with just that attribute.
-        Owned<IFileContents> contents = createFileContentsFromText(value, NULL);
+        Owned<IFileContents> contents = createFileContentsFromText(value, NULL, false);
         repositories.append(*createSingleDefinitionEclRepository(module, attr, contents));
     }
 }
@@ -1410,7 +1410,7 @@ void EclCC::processXmlFile(EclCompileInstance & instance, const char *archiveXML
     {
         const char * sourceFilename = archiveTree->queryProp("Query/@originalFilename");
         Owned<ISourcePath> sourcePath = createSourcePath(sourceFilename);
-        contents.setown(createFileContentsFromText(queryText, sourcePath));
+        contents.setown(createFileContentsFromText(queryText, sourcePath, false));
         if (queryAttributePath && queryText && *queryText)
         {
             Owned<IEclSourceCollection> inputFileCollection = createSingleDefinitionEclCollection(queryAttributePath, contents);
@@ -1431,7 +1431,7 @@ void EclCC::processXmlFile(EclCompileInstance & instance, const char *archiveXML
         queryAttributePath = fullPath.str();
 
         //Create a repository with just that attribute, and place it before the archive in the resolution order.
-        Owned<IFileContents> contents = createFileContentsFromText(queryText, NULL);
+        Owned<IFileContents> contents = createFileContentsFromText(queryText, NULL, false);
         repositories.append(*createSingleDefinitionEclRepository(syntaxCheckModule, syntaxCheckAttribute, contents));
     }
 
@@ -1455,7 +1455,7 @@ void EclCC::processFile(EclCompileInstance & instance)
     assertex(curFilename);
 
     Owned<ISourcePath> sourcePath = optNoSourcePath ? NULL : createSourcePath(curFilename);
-    Owned<IFileContents> queryText = createFileContentsFromFile(curFilename, sourcePath);
+    Owned<IFileContents> queryText = createFileContentsFromFile(curFilename, sourcePath, false);
     const char * queryTxt = queryText->getText();
     if (optArchive || optGenerateDepend)
         instance.archive.setown(createAttributeArchive());

ecl/hql/hqlcollect.cpp

@@ -144,7 +144,7 @@ void CEclCollection::ensureDefinition()
 class FileSystemFile : public CEclSource
 {
 public:
-    FileSystemFile(EclSourceType _type, IFile & _file);
+    FileSystemFile(EclSourceType _type, IFile & _file, bool _allowPlugins);
 
 //interface IEclSource
     virtual IProperties * getProperties();
@@ -243,11 +243,11 @@ static IIdAtom * deriveEclName(const char * filename)
 
 //---------------------------------------------------------------------------------------
 
-FileSystemFile::FileSystemFile(EclSourceType _type, IFile & _file)
+FileSystemFile::FileSystemFile(EclSourceType _type, IFile & _file, bool _allowPlugins)
 : CEclSource(deriveEclName(_file.queryFilename()), _type), file(&_file)
 {
     Owned<ISourcePath> path = createSourcePath(file->queryFilename());
-    fileContents.setown(createFileContents(file, path));
+    fileContents.setown(createFileContents(file, path, _allowPlugins));
     extraFlags = 0;
     switch (type)
     {
@@ -289,7 +289,7 @@ bool FileSystemFile::checkValid()
                         version.set(pb.version);
 
                         Owned<ISourcePath> pluginPath = createSourcePath(pb.moduleName);
-                        fileContents.setown(createFileContentsFromText(pb.ECL, pluginPath));
+                        fileContents.setown(createFileContentsFromText(pb.ECL, pluginPath, true));
 
                         //if (traceMask & PLUGIN_DLL_MODULE)
                         DBGLOG("Loading plugin %s[%s] version = %s", filename, pb.moduleName, version.get());
@@ -368,7 +368,7 @@ void FileSystemDirectory::addFile(IFile &file, bool allowPlugins)
             EclSourceType type = getEclSourceType(tail);
             if (type && ((type != ESTplugin) || allowPlugins))
             {
-                Owned<FileSystemFile> newFile = new FileSystemFile(type, file);
+                Owned<FileSystemFile> newFile = new FileSystemFile(type, file, allowPlugins);
                 if (allowPlugins)
                     newFile->extraFlags |= SOURCEFILE_PLUGIN;
                 if (newFile->checkValid())
@@ -665,7 +665,7 @@ IFileContents * CXmlEclElement::queryFileContents()
                 getFullName(defaultName);
                 sourcePath.setown(createSourcePath(defaultName));
             }
-            fileContents.setown(createFileContentsFromText(text, sourcePath));
+            fileContents.setown(createFileContentsFromText(text, sourcePath, false));
         }
     }
     return fileContents;

ecl/hql/hqlcollect.hpp

@@ -56,8 +56,6 @@ interface IEclSource : public IInterface
         EclSourceType type = queryType();
         return (type==ESTlibrary) || (type==ESTplugin);
     }
-    //virtual bool isPluginDllScope(IHqlScope * scope)
-    //virtual bool allowImplicitImport(IHqlScope * scope)
 };
 
 typedef IIteratorOf<IEclSource> IEclSourceIterator;

ecl/hql/hqlexpr.cpp

@@ -7552,7 +7552,8 @@ IHqlExpression * createJavadocAnnotation(IHqlExpression * _ownedBody, IPropertyT
 
 //==============================================================================================================
 
-CFileContents::CFileContents(IFile * _file, ISourcePath * _sourcePath) : file(_file), sourcePath(_sourcePath)
+CFileContents::CFileContents(IFile * _file, ISourcePath * _sourcePath, bool _isSigned)
+  : file(_file), sourcePath(_sourcePath), implicitlySigned(_isSigned)
 {
     delayedRead = false;
     if (!preloadFromFile())
@@ -7628,8 +7629,8 @@ void CFileContents::ensureLoaded()
         throw MakeStringException(1, "File %s only read %u of %u bytes", file->queryFilename(), sizeRead, sizeToRead);
 }
 
-CFileContents::CFileContents(const char *query, ISourcePath * _sourcePath) 
-: sourcePath(_sourcePath)
+CFileContents::CFileContents(const char *query, ISourcePath * _sourcePath, bool _isSigned)
+: sourcePath(_sourcePath), implicitlySigned(_isSigned)
 {
     if (query)
         setContents(strlen(query), query);
@@ -7637,8 +7638,8 @@ CFileContents::CFileContents(const char *query, ISourcePath * _sourcePath)
     delayedRead = false;
 }
 
-CFileContents::CFileContents(unsigned len, const char *query, ISourcePath * _sourcePath) 
-: sourcePath(_sourcePath)
+CFileContents::CFileContents(unsigned len, const char *query, ISourcePath * _sourcePath, bool _isSigned)
+: sourcePath(_sourcePath), implicitlySigned(_isSigned)
 {
     setContents(len, query);
     delayedRead = false;
@@ -7660,26 +7661,26 @@ void CFileContents::setContentsOwn(MemoryBuffer & buffer)
     fileContents.setOwn(len, buffer.detach());
 }
 
-IFileContents * createFileContentsFromText(unsigned len, const char * text, ISourcePath * sourcePath)
+IFileContents * createFileContentsFromText(unsigned len, const char * text, ISourcePath * sourcePath, bool isSigned)
 {
-    return new CFileContents(len, text, sourcePath);
+    return new CFileContents(len, text, sourcePath, isSigned);
 }
 
-IFileContents * createFileContentsFromText(const char * text, ISourcePath * sourcePath)
+IFileContents * createFileContentsFromText(const char * text, ISourcePath * sourcePath, bool isSigned)
 {
     //MORE: Treatment of nulls?
-    return new CFileContents(text, sourcePath);
+    return new CFileContents(text, sourcePath, isSigned);
 }
 
-IFileContents * createFileContentsFromFile(const char * filename, ISourcePath * sourcePath)
+IFileContents * createFileContentsFromFile(const char * filename, ISourcePath * sourcePath, bool isSigned)
 {
     Owned<IFile> file = createIFile(filename);
-    return new CFileContents(file, sourcePath);
+    return new CFileContents(file, sourcePath, isSigned);
 }
 
-IFileContents * createFileContents(IFile * file, ISourcePath * sourcePath)
+IFileContents * createFileContents(IFile * file, ISourcePath * sourcePath, bool isSigned)
 {
-    return new CFileContents(file, sourcePath);
+    return new CFileContents(file, sourcePath, isSigned);
 }
 
 class CFileContentsSubset : public CInterfaceOf<IFileContents>
@@ -7694,7 +7695,7 @@ public:
     virtual ISourcePath * querySourcePath() { return contents->querySourcePath(); }
     virtual const char *getText() { return contents->getText() + offset; }
     virtual size32_t length() { return len; }
-
+    virtual bool isImplicitlySigned() { return contents->isImplicitlySigned(); }
 protected:
     Linked<IFileContents> contents;
     size32_t offset;

ecl/hql/hqlexpr.hpp

@@ -836,6 +836,7 @@ public:
     virtual ISourcePath * querySourcePath() = 0;
     virtual const char * getText() = 0;
     virtual size32_t length() = 0;
+    virtual bool isImplicitlySigned() = 0;
 };
 
 //This class ensures that the pointer to the owner is cleared before both links are released, which allows
@@ -1840,11 +1841,11 @@ extern HQL_API ITypeInfo * getPromotedECLCompareType(ITypeInfo * lType, ITypeInf
 extern HQL_API void extendAdd(SharedHqlExpr & value, IHqlExpression * expr);
 inline bool isOmitted(IHqlExpression * actual) { return !actual || actual->getOperator() == no_omitted; }
 
-extern HQL_API IFileContents * createFileContentsFromText(unsigned len, const char * text, ISourcePath * sourcePath);
-extern HQL_API IFileContents * createFileContentsFromText(const char * text, ISourcePath * sourcePath);
-extern HQL_API IFileContents * createFileContentsFromFile(const char * filename, ISourcePath * sourcePath);
+extern HQL_API IFileContents * createFileContentsFromText(unsigned len, const char * text, ISourcePath * sourcePath, bool isSigned);
+extern HQL_API IFileContents * createFileContentsFromText(const char * text, ISourcePath * sourcePath, bool isSigned);
+extern HQL_API IFileContents * createFileContentsFromFile(const char * filename, ISourcePath * sourcePath, bool isSigned);
 extern HQL_API IFileContents * createFileContentsSubset(IFileContents * contents, size32_t offset, size32_t len);
-extern HQL_API IFileContents * createFileContents(IFile * file, ISourcePath * sourcePath);
+extern HQL_API IFileContents * createFileContents(IFile * file, ISourcePath * sourcePath, bool isSigned);
 
 void addForwardDefinition(IHqlScope * scope, IIdAtom * symbolName, IIdAtom * moduleName, IFileContents * contents,
                           unsigned symbolFlags, bool isExported, unsigned startLine, unsigned startColumn);

ecl/hql/hqlexpr.ipp

@@ -461,11 +461,12 @@ private:
     MemoryAttr fileContents;
     Linked<ISourcePath> sourcePath;
     bool delayedRead;
+    bool implicitlySigned;
 
 public:
-    CFileContents(IFile * _file, ISourcePath * _sourcePath);
-    CFileContents(const char *query, ISourcePath * _sourcePath);
-    CFileContents(unsigned len, const char *query, ISourcePath * _sourcePath);
+    CFileContents(IFile * _file, ISourcePath * _sourcePath, bool _isSigned);
+    CFileContents(const char *query, ISourcePath * _sourcePath, bool _isSigned);
+    CFileContents(unsigned len, const char *query, ISourcePath * _sourcePath, bool _isSigned);
 
     virtual IFile * queryFile() { return file; }
     virtual ISourcePath * querySourcePath() { return sourcePath; }
@@ -480,7 +481,10 @@ public:
         ensureLoaded();
         return (size32_t)(fileContents.length()-1);
     }
-
+    virtual bool isImplicitlySigned()
+    {
+        return implicitlySigned;
+    }
 private:
     bool preloadFromFile();
     void ensureLoaded();

ecl/hql/hqlgram2.cpp

@@ -343,7 +343,8 @@ HqlGram::HqlGram(IHqlScope * _globalScope, IHqlScope * _containerScope, IFileCon
     moduleName = _containerScope->queryId();
     forceResult = false;
     parsingTemplateAttribute = false;
-    inSignedModule = false;
+    inSignedModule = _text->isImplicitlySigned();
+;
 
     lexObject = new HqlLex(this, _text, xmlScope, NULL);
 
@@ -369,7 +370,7 @@ HqlGram::HqlGram(HqlGramCtx & parent, IHqlScope * _containerScope, IFileContents
     for (unsigned i=0;i<parent.defaultScopes.length();i++)
         defaultScopes.append(*LINK(&parent.defaultScopes.item(i)));
     sourcePath.set(parent.sourcePath);
-    inSignedModule = parent.inSignedModule;
+    inSignedModule = parent.inSignedModule || _text->isImplicitlySigned();
     errorHandler = lookupCtx.errs;
     moduleName = containerScope->queryId();
 
@@ -11619,7 +11620,7 @@ IHqlExpression * reparseTemplateFunction(IHqlExpression * funcdef, IHqlScope *sc
     text.append("=>").append(contents->length(), contents->getText());
 
     //Could use a merge string implementation of IFileContents instead of expanding...
-    Owned<IFileContents> parseContents = createFileContentsFromText(text.str(), contents->querySourcePath());
+    Owned<IFileContents> parseContents = createFileContentsFromText(text.str(), contents->querySourcePath(), contents->isImplicitlySigned());
     HqlGram parser(scope, scope, parseContents, ctx, NULL, hasFieldMap, true);
     unsigned startLine = funcdef->getStartLine();
 
@@ -11743,7 +11744,7 @@ extern HQL_API IHqlExpression * parseQuery(const char * text, IErrorReceiver * e
 {
     Owned<IHqlScope> scope = createScope();
     HqlDummyLookupContext ctx(errs);
-    Owned<IFileContents> contents = createFileContentsFromText(text, NULL);
+    Owned<IFileContents> contents = createFileContentsFromText(text, NULL, false);
     return parseQuery(scope, contents, ctx, NULL, NULL, true);
 }
 

ecl/hql/hqllex.l

@@ -32,6 +32,7 @@
 #include "hqlgram.h"
 
 #define YY_DECL int HqlLex::doyyFlex(YYSTYPE & returnToken, yyscan_t yyscanner, HqlLex * lexer, bool lookup, const short * activeState)
+#define register
 
 //#define DEBUG_TOKEN 
 
@@ -1456,7 +1457,7 @@ FUNCTIONMACRO|MACRO {
                         Owned<IFileContents> macroContents;
                         if (!isFunctionMacro)
                         {
-                            macroContents.setown(createFileContentsFromText(len, lexer->yyBuffer+startpos, lexer->sourcePath));
+                            macroContents.setown(createFileContentsFromText(len, lexer->yyBuffer+startpos, lexer->sourcePath, lexer->yyParser->inSignedModule));
                         }
                         else
                         {
@@ -1470,7 +1471,7 @@ FUNCTIONMACRO|MACRO {
                             memcpy(macroBuf, prefix, lenPrefix);
                             memcpy(macroBuf + lenPrefix, lexer->yyBuffer+startpos, len);
                             memcpy(macroBuf + lenPrefix + len, suffix, lenSuffix);
-                            macroContents.setown(createFileContentsFromText(lenSuffix + len + lenPrefix, macroBuf, lexer->sourcePath));
+                            macroContents.setown(createFileContentsFromText(lenSuffix + len + lenPrefix, macroBuf, lexer->sourcePath,lexer->yyParser->inSignedModule));
                             free(macroBuf);
                         }
 

ecl/hql/hqlparse.cpp

@@ -317,7 +317,7 @@ void HqlLex::pushText(IFileContents * text, int startLineNo, int startColumn)
 
 void HqlLex::pushText(const char *s, int startLineNo, int startColumn)
 {
-    Owned<IFileContents> macroContents = createFileContentsFromText(s, sourcePath);
+    Owned<IFileContents> macroContents = createFileContentsFromText(s, sourcePath, yyParser->inSignedModule);
     pushText(macroContents, startLineNo, startColumn);
 }
 
@@ -327,7 +327,7 @@ void HqlLex::pushText(const char *s)
 #ifdef TIMING_DEBUG
     MTIME_SECTION(timer, "HqlLex::pushText");
 #endif
-    Owned<IFileContents> macroContents = createFileContentsFromText(s, sourcePath);
+    Owned<IFileContents> macroContents = createFileContentsFromText(s, sourcePath, yyParser->inSignedModule);
     inmacro = new HqlLex(yyParser, macroContents, NULL, NULL);
     inmacro->set_yyLineNo(yyLineNo);
     inmacro->set_yyColumn(yyColumn);
@@ -659,7 +659,7 @@ void HqlLex::processEncrypted()
     decryptEclAttribute(decrypted, encoded64.str());
     decrypted.append(0);    // add a null terminator to the string...
     Owned<ISourcePath> sourcePath = createSourcePath("<encrypted>");
-    Owned<IFileContents> decryptedContents = createFileContentsFromText((const char *)decrypted.toByteArray(), sourcePath);
+    Owned<IFileContents> decryptedContents = createFileContentsFromText((const char *)decrypted.toByteArray(), sourcePath, yyParser->inSignedModule);
     inmacro = new HqlLex(yyParser, decryptedContents, NULL, NULL);
     inmacro->setParentLex(this);
     inmacro->encrypted = true;
@@ -1077,7 +1077,7 @@ void HqlLex::doExport(YYSTYPE & returnToken, bool toXml)
         try
         {
             HqlLookupContext ctx(yyParser->lookupCtx);
-            Owned<IFileContents> exportContents = createFileContentsFromText(curParam.str(), sourcePath);
+            Owned<IFileContents> exportContents = createFileContentsFromText(curParam.str(), sourcePath, yyParser->inSignedModule);
             expr.setown(parseQuery(scope, exportContents, ctx, xmlScope, NULL, true));
 
             if (expr && (expr->getOperator() == no_sizeof))
@@ -1219,8 +1219,8 @@ void HqlLex::doFor(YYSTYPE & returnToken, bool doAll)
 
     forLoop = getSubScopes(returnToken, str(name), doAll);
     if (forFilterText.length())
-        forFilter.setown(createFileContentsFromText(forFilterText, sourcePath));
-    forBody.setown(createFileContentsFromText(forBodyText, sourcePath));
+        forFilter.setown(createFileContentsFromText(forFilterText, sourcePath, yyParser->inSignedModule));
+    forBody.setown(createFileContentsFromText(forBodyText, sourcePath, yyParser->inSignedModule));
 
     loopTimes = 0;
     if (forLoop && forLoop->first()) // more - check filter
@@ -1269,7 +1269,7 @@ void HqlLex::doLoop(YYSTYPE & returnToken)
     ::Release(forLoop);
     forLoop = new CDummyScopeIterator(ensureTopXmlScope());
     forFilter.clear();
-    forBody.setown(createFileContentsFromText(forBodyText, sourcePath));
+    forBody.setown(createFileContentsFromText(forBodyText, sourcePath, yyParser->inSignedModule));
     loopTimes = 0;
     if (forLoop->first()) // more - check filter
         checkNextLoop(returnToken, true,startLine,startCol);
@@ -1510,7 +1510,7 @@ void HqlLex::doIsValid(YYSTYPE & returnToken)
     {
         HqlLookupContext ctx(yyParser->lookupCtx);
         ctx.errs.clear();   //Deliberately ignore any errors
-        Owned<IFileContents> contents = createFileContentsFromText(curParam.str(), sourcePath);
+        Owned<IFileContents> contents = createFileContentsFromText(curParam.str(), sourcePath, yyParser->inSignedModule);
         expr = parseQuery(scope, contents, ctx, xmlScope, NULL, true);
 
         if(expr)
@@ -1788,7 +1788,7 @@ IHqlExpression *HqlLex::parseECL(IFileContents * contents, IXmlScope *xmlScope,
 
 IHqlExpression *HqlLex::parseECL(const char * text, IXmlScope *xmlScope, int startLine, int startCol)
 {
-    Owned<IFileContents> contents = createFileContentsFromText(text, querySourcePath());
+    Owned<IFileContents> contents = createFileContentsFromText(text, querySourcePath(), yyParser->inSignedModule);
     return parseECL(contents, xmlScope, startLine, startCol);
 }
 

ecl/hql/hqlutil.cpp

@@ -8710,7 +8710,7 @@ IHqlExpression * expandMacroDefinition(IHqlExpression * expr, HqlLookupContext &
     //This might be cleaner if it was implemented by parsing the text myModule.myAttribute().
     //It would make implementing default parameters easy.  However it could introduce other problems
     //with implicitly importing myModule.
-    Owned<IFileContents> mappedContents = createFileContentsFromText(macroText.length(), macroText.str(), macroContents->querySourcePath());
+    Owned<IFileContents> mappedContents = createFileContentsFromText(macroText.length(), macroText.str(), macroContents->querySourcePath(), false);
     Owned<IHqlScope> scope = createPrivateScope();
     if (queryLegacyImportSemantics())
         importRootModulesToScope(scope, ctx);

ecl/hqlcpp/hqlcpp.cpp

@@ -1396,7 +1396,7 @@ HqlCppTranslator::HqlCppTranslator(IErrorReceiver * _errors, const char * _soNam
             HqlDummyLookupContext ctx(&errs);
             cppSystemScope = createScope();
             Owned<ISourcePath> sysPath = createSourcePath("<system-definitions>");
-            Owned<IFileContents> systemContents = createFileContentsFromText(systemText.str(), sysPath);
+            Owned<IFileContents> systemContents = createFileContentsFromText(systemText.str(), sysPath, true);
             OwnedHqlExpr query = parseQuery(cppSystemScope, systemContents, ctx, NULL, NULL, false);
             if (errs.errCount())
             {