Merge pull request #682 from ghalliday/issue642

Fix h-642 ensure correct file is compiled

Reviewed-By: Richard Chapman <rchapman@hpccsystems.com>

ecl/eclcc/eclcc.cpp

@@ -199,8 +199,8 @@ protected:
     void generateOutput(EclCompileInstance & instance);
     void instantECL(EclCompileInstance & instance, IWorkUnit *wu, IHqlExpression * query, const char * queryFullName, IErrorReceiver *errs, const char * outputFile);
     void getComplexity(IWorkUnit *wu, IHqlExpression * query, IErrorReceiver *errs);
-    void processSingleQuery(EclCompileInstance & instance, IEclRepository * dataServer, const char * sourcePathname,
-                               const char * queryText,
+    void processSingleQuery(EclCompileInstance & instance, IEclRepository * dataServer,
+                               IFileContents * queryContents,
                                const char * queryAttributePath);
     void processXmlFile(EclCompileInstance & instance, const char *archiveXML);
     void processFile(EclCompileInstance & info);
@@ -217,7 +217,6 @@ protected:
     Owned<IEclRepository> pluginsRepository;
     Owned<IEclRepository> libraryRepository;
     Owned<IEclRepository> includeRepository;
-    Owned<IEclRepository> searchRepository;
     const char * programName;
 
     StringBuffer pluginsPath;
@@ -582,23 +581,18 @@ void EclCC::getComplexity(IWorkUnit *wu, IHqlExpression * query, IErrorReceiver
 
 //=========================================================================================
 
-inline bool endsWith(unsigned lenSrc, const char * src, unsigned lenSuffix, const char * suffix)
-{
-    return (lenSrc >= lenSuffix) && (memcmp(src+lenSrc-lenSuffix, suffix, lenSuffix) == 0);
-}
-
 static bool convertPathToModule(StringBuffer & out, const char * filename)
 {
-    const unsigned len = strlen(filename);
-    unsigned trail;
-    if (endsWith(len, filename, 4, ".ecl"))
-        trail = 4;
-    else if (endsWith(len, filename, 7, ".eclmod"))
-        trail = 7;
+    const char * dot = strrchr(filename, '.');
+    if (dot)
+    {
+        if (!strieq(dot, ".ecl") && !strieq(dot, ".hql") && !strieq(dot, ".eclmod") && !strieq(dot, ".eclattr"))
+            return false;
+    }
     else
         return false;
 
-    const unsigned copyLen = len-trail;
+    const unsigned copyLen = dot-filename;
     if (copyLen == 0)
         return false;
 
@@ -672,8 +666,8 @@ bool EclCC::checkWithinRepository(StringBuffer & attributePath, const char * sou
 }
 
 
-void EclCC::processSingleQuery(EclCompileInstance & instance, IEclRepository * dataServer, const char * sourcePathname,
-                               const char * queryText,
+void EclCC::processSingleQuery(EclCompileInstance & instance, IEclRepository * dataServer,
+                               IFileContents * queryContents,
                                const char * queryAttributePath)
 {
     Owned<IErrorReceiver> wuErrs = new WorkUnitErrorReceiver(instance.wu, "eclcc");
@@ -689,44 +683,14 @@ void EclCC::processSingleQuery(EclCompileInstance & instance, IEclRepository * d
     if (optTargetCompiler != DEFAULT_COMPILER)
         instance.wu->setDebugValue("targetCompiler", compilerTypeText[optTargetCompiler], true);
 
-    StringBuffer attributePath;
-    bool withinRepository = false;
-    if (queryAttributePath)
-    {
-        withinRepository = true;
-        attributePath.append(queryAttributePath);
-    }
-    else if (!instance.fromArchive)
-    {
-        withinRepository = checkWithinRepository(attributePath, sourcePathname);
-
-        //Disable this for the moment when running the regression suite.
-        //It also doesn't work in batch mode for files in nested directories since filename is reduced to basename
-        if (!optBatchMode && !withinRepository)
-        {
-            StringBuffer expandedSourceName;
-            makeAbsolutePath(sourcePathname, expandedSourceName);
-
-            StringBuffer thisDirectory;
-            StringBuffer thisTail;
-            splitFilename(expandedSourceName, &thisDirectory, &thisDirectory, &thisTail, &thisTail);
-
-            Owned<IEclRepository> localRepository =  createNewSourceFileEclRepository(errs, thisDirectory, 0, 0);
-            Owned<IEclRepository> compound = createCompoundRepositoryF(repository.get(), localRepository.get(), NULL);
-            repository.set(compound);
-
-            if (convertPathToModule(attributePath, thisTail))
-                withinRepository = true;
-        }
-    }
-
+    bool withinRepository = (queryAttributePath && *queryAttributePath);
     Owned<IHqlScope> scope;
-    Owned<ISourcePath> sourcePath = createSourcePath(sourcePathname);
     bool syntaxChecking = instance.wu->getDebugValueBool("syntaxCheck", false);
     size32_t prevErrs = errs->errCount();
     unsigned startTime = msTick();
     OwnedHqlExpr qquery;
-    const char * defaultErrorPathname = sourcePathname ? sourcePathname : attributePath.str();
+    const char * sourcePathname = queryContents ? queryContents->querySourcePath()->str() : NULL;
+    const char * defaultErrorPathname = sourcePathname ? sourcePathname : queryAttributePath;
 
     {
         //Minimize the scope of the parse context to reduce lifetime of cached items.
@@ -747,24 +711,22 @@ void EclCC::processSingleQuery(EclCompileInstance & instance, IEclRepository * d
                 if (instance.archive)
                 {
                     instance.archive->setProp("Query", "");
-                    instance.archive->setProp("Query/@attributePath", attributePath);
+                    instance.archive->setProp("Query/@attributePath", queryAttributePath);
                 }
-                qquery.setown(getResolveAttributeFullPath(attributePath, LSFpublic, ctx));
+                qquery.setown(getResolveAttributeFullPath(queryAttributePath, LSFpublic, ctx));
                 if (!qquery && !syntaxChecking && (errs->errCount() == prevErrs))
                 {
                     StringBuffer msg;
-                    msg.append("Could not resolve attribute ").append(attributePath.str());
+                    msg.append("Could not resolve attribute ").append(queryAttributePath);
                     errs->reportError(3, msg.str(), defaultErrorPathname, 0, 0, 0);
                 }
             }
             else
             {
-                Owned<IFileContents> contents = createFileContentsFromText(queryText, sourcePath);
-
                 if (instance.legacyMode)
                     importRootModulesToScope(scope, ctx);
 
-                qquery.setown(parseQuery(scope, contents, ctx, NULL, true));
+                qquery.setown(parseQuery(scope, queryContents, ctx, NULL, true));
             }
 
             gatherWarnings(ctx.errs, qquery);
@@ -880,7 +842,10 @@ void EclCC::processXmlFile(EclCompileInstance & instance, const char *archiveXML
     {
         const char * sourceFilename = archiveTree->queryProp("Query/@originalFilename");
         Owned<IEclRepository> dataServer = createCompoundRepositoryF(pluginsRepository.get(), archiveServer.get(), NULL);
-        processSingleQuery(instance, dataServer, sourceFilename, queryText, queryAttributePath);
+
+        Owned<ISourcePath> sourcePath = createSourcePath(sourceFilename);
+        Owned<IFileContents> contents = createFileContentsFromText(queryText, sourcePath);
+        processSingleQuery(instance, dataServer, contents, queryAttributePath);
     }
     else
     {
@@ -896,9 +861,10 @@ void EclCC::processXmlFile(EclCompileInstance & instance, const char *archiveXML
         fullPath.append(syntaxCheckModule).append('.').append(syntaxCheckAttribute);
 
         //Create a repository with just that attribute, and place it before the archive in the resolution order.
-        Owned<IEclRepository> syntaxCheckRepository = createSingleDefinitionEclRepository(syntaxCheckModule, syntaxCheckAttribute, queryText);
+        Owned<IFileContents> contents = createFileContentsFromText(queryText, NULL);
+        Owned<IEclRepository> syntaxCheckRepository = createSingleDefinitionEclRepository(syntaxCheckModule, syntaxCheckAttribute, contents);
         Owned<IEclRepository> dataServer = createCompoundRepositoryF(pluginsRepository.get(), syntaxCheckRepository.get(), archiveServer.get(), NULL);
-        processSingleQuery(instance, dataServer, NULL, NULL, fullPath.str());
+        processSingleQuery(instance, dataServer, NULL, fullPath.str());
     }
 }
 
@@ -909,13 +875,8 @@ void EclCC::processFile(EclCompileInstance & instance)
 {
     const char * curFilename = instance.inputFile->queryFilename();
     assertex(curFilename);
-    StringBuffer fname;
-    if (optBatchMode)
-        splitFilename(curFilename, NULL, NULL, &fname, &fname);
-    else
-        fname.append(curFilename);
 
-    Owned<ISourcePath> sourcePath = createSourcePath(fname.str());
+    Owned<ISourcePath> sourcePath = createSourcePath(curFilename);
     Owned<IFileContents> queryText = createFileContentsFromFile(curFilename, sourcePath);
     const char * queryTxt = queryText->getText();
     if (optArchive)
@@ -934,7 +895,55 @@ void EclCC::processFile(EclCompileInstance & instance)
         processXmlFile(instance, queryTxt);
     }
     else
-        processSingleQuery(instance, searchRepository, fname.str(), queryTxt, NULL);
+    {
+        StringBuffer attributePath;
+        bool withinRepository = checkWithinRepository(attributePath, curFilename);
+
+        StringBuffer expandedSourceName;
+        makeAbsolutePath(curFilename, expandedSourceName);
+
+        EclRepositoryArray repositories;
+        repositories.append(*LINK(pluginsRepository));
+        repositories.append(*LINK(libraryRepository));
+
+        //Ensure that this source file is used as the definition (in case there are potential clashes)
+        //Note, this will not override standard library files.
+        if (withinRepository)
+        {
+            Owned<IEclSourceCollection> inputFileCollection = createSingleDefinitionEclCollection(attributePath, queryText);
+            repositories.append(*createRepository(inputFileCollection));
+        }
+        else
+        {
+            //Ensure that $ is valid for any file submitted - even if it isn't in the include direcotories
+            //Disable this for the moment when running the regression suite.
+            if (!optBatchMode && !withinRepository)
+            {
+                //Associate the contents of the directory with an internal module called _local_directory_
+                //(If it was root it might override existing root symbols).  $ is the only public way to get at the symbol
+                const char * moduleName = "_local_directory_";
+                _ATOM moduleNameAtom = createAtom(moduleName);
+
+                StringBuffer thisDirectory;
+                StringBuffer thisTail;
+                splitFilename(expandedSourceName, &thisDirectory, &thisDirectory, &thisTail, NULL);
+                attributePath.append(moduleName).append(".").append(thisTail);
+
+                Owned<IEclSourceCollection> inputFileCollection = createSingleDefinitionEclCollection(attributePath, queryText);
+                repositories.append(*createRepository(inputFileCollection));
+
+                Owned<IEclSourceCollection> directory = createFileSystemEclCollection(instance.errs, thisDirectory, 0, 0);
+                Owned<IEclRepository> directoryRepository = createRepository(directory, moduleName);
+                Owned<IEclRepository> nested = createNestedRepository(moduleNameAtom, directoryRepository);
+                repositories.append(*LINK(nested));
+            }
+        }
+
+        repositories.append(*LINK(includeRepository));
+
+        Owned<IEclRepository> searchRepository = createCompoundRepository(repositories);
+        processSingleQuery(instance, searchRepository, queryText, attributePath.str());
+    }
 
     if (instance.reportErrorSummary() && !instance.archive)
         return;
@@ -947,7 +956,7 @@ void EclCC::processFile(EclCompileInstance & instance)
             if (0 == strncmp(p, (const char *)UTF8_BOM,3))
                 p += 3;
             instance.archive->setProp("Query", p );
-            instance.archive->setProp("Query/@originalFilename", fname.str());
+            instance.archive->setProp("Query/@originalFilename", curFilename);
         }
     }
 
@@ -996,7 +1005,8 @@ void EclCC::processReference(EclCompileInstance & instance, const char * queryAt
 
     instance.wu.setown(createLocalWorkUnit());
 
-    processSingleQuery(instance, searchRepository, NULL, NULL, queryAttributePath);
+    Owned<IEclRepository> searchRepository = createCompoundRepositoryF(pluginsRepository.get(), libraryRepository.get(), includeRepository.get(), NULL);
+    processSingleQuery(instance, searchRepository, NULL, queryAttributePath);
 
     if (instance.reportErrorSummary())
         return;
@@ -1015,7 +1025,6 @@ bool EclCC::processFiles()
     pluginsRepository.setown(createNewSourceFileEclRepository(errs, pluginsPath.str(), ESFallowplugins, logVerbose ? PLUGIN_DLL_MODULE : 0));
     libraryRepository.setown(createNewSourceFileEclRepository(errs, eclLibraryPath.str(), 0, 0));
     includeRepository.setown(createNewSourceFileEclRepository(errs, searchPath.str(), 0, 0));
-    searchRepository.setown(createCompoundRepositoryF(pluginsRepository.get(), libraryRepository.get(), includeRepository.get(), NULL));
 
     //Ensure symbols for plugins are initialised - see comment before CHqlMergedScope...
 //    lookupAllRootDefinitions(pluginsRepository);

ecl/hql/hql.hpp

@@ -176,7 +176,6 @@ interface IFileContents;
 interface IEclRepository: public IInterface
 {
     virtual IHqlScope * queryRootScope() = 0;
-    virtual void checkCacheValid() = 0;
 };
 
 //MORE: Make this more private

ecl/hql/hqlcollect.cpp

@@ -797,7 +797,7 @@ extern HQL_API IEclSourceCollection * createArchiveEclCollection(IPropertyTree *
 
 //---------------------------------------------------------------------------------------
 
-extern HQL_API IEclSourceCollection * createSingleDefinitionEclCollection(const char * moduleName, const char * attrName, const char * text)
+IEclSourceCollection * createSingleDefinitionEclCollection(const char * moduleName, const char * attrName, IFileContents * contents)
 {
     //Create an archive with a single module/
     Owned<IPropertyTree> archive = createPTree("Archive");
@@ -805,10 +805,27 @@ extern HQL_API IEclSourceCollection * createSingleDefinitionEclCollection(const
     module->setProp("@name", moduleName);
     IPropertyTree * attr = module->addPropTree("Attribute", createPTree("Attribute"));
     attr->setProp("@name", attrName);
-    attr->setProp("", text);
+    const char * filename = contents->querySourcePath()->str();
+    if (filename)
+        attr->setProp("@sourcePath", filename);
+
+    StringBuffer temp;
+    temp.append(contents->length(), contents->getText());
+    attr->setProp("", temp.str());
     return createArchiveEclCollection(archive);
 }
 
+IEclSourceCollection * createSingleDefinitionEclCollection(const char * attrName, IFileContents * contents)
+{
+    const char * dot = strrchr(attrName, '.');
+    if (dot)
+    {
+        StringAttr module(attrName, dot-attrName);
+        return createSingleDefinitionEclCollection(module, attrName, contents);
+    }
+    return createSingleDefinitionEclCollection("", attrName, contents);
+}
+
 //---------------------------------------------------------------------------------------
 
 static void setKeyAttribute(IPropertyTree * elem)

ecl/hql/hqlcollect.hpp

@@ -80,7 +80,8 @@ enum EclSourceCollectionFlags {
 
 extern HQL_API IEclSourceCollection * createFileSystemEclCollection(IErrorReceiver *errs, const char * path, unsigned flags, unsigned trace);
 extern HQL_API IEclSourceCollection * createArchiveEclCollection(IPropertyTree * tree);
-extern HQL_API IEclSourceCollection * createSingleDefinitionEclCollection(const char * moduleName, const char * attrName, const char * text);
+extern HQL_API IEclSourceCollection * createSingleDefinitionEclCollection(const char * moduleName, const char * attrName, IFileContents * contents);
+extern HQL_API IEclSourceCollection * createSingleDefinitionEclCollection(const char * attrName, IFileContents * contents);
 extern HQL_API IEclSourceCollection * createRemoteXmlEclCollection(IEclUser * user, IXmlEclRepository & repository, const char * snapshot, bool useSandbox);
 
 extern HQL_API IXmlEclRepository * createArchiveXmlEclRepository(IPropertyTree * archive);

ecl/hql/hqlrepository.cpp

@@ -171,7 +171,6 @@ public:
     void addRepository(IEclRepository & _repository);
 
     virtual IHqlScope * queryRootScope() { return rootScope; }
-    virtual void checkCacheValid();
 
 protected:
     IArrayOf<IEclRepository> repositories;
@@ -186,13 +185,6 @@ void CompoundEclRepository::addRepository(IEclRepository & _repository)
 
 //-------------------------------------------------------------------------------------------------------------------
 
-void CompoundEclRepository::checkCacheValid()
-{
-    ForEachItemIn(i, repositories)
-        repositories.item(i).checkCacheValid();
-}
-
-
 extern HQL_API IEclRepository * createCompoundRepositoryF(IEclRepository * repository, ...)
 {
     Owned<CompoundEclRepository> compound = new CompoundEclRepository;
@@ -221,17 +213,57 @@ extern HQL_API IEclRepository * createCompoundRepository(EclRepositoryArray & re
 
 //-------------------------------------------------------------------------------------------------------------------
 
+class HQL_API NestedEclRepository : public CInterface, implements IEclRepository
+{
+public:
+    NestedEclRepository(_ATOM name, IEclRepository * _repository) : repository(_repository)
+    {
+        rootScope.setown(createScope());
+        IHqlExpression * scope = repository->queryRootScope()->queryExpression();
+
+        rootScope->defineSymbol(name, NULL, LINK(scope), true, false, 0, NULL, 0, 0, 0);
+    }
+
+    IMPLEMENT_IINTERFACE;
+
+    virtual IHqlScope * queryRootScope() { return rootScope; }
+
+protected:
+    Linked<IEclRepository> repository;
+    Owned<IHqlScope> rootScope;
+};
+
+//-------------------------------------------------------------------------------------------------------------------
+
+extern HQL_API IEclRepository * createNestedRepository(_ATOM name, IEclRepository * repository)
+{
+    if (!repository)
+        return NULL;
+    return new NestedEclRepository(name, repository);
+}
+
+//-------------------------------------------------------------------------------------------------------------------
+
+static _ATOM queryModuleFromFullName(const char * name)
+{
+    if (!name)
+        return NULL;
+    const char * dot = strrchr(name, '.');
+    if (dot)
+        return createAtom(dot+1);
+    return createAtom(name);
+}
+
 class HQL_API CNewEclRepository : public CInterface, implements IEclRepositoryCallback
 {
 public:
-    CNewEclRepository(IEclSourceCollection * _collection) : collection(_collection)
+    CNewEclRepository(IEclSourceCollection * _collection, const char * rootScopeFullName) : collection(_collection)
     {
-        rootScope.setown(createRemoteScope(NULL, NULL, this, NULL, NULL, true, NULL));
+        rootScope.setown(createRemoteScope(queryModuleFromFullName(rootScopeFullName), rootScopeFullName, this, NULL, NULL, true, NULL));
     }
     IMPLEMENT_IINTERFACE
 
     virtual IHqlScope * queryRootScope() { return rootScope->queryScope(); }
-    virtual void checkCacheValid() { collection->checkCacheValid(); }
     virtual bool loadModule(IHqlRemoteScope *scope, IErrorReceiver *errs, bool forceAll);
     virtual IHqlExpression * loadSymbol(IHqlRemoteScope *scope, IAtom * searchName);
 
@@ -335,9 +367,9 @@ IHqlExpression * CNewEclRepository::createSymbol(IHqlRemoteScope * rScope, IEclS
 }
 
 
-extern HQL_API IEclRepository * createRepository(IEclSourceCollection * source)
+extern HQL_API IEclRepository * createRepository(IEclSourceCollection * source, const char * rootScopeFullName)
 {
-    return new CNewEclRepository(source);
+    return new CNewEclRepository(source, rootScopeFullName);
 }
 
 extern HQL_API IEclRepository * createRepository(EclSourceCollectionArray & sources)
@@ -363,8 +395,8 @@ extern HQL_API IEclRepository * createNewSourceFileEclRepository(IErrorReceiver
     return createRepository(source);
 }
 
-extern HQL_API IEclRepository * createSingleDefinitionEclRepository(const char * moduleName, const char * attrName, const char * text)
+extern HQL_API IEclRepository * createSingleDefinitionEclRepository(const char * moduleName, const char * attrName, IFileContents * contents)
 {
-    Owned<IEclSourceCollection> source = createSingleDefinitionEclCollection(moduleName, attrName, text);
+    Owned<IEclSourceCollection> source = createSingleDefinitionEclCollection(moduleName, attrName, contents);
     return createRepository(source);
 }

ecl/hql/hqlrepository.hpp

@@ -25,12 +25,13 @@
 typedef IArrayOf<IEclRepository> EclRepositoryArray;
 
 extern HQL_API IEclRepository * createNewSourceFileEclRepository(IErrorReceiver *err, const char * pluginPath, unsigned flags, unsigned traceMask);
-extern HQL_API IEclRepository * createSingleDefinitionEclRepository(const char * moduleName, const char * attrName, const char * text);
+extern HQL_API IEclRepository * createSingleDefinitionEclRepository(const char * moduleName, const char * attrName, IFileContents * contents);
 
 extern HQL_API IEclRepository * createCompoundRepositoryF(IEclRepository * repository, ...);
 extern HQL_API IEclRepository * createCompoundRepository(EclRepositoryArray & repositories);
-extern HQL_API IEclRepository * createRepository(IEclSourceCollection * source);
+extern HQL_API IEclRepository * createRepository(IEclSourceCollection * source, const char * rootScopeFullName = NULL);
 extern HQL_API IEclRepository * createRepository(EclSourceCollectionArray & sources);
+extern HQL_API IEclRepository * createNestedRepository(_ATOM name, IEclRepository * root);
 
 extern HQL_API void getRootScopes(HqlScopeArray & rootScopes, IHqlScope * scope, HqlLookupContext & ctx);
 extern HQL_API void getRootScopes(HqlScopeArray & rootScopes, IEclRepository * repository, HqlLookupContext & ctx);