Merge pull request #5324 from ghalliday/issue10649

HPCC-10649 Allow legacy import and when to be configured separately

Reviewed-By: Jamie Noss <james.noss@lexisnexis.com>
Reviewed-By: Richard Chapman <rchapman@hpccsystems.com>

ecl/eclcc/eclcc.cpp

@@ -194,10 +194,11 @@ static bool getHomeFolder(StringBuffer & homepath)
 struct EclCompileInstance
 {
 public:
-    EclCompileInstance(IFile * _inputFile, IErrorReceiver & _errs, FILE * _errout, const char * _outputFilename, bool _legacyMode) :
+    EclCompileInstance(IFile * _inputFile, IErrorReceiver & _errs, FILE * _errout, const char * _outputFilename, bool _legacyImport, bool _legacyWhen) :
       inputFile(_inputFile), errs(&_errs), errout(_errout), outputFilename(_outputFilename)
     {
-        legacyMode = _legacyMode;
+        legacyImport = _legacyImport;
+        legacyWhen = _legacyWhen;
         ignoreUnknownImport = false;
         fromArchive = false;
         stats.parseTime = 0;
@@ -221,7 +222,8 @@ public:
     FILE * errout;
     Owned<IPropertyTree> srcArchive;
     Owned<IPropertyTree> generatedMeta;
-    bool legacyMode;
+    bool legacyImport;
+    bool legacyWhen;
     bool fromArchive;
     bool ignoreUnknownImport;
     struct {
@@ -248,7 +250,8 @@ public:
         optGenerateMeta = false;
         optGenerateDepend = false;
         optIncludeMeta = false;
-        optLegacy = false;
+        optLegacyImport = false;
+        optLegacyWhen = false;
         optShared = false;
         optWorkUnit = false;
         optNoCompile = false;
@@ -371,7 +374,8 @@ protected:
     bool optShared;
     bool optOnlyCompile;
     bool optSaveQueryText;
-    bool optLegacy;
+    bool optLegacyImport;
+    bool optLegacyWhen;
     bool optGenerateHeader;
     bool optShowPaths;
     int argc;
@@ -492,8 +496,13 @@ void EclCC::loadManifestOptions()
             makeAbsolutePath(ecl->queryProp("@filename"), dir.str(), abspath);
             processArgvFilename(inputFiles, abspath.str());
         }
-        if (!optLegacy)
-            optLegacy = ecl->getPropBool("@legacy");
+        if (!optLegacyImport && !optLegacyWhen)
+        {
+            bool optLegacy = ecl->getPropBool("@legacy");
+            optLegacyImport = ecl->getPropBool("@legacyImport", optLegacy);
+            optLegacyWhen = ecl->getPropBool("@legacyWhen", optLegacy);
+        }
+
         if (!optQueryRepositoryReference && ecl->hasProp("@main"))
             optQueryRepositoryReference.set(ecl->queryProp("@main"));
 
@@ -982,7 +991,7 @@ void EclCC::processSingleQuery(EclCompileInstance & instance,
                                const char * queryAttributePath)
 {
 #ifdef TEST_LEGACY_DEPENDENCY_CODE
-    setLegacyEclSemantics(instance.legacyMode);
+    setLegacyEclSemantics(instance.legacyImportMode, instance.legacyWhenMode);
     Owned<IPropertyTree> dependencies = gatherAttributeDependencies(instance.dataServer, "");
     if (dependencies)
         saveXML("depends.xml", dependencies);
@@ -1025,9 +1034,12 @@ void EclCC::processSingleQuery(EclCompileInstance & instance,
             parseCtx.setGatherMeta(options);
         }
 
-        setLegacyEclSemantics(instance.legacyMode);
+        setLegacyEclSemantics(instance.legacyImport, instance.legacyWhen);
         if (instance.archive)
-            instance.archive->setPropBool("@legacyMode", instance.legacyMode);
+        {
+            instance.archive->setPropBool("@legacyImport", instance.legacyImport);
+            instance.archive->setPropBool("@legacyWhen", instance.legacyWhen);
+        }
 
         parseCtx.ignoreUnknownImport = instance.ignoreUnknownImport;
 
@@ -1054,7 +1066,7 @@ void EclCC::processSingleQuery(EclCompileInstance & instance,
             else
             {
                 Owned<IHqlScope> scope = createPrivateScope();
-                if (instance.legacyMode)
+                if (instance.legacyImport)
                     importRootModulesToScope(scope, ctx);
 
                 instance.query.setown(parseQuery(scope, queryContents, ctx, NULL, NULL, true));
@@ -1176,7 +1188,10 @@ void EclCC::processXmlFile(EclCompileInstance & instance, const char *archiveXML
         queryAttributePath = optQueryRepositoryReference;
 
     //The legacy mode (if specified) in the archive takes precedence - it needs to match to compile.
-    instance.legacyMode = archiveTree->getPropBool("@legacyMode", instance.legacyMode);
+    instance.legacyImport = archiveTree->getPropBool("@legacyMode", instance.legacyImport);
+    instance.legacyWhen = archiveTree->getPropBool("@legacyMode", instance.legacyWhen);
+    instance.legacyImport = archiveTree->getPropBool("@legacyImport", instance.legacyImport);
+    instance.legacyWhen = archiveTree->getPropBool("@legacyWhen", instance.legacyWhen);
 
     //Some old archives contained imports, but no definitions of the module.  This option is to allow them to compile.
     //It shouldn't be needed for new archives in non-legacy mode. (But neither should it cause any harm.)
@@ -1319,7 +1334,7 @@ void EclCC::processFile(EclCompileInstance & instance)
         {
             //Ensure that $ is valid for any file submitted - even if it isn't in the include direcotories
             //Disable this for the moment when running the regression suite.
-            if (!optBatchMode && !withinRepository && !inputFromStdIn && !optLegacy)
+            if (!optBatchMode && !withinRepository && !inputFromStdIn && !optLegacyImport)
             {
                 //Associate the contents of the directory with an internal module called _local_directory_
                 //(If it was root it might override existing root symbols).  $ is the only public way to get at the symbol
@@ -1599,7 +1614,7 @@ bool EclCC::processFiles()
     else if (inputFiles.ordinality() == 0)
     {
         assertex(optQueryRepositoryReference);
-        EclCompileInstance info(NULL, *errs, stderr, optOutputFilename, optLegacy);
+        EclCompileInstance info(NULL, *errs, stderr, optOutputFilename, optLegacyImport, optLegacyWhen);
         processReference(info, optQueryRepositoryReference);
         ok = (errs->errCount() == 0);
 
@@ -1607,7 +1622,7 @@ bool EclCC::processFiles()
     }
     else
     {
-        EclCompileInstance info(&inputFiles.item(0), *errs, stderr, optOutputFilename, optLegacy);
+        EclCompileInstance info(&inputFiles.item(0), *errs, stderr, optOutputFilename, optLegacyImport, optLegacyWhen);
         processFile(info);
         ok = (errs->errCount() == 0);
 
@@ -1765,7 +1780,15 @@ bool EclCC::parseCommandLineOptions(int argc, const char* argv[])
         {
             libraryPaths.append(tempArg);
         }
-        else if (iter.matchFlag(optLegacy, "-legacy"))
+        else if (iter.matchFlag(tempBool, "-legacy"))
+        {
+            optLegacyImport = tempBool;
+            optLegacyWhen = tempBool;
+        }
+        else if (iter.matchFlag(optLegacyImport, "-legacyimport"))
+        {
+        }
+        else if (iter.matchFlag(optLegacyWhen, "-legacywhen"))
         {
         }
         else if (iter.matchOption(optLogfile, "--logfile"))
@@ -1987,7 +2010,9 @@ const char * const helpText[] = {
     "    -help, --help Display this message",
     "    -help -v      Display verbose help message",
     "!   -internal     Run internal tests",
-    "!   -legacy       Use legacy import semantics (deprecated)",
+    "!   -legacy       Use legacy import and when semantics (deprecated)",
+    "!   -legacyimport Use legacy import semantics (deprecated)",
+    "!   -legacywhen   Use legacy when/side-effects semantics (deprecated)",
     "    --logfile <file> Write log to specified file",
     "!   --logdetail=n Set the level of detail in the log file",
     "!   --nologfile   Do not write any logfile",
@@ -2080,7 +2105,7 @@ void EclCC::processBatchedFile(IFile & file, bool multiThreaded)
             }
 
             Owned<IErrorReceiver> localErrs = createFileErrorReceiver(logFile);
-            EclCompileInstance info(&file, *localErrs, logFile, outFilename, optLegacy);
+            EclCompileInstance info(&file, *localErrs, logFile, outFilename, optLegacyImport, optLegacyWhen);
             processFile(info);
             //Following only produces output if the system has been compiled with TRANSFORM_STATS defined
             dbglogTransformStats(true);

ecl/hql/hqlexpr.cpp

@@ -15754,14 +15754,20 @@ IHqlExpression * closeAndLink(IHqlExpression * expr)
 }
 
 //MORE: This should probably be handled via the Lookup context instead (which shoudl be renamed parse
-static bool legacyEclMode = false;
-extern HQL_API void setLegacyEclSemantics(bool _value)
+static bool legacyImportMode = false;
+static bool legacyWhenMode = false;
+extern HQL_API void setLegacyEclSemantics(bool _legacyImport, bool _legacyWhen)
 {
-    legacyEclMode = _value;
+    legacyImportMode = _legacyImport;
+    legacyWhenMode = _legacyWhen;
 }
-extern HQL_API bool queryLegacyEclSemantics()
+extern HQL_API bool queryLegacyImportSemantics()
 {
-    return legacyEclMode;
+    return legacyImportMode;
+}
+extern HQL_API bool queryLegacyWhenSemantics()
+{
+    return legacyWhenMode;
 }
 
 

ecl/hql/hqlexpr.hpp

@@ -1861,8 +1861,9 @@ extern HQL_API void ensureSymbolsDefined(IHqlExpression * scope, HqlLookupContex
 extern HQL_API void ensureSymbolsDefined(IHqlScope * scope, HqlLookupContext & ctx);
 extern HQL_API bool getBoolAttribute(IHqlExpression * expr, IAtom * name, bool dft=false);
 
-extern HQL_API void setLegacyEclSemantics(bool _value);
-extern HQL_API bool queryLegacyEclSemantics();
+extern HQL_API void setLegacyEclSemantics(bool _legacyImport, bool _legacyWhen);
+extern HQL_API bool queryLegacyImportSemantics();
+extern HQL_API bool queryLegacyWhenSemantics();
 void exportSymbols(IPropertyTree* data, IHqlScope * scope, HqlLookupContext & ctx);
 
 #endif

ecl/hql/hqlgram.hpp

@@ -831,7 +831,8 @@ protected:
     bool resolveSymbols;
     bool forceResult;
     bool associateWarnings;
-    bool legacyEclSemantics;
+    bool legacyImportSemantics;
+    bool legacyWhenSemantics;
     bool isQuery;
     bool parseConstantText;
     unsigned m_maxErrorsAllowed;

ecl/hql/hqlgram.y

@@ -667,7 +667,7 @@ importItem
                         }
     | importSelectorList AS '*'
                         {
-                            if (queryLegacyEclSemantics())
+                            if (queryLegacyImportSemantics())
                                 parser->reportWarning(ERR_DEPRECATED, $1.pos, "IMPORT <module> AS * is deprecated, use IMPORT * FROM <module>");
                             else
                                 parser->reportError(ERR_DEPRECATED, $1.pos, "IMPORT <module> AS * is deprecated, use IMPORT * FROM <module>");
@@ -6038,10 +6038,7 @@ primexpr1
                         }
     | COUNT             {
                             $$.setExpr(parser->getActiveCounter($1));
-                            if (queryLegacyEclSemantics())
-                                parser->reportWarning(ERR_COUNTER_NOT_COUNT, $1.pos, "Use of COUNT instead of COUNTER is deprecated");
-                            else
-                                parser->reportError(ERR_COUNTER_NOT_COUNT, $1.pos, "Use of COUNT instead of COUNTER is deprecated");
+                            parser->reportError(ERR_COUNTER_NOT_COUNT, $1.pos, "Use of COUNT instead of COUNTER is deprecated");
                         }
     | COUNTER               {
                             $$.setExpr(parser->getActiveCounter($1));

ecl/hql/hqlgram2.cpp

@@ -287,7 +287,7 @@ HqlGram::HqlGram(IHqlScope * _globalScope, IHqlScope * _containerScope, IFileCon
     forceResult = false;
     lexObject = new HqlLex(this, _text, xmlScope, NULL);
 
-    if(lookupCtx.queryRepository() && loadImplicit && legacyEclSemantics)
+    if(lookupCtx.queryRepository() && loadImplicit && legacyImportSemantics)
     {
         HqlScopeArray scopes;
         getImplicitScopes(scopes, lookupCtx.queryRepository(), _containerScope, lookupCtx);
@@ -356,7 +356,8 @@ void HqlGram::init(IHqlScope * _globalScope, IHqlScope * _containerScope)
 {
     minimumScopeIndex = 0;
     isQuery = false;
-    legacyEclSemantics = queryLegacyEclSemantics();
+    legacyImportSemantics = queryLegacyImportSemantics();
+    legacyWhenSemantics = queryLegacyWhenSemantics();
     current_id = NULL;
     lexObject = NULL;
     expectedAttribute = NULL;
@@ -374,7 +375,7 @@ void HqlGram::init(IHqlScope * _globalScope, IHqlScope * _containerScope)
     globalScope = _globalScope;
     parseScope.setown(createPrivateScope(_containerScope));
     transformScope = NULL;
-    if (globalScope->queryName() && legacyEclSemantics)
+    if (globalScope->queryName() && legacyImportSemantics)
         parseScope->defineSymbol(globalScope->queryId(), NULL, LINK(queryExpression(globalScope)), false, false, ob_import);
 
     boolType = makeBoolType();
@@ -3382,7 +3383,7 @@ IHqlExpression *HqlGram::lookupSymbol(IIdAtom * searchName, const attribute& err
             return recordLookupInTemplateContext(searchName, ret, templateScope);
 
         // finally comes the local scope
-        if (legacyEclSemantics && searchName->lower()==globalScope->queryName())
+        if (legacyImportSemantics && searchName->lower()==globalScope->queryName())
             return LINK(recordLookupInTemplateContext(searchName, queryExpression(globalScope), templateScope));
 
         ForEachItemIn(idx2, defaultScopes)
@@ -8580,7 +8581,7 @@ void HqlGram::checkNotAlreadyDefined(IIdAtom * name, IHqlScope * scope, const at
     OwnedHqlExpr expr = scope->lookupSymbol(name, LSFsharedOK|LSFignoreBase, lookupCtx);
     if (expr)
     {
-        if (legacyEclSemantics && isImport(expr))
+        if (legacyImportSemantics && isImport(expr))
             reportWarning(ERR_ID_REDEFINE, idattr.pos, "Identifier '%s' hides previous import", name->str());
         else
             reportError(ERR_ID_REDEFINE, idattr, "Identifier '%s' is already defined", name->str());
@@ -8868,7 +8869,7 @@ IHqlExpression * HqlGram::associateSideEffects(IHqlExpression * expr, const ECLl
 {
     if (sideEffectsPending())
     {
-        if (legacyEclSemantics)
+        if (legacyWhenSemantics)
         {
             if (okToAddSideEffects(expr))
                 return addSideEffects(expr);
@@ -8933,7 +8934,7 @@ void HqlGram::doDefineSymbol(DefineIdSt * defineid, IHqlExpression * _expr, IHql
             if (isQuery && !insideNestedScope())
             {
                 //If this is a global query, and not inside a nested attribute, then keep any actions on the global list of results
-                if (!legacyEclSemantics)
+                if (!legacyWhenSemantics)
                 {
                     //Should we give a warning here?? export/shared would not be legal if this was within the repository
                 }
@@ -11568,7 +11569,7 @@ IHqlExpression *HqlGram::doParse()
     {
         if (queryExpression(containerScope)->getOperator() == no_forwardscope)
             enterScope(containerScope, true);
-        if (legacyEclSemantics)
+        if (legacyImportSemantics)
             enterScope(globalScope, true);
 
         //If expecting a particular attribute, add symbols to a private scope, and then copy result across

ecl/hql/hqllex.l

@@ -977,7 +977,7 @@ __ECL_VERSION_SUBMINOR__ {
                     }
 __ECL_LEGACY_MODE__ {
                         setupdatepos;
-                        returnToken.setExpr(createConstant(queryLegacyEclSemantics()));
+                        returnToken.setExpr(createConstant(queryLegacyImportSemantics()));
                         return (BOOL_CONST);
                     }
 __OS__              { 

ecl/hql/hqlutil.cpp

@@ -8231,7 +8231,7 @@ IHqlExpression * expandMacroDefinition(IHqlExpression * expr, HqlLookupContext &
     //with implicitly importing myModule.
     Owned<IFileContents> mappedContents = createFileContentsFromText(macroText.length(), macroText.str(), macroContents->querySourcePath());
     Owned<IHqlScope> scope = createPrivateScope();
-    if (queryLegacyEclSemantics())
+    if (queryLegacyImportSemantics())
         importRootModulesToScope(scope, ctx);
     return parseQuery(scope, mappedContents, ctx, NULL, macroParms, true);
 }

ecl/hqlcpp/hqlcpp.cpp

@@ -1699,7 +1699,7 @@ void HqlCppTranslator::cacheOptions()
         DebugOption(options.precalculateFieldOffsets,"precalculateFieldOffsets", false),
         DebugOption(options.generateStaticInlineTables,"generateStaticInlineTables", true),
         DebugOption(options.staticRowsUseStringInitializer,"staticRowsUseStringInitializer", true),
-        DebugOption(options.convertWhenExecutedToCompound,"convertWhenExecutedToCompound", queryLegacyEclSemantics()),
+        DebugOption(options.convertWhenExecutedToCompound,"convertWhenExecutedToCompound", queryLegacyWhenSemantics()),
         DebugOption(options.standAloneExe,"standAloneExe", false),
         DebugOption(options.enableCompoundCsvRead,"enableCompoundCsvRead", true),
         // The following works 99% of the time, but disabled due to potential problems with the ambiguity of LEFT

ecl/regress/legacy2b.eclxml

@@ -0,0 +1,29 @@
+<Archive legacyImport="true">
+<!--
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+-->
+ <Module name="common">
+  <Attribute name="m1">
+export m1 := m2+10;
+  </Attribute>
+  <Attribute name="m2">
+export m2 := 20;
+  </Attribute>
+ </Module>
+ <Query>
+ output(common.m1);
+ </Query>
+</Archive>

ecl/regress/legacy2c.eclxml

@@ -0,0 +1,31 @@
+<Archive legacyWhen="true">
+<!--
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+-->
+ <Module name="common">
+  <Attribute name="m1">
+output('hello');
+export m1 := $.m2+10;
+  </Attribute>
+  <Attribute name="m2">
+export m2 := 20;
+  </Attribute>
+ </Module>
+ <Query>
+     import common;
+ output(common.m1);
+ </Query>
+</Archive>