Merge pull request #10810 from ghalliday/issue18875

HPCC-18875 Add support for __TARGET_PLATFORM__

Reviewed By: Shamser Ahmed <shamser.ahmed@lexisnexis.co.uk>
Reviewed-By: Richard Chapman <rchapman@hpccsystems.com>

ecl/eclcc/eclcc.cpp

@@ -175,11 +175,12 @@ static bool getHomeFolder(StringBuffer & homepath)
     return true;
 }
 
-struct EclCompileInstance
+class EclCC;
+struct EclCompileInstance : public CInterfaceOf<ICodegenContextCallback>
 {
 public:
-    EclCompileInstance(IFile * _inputFile, IErrorReceiver & _errorProcessor, FILE * _errout, const char * _outputFilename, bool _legacyImport, bool _legacyWhen, bool _ignoreSignatures, bool _optXml) :
-      inputFile(_inputFile), errorProcessor(&_errorProcessor), errout(_errout), outputFilename(_outputFilename), legacyImport(_legacyImport), legacyWhen(_legacyWhen), ignoreSignatures(_ignoreSignatures), optXml(_optXml)
+    EclCompileInstance(EclCC & _eclcc, IFile * _inputFile, IErrorReceiver & _errorProcessor, FILE * _errout, const char * _outputFilename, bool _legacyImport, bool _legacyWhen, bool _ignoreSignatures, bool _optXml) :
+      eclcc(_eclcc), inputFile(_inputFile), errorProcessor(&_errorProcessor), errout(_errout), outputFilename(_outputFilename), legacyImport(_legacyImport), legacyWhen(_legacyWhen), ignoreSignatures(_ignoreSignatures), optXml(_optXml)
 {
         stats.parseTime = 0;
         stats.generateTime = 0;
@@ -192,8 +193,17 @@ public:
     bool reportErrorSummary();
     inline IErrorReceiver & queryErrorProcessor() { return *errorProcessor; }
 
+// interface ICodegenContextCallback
+    virtual void noteCluster(const char *clusterName) override;
+    virtual void pushCluster(const char *clusterName) override;
+    virtual void popCluster() override;
+    virtual bool allowAccess(const char * category, bool isSigned) override;
+    virtual IHqlExpression *lookupDFSlayout(const char *filename, IErrorReceiver &errs, const ECLlocation &location, bool isOpt) const override;
+    virtual unsigned lookupClusterSize() const override;
+    virtual void getTargetPlatform(StringBuffer & result) override;
 
 public:
+    EclCC & eclcc;
     Linked<IFile> inputFile;
     Linked<IPropertyTree> archive;
     Linked<IWorkUnit> wu;
@@ -223,7 +233,7 @@ protected:
     Linked<IErrorReceiver> errorProcessor;
 };
 
-class EclCC : public CInterfaceOf<ICodegenContextCallback>
+class EclCC
 {
 public:
     EclCC(int _argc, const char **_argv)
@@ -257,12 +267,12 @@ public:
 
     // interface ICodegenContextCallback
 
-    virtual void noteCluster(const char *clusterName) override;
-    virtual void pushCluster(const char *clusterName) override;
-    virtual void popCluster() override;
-    virtual bool allowAccess(const char * category, bool isSigned) override;
-    virtual IHqlExpression *lookupDFSlayout(const char *filename, IErrorReceiver &errs, const ECLlocation &location, bool isOpt) const override;
-    virtual unsigned lookupClusterSize() const override;
+    void pushCluster(const char *clusterName);
+    void popCluster();
+    bool allowAccess(const char * category, bool isSigned);
+    IHqlExpression *lookupDFSlayout(const char *filename, IErrorReceiver &errs, const ECLlocation &location, bool isOpt) const;
+    unsigned lookupClusterSize() const;
+    void getTargetPlatform(StringBuffer & result);
 
 protected:
     bool checkDaliConnected() const;
@@ -775,7 +785,7 @@ void EclCC::instantECL(EclCompileInstance & instance, IWorkUnit *wu, const char
             bool optSaveCpp = optSaveTemps || optNoCompile || wu->getDebugValueBool("saveCppTempFiles", false) || wu->getDebugValueBool("saveCpp", false);
             //New scope - testing things are linked correctly
             {
-                Owned<IHqlExprDllGenerator> generator = createDllGenerator(&errorProcessor, processName.str(), NULL, wu, templateDir, optTargetClusterType, this, false, false);
+                Owned<IHqlExprDllGenerator> generator = createDllGenerator(&errorProcessor, processName.str(), NULL, wu, templateDir, optTargetClusterType, &instance, false, false);
 
                 setWorkunitHash(wu, instance.query);
                 if (!optShared)
@@ -1135,7 +1145,7 @@ void EclCC::processSingleQuery(EclCompileInstance & instance,
 
     {
         //Minimize the scope of the parse context to reduce lifetime of cached items.
-        HqlParseContext parseCtx(instance.dataServer, this, instance.archive);
+        HqlParseContext parseCtx(instance.dataServer, &instance, instance.archive);
         if (optFastSyntax)
             parseCtx.setFastSyntax();
         unsigned maxErrorsDebugOption = instance.wu->getDebugValueInt("maxErrors", 0);
@@ -1924,7 +1934,7 @@ bool EclCC::processFiles()
     else if (inputFiles.ordinality() == 0)
     {
         assertex(optQueryRepositoryReference);
-        EclCompileInstance info(NULL, *errs, stderr, optOutputFilename, optLegacyImport, optLegacyWhen, optIgnoreSignatures, optXml);
+        EclCompileInstance info(*this, NULL, *errs, stderr, optOutputFilename, optLegacyImport, optLegacyWhen, optIgnoreSignatures, optXml);
         processReference(info, optQueryRepositoryReference);
         ok = (errs->errCount() == 0);
 
@@ -1932,7 +1942,7 @@ bool EclCC::processFiles()
     }
     else
     {
-        EclCompileInstance info(&inputFiles.item(0), *errs, stderr, optOutputFilename, optLegacyImport, optLegacyWhen, optIgnoreSignatures, optXml);
+        EclCompileInstance info(*this, &inputFiles.item(0), *errs, stderr, optOutputFilename, optLegacyImport, optLegacyWhen, optIgnoreSignatures, optXml);
         processFile(info);
         ok = (errs->errCount() == 0);
 
@@ -2014,10 +2024,44 @@ bool EclCompileInstance::reportErrorSummary()
     return errorProcessor->errCount() != 0;
 }
 
-//=========================================================================================
+void EclCompileInstance::noteCluster(const char *clusterName)
+{
+}
+
+void EclCompileInstance::pushCluster(const char *clusterName)
+{
+    eclcc.pushCluster(clusterName);
+}
 
-void EclCC::noteCluster(const char *clusterName)
+void EclCompileInstance::popCluster()
 {
+    eclcc.popCluster();
+}
+
+unsigned EclCompileInstance::lookupClusterSize() const
+{
+    return eclcc.lookupClusterSize();
+}
+
+bool EclCompileInstance::allowAccess(const char * category, bool isSigned)
+{
+    return eclcc.allowAccess(category, isSigned);
+}
+
+IHqlExpression * EclCompileInstance::lookupDFSlayout(const char *filename, IErrorReceiver &errs, const ECLlocation &location, bool isOpt) const
+{
+    return eclcc.lookupDFSlayout(filename, errs, location, isOpt);
+}
+
+void EclCompileInstance::getTargetPlatform(StringBuffer & result)
+{
+    SCMStringBuffer targetText;
+    wu->getDebugValue("targetClusterType", targetText);
+    ClusterType clusterType = getClusterType(targetText.s.str());
+    if (clusterType != NoCluster)
+        result.append(clusterTypeString(clusterType, true));
+    else
+        return eclcc.getTargetPlatform(result);
 }
 
 void EclCC::pushCluster(const char *clusterName)
@@ -2229,6 +2273,11 @@ bool EclCC::allowAccess(const char * category, bool isSigned)
     return defaultAllowed[isSigned];
 }
 
+void EclCC::getTargetPlatform(StringBuffer & result)
+{
+    result.append(clusterTypeString(optTargetClusterType, true));
+}
+
 //=========================================================================================
 int EclCC::parseCommandLineOptions(int argc, const char* argv[])
 {
@@ -2679,7 +2728,7 @@ void EclCC::processBatchedFile(IFile & file, bool multiThreaded)
             }
 
             Owned<IErrorReceiver> localErrs = createFileErrorReceiver(logFile);
-            EclCompileInstance info(&file, *localErrs, logFile, outFilename, optLegacyImport, optLegacyWhen, optIgnoreSignatures, optXml);
+            EclCompileInstance info(*this, &file, *localErrs, logFile, outFilename, optLegacyImport, optLegacyWhen, optIgnoreSignatures, optXml);
             info.metaOutputFilename.set(metaFilename);
             processFile(info);
             if (info.wu &&

ecl/hql/hql.hpp

@@ -211,6 +211,10 @@ interface ICodegenContextCallback : public IInterface
      *
      */
     virtual unsigned lookupClusterSize() const = 0;
+    /*
+     * Which platform was this query originally targeted to?
+     */
+    virtual void getTargetPlatform(StringBuffer & result) = 0;
 };
 
 

ecl/hql/hqlgram.hpp

@@ -528,6 +528,7 @@ public:
     void ensureType(attribute &atr, ITypeInfo * type);
     void inheritRecordMaxLength(IHqlExpression * dataset, SharedHqlExpr & record);
 
+    IHqlExpression * getTargetPlatformExpr();
     void normalizeExpression(attribute & expr);
     void normalizeExpression(attribute & expr, type_t expectedType, bool isConstant);
 

ecl/hql/hqlgram.y

@@ -439,6 +439,7 @@ static void eclsyntaxerror(HqlGram * parser, const char * s, short yystate, int
   TABLE
   TAN
   TANH
+  __TARGET_PLATFORM__
   TERMINATOR
   THEN
   THISNODE
@@ -6942,6 +6943,10 @@ primexpr1
                             OwnedHqlExpr option = createConstant("targetClusterType");
                             $$.setExpr(createValue(no_debug_option_value, makeStringType(UNKNOWN_LENGTH, NULL), option.getClear()), $1);
                         }
+    | __TARGET_PLATFORM__
+                        {
+                            $$.setExpr(parser->getTargetPlatformExpr(), $1);
+                        }
     ;
 
 optCount

ecl/hql/hqlgram2.cpp

@@ -4525,6 +4525,14 @@ ITypeInfo * HqlGram::queryElementType(const attribute & errpos, IHqlExpression *
 }
 
 
+IHqlExpression * HqlGram::getTargetPlatformExpr()
+{
+    StringBuffer platform;
+    lookupCtx.queryParseContext().codegenCtx->getTargetPlatform(platform);
+    return createConstant(platform);
+}
+
+
 void HqlGram::setDefaultString(attribute &a)
 {
     a.release();
@@ -11142,6 +11150,7 @@ static void getTokenText(StringBuffer & msg, int token)
     case TABLE: msg.append("TABLE"); break;
     case TAN: msg.append("TAN"); break;
     case TANH: msg.append("TANH"); break;
+    case __TARGET_PLATFORM__: msg.append("__TARGET_PLATFORM__"); break;
     case TERMINATOR: msg.append("TERMINATOR"); break;
     case THEN: msg.append("THEN"); break;
     case THISNODE: msg.append("THISNODE"); break;

ecl/hql/hqllex.l

@@ -1041,6 +1041,7 @@ __STAND_ALONE__     {
                         setupdatepos; 
                         return (__STAND_ALONE__);
                     }
+__TARGET_PLATFORM__ { RETURNSYM(__TARGET_PLATFORM__); }
                         
 {percent}{alphanumcolon}*{percent} {
                         setupdatepos;

ecl/hql/reservedwords.cpp

@@ -62,6 +62,7 @@ static const char * eclReserved2[] = { //HPCC and OS environment settings
     "__platform__",
     "__set_debug_option__",
     "__stand_alone__",
+    "__target_platform__",
 
     "clustersize",
     "getenv",

ecl/hqlcpp/hqlecl.cpp

@@ -50,6 +50,8 @@
 
 class NullContextCallback : implements ICodegenContextCallback, public CInterface
 {
+public:
+    NullContextCallback(IWorkUnit * _wu) : workunit(_wu) {}
     IMPLEMENT_IINTERFACE
 
     virtual void noteCluster(const char *clusterName) override {}
@@ -58,6 +60,12 @@ class NullContextCallback : implements ICodegenContextCallback, public CInterfac
     virtual bool allowAccess(const char * category, bool isSigned) override { return true; }
     virtual IHqlExpression *lookupDFSlayout(const char *filename, IErrorReceiver &errs, const ECLlocation &location, bool isOpt) const override { return nullptr; }
     virtual unsigned lookupClusterSize() const override { return 0; }
+    virtual void getTargetPlatform(StringBuffer & result) override
+    {
+        workunit->getDebugValue("targetClusterType", StringBufferAdaptor(result));
+    }
+protected:
+    Linked<IWorkUnit> workunit;
 };
 
 class HqlDllGenerator : implements IHqlExprDllGenerator, implements IAbortRequestCallback, public CInterface
@@ -67,7 +75,7 @@ public:
         errs(_errs), wuname(_wuname), targetDir(_targetdir), wu(_wu), template_dir(_template_dir), targetClusterType(_targetClusterType), ctxCallback(_ctxCallback), checkForLocalFileUploads(_checkForLocalFileUploads), okToAbort(_okToAbort)
     {
         if (!ctxCallback)
-            ctxCallback.setown(new NullContextCallback);
+            ctxCallback.setown(new NullContextCallback(_wu));
         noOutput = true;
         defaultMaxCompileThreads = 1;
         generateTarget = EclGenerateNone;

ecl/regress/platform1.eclxml

@@ -0,0 +1,21 @@
+<Archive>
+    <!--
+
+    HPCC SYSTEMS software Copyright (C) 2018 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+-->
+<Query>
+__TARGET_PLATFORM__;
+</Query>
+</Archive>

ecl/regress/platform2.eclxml

@@ -0,0 +1,22 @@
+<Archive>
+    <!--
+
+    HPCC SYSTEMS software Copyright (C) 2018 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+-->
+<Option name="targetClusterType" value="roxie"/>
+<Query>
+__TARGET_PLATFORM__ = 'roxie';
+</Query>
+</Archive>

ecl/regress/platform3.eclxml

@@ -0,0 +1,22 @@
+<Archive>
+    <!--
+
+    HPCC SYSTEMS software Copyright (C) 2018 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+-->
+<Option name="targetClusterType" value="hthor"/>
+<Query>
+__TARGET_PLATFORM__ = 'hthor';
+</Query>
+</Archive>

ecl/regress/platform4.eclxml

@@ -0,0 +1,23 @@
+<Archive>
+    <!--
+
+    HPCC SYSTEMS software Copyright (C) 2018 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+-->
+<Option name="targetClusterType" value="thorlcr"/>
+<Query>
+__TARGET_PLATFORM__;
+__TARGET_PLATFORM__ = 'thorlcr' OR __TARGET_PLATFORM__ = 'thor';
+</Query>
+</Archive>

ecl/regress/platform5.eclxml

@@ -0,0 +1,23 @@
+<Archive>
+    <!--
+
+    HPCC SYSTEMS software Copyright (C) 2018 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+-->
+<Option name="targetClusterType" value="thor"/>
+<Query>
+__TARGET_PLATFORM__;
+__TARGET_PLATFORM__ = 'thorlcr' OR __TARGET_PLATFORM__ = 'thor';
+</Query>
+</Archive>

testing/regress/ecl/key/targetplatform.xml

@@ -0,0 +1,3 @@
+<Dataset name='Result 1'>
+ <Row><Result_1>true</Result_1></Row>
+</Dataset>

testing/regress/ecl/targetplatform.ecl

@@ -0,0 +1,24 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2018 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#option ('pickBestEngine', false);
+//Ensure that Target platform symbol can be constant folded at compile time
+#if (__TARGET_PLATFORM__ = 'hadoop')
+'Really??';
+#end
+//Check both symbols are consistently implemented
+output(__TARGET_PLATFORM__ = __PLATFORM__);