Merge pull request #1442 from ghalliday/linkoption

Fix gh-1426 Implement -Wl, and related changes

Reviewed-By: Richard Chapman <rchapman@hpccsystems.com>
Reviewed-By: Renato Golin <rengolin@hpccsystems.com>

ecl/eclcc/eclcc.cpp

@@ -244,6 +244,7 @@ protected:
     IFileArray inputFiles;
     StringArray debugOptions;
     StringArray compileOptions;
+    StringArray linkOptions;
     StringArray libraryPaths;
 
     ClusterType optTargetClusterType;
@@ -438,6 +439,9 @@ ICppCompiler * EclCC::createCompiler(const char * coreName)
     ForEachItemIn(iComp, compileOptions)
         compiler->addCompileOption(compileOptions.item(iComp));
 
+    ForEachItemIn(iLink, linkOptions)
+        compiler->addLinkOption(linkOptions.item(iLink));
+
     ForEachItemIn(iLib, libraryPaths)
         compiler->addLibraryPath(libraryPaths.item(iLib));
     
@@ -1357,9 +1361,14 @@ bool EclCC::parseCommandLineOptions(int argc, const char* argv[])
         {
             expandCommaList(compileOptions, arg+4);
         }
-        else if (startsWith(arg, "-Wl,") || startsWith(arg, "-Wp,") || startsWith(arg, "-Wa,"))
+        else if (startsWith(arg, "-Wl,"))
+        {
+            //Pass these straight through to the linker - with -Wl, prefix removed
+            linkOptions.append(arg+4);
+        }
+        else if (startsWith(arg, "-Wp,") || startsWith(arg, "-Wa,"))
         {
-            //Pass these straigh through to the gcc compiler
+            //Pass these straight through to the gcc compiler
             compileOptions.append(arg);
         }
         else if (iter.matchFlag(optWorkUnit, "-wu"))
@@ -1417,6 +1426,7 @@ void EclCC::usage()
            "                  executable, or stdout)\n"
            "    -P <path>     Specify the path of the output files\n"
            "    -Wc,xx        Supply option for the c++ compiler\n"
+           "    -Wl,xx        Supply option for the linker\n"
            "    -save-cpps    Do not delete generated c++ files (implied if -g)\n"
            "    -save-temps   Do not delete intermediate files\n"
            "    -manifest     Specify path to manifest file listing resources to add\n"

ecl/hql/hqlexpr.cpp

@@ -3416,8 +3416,17 @@ void CHqlExpression::updateFlagsAfterOperands()
     case no_call:
         {
             IHqlExpression * funcdef = queryBody()->queryFunctionDefinition();
-            if (funcdef->getOperator() == no_funcdef && funcdef->queryChild(0)->getOperator() == no_outofline)
+            IHqlExpression * body = funcdef->queryChild(0);
+            if ((funcdef->getOperator() == no_funcdef) && (body->getOperator() == no_outofline))
+            {
                 infoFlags2 |= HEF2containsCall;
+                IHqlExpression * bodycode = body->queryChild(0);
+                if (bodycode->getOperator() == no_cppbody)
+                {
+                    if (bodycode->queryProperty(actionAtom))
+                        infoFlags |= HEFvolatile;
+                }
+            }
             else
                 infoFlags2 |= (HEF2containsCall|HEF2containsDelayedCall);
             break;
@@ -3429,6 +3438,12 @@ void CHqlExpression::updateFlagsAfterOperands()
                 infoFlags |= HEFvolatile;
             break;
         }
+    case no_cppbody:
+        {
+            if (queryProperty(actionAtom))
+                infoFlags |= HEFvolatile;
+            break;
+        }
     }
 
 #ifdef VERIFY_EXPR_INTEGRITY

ecl/hql/hqlgram2.cpp

@@ -838,7 +838,8 @@ IHqlExpression * HqlGram::convertToOutOfLineFunction(const ECLlocation & errpos,
 IHqlExpression * HqlGram::processCppBody(const attribute & errpos, IHqlExpression * cpp)
 {
     HqlExprArray args;
-    args.append(*LINK(cpp));
+    cpp->unwindList(args, no_comma);
+
     Linked<ITypeInfo> type = current_type;
     if (!type)
         type.setown(makeVoidType());
@@ -847,7 +848,7 @@ IHqlExpression * HqlGram::processCppBody(const attribute & errpos, IHqlExpressio
     OwnedHqlExpr result;
     if (record)
     {
-        args.append(*LINK(record));
+        args.add(*LINK(record),1);
         if (hasLinkCountedModifier(type))
             args.append(*getLinkCountedAttr());
         if (hasStreamedModifier(type))

ecl/hql/hqllex.l

@@ -1478,7 +1478,9 @@ FUNCTIONMACRO|MACRO {
                         lexer->yyPosition -= delta;
                         lexer->yyColumn -= delta;
                         OwnedHqlExpr cppText = createConstant(createStringValue(lexer->yyBuffer+startpos, len));
-                        returnToken.setExpr(createLocationAnnotation(cppText.getClear(), returnToken.pos));
+                        OwnedHqlExpr annotated = createLocationAnnotation(cppText.getClear(), returnToken.pos);
+                        OwnedHqlExpr options = extractCppBodyAttrs(len, lexer->yyBuffer+startpos);
+                        returnToken.setExpr(createComma(annotated.getClear(), options.getClear()));
                         return CPPBODY;
                     }
 <CPP>[^\n]+         { updatepos1; }

ecl/hql/hqlutil.cpp

@@ -4768,13 +4768,53 @@ bool isConstantDataset(IHqlExpression * expr)
 
 inline bool iseol(char c) { return c == '\r' || c == '\n'; }
 
-#define MATCHOPTION(len, text) (((end - start) == len) && (memicmp(buffer+start, text, len) == 0))
+static unsigned skipSpace(unsigned start, unsigned len, const char * buffer)
+{
+    while (start < len && isspace((byte)buffer[start]))
+        start++;
+    return start;
+}
+
+static unsigned trimSpace(unsigned len, const char * buffer)
+{
+    while (len && isspace((byte)buffer[len-1]))
+        len--;
+    return len;
+}
 
-IHqlExpression * extractCppBodyAttrs(unsigned len, const char * buffer)
+static void stripQuotes(unsigned & start, unsigned & end, const char * buffer)
+{
+    if (end - start >= 2)
+    {
+        if (buffer[start] == '\'' && buffer[end-1] == '\'')
+        {
+            start++;
+            end--;
+        }
+    }
+}
+
+static bool matchOption(unsigned & cur, unsigned max, const char * buffer, unsigned lenMatch, const char * match)
+{
+    if (cur + lenMatch > max)
+        return false;
+    if (memicmp(buffer+cur, match, lenMatch) != 0)
+        return false;
+    if (cur + lenMatch < max)
+    {
+        if (isalnum(buffer[cur+lenMatch]))
+            return false;
+    }
+    cur = skipSpace(cur+lenMatch, max, buffer);
+    return true;
+}
+
+
+IHqlExpression * extractCppBodyAttrs(unsigned lenBuffer, const char * buffer)
 {
     OwnedHqlExpr attrs;
     unsigned prev = '\n';
-    for (unsigned i=0; i < len; i++)
+    for (unsigned i=0; i < lenBuffer; i++)
     {
         char next = buffer[i];
         switch (next)
@@ -4785,20 +4825,32 @@ IHqlExpression * extractCppBodyAttrs(unsigned len, const char * buffer)
         case '#':
             if (prev == '\n')
             {
-                if ((i + 1 + 6 < len) && memicmp(buffer+i+1, "option", 6) == 0)
+                if ((i + 1 + 6 < lenBuffer) && memicmp(buffer+i+1, "option", 6) == 0)
                 {
-                    unsigned start = i+1+6;
-                    while (start < len && isspace((byte)buffer[start]))
-                        start++;
+                    unsigned start = skipSpace(i+1+6, lenBuffer, buffer);
                     unsigned end = start;
-                    while (end < len && !iseol((byte)buffer[end]))
+                    while (end < lenBuffer && !iseol((byte)buffer[end]))
                         end++;
-                    if (MATCHOPTION(4, "pure"))
+                    end = trimSpace(end, buffer);
+                    if (matchOption(start, lenBuffer, buffer, 4, "pure"))
                         attrs.setown(createComma(attrs.getClear(), createAttribute(pureAtom)));
-                    else if (MATCHOPTION(4, "once"))
+                    else if (matchOption(start, lenBuffer, buffer, 4, "once"))
                         attrs.setown(createComma(attrs.getClear(), createAttribute(onceAtom)));
-                    else if (MATCHOPTION(6, "action"))
+                    else if (matchOption(start, lenBuffer, buffer, 6, "action"))
                         attrs.setown(createComma(attrs.getClear(), createAttribute(actionAtom)));
+                    else if (matchOption(start, lenBuffer, buffer, 7, "library"))
+                    {
+                        stripQuotes(start, end, buffer);
+                        Owned<IValue> restOfLine = createUtf8Value(end-start, buffer+start, makeUtf8Type(UNKNOWN_LENGTH, NULL));
+                        OwnedHqlExpr arg = createConstant(restOfLine.getClear());
+                        attrs.setown(createComma(attrs.getClear(), createAttribute(libraryAtom, arg.getClear())));
+                    }
+                    else if (matchOption(start, lenBuffer, buffer, 4, "link"))
+                    {
+                        Owned<IValue> restOfLine = createUtf8Value(end-start, buffer+start, makeUtf8Type(UNKNOWN_LENGTH, NULL));
+                        OwnedHqlExpr arg = createConstant(restOfLine.getClear());
+                        attrs.setown(createComma(attrs.getClear(), createAttribute(linkAtom, arg.getClear())));
+                    }
                 }
             }
             //fallthrough

ecl/hqlcpp/hqlcpp.cpp

@@ -7115,13 +7115,38 @@ void HqlCppTranslator::doBuildAssignToXml(BuildCtx & ctx, const CHqlBoundTarget
 
 //---------------------------------------------------------------------------
 
+void HqlCppTranslator::processCppBodyDirectives(IHqlExpression * expr)
+{
+    ForEachChild(i, expr)
+    {
+        IHqlExpression * cur = expr->queryChild(i);
+        if (cur->isAttribute())
+        {
+            _ATOM name = cur->queryName();
+            if (name == linkAtom)
+            {
+                //MORE: Add code to add the argument to the linker options.
+            }
+            else if (name == libraryAtom)
+            {
+                StringBuffer libraryName;
+                getStringValue(libraryName, cur->queryChild(0));
+                if (libraryName.length())
+                    useLibrary(libraryName.str());
+            }
+        }
+    }
+}
+
 void HqlCppTranslator::doBuildExprCppBody(BuildCtx & ctx, IHqlExpression * expr, CHqlBoundExpr * tgt)
 {
     if (!allowEmbeddedCpp())
         throwError(HQLERR_EmbeddedCppNotAllowed);
 
+    processCppBodyDirectives(expr);
     StringBuffer text;
     expr->queryChild(0)->queryValue()->getStringValue(text);
+    text.setLength(cleanupEmbeddedCpp(text.length(), (char*)text.str()));
     OwnedHqlExpr quoted = createQuoted(text.str(), expr->getType());
 
     if (tgt)
@@ -11237,6 +11262,7 @@ void HqlCppTranslator::buildFunctionDefinition(IHqlExpression * funcdef)
         if (!allowEmbeddedCpp())
             throwError(HQLERR_EmbeddedCppNotAllowed);
 
+        processCppBodyDirectives(bodyCode);
         IHqlExpression * location = queryLocation(bodyCode);
         const char * locationFilename = location ? location->querySourcePath()->str() : NULL;
         unsigned startLine = location ? location->getStartLine() : 0;
@@ -11246,7 +11272,7 @@ void HqlCppTranslator::buildFunctionDefinition(IHqlExpression * funcdef)
 
         StringBuffer text;
         cppBody->queryValue()->getStringValue(text);
-        //remove /r so we don't end up with mixed format end of lines.
+        //remove #option, and remove /r so we don't end up with mixed format end of lines.
         text.setLength(cleanupEmbeddedCpp(text.length(), (char*)text.str()));
 
         const char * start = text.str();

ecl/hqlcpp/hqlcpp.ipp

@@ -1748,6 +1748,7 @@ protected:
     IHqlExpression * spotGlobalCSE(IHqlExpression * _expr);
     void spotGlobalCSE(HqlExprArray & exprs);
     IHqlExpression * extractGlobalCSE(IHqlExpression * expr);
+    void processCppBodyDirectives(IHqlExpression * expr);
 
 
     void markThorBoundaries(WorkflowArray & array);

ecl/hqlcpp/hqlecl.cpp

@@ -37,6 +37,12 @@
 #include "workunit.hpp"
 #include "thorplugin.hpp"
 
+#ifdef _DEBUG
+#define IS_DEBUG_BUILD          true
+#else
+#define IS_DEBUG_BUILD          false
+#endif
+
 #define MAIN_MODULE_TEMPLATE        "thortpl.cpp"
 #define HEADER_TEMPLATE             "thortpl.hpp"
 #define CHILD_MODULE_TEMPLATE       "childtpl.cpp"
@@ -366,8 +372,9 @@ bool HqlDllGenerator::doCompile(ICppCompiler * compiler)
     compiler->setMaxCompileThreads(maxThreads);
 
     bool debug = wu->getDebugValueBool("debugQuery", false);
+    bool debugLibrary = debug;  // should be wu->getDebugValueBool("debugLibrary", IS_DEBUG_BUILD); change for 3.8
     compiler->setDebug(debug);
-    compiler->setDebugLibrary(debug);
+    compiler->setDebugLibrary(debugLibrary);
     if (!debug)
     {
         int optimizeLevel = wu->getDebugValueInt("optimizeLevel", targetClusterType == RoxieCluster ? 3 : -1);

ecl/regress/cppbody6.ecl

@@ -0,0 +1,40 @@
+/*##############################################################################
+
+    Copyright (C) 2011 HPCC Systems.
+
+    All rights reserved. This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as
+    published by the Free Software Foundation, either version 3 of the
+    License, or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+############################################################################## */
+
+
+
+integer4 add1(integer4 x, integer4 y) :=
+BEGINC++
+return x + y;
+ENDC++;
+
+integer4 add2(integer4 x, integer4 y) :=
+BEGINC++
+#option action
+return x + y;
+ENDC++;
+
+integer4 add3(integer4 x, integer4 y) :=
+BEGINC++
+#option pure
+return x + y;
+ENDC++;
+
+output(add1(10,20) * add1(10,20));
+output(add2(10,20) * add2(10,20));
+output(add3(10,20) * add3(10,20));

ecl/regress/cppbody7.ecl

@@ -0,0 +1,40 @@
+/*##############################################################################
+
+    Copyright (C) 2011 HPCC Systems.
+
+    All rights reserved. This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as
+    published by the Free Software Foundation, either version 3 of the
+    License, or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+############################################################################## */
+
+
+
+integer4 mkRandom1 :=
+BEGINC++
+rtlRandom()
+ENDC++;
+
+integer4 mkRandom2 :=
+BEGINC++
+#option pure
+rtlRandom()
+ENDC++;
+
+integer4 mkRandom3 :=
+BEGINC++
+#option action
+rtlRandom()
+ENDC++;
+
+output(mkRandom1 * mkRandom1);
+output(mkRandom2 * mkRandom2);
+output(mkRandom3 * mkRandom3);

ecl/regress/cppbody8.ecl

@@ -0,0 +1,27 @@
+/*##############################################################################
+
+    Copyright (C) 2011 HPCC Systems.
+
+    All rights reserved. This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as
+    published by the Free Software Foundation, either version 3 of the
+    License, or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+############################################################################## */
+
+
+
+integer4 add1(integer4 x, integer4 y) :=
+BEGINC++
+#option library 'MySpecialLibrary'
+return x + y;
+ENDC++;
+
+output(add1(10,20) * add1(10,20));

system/jlib/jcomp.cpp

@@ -442,7 +442,12 @@ bool CppCompiler::compileFile(IThreadPool * pool, const char * filename, Semapho
     cmdline.append(" ").append(sourceDir);
     cmdline.append(fullFileName);
     expandCompileOptions(cmdline);
-    cmdline.append(" ").append(libraryOptions);
+
+    if (useDebugLibrary)
+        cmdline.append(" ").append(LIBFLAG_DEBUG[targetCompiler]);
+    else
+        cmdline.append(" ").append(LIBFLAG_RELEASE[targetCompiler]);
+
     _addInclude(cmdline, stdIncludes);
     
     if (targetCompiler == Vs6CppCompiler)
@@ -484,7 +489,19 @@ void CppCompiler::expandCompileOptions(StringBuffer & target)
 bool CppCompiler::doLink()
 {
     StringBuffer cmdline;
-    cmdline.append(LINK_NAME[targetCompiler]).append(LINK_SEPARATOR[targetCompiler]).append(linkerOptions);
+    cmdline.append(LINK_NAME[targetCompiler]).append(LINK_SEPARATOR[targetCompiler]);
+
+    cmdline.append(" ");
+    if (targetDebug)
+        cmdline.append(createDLL ? DLL_LINK_OPTION_DEBUG[targetCompiler] : EXE_LINK_OPTION_DEBUG[targetCompiler]);
+    else
+        cmdline.append(createDLL ? DLL_LINK_OPTION_RELEASE[targetCompiler] : EXE_LINK_OPTION_RELEASE[targetCompiler]);
+    cmdline.append(" ");
+
+    if (createDLL)
+        cmdline.append(" ").append(LINK_OPTION_CORE[targetCompiler]);
+    cmdline.append(stdLibs);
+    cmdline.append(linkerOptions);
 
     ForEachItemIn(i0, allSources)
         cmdline.append(" ").append("\"").append(targetDir).append(allSources.item(i0)).append(".").append(OBJECT_FILE_EXT[targetCompiler]).append("\"");
@@ -562,42 +579,16 @@ void CppCompiler::removeTemporaries()
 void CppCompiler::setDebug(bool _debug)
 {
     targetDebug = _debug;
-    resetLinkOptions();
 }
 
-void CppCompiler::resetLinkOptions()
-{
-    if (targetDebug)
-    {
-        setLinkOptions(createDLL ? DLL_LINK_OPTION_DEBUG[targetCompiler] : EXE_LINK_OPTION_DEBUG[targetCompiler]);
-    }
-    else
-    {
-        setLinkOptions(createDLL ? DLL_LINK_OPTION_RELEASE[targetCompiler] : EXE_LINK_OPTION_RELEASE[targetCompiler]);
-    }
-}
-
-
 void CppCompiler::setDebugLibrary(bool debug)
 {
-    if (debug)
-        libraryOptions.set(LIBFLAG_DEBUG[targetCompiler]);
-    else
-        libraryOptions.set(LIBFLAG_RELEASE[targetCompiler]);
-}
-
-void CppCompiler::setLinkOptions(const char * option)
-{
-    linkerOptions.clear().append(" ").append(option).append(" ");
-    if (createDLL)
-        linkerOptions.append(" ").append(LINK_OPTION_CORE[targetCompiler]);
-    linkerOptions.append(stdLibs);
+    useDebugLibrary = debug;
 }
 
 void CppCompiler::setCreateExe(bool _createExe) 
 { 
     createDLL = !_createExe; 
-    resetLinkOptions();
 }
 
 

system/jlib/jcomp.hpp

@@ -55,7 +55,6 @@ public:
     virtual bool compile() = 0;
     virtual void setDebug(bool _debug) = 0;
     virtual void setDebugLibrary(bool _debug) = 0;
-    virtual void setLinkOptions(const char * option) = 0;
     virtual void setOnlyCompile(bool _onlyCompile) = 0;
     virtual void setCreateExe(bool _createExe) = 0;
     virtual void setOptimizeLevel(unsigned level) = 0;

system/jlib/jcomp.ipp

@@ -39,7 +39,6 @@ public:
     virtual bool compile();
     virtual void setDebug(bool _debug);
     virtual void setDebugLibrary(bool _debug);
-    virtual void setLinkOptions(const char * option);
     virtual void setOnlyCompile(bool _onlyCompile) { onlyCompile = _onlyCompile; }
     virtual void setCreateExe(bool _createExe); 
     virtual void setOptimizeLevel(unsigned level);
@@ -55,7 +54,6 @@ protected:
     void expandRootDirectory(StringBuffer & expanded, StringBuffer & in);
     StringBuffer & getObjectName(StringBuffer & out, const char * filename);
     void removeTemporaries();
-    void resetLinkOptions();
     bool compileFile(IThreadPool * pool, const char * filename, Semaphore & finishedCompiling);
     bool doLink();
     void writeLogFile(const char* filepath, StringBuffer& log);
@@ -65,7 +63,6 @@ public:
 
 protected:
     StringBuffer    compilerOptions;
-    StringAttr      libraryOptions;
     StringBuffer    linkerOptions;
     StringBuffer    linkerLibraries;
     StringAttr      sourceDir;
@@ -78,6 +75,7 @@ protected:
     bool            onlyCompile;
     bool            createDLL;
     bool            targetDebug;
+    bool            useDebugLibrary;
     bool            verbose;
     void _addInclude(StringBuffer &s, const char *paths);
     bool            saveTemps;