/*##############################################################################
Copyright (C) 2011 HPCC Systems.
All rights reserved. This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see .
############################################################################## */
#include "platform.h"
#include "jlib.hpp"
#include "jmisc.hpp"
#include "jcomp.hpp"
#include "jsem.hpp"
#include "jexcept.hpp"
#ifdef _WIN32
#include
#include
#include
#else
#include
#endif
#include
#include
#include
#include
#include "jfile.hpp"
#include "jdebug.hpp"
#include "jcomp.ipp"
#define CC_EXTRA_OPTIONS ""
#ifdef GENERATE_LISTING
#undef CC_EXTRA_OPTIONS
#define CC_EXTRA_OPTIONS " /FAs"
#endif
#ifdef _WIN32
#define DEFAULT_COMPILER Vs6CppCompiler
#else
#define DEFAULT_COMPILER GccCppCompiler
#endif
//---------------------------------------------------------------------------
#define BASE_ADDRESS "0x00480000"
//#define BASE_ADDRESS "0x10000000"
static const char * CC_NAME[] = { "\"#" PATHSEPSTR "bin" PATHSEPSTR "cl.bat\"", "\"#" PATHSEPSTR "bin" PATHSEPSTR "g++\"" };
static const char * LINK_NAME[] = { "\"#" PATHSEPSTR "bin" PATHSEPSTR "link.bat\"", "\"#" PATHSEPSTR "bin" PATHSEPSTR "g++\"" };
static const char * LIB_DIR[] = { "\"#\\lib\"", "\"#/lib\"" };
static const char * USE_LIBPATH_FLAG[] = { "/libpath:\"", "-L" };
static const char * USE_LIBPATH_TAIL[] = { "\"", "" };
static const char * USE_LIBRPATH_FLAG[] = { NULL, "-Wl,-rpath -Wl," };
static const char * USE_LIB_FLAG[] = { "", "-l" };
static const char * USE_LIB_TAIL[] = { ".lib", "" };
static const char * USE_INCLUDE_FLAG[] = { "/I\"", "\"-I" };
static const char * USE_DEFINE_FLAG[] = { "/D", "-D" };
static const char * USE_INCLUDE_TAIL[] = { "\"", "\"" };
static const char * INCLUDEPATH[] = { "\"#\\include\"", "\"#/include\"" };
static const char * LINK_SEPARATOR[] = { " /link ", " " };
static const char * OBJECT_FILE_EXT[] = { "obj", "o" };
static const char * LIBFLAG_DEBUG[] = { "/MDd", "" };
static const char * LIBFLAG_RELEASE[] = { "/MD", "" };
static const char * COMPILE_ONLY[] = { "/c", "-c" };
static const char * CC_OPTION_CORE[] = { "", "-fvisibility=hidden -DUSE_VISIBILITY=1" };
static const char * LINK_OPTION_CORE[] = { "/DLL /libpath:." , "" };
static const char * CC_OPTION_DEBUG[] = { "/Zm500 /EHsc /GR /Zi /nologo /bigobj", "-g -fPIC -pipe -O0" };
static const char * DLL_LINK_OPTION_DEBUG[] = { "/BASE:" BASE_ADDRESS " /NOLOGO /LARGEADDRESSAWARE /INCREMENTAL:NO /DEBUG /DEBUGTYPE:CV", "-g -shared -L. -fPIC -pipe -O0" };
static const char * EXE_LINK_OPTION_DEBUG[] = { "/BASE:" BASE_ADDRESS " /NOLOGO /LARGEADDRESSAWARE /INCREMENTAL:NO /DEBUG /DEBUGTYPE:CV", "-g -L. -Wl,-E -fPIC -pipe -O0" };
static const char * CC_OPTION_RELEASE[] = { "/Zm500 /EHsc /GR /Oi /Ob1 /GF /nologo /bigobj", "-fPIC -pipe -O0" };
static const char * DLL_LINK_OPTION_RELEASE[] = { "/BASE:" BASE_ADDRESS " /NOLOGO /LARGEADDRESSAWARE /INCREMENTAL:NO", "-shared -L. -fPIC -pipe -O0" };
static const char * EXE_LINK_OPTION_RELEASE[] = { "/BASE:" BASE_ADDRESS " /NOLOGO /LARGEADDRESSAWARE /INCREMENTAL:NO", "-L. -Wl,-E -fPIC -pipe -O0" };
static const char * LINK_TARGET[] = { " /out:", " -o " };
static const char * DEFAULT_CC_LOCATION[] = { ".", "." };
//===========================================================================
static StringAttr compilerRoot;
static StringAttr stdIncludes;
static StringBuffer stdLibs;
static StringBuffer &dequote(StringBuffer &in)
{
if (in.length() >= 2 && in.charAt(0)=='"' && in.charAt(in.length()-1)=='"')
{
in.remove(in.length()-1, 1);
in.remove(0, 1);
}
return in;
}
static void doSetCompilerPath(const char * path, const char * includes, const char * libs, const char * tmpdir, unsigned targetCompiler, bool verbose)
{
if (!includes)
includes = INCLUDEPATH[targetCompiler];
if (!libs)
libs = LIB_DIR[targetCompiler];
if (verbose)
{
PrintLog("Include directory set to %s", includes);
PrintLog("Library directory set to %s", libs);
}
compilerRoot.set(path ? path : targetCompiler==GccCppCompiler ? "/usr" : ".\\CL");
stdIncludes.set(includes);
stdLibs.clear();
for (;;)
{
StringBuffer thislib;
while (*libs && *libs != ';')
thislib.append(*libs++);
if (thislib.length())
{
stdLibs.append(" ").append(USE_LIBPATH_FLAG[targetCompiler]).append(thislib).append(USE_LIBPATH_TAIL[targetCompiler]);
if (USE_LIBRPATH_FLAG[targetCompiler])
stdLibs.append(" ").append(USE_LIBRPATH_FLAG[targetCompiler]).append(thislib);
}
if (!*libs)
break;
libs++;
}
StringBuffer fname;
if (path)
{
const char *finger = CC_NAME[targetCompiler];
while (*finger)
{
if (*finger == '#')
fname.append(path);
else
fname.append(*finger);
finger++;
}
#if defined(__linux__)
StringBuffer clbin_dir;
const char* dir_end = strrchr(fname, '/');
if(dir_end == NULL)
clbin_dir.append(".");
else
clbin_dir.append((dir_end - fname.str()) + 1, fname.str());
StringBuffer pathenv(clbin_dir.str());
const char* oldpath = getenv("PATH");
if(oldpath != NULL && *oldpath != '\0')
pathenv.append(":").append(oldpath);
setenv("PATH", pathenv.str(), 1);
#endif
}
else
{
fname.append(compilerRoot).append(CC_NAME[targetCompiler]);
fname.replaceString("#",NULL);
}
if (verbose)
PrintLog("Compiler path set to %s", fname.str());
dequote(fname);
#ifdef _WIN32
if (_access(fname.str(), 4))
{
#else
#if defined(__linux__) || defined(__FreeBSD__) || defined(__APPLE__)
struct stat filestatus;
int r = stat(fname.str(), &filestatus);
if ( (r != 0)
|| (!S_ISREG(filestatus.st_mode))
|| (filestatus.st_mode&(S_IXOTH|S_IXGRP|S_IXUSR)==0))
{
if (r == -1) errno = ENOENT;
#endif
#endif
if (verbose)
PrintLog("SetCompilerPath - no compiler found");
throw MakeOsException(GetLastError(), "setCompilerPath could not locate compiler %s", fname.str());
}
if(tmpdir && *tmpdir)
{
//MORE: this should be done for the child process instead of the parent but invoke does not let me do it
#if defined(__linux__)
setenv("TMPDIR", tmpdir, 1);
#endif
#ifdef _WIN32
StringBuffer tmpbuf;
tmpbuf.append("TMP=").append(tmpdir);
_putenv(tmpbuf.str());
#endif
}
}
//===========================================================================
class CCompilerThreadParam : CInterface
{
public:
IMPLEMENT_IINTERFACE;
CCompilerThreadParam(const StringBuffer & _cmdline, Semaphore & _finishedCompiling, const StringBuffer & _logfile) : cmdline(_cmdline), finishedCompiling(_finishedCompiling), logfile(_logfile) {};
StringBuffer cmdline;
StringBuffer logfile;
Semaphore & finishedCompiling;
};
//===========================================================================
static void setDirectoryPrefix(StringAttr & target, const char * source)
{
if (source && *source)
{
StringBuffer temp;
target.set(addDirectoryPrefix(temp, source));
}
}
CppCompiler::CppCompiler(const char * _coreName, const char * _sourceDir, const char * _targetDir, unsigned _targetCompiler, bool _verbose)
{
#define CORE_NAME allSources.item(0)
if (_coreName)
addSourceFile(_coreName);
targetCompiler = _targetCompiler;
createDLL = true;
#ifdef _DEBUG
setDebug(true);
setDebugLibrary(true);
#else
setDebug(false);
setDebugLibrary(false);
#endif
setDirectoryPrefix(sourceDir, _sourceDir);
setDirectoryPrefix(targetDir, _targetDir);
maxCompileThreads = 1;
onlyCompile = false;
verbose = _verbose;
saveTemps = false;
abortChecker = NULL;
}
void CppCompiler::addCompileOption(const char * option)
{
compilerOptions.append(' ').append(option);
}
void CppCompiler::addDefine(const char * symbolName, const char * value)
{
compilerOptions.append(" ").append(USE_DEFINE_FLAG[targetCompiler]).append(symbolName);
if (value)
compilerOptions.append('=').append(value);
}
void CppCompiler::addLibrary(const char * libName)
{
if (verbose)
PrintLog("addLibrary %s", libName);
const char* lname = libName;
const char * quote;
StringBuffer path, tail; // NOTE - because of the (hacky) code below that sets lname to point within tail.str(), this must NOT be moved inside the if block
if (targetCompiler == GccCppCompiler)
{
// It seems gcc compiler doesn't like things like -lmydir/libx.so
splitFilename(libName, &path, &path, &tail, &tail);
if(path.length())
{
addLibraryPath(path);
// HACK - make it work with plugins. This should be handled at caller end!
if (strncmp(tail.str(), "lib", 3) == 0)
lname = tail.str() + 3;
}
quote = NULL; //quoting lib names with gcc causes link error (lib not found)
}
else
{
quote = "\"";
}
linkerOptions.append(" ").append(USE_LIB_FLAG[targetCompiler]).append(quote).append(lname).append(USE_LIB_TAIL[targetCompiler]).append(quote);
}
void CppCompiler::addLibraryPath(const char * libPath)
{
linkerOptions.append(" ").append(USE_LIBPATH_FLAG[targetCompiler]).append(libPath).append(USE_LIBPATH_TAIL[targetCompiler]);
if (USE_LIBRPATH_FLAG[targetCompiler])
linkerOptions.append(" ").append(USE_LIBRPATH_FLAG[targetCompiler]).append(libPath);
}
void CppCompiler::_addInclude(StringBuffer &s, const char * paths)
{
if (!paths)
return;
StringBuffer includePath;
for (;;)
{
while (*paths && *paths != ENVSEPCHAR)
includePath.append(*paths++);
if (includePath.length())
s.append(" ").append(USE_INCLUDE_FLAG[targetCompiler]).append(includePath).append(USE_INCLUDE_TAIL[targetCompiler]);
if (!*paths)
break;
paths++;
includePath.clear();
}
}
void CppCompiler::addInclude(const char * paths)
{
_addInclude(compilerOptions, paths);
}
void CppCompiler::addLinkOption(const char * option)
{
linkerOptions.append(' ').append(option);
}
void CppCompiler::addSourceFile(const char * filename)
{
allSources.append(filename);
}
void CppCompiler::writeLogFile(const char* filepath, StringBuffer& log)
{
if(!filepath || !*filepath || !log.length())
return;
Owned f = createIFile(filepath);
if(f->exists())
f->remove();
Owned fio = f->open(IFOcreaterw);
if(fio.get())
fio->write(0, log.length(), log.str());
}
bool CppCompiler::compile()
{
if (abortChecker && abortChecker->abortRequested())
return false;
TIME_SECTION(!verbose ? NULL : onlyCompile ? "compile" : "compile/link");
Owned pool = createThreadPool("CCompilerWorker", this, NULL, maxCompileThreads?maxCompileThreads:1, INFINITE);
addCompileOption(COMPILE_ONLY[targetCompiler]);
bool ret = false;
Semaphore finishedCompiling;
int numSubmitted = 0;
atomic_set(&numFailed, 0);
ForEachItemIn(i0, allSources)
{
ret = compileFile(pool, allSources.item(i0), finishedCompiling);
if (!ret)
break;
++numSubmitted;
}
for (int idx = 0; idx < numSubmitted; idx++)
{
if (abortChecker)
{
while (!finishedCompiling.wait(5*1000))
{
if (abortChecker && abortChecker->abortRequested())
{
PrintLog("Aborting compilation");
pool->stopAll(true);
if (!pool->joinAll(true, 10*1000))
WARNLOG("CCompilerWorker; timed out waiting for threads in pool");
return false;
}
}
}
else
finishedCompiling.wait();
}
if (atomic_read(&numFailed) > 0)
ret = false;
else if (!onlyCompile)
ret = doLink();
if (!saveTemps)
{
removeTemporaries();
StringBuffer temp;
ForEachItemIn(i2, allSources)
remove(getObjectName(temp.clear(), allSources.item(i2)).str());
}
//Combine logfiles
const char* cclog = ccLogPath.get();
if(!cclog||!*cclog)
cclog = queryCcLogName();
Owned dstfile = createIFile(cclog);
dstfile->remove();
Owned dstIO = dstfile->open(IFOwrite);
ForEachItemIn(i2, logFiles)
{
Owned srcfile = createIFile(logFiles.item(i2));
if (srcfile->exists())
{
dstIO->appendFile(srcfile);
srcfile->remove();
}
}
pool->joinAll(true, 1000);
return ret;
}
bool CppCompiler::compileFile(IThreadPool * pool, const char * filename, Semaphore & finishedCompiling)
{
if (!filename || *filename == 0)
return false;
StringBuffer fullFileName;
fullFileName.clear().append("\"").append(filename).append(".cpp").append("\"").append(" ");
StringBuffer cmdline;
cmdline.append(CC_NAME[targetCompiler]);
cmdline.append(" ").append(sourceDir);
cmdline.append(fullFileName);
expandCompileOptions(cmdline);
cmdline.append(" ").append(libraryOptions);
_addInclude(cmdline, stdIncludes);
if (targetCompiler == Vs6CppCompiler)
{
if (targetDir.get())
cmdline.append(" /Fo").append("\"").append(targetDir).append("\"");
cmdline.append(" /Fd").append("\"").append(targetDir).append(createDLL ? SharedObjectPrefix : NULL).append(filename).append(".pdb").append("\"");//MORE: prefer create a single pdb file using coreName
}
StringBuffer expanded;
expandRootDirectory(expanded, cmdline);
StringBuffer logFile;
logFile.append(filename).append(".log");
logFiles.append(logFile);
Owned parm;
if (verbose)
PrintLog("%s", expanded.toCharArray());
parm.setown(new CCompilerThreadParam(expanded, finishedCompiling, logFile));
pool->start(parm.get());
return true;
}
void CppCompiler::expandCompileOptions(StringBuffer & target)
{
target.append(" ").append(CC_OPTION_CORE[targetCompiler]).append(" ");
if (targetDebug)
target.append(CC_OPTION_DEBUG[targetCompiler]);
else
target.append(CC_OPTION_RELEASE[targetCompiler]);
target.append(compilerOptions).append(CC_EXTRA_OPTIONS);
}
bool CppCompiler::doLink()
{
StringBuffer cmdline;
cmdline.append(LINK_NAME[targetCompiler]).append(LINK_SEPARATOR[targetCompiler]).append(linkerOptions);
ForEachItemIn(i0, allSources)
cmdline.append(" ").append("\"").append(targetDir).append(allSources.item(i0)).append(".").append(OBJECT_FILE_EXT[targetCompiler]).append("\"");
StringBuffer outName;
outName.append(createDLL ? SharedObjectPrefix : NULL).append(CORE_NAME).append(createDLL ? SharedObjectExtension : ProcessExtension);
cmdline.append(LINK_TARGET[targetCompiler]).append("\"").append(targetDir).append(outName).append("\"");
StringBuffer temp;
remove(temp.clear().append(targetDir).append(outName).str());
StringBuffer expanded;
expandRootDirectory(expanded, cmdline);
DWORD runcode = 0;
if (verbose)
PrintLog("%s", expanded.toCharArray());
StringBuffer logFile = StringBuffer(CORE_NAME).append("_link.log");
logFiles.append(logFile);
bool ret = invoke_program(expanded.toCharArray(), runcode, true, logFile) && (runcode == 0);
return ret;
}
void CppCompiler::expandRootDirectory(StringBuffer & expanded, StringBuffer & in)
{
unsigned len = in.length();
unsigned i;
for (i = 0; i < len; i++)
{
char c = in.charAt(i);
if (c == '#')
{
if (compilerRoot)
expanded.append(compilerRoot);
else
expanded.append(DEFAULT_CC_LOCATION[targetCompiler]);
}
else
expanded.append(c);
}
}
StringBuffer & CppCompiler::getObjectName(StringBuffer & out, const char * filename)
{
#ifdef _WIN32
out.append(targetDir);
splitFilename(filename, NULL, NULL, &out, &out);
#else
//MORE: Not sure where gcc puts the object files... current directory? Same as cpp file?
out.append(targetDir);
splitFilename(filename, NULL, NULL, &out, &out);
#endif
return out.append(".").append(OBJECT_FILE_EXT[targetCompiler]);
}
void CppCompiler::removeTemporaries()
{
switch (targetCompiler)
{
case Vs6CppCompiler:
case GccCppCompiler:
{
StringBuffer temp;
remove(temp.clear().append(targetDir).append(CORE_NAME).append(".exp").str());
remove(getObjectName(temp.clear(), CORE_NAME).str());
remove(temp.clear().append(targetDir).append(CORE_NAME).append(".lib").str());
#ifdef _WIN32
remove(temp.clear().append(targetDir).append(CORE_NAME).append(".res.lib").str());
#else
remove(temp.clear().append(targetDir).append(CORE_NAME).append(".res.o.lib").str());
remove(temp.clear().append(targetDir).append("lib").append(CORE_NAME).append(".res.o.so").str());
#endif
break;
}
}
}
void CppCompiler::setDebug(bool _debug)
{
targetDebug = _debug;
resetLinkOptions();
}
void CppCompiler::resetLinkOptions()
{
if (targetDebug)
{
setLinkOptions(createDLL ? DLL_LINK_OPTION_DEBUG[targetCompiler] : EXE_LINK_OPTION_DEBUG[targetCompiler]);
}
else
{
setLinkOptions(createDLL ? DLL_LINK_OPTION_RELEASE[targetCompiler] : EXE_LINK_OPTION_RELEASE[targetCompiler]);
}
}
void CppCompiler::setDebugLibrary(bool debug)
{
if (debug)
libraryOptions.set(LIBFLAG_DEBUG[targetCompiler]);
else
libraryOptions.set(LIBFLAG_RELEASE[targetCompiler]);
}
void CppCompiler::setLinkOptions(const char * option)
{
linkerOptions.clear().append(" ").append(option).append(" ");
if (createDLL)
linkerOptions.append(" ").append(LINK_OPTION_CORE[targetCompiler]);
linkerOptions.append(stdLibs);
}
void CppCompiler::setCreateExe(bool _createExe)
{
createDLL = !_createExe;
resetLinkOptions();
}
void CppCompiler::setOptimizeLevel(unsigned level)
{
const char * option = NULL;
switch (targetCompiler)
{
case Vs6CppCompiler:
switch (level)
{
case 0: option = "/Od"; break;
case 1: option = "/O1"; break;
case 2: option = "/O2"; break;
default: // i.e. 3 or higher
option = "/Ob1gty /G6"; break;
}
break;
case GccCppCompiler:
switch (level)
{
case 0: option = "-O0"; break; // do not optimize
case 1: option = "-O1"; break;
case 2: option = "-O2"; break;
default: // i.e. 3 or higher
option = "-O3"; break;
}
break;
}
if (option)
addCompileOption(option);
}
void CppCompiler::setTargetBitLength(unsigned bitlength)
{
const char * option = NULL;
switch (targetCompiler)
{
case Vs6CppCompiler:
switch (bitlength)
{
case 32: break; // 64-bit windows TBD at present....
default:
throwUnexpected();
}
break;
case GccCppCompiler:
switch (bitlength)
{
case 32: option = "-m32"; break;
case 64: option = "-m64"; break;
default:
throwUnexpected();
}
break;
}
if (option)
addCompileOption(option);
}
void CppCompiler::setCCLogPath(const char* path)
{
if(path && *path)
ccLogPath.set(path);
}
//===========================================================================
bool fileIsOlder(const char *dest, const char *src)
{
int h1 = _open(dest, _O_RDONLY);
if (h1 == -1)
return true;
int h2 = _open(src, _O_RDONLY);
assertex(h2 != -1);
struct _stat stat1;
struct _stat stat2;
_fstat(h1, &stat1);
_fstat(h2, &stat2);
_close(h1);
_close(h2);
return stat1.st_mtime < stat2.st_mtime;
}
//===========================================================================
void setCompilerPath(const char * path, const char * includes, const char * libs, const char * tmpdir, bool verbose)
{
doSetCompilerPath(path, includes, libs, tmpdir, DEFAULT_COMPILER, verbose);
}
void setCompilerPath(const char * path, const char * includes, const char * libs, const char * tmpdir, CompilerType targetCompiler, bool verbose)
{
doSetCompilerPath(path, includes, libs, tmpdir, targetCompiler, verbose);
}
ICppCompiler * createCompiler(const char * coreName, const char * sourceDir, const char * targetDir, bool verbose)
{
return new CppCompiler(coreName, sourceDir, targetDir, DEFAULT_COMPILER, verbose);
}
ICppCompiler * createCompiler(const char * coreName, const char * sourceDir, const char * targetDir, CompilerType targetCompiler, bool verbose)
{
return new CppCompiler(coreName, sourceDir, targetDir, targetCompiler, verbose);
}
//===========================================================================
class CCompilerWorker : public CInterface, implements IPooledThread
{
public:
IMPLEMENT_IINTERFACE;
CCompilerWorker(CppCompiler * _compiler) : compiler(_compiler) {}
bool canReuse() { return true; }
bool stop() { return true; }
void init(void *_params) { params.set((CCompilerThreadParam *)_params); }
void main()
{
DWORD runcode = 0;
bool success;
try
{
success = invoke_program(params->cmdline, runcode, true, params->logfile) && (runcode == 0);
}
catch(IException* e)
{
StringBuffer sb;
e->errorMessage(sb);
e->Release();
if (sb.length())
PrintLog("%s", sb.str());
success = false;
}
if (!success)
atomic_inc(&compiler->numFailed);
params->finishedCompiling.signal();
return;
}
private:
CppCompiler * compiler;
Owned params;
};
IPooledThread *CppCompiler::createNew()
{
return new CCompilerWorker(this);
}