Big-Data-HPC-Platform/ecl/hqlcpp/hqlecl.cpp

/*##############################################################################

    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.

    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.
    You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

    Unless required by applicable law or agreed to in writing, software
    distributed under the License is distributed on an "AS IS" BASIS,
    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    See the License for the specific language governing permissions and
    limitations under the License.
############################################################################## */

#include "build-config.h"
#include "jliball.hpp"
#include "jmisc.hpp"
#include "jstream.hpp"

#include "hql.hpp"
#include "hqlexpr.hpp"
#include "hqlecl.hpp"
#include "hqlthql.hpp"
#include "hqlcerrors.hpp"
#include "hqlcatom.hpp"
#include "hqllib.ipp"
#include "hqlutil.hpp"
#include "hqlhtcpp.ipp"
#include "hqlwcpp.hpp"
#include "hqllib.ipp"
#include "hqlttcpp.ipp"

#include "workunit.hpp"
#include "thorplugin.hpp"

#ifdef _DEBUG
#define IS_DEBUG_BUILD          true
#else
#define IS_DEBUG_BUILD          false
#endif

#define MAIN_MODULE_TEMPLATE        "thortpl.cpp"
#define HEADER_TEMPLATE             "thortpl.hpp"
#define CHILD_MODULE_TEMPLATE       "childtpl.cpp"

#define PROTO_TEMPLATE "prototpl.cpp"

class NullContextCallback : public CInterface, implements ICodegenContextCallback
{
    IMPLEMENT_IINTERFACE

    virtual void noteCluster(const char *clusterName) {}
    virtual void registerFile(const char * filename, const char * description) {}
    virtual bool allowAccess(const char * category) { return true; }
};

class HqlDllGenerator : public CInterface, implements IHqlExprDllGenerator, implements IAbortRequestCallback
{
public:
    HqlDllGenerator(IErrorReceiver * _errs, const char * _wuname, const char * _targetdir, IWorkUnit * _wu, const char * _template_dir, ClusterType _targetClusterType, ICodegenContextCallback * _ctxCallback, bool _checkForLocalFileUploads) :
        errs(_errs), wuname(_wuname), targetDir(_targetdir), wu(_wu), template_dir(_template_dir), targetClusterType(_targetClusterType), ctxCallback(_ctxCallback), checkForLocalFileUploads(_checkForLocalFileUploads)
    {
        if (!ctxCallback)
            ctxCallback.setown(new NullContextCallback);
        noOutput = true;
        defaultMaxCompileThreads = 1;
        generateTarget = EclGenerateNone;
        code.setown(createCppInstance(wu, wuname));
        deleteGenerated = false;
        totalGeneratedSize = 0;
    }
    IMPLEMENT_IINTERFACE

    virtual void addLibrary(const char * name);
    virtual bool processQuery(OwnedHqlExpr & parsedQuery, EclGenerateTarget _generateTarget);
    virtual bool generateDll(ICppCompiler * compiler);
    virtual bool generateExe(ICppCompiler * compiler);
    virtual bool generatePackage(const char * packageName);
    virtual void setMaxCompileThreads(unsigned value) { defaultMaxCompileThreads = value; }
    virtual void addManifest(const char *filename) { code->addManifest(filename); }
    virtual void addManifestFromArchive(IPropertyTree *archive) { code->addManifestFromArchive(archive); }
    virtual void addWebServiceInfo(IPropertyTree *wsinfo){ code->addWebServiceInfo(wsinfo); }

    virtual double getECLcomplexity(IHqlExpression * exprs);
    virtual void setSaveGeneratedFiles(bool value) { deleteGenerated = !value; }

protected:
    void addCppName(const char * filename);
    void addLibrariesToCompiler();
    void addWorkUnitAsResource();
    void calculateHash(IHqlExpression * expr);
    bool doCompile(ICppCompiler * compiler);
    void doExpand(HqlCppTranslator & translator);
    void expandCode(const char * templateName, const char * ext, IHqlCppInstance * code, bool multiFile, unsigned pass, CompilerType compiler);
    void flushResources();
    bool generateCode(HqlQueryContext & query);
    bool generateFullFieldUsageStatistics(HqlCppTranslator & translator, IHqlExpression * query);
    IPropertyTree * generateSingleFieldUsageStatistics(IHqlExpression * expr, const char * variety, const IPropertyTree * exclude);
    offset_t getGeneratedSize() const;
    void insertStandAloneCode();
    void setWuState(bool ok);
    inline bool abortRequested();

protected:
    Linked<IErrorReceiver> errs;
    const char * wuname;
    StringAttr targetDir;
    Linked<IWorkUnit> wu;
    Owned<IHqlCppInstance> code;
    const char * template_dir;
    ClusterType targetClusterType;
    Linked<ICodegenContextCallback> ctxCallback;
    unsigned defaultMaxCompileThreads;
    StringArray sourceFiles;
    StringArray libraries;
    offset_t totalGeneratedSize;
    bool checkForLocalFileUploads;
    bool noOutput;
    EclGenerateTarget generateTarget;
    bool deleteGenerated;
};


//---------------------------------------------------------------------------

static void processMetaCommands(HqlCppTranslator & translator, IWorkUnit * wu, HqlQueryContext & query, ICodegenContextCallback *ctxCallback);

void HqlDllGenerator::addCppName(const char * filename)
{
    if (wu)
    {
        Owned<IWUQuery> query = wu->updateQuery();
        associateLocalFile(query, FileTypeCpp, filename, pathTail(filename), 0);
        ctxCallback->registerFile(filename, "Workunit CPP");
    }
}

void HqlDllGenerator::addLibrary(const char * name)
{
    libraries.append(name);
}


void HqlDllGenerator::addLibrariesToCompiler()
{
    unsigned idx=0;
    loop
    {
        const char * lib = code->queryLibrary(idx);
        if (!lib)
            break;

        PrintLog("Adding library: %s", lib);
        addLibrary(lib);
        idx++;
    }
}


void HqlDllGenerator::expandCode(const char * templateName, const char * ext, IHqlCppInstance * code, bool multiFile, unsigned pass, CompilerType compiler)
{
    StringBuffer fullname;
    addDirectoryPrefix(fullname, targetDir).append(wuname).append(ext);

    Owned<IFile> out = createIFile(fullname.str());
    Owned<ITemplateExpander> expander = createTemplateExpander(out, templateName, template_dir);
    if (!expander)
        throwError2(HQLERR_CouldNotOpenTemplateXatY, templateName, template_dir ? template_dir : ".");

    Owned<ISectionWriter> writer = createCppWriter(*code, compiler);
    Owned<IProperties> props = createProperties(true);
    if (multiFile)
    {
        props->setProp("multiFile", true);
        props->setProp("pass", pass);
    }
    StringBuffer headerName;
    headerName.append(wuname).append(".hpp");
    props->setProp("headerName", headerName.str());
    props->setProp("outputName", fullname.str());

    expander->generate(*writer, pass, props);

    totalGeneratedSize += out->size();

    if (!deleteGenerated)
        addCppName(fullname);
}


//---------------------------------------------------------------------------

bool HqlDllGenerator::processQuery(OwnedHqlExpr & parsedQuery, EclGenerateTarget _generateTarget)
{
    generateTarget = _generateTarget;
    assertex(wu->getHash());
    unsigned prevCount = errs->errCount();

    HqlQueryContext query;
    query.expr.setown(parsedQuery.getClear());
    bool ok = generateCode(query);
    code->flushHints();
    wu->commit();

    if (!ok)
        return false;

    if (errs->errCount() != prevCount)
        return false;

    switch (generateTarget)
    {
    case EclGenerateNone:
    case EclGenerateCpp:
        return true;
    }
    
    flushResources();
    addLibrariesToCompiler();

    // Free up memory before the c++ compile occurs
    code.clear();
    return true;
}

bool HqlDllGenerator::generateDll(ICppCompiler * compiler)
{
    if (noOutput || !compiler)
        return true;

    bool ok = doCompile(compiler);
    setWuState(ok);
    return ok;
}

bool HqlDllGenerator::generateExe(ICppCompiler * compiler)
{
    if (noOutput || !compiler)
        return true;

    compiler->setCreateExe(true);

    bool ok = doCompile(compiler);
    setWuState(ok);
    return ok;
}

bool HqlDllGenerator::generatePackage(const char * packageName)
{
    return false;
}

static bool isSetMeta(IHqlExpression * expr)
{
    switch (expr->getOperator())
    {
    case no_compound:
    case no_comma:
        return isSetMeta(expr->queryChild(0)) && isSetMeta(expr->queryChild(1));
    case no_setmeta:
        return true;
    default:
        return false;
    }
}

IPropertyTree * HqlDllGenerator::generateSingleFieldUsageStatistics(IHqlExpression * expr, const char * variety, const IPropertyTree * exclude)
{
    //Generate each into a new workunit
//    Owned<IWorkUnit> localWu = createLocalWorkUnit();
    IWorkUnit * localWu = wu;
    //Generate the code into a new instance each time so it doesn't hang around eating memory
    Owned<IHqlCppInstance> localCode = createCppInstance(localWu, wuname);


    HqlQueryContext query;
    query.expr.set(expr);

    resetUniqueId();
    wu->resetBeforeGeneration();
    HqlCppTranslator translator(errs, wuname, localCode, targetClusterType, ctxCallback);

    try
    {
        translator.buildCpp(*localCode, query);
        return translator.gatherFieldUsage(variety, exclude);
    }
    catch (IException * e)
    {
        if (e->errorCode() != HQLERR_ErrorAlreadyReported)
        {
            unsigned errcode = e->errorCode() ? e->errorCode() : 1;
            StringBuffer s;
            errs->reportError(errcode, e->errorMessage(s).str(), NULL, 0, 0, 1);
        }
        e->Release();
        return NULL;
    }
    catch (RELEASE_CATCH_ALL)
    {
        errs->reportError(99, "Unknown error", NULL, 0, 0, 1);
        return NULL;
    }
}

bool HqlDllGenerator::generateFullFieldUsageStatistics(HqlCppTranslator & translator, IHqlExpression* query)
{
    LinkedHqlExpr savedQuery = query;

    //Strip any #options etc. and check that the query consists of a single OUTPUT() action
    while ((savedQuery->getOperator() == no_compound) && isSetMeta(savedQuery->queryChild(0)))
        savedQuery.set(savedQuery->queryChild(1));

    if (savedQuery->getOperator() != no_output)
        throw MakeStringException(0, "#option('GenerateFullFieldUsage') requires the query to be a single OUTPUT()");

    Owned<IPropertyTree> allFields = createPTree("usage");
    IHqlExpression * dataset = savedQuery->queryChild(0);

    //First calculate which files are used if no fields are extracted from the output i.e. shared by all output fields
    //Use COUNT(dataset) as the query to test that
    OwnedHqlExpr countDataset = createValue(no_count, makeIntType(8, false), LINK(dataset));
    Owned<IPropertyTree> countFilesUsed = generateSingleFieldUsageStatistics(countDataset, "__shared__", NULL);
    if (countFilesUsed)
        allFields->addPropTree(countFilesUsed->queryName(), LINK(countFilesUsed));

    //Now project the output dataset down to a single field, and generate the file/field dependencies for each of those
    OwnedHqlExpr selSeq = createUniqueSelectorSequence();
    OwnedHqlExpr left = createSelector(no_left, dataset, selSeq);
    RecordSelectIterator iter(dataset->queryRecord(), left);
    ForEach(iter)
    {
        IHqlExpression * cur = iter.query();
        IHqlExpression * field = cur->queryChild(1);
        OwnedHqlExpr record = createRecord(field);
        OwnedHqlExpr self = getSelf(record);
        OwnedHqlExpr assign = createAssign(createSelectExpr(LINK(self), LINK(field)), LINK(cur));
        OwnedHqlExpr transform = createValue(no_transform, makeTransformType(record->getType()), LINK(assign), NULL);

        HqlExprArray args;
        args.append(*LINK(dataset));
        args.append(*LINK(transform));
        args.append(*LINK(selSeq));
        OwnedHqlExpr project = createDataset(no_hqlproject,args);
        OwnedHqlExpr projectedOutput = replaceChildDataset(savedQuery, project, 0);

        StringBuffer variety;
        getExprIdentifier(variety, cur);

        //Generate each into a new property tree, and only include fields not shared by all other output fields.
        Owned<IPropertyTree> filesUsed = generateSingleFieldUsageStatistics(projectedOutput, variety.str(), countFilesUsed);
        if (filesUsed)
            allFields->addPropTree(filesUsed->queryName(), LINK(filesUsed));

        //Generate progress so far
        translator.writeFieldUsage(targetDir, allFields, NULL);
    }

    translator.writeFieldUsage(targetDir, allFields, NULL);
    return false;
}

bool HqlDllGenerator::generateCode(HqlQueryContext & query)
{
    noOutput = true;
    {
        // ensure warnings/errors are available before we do the processing...
        wu->commit();

        MTIME_SECTION (timer, "Generate_code");
        unsigned time = msTick();
        HqlCppTranslator translator(errs, wuname, code, targetClusterType, ctxCallback);
        processMetaCommands(translator, wu, query, ctxCallback);

        if (wu->getDebugValueBool("generateFullFieldUsage", false))
        {
            //Ensure file information is generated.  It only partially works with field information at the moment.
            //(The merging/difference logic would need to improve, but would be relatively simple if it was useful.)
            wu->setDebugValueInt("reportFileUsage", 1, true);
            return generateFullFieldUsageStatistics(translator, query.expr);
        }

        bool ok = false;
        try
        {
            if (!translator.buildCpp(*code, query))
            {
                wu->setState(WUStateCompleted);
                return true;
            }
            translator.generateStatistics(targetDir, NULL);
            translator.finalizeResources();
            translator.expandFunctions(true);
        }
        catch (IECLError * e)
        {
            if (e->errorCode() != HQLERR_ErrorAlreadyReported)
            {
                StringBuffer s;
                errs->reportError(e->errorCode(), e->errorMessage(s).str(), e->getFilename(), e->getLine(), e->getColumn(), e->getPosition());
            }
            e->Release();
            return false;
        }
        catch (IException * e)
        {
            if (e->errorCode() != HQLERR_ErrorAlreadyReported)
            {
                unsigned errcode = e->errorCode() ? e->errorCode() : 1;
                StringBuffer s;
                errs->reportError(errcode, e->errorMessage(s).str(), NULL, 0, 0, 1);
            }
            e->Release();
            return false;
        }
        catch (RELEASE_CATCH_ALL)
        {
            errs->reportError(99, "Unknown error", NULL, 0, 0, 1);
            return false;
        }

        if (generateTarget == EclGenerateExe)
            insertStandAloneCode();

        wu->commit();
        
        //Commit work unit so can view graphs etc. while compiling the C++
        if ((generateTarget == EclGenerateNone) || wu->getDebugValueBool("OnlyCheckQuery", false))
        {
            wu->setState(WUStateCompleted);
            return false;
        }

        doExpand(translator);
        if (wu->getDebugValueBool("addTimingToWorkunit", true))
            wu->setTimerInfo("EclServer: generate code", NULL, msTick()-time, 1, 0);

        wu->commit();
        addWorkUnitAsResource();
    }
    noOutput = false;
    return true;
}

void HqlDllGenerator::addWorkUnitAsResource()
{
    SCMStringBuffer wuXML;
    exportWorkUnitToXML(wu, wuXML, false);
    code->addCompressResource("WORKUNIT", wuXML.length(), wuXML.str(), NULL, 1000);
}


void HqlDllGenerator::insertStandAloneCode()
{
    BuildCtx ctx(static_cast<HqlCppInstance &>(*code), goAtom);
    ctx.addQuotedCompound("int main(int argc, const char *argv[])");
    ctx.addQuoted("return start_query(argc, argv);\n");
}


void HqlDllGenerator::doExpand(HqlCppTranslator & translator)
{
    unsigned startExpandTime = msTick();
    unsigned numExtraFiles = translator.getNumExtraCppFiles();
    bool isMultiFile = translator.spanMultipleCppFiles() && (numExtraFiles != 0);
    expandCode(MAIN_MODULE_TEMPLATE, ".cpp", code, isMultiFile, 0, translator.queryOptions().targetCompiler);
    if (isMultiFile)
    {
        expandCode(HEADER_TEMPLATE, ".hpp", code, true, 0, translator.queryOptions().targetCompiler);
        for (unsigned i= 0; i < translator.getNumExtraCppFiles(); i++)
        {
            StringBuffer fullext;
            fullext.append("_").append(i+1).append(".cpp");
            expandCode(CHILD_MODULE_TEMPLATE, fullext, code, true, i+1, translator.queryOptions().targetCompiler);

            StringBuffer fullname;
            fullname.append(wuname).append("_").append(i+1);
            sourceFiles.append(fullname);
        }
    }

    unsigned endExpandTime = msTick();
    if (wu->getDebugValueBool("addTimingToWorkunit", true))
        wu->setTimerInfo("EclServer: write c++", NULL, endExpandTime-startExpandTime, 1, 0);
}

bool HqlDllGenerator::abortRequested()
{
    return (wu && wu->aborting());
}

bool HqlDllGenerator::doCompile(ICppCompiler * compiler)
{
    ForEachItemIn(i, sourceFiles)
        compiler->addSourceFile(sourceFiles.item(i));

    unsigned maxThreads = wu->getDebugValueInt("maxCompileThreads", defaultMaxCompileThreads);
    compiler->setMaxCompileThreads(maxThreads);

    bool debug = wu->getDebugValueBool("debugQuery", false);
    bool debugLibrary = debug;  // should be wu->getDebugValueBool("debugLibrary", IS_DEBUG_BUILD); change for 3.8
    compiler->setDebug(debug);
    compiler->setDebugLibrary(debugLibrary);
    if (!debug)
    {
        int optimizeLevel = wu->getDebugValueInt("optimizeLevel", targetClusterType == RoxieCluster ? 3 : -1);
        if (optimizeLevel != -1)
            compiler->setOptimizeLevel(optimizeLevel);
    }
#ifdef __64BIT__
    bool target64bit = wu->getDebugValueBool("target64bit", true);
#else
    bool target64bit = wu->getDebugValueBool("target64bit", false);
#endif
    compiler->setTargetBitLength(target64bit ? 64 : 32);

    ForEachItemIn(idx, libraries)
        compiler->addLibrary(libraries.item(idx));

    StringBuffer options;
    StringBufferAdaptor linkOptionAdaptor(options);
    wu->getDebugValue("linkOptions", linkOptionAdaptor);
    compiler->addLinkOption(options.str());
    options.clear();
    StringBufferAdaptor optionAdaptor(options);
    wu->getDebugValue("compileOptions", optionAdaptor);
    compiler->addCompileOption(options.str());

    compiler->setAbortChecker(this);

    MTIME_SECTION (timer, "Compile_code");
    unsigned time = msTick();
    PrintLog("Compiling %s", wuname);
    bool ok = compiler->compile();
    if(ok)
        PrintLog("Compiled %s", wuname);
    else
        PrintLog("Failed to compile %s", wuname);
    time = msTick()-time;
    if (wu->getDebugValueBool("addTimingToWorkunit", true))
        wu->setTimerInfo("EclServer: compile code", NULL, time, 1, 0);

    //Keep the files if there was a compile error.
    if (ok && deleteGenerated)
    {
        StringBuffer temp;
        remove(temp.clear().append(wuname).append(".cpp").str());
        remove(temp.clear().append(wuname).append(".hpp").str());
        ForEachItemIn(i, sourceFiles)
        {
            temp.clear().append(sourceFiles.item(i)).append(".cpp");
            remove(temp.str());
        }
    }
    return ok;
}

void HqlDllGenerator::flushResources()
{
    StringBuffer resname(wuname);
#ifdef _WIN32
    resname.append(".res");
#else
    resname.append(".res.o");
#endif
    if (code)
        code->flushResources(resname.str(), ctxCallback);
}


void HqlDllGenerator::setWuState(bool ok)
{
    if(ok)
    {
        if(checkForLocalFileUploads && wu->requiresLocalFileUpload())
            wu->setState(WUStateUploadingFiles);
        else
            wu->setState(WUStateCompiled);
    }
    else
        wu->setState(WUStateFailed);
}


double HqlDllGenerator::getECLcomplexity(IHqlExpression * exprs)
{
    Owned<IHqlCppInstance> code = createCppInstance(wu, NULL);

    HqlCppTranslator translator(errs, "temp", code, targetClusterType, NULL);

    HqlQueryContext query;
    query.expr.set(exprs);
    processMetaCommands(translator, wu, query, ctxCallback);

    return translator.getComplexity(*code, query.expr);
}

offset_t HqlDllGenerator::getGeneratedSize() const
{
    return totalGeneratedSize;
}

extern HQLCPP_API double getECLcomplexity(IHqlExpression * exprs, IErrorReceiver * errs, IWorkUnit *wu, ClusterType targetClusterType)
{
    HqlDllGenerator generator(errs, "unknown", NULL, wu, NULL, targetClusterType, NULL, false);
    return generator.getECLcomplexity(exprs);
}


extern HQLCPP_API IHqlExprDllGenerator * createDllGenerator(IErrorReceiver * errs, const char *wuname, const char * targetdir, IWorkUnit *wu, const char * template_dir, ClusterType targetClusterType, ICodegenContextCallback *ctxCallback, bool checkForLocalFileUploads)
{
    return new HqlDllGenerator(errs, wuname, targetdir, wu, template_dir, targetClusterType, ctxCallback, checkForLocalFileUploads);
}

/*
extern HQLCPP_API void addLibraryReference(IWorkUnit * wu, IHqlLibraryInfo * library, const char * lid)
{
    StringBuffer libraryName;
    library->getName(libraryName);

    Owned<IWULibrary> entry = wu->updateLibraryByName(libraryName.str());
    entry->setMajorVersion(library->queryMajorVersion());
    entry->setMinorVersion(library->queryMinorVersion());
    entry->setCrc(library->queryInterfaceCrc());
    if (lid && *lid)
        entry->setLID(lid);
}
*/

extern HQLCPP_API ClusterType queryClusterType(IConstWorkUnit * wu, ClusterType prevType)
{
    ClusterType targetClusterType = prevType;
    if (wu->getDebugValueBool("forceRoxie", false))
        targetClusterType = RoxieCluster;
    else if (prevType == NoCluster)
        targetClusterType = ThorLCRCluster;

    SCMStringBuffer targetText;
    wu->getDebugValue("targetClusterType", targetText);
    targetClusterType = getClusterType(targetText.s.str(), targetClusterType);

    if ((targetClusterType != RoxieCluster) && wu->getDebugValueBool("forceFakeThor", false))
        targetClusterType = HThorCluster;

    return targetClusterType;
}

static void processMetaCommands(HqlCppTranslator & translator, IWorkUnit * wu, HqlQueryContext & query, ICodegenContextCallback *ctxCallback)
{
    NewThorStoredReplacer transformer(translator, wu, ctxCallback);

    translator.traceExpression("before process meta commands", query.expr);

    transformer.analyse(query.expr);
    if (transformer.needToTransform())
        query.expr.setown(transformer.transform(query.expr));
}


extern HQLCPP_API unsigned getLibraryCRC(IHqlExpression * library)
{
    assertex(library->getOperator() == no_funcdef);
    return getExpressionCRC(library->queryChild(0));
}


void setWorkunitHash(IWorkUnit * wu, IHqlExpression * expr)
{
    //Assuming builds come from different branches this will change the crc for each one.
    unsigned cacheCRC = crc32(BUILD_TAG, strlen(BUILD_TAG), ACTIVITY_INTERFACE_VERSION);
    cacheCRC += getExpressionCRC(expr);
#ifdef _WIN32
    cacheCRC++; // make sure CRC is different in windows/linux
#endif
#ifdef __64BIT__
    cacheCRC += 2; // make sure CRC is different for different host platform (shouldn't really matter if cross-compiling working properly, but fairly harmless)
#endif
    IExtendedWUInterface *ewu = queryExtendedWU(wu);
    cacheCRC = ewu->calculateHash(cacheCRC);
    wu->setHash(cacheCRC);
}