HPCC-15103 Add information about which c++ file contains an activity

Signed-off-by: Gavin Halliday <gavin.halliday@lexisnexis.com>

common/workunit/workunit.cpp

@@ -1621,6 +1621,8 @@ public:
     virtual IStringVal & getName(IStringVal & ret) const;
     virtual IStringVal & getNameTail(IStringVal & ret) const;
     virtual unsigned getCrc() const;
+    virtual unsigned getMinActivityId() const;
+    virtual unsigned getMaxActivityId() const;
 };
 
 class CLocalWUQuery : public CInterface, implements IWUQuery
@@ -1655,7 +1657,7 @@ public:
     virtual void        setQueryText(const char *pstr);
     virtual void        setQueryName(const char *);
     virtual void        setQueryMainDefinition(const char * str);
-    virtual void        addAssociatedFile(WUFileType type, const char * name, const char * ip, const char * desc, unsigned crc);
+    virtual void        addAssociatedFile(WUFileType type, const char * name, const char * ip, const char * desc, unsigned crc, unsigned minActivity, unsigned maxActivity);
     virtual void        removeAssociatedFiles();
     virtual void        removeAssociatedFile(WUFileType type, const char * name, const char * desc);
 };
@@ -6967,6 +6969,15 @@ unsigned CLocalWUAssociated::getCrc() const
     return p->getPropInt("@crc", 0);
 }
 
+unsigned CLocalWUAssociated::getMinActivityId() const
+{
+    return p->getPropInt("@minActivity", 0);
+}
+
+unsigned CLocalWUAssociated::getMaxActivityId() const
+{
+    return p->getPropInt("@maxActivity", 0);
+}
 
 
 //=================================================================================================
@@ -7110,7 +7121,7 @@ void CLocalWUQuery::setQueryMainDefinition(const char * str)
     p->setProp("@main", str);
 }
 
-void CLocalWUQuery::addAssociatedFile(WUFileType type, const char * name, const char * ip, const char * desc, unsigned crc)
+void CLocalWUQuery::addAssociatedFile(WUFileType type, const char * name, const char * ip, const char * desc, unsigned crc, unsigned minActivity, unsigned maxActivity)
 {
     CriticalBlock block(crit);
     loadAssociated();
@@ -7125,6 +7136,10 @@ void CLocalWUQuery::addAssociatedFile(WUFileType type, const char * name, const
 
     if (crc)
         s->setPropInt("@crc", crc);
+    if (minActivity)
+        s->setPropInt("@minActivity", minActivity);
+    if (maxActivity)
+        s->setPropInt("@maxActivity", maxActivity);
     IConstWUAssociatedFile * q = new CLocalWUAssociated(LINK(s)); 
     associated.append(*q);
 }
@@ -10236,14 +10251,14 @@ IPropertyTree * resolveDefinitionInArchive(IPropertyTree * archive, const char *
     return module->queryPropTree(xpath);
 }
 
-extern WORKUNIT_API void associateLocalFile(IWUQuery * query, WUFileType type, const char * name, const char * description, unsigned crc)
+extern WORKUNIT_API void associateLocalFile(IWUQuery * query, WUFileType type, const char * name, const char * description, unsigned crc, unsigned minActivity, unsigned maxActivity)
 {
     StringBuffer hostname;
     queryHostIP().getIpText(hostname);
 
     StringBuffer fullPathname;
     makeAbsolutePath(name, fullPathname);
-    query->addAssociatedFile(type, fullPathname, hostname, description, crc);
+    query->addAssociatedFile(type, fullPathname, hostname, description, crc, minActivity, maxActivity);
 }
 
 extern WORKUNIT_API void descheduleWorkunit(char const * wuid)

common/workunit/workunit.hpp

@@ -366,6 +366,8 @@ interface IConstWUAssociatedFile : extends IInterface
     virtual IStringVal & getName(IStringVal & ret) const = 0;
     virtual IStringVal & getNameTail(IStringVal & ret) const = 0;
     virtual unsigned getCrc() const = 0;
+    virtual unsigned getMinActivityId() const = 0;
+    virtual unsigned getMaxActivityId() const = 0;
 };
 
 
@@ -400,7 +402,7 @@ interface IWUQuery : extends IConstWUQuery
     virtual void setQueryType(WUQueryType qt) = 0;
     virtual void setQueryText(const char * pstr) = 0;
     virtual void setQueryName(const char * pstr) = 0;
-    virtual void addAssociatedFile(WUFileType type, const char * name, const char * ip, const char * desc, unsigned crc) = 0;
+    virtual void addAssociatedFile(WUFileType type, const char * name, const char * ip, const char * desc, unsigned crc, unsigned minActivity, unsigned maxActivity) = 0;
     virtual void removeAssociatedFiles() = 0;
     virtual void setQueryMainDefinition(const char * str) = 0;
     virtual void removeAssociatedFile(WUFileType type, const char * name, const char * desc) = 0;
@@ -1459,7 +1461,8 @@ extern WORKUNIT_API void removeQuerySetAliasesFromNamedQuery(const char *querySe
 extern WORKUNIT_API void setQueryCommentForNamedQuery(const char *querySetName, const char *id, const char *comment);
 extern WORKUNIT_API void gatherLibraryNames(StringArray &names, StringArray &unresolved, IWorkUnitFactory &workunitFactory, IConstWorkUnit &cw, IPropertyTree *queryset);
 
-extern WORKUNIT_API void associateLocalFile(IWUQuery * query, WUFileType type, const char * name, const char * description, unsigned crc);
+//If we add any more parameters we should consider returning an object that can be updated
+extern WORKUNIT_API void associateLocalFile(IWUQuery * query, WUFileType type, const char * name, const char * description, unsigned crc, unsigned minActivity=0, unsigned maxActivity=0);
 
 interface ITimeReporter;
 extern WORKUNIT_API void updateWorkunitTimeStat(IWorkUnit * wu, StatisticScopeType scopeType, const char * scope, StatisticKind kind, const char * description, unsigned __int64 value);

ecl/hqlcpp/hqlcpp.cpp

@@ -1206,6 +1206,20 @@ void HqlCppInstance::addPluginsAsResource()
 }
 
 
+void HqlCppInstance::getActivityRange(unsigned cppIndex, unsigned & minActivityId, unsigned & maxActivityId)
+{
+    if (cppInfo.isItem(cppIndex))
+    {
+        minActivityId = cppInfo.item(cppIndex).minActivityId;
+        maxActivityId = cppInfo.item(cppIndex).maxActivityId;
+    }
+    else
+    {
+        minActivityId = 0;
+        maxActivityId = 0;
+    }
+}
+
 bool HqlCppInstance::useFunction(IHqlExpression * func)
 {
     assertex(func);
@@ -1804,10 +1818,12 @@ void HqlCppTranslator::cacheOptions()
     //Or where one debug options sets more than one option
     if (options.spanMultipleCpp)
     {
+        code->cppInfo.append(* new CppFileInfo(0)); // Add an entry for the main file which contains no activities
         options.activitiesPerCpp = wu()->getDebugValueInt("activitiesPerCpp", DEFAULT_ACTIVITIES_PER_CPP);
         curCppFile = 1;
     }
 
+    code->cppInfo.append(* new CppFileInfo(0));
     options.targetCompiler = DEFAULT_COMPILER;
     if (wu()->hasDebugValue("targetGcc"))
         options.targetCompiler = wu()->getDebugValueBool("targetGcc", false) ? GccCppCompiler : Vs6CppCompiler;
@@ -11934,7 +11950,7 @@ void HqlCppTranslator::buildFunctionDefinition(IHqlExpression * funcdef)
     if (options.spanMultipleCpp)
     {
         const bool inChildActivity = true;  // assume the worst
-        OwnedHqlExpr pass = getSizetConstant(cppIndexNextActivity(inChildActivity));
+        OwnedHqlExpr pass = getSizetConstant(beginFunctionGetCppIndex(0, inChildActivity));
         funcctx.addGroupPass(pass);
     }
     expandFunctionPrototype(proto, funcdef);

ecl/hqlcpp/hqlcpp.hpp

@@ -117,6 +117,7 @@ public:
     virtual void addWebServiceInfo(IPropertyTree *wsinfo) = 0;
     virtual void flushHints() = 0;
     virtual void flushResources(const char *filename, ICodegenContextCallback * ctxCallback) = 0;
+    virtual void getActivityRange(unsigned cppIndex, unsigned & minActivityId, unsigned & maxActivityId) = 0;
 };
 
 // A class used to hold the context about the expression being processed at this point in time.

ecl/hqlcpp/hqlcpp.ipp

@@ -120,6 +120,18 @@ public:
 };
 
 
+class CppFileInfo : public CInterface
+{
+public:
+    explicit CppFileInfo(unsigned activityId) : minActivityId(activityId), maxActivityId(activityId)
+    {
+    }
+
+public:
+    unsigned minActivityId;
+    unsigned maxActivityId;
+};
+
 class HQLCPP_API HqlCppInstance : public CInterface, public IHqlCppInstance
 {
 public:
@@ -138,6 +150,7 @@ public:
     virtual void addManifest(const char *filename){resources.addManifest(filename);}
     virtual void addManifestFromArchive(IPropertyTree *archive){resources.addManifestFromArchive(archive);}
     virtual void addWebServiceInfo(IPropertyTree *wsinfo){resources.addWebServiceInfo(wsinfo);}
+    virtual void getActivityRange(unsigned cppIndex, unsigned & minActivityId, unsigned & maxActivityId);
     
     bool useFunction(IHqlExpression * funcdef);
     void useInclude(const char * include);
@@ -167,6 +180,7 @@ public:
     StringAttr          wupathname;
     Owned<IPropertyTree> plugins;
     Owned<IFileIOStream> hintFile;
+    CIArrayOf<CppFileInfo> cppInfo;
 };
 
 //---------------------------------------------------------------------------
@@ -1122,7 +1136,7 @@ public:
     void useInclude(const char * name)                      { code->useInclude(name); }
     HqlCppInstance * queryCode() const                      { return code; }
     unsigned curSubGraphId(BuildCtx & ctx);
-    unsigned cppIndexNextActivity(bool isChildActivity);
+    unsigned beginFunctionGetCppIndex(unsigned activityId, bool isChildActivity);
 
     void buildAssignToTemp(BuildCtx & ctx, IHqlExpression * variable, IHqlExpression * expr);       // create a bound target for the variable and assign
     void queryAddResultDependancy(ABoundActivity & whoAmIActivity, IHqlExpression * seq, IHqlExpression * name);

ecl/hqlcpp/hqlecl.cpp

@@ -87,13 +87,13 @@ public:
     virtual void setSaveGeneratedFiles(bool value) { deleteGenerated = !value; }
 
 protected:
-    void addCppName(const char * filename);
+    void addCppName(const char * filename, unsigned minActivity, unsigned maxActivity);
     void addLibrariesToCompiler();
     void addWorkUnitAsResource();
     void calculateHash(IHqlExpression * expr);
     bool doCompile(ICppCompiler * compiler);
     void doExpand(HqlCppTranslator & translator);
-    void expandCode(const char * templateName, const char * ext, IHqlCppInstance * code, bool multiFile, unsigned pass, CompilerType compiler);
+    void expandCode(StringBuffer & filename, const char * templateName, bool isHeader, IHqlCppInstance * code, bool multiFile, unsigned pass, CompilerType compiler);
     void flushResources();
     bool generateCode(HqlQueryContext & query);
     bool generateFullFieldUsageStatistics(HqlCppTranslator & translator, IHqlExpression * query);
@@ -129,12 +129,12 @@ protected:
 
 static void processMetaCommands(HqlCppTranslator & translator, IWorkUnit * wu, HqlQueryContext & query, ICodegenContextCallback *ctxCallback);
 
-void HqlDllGenerator::addCppName(const char * filename)
+void HqlDllGenerator::addCppName(const char * filename, unsigned minActivity, unsigned maxActivity)
 {
     if (wu)
     {
         Owned<IWUQuery> query = wu->updateQuery();
-        associateLocalFile(query, FileTypeCpp, filename, pathTail(filename), 0);
+        associateLocalFile(query, FileTypeCpp, filename, pathTail(filename), 0, minActivity, maxActivity);
     }
 }
 
@@ -179,10 +179,16 @@ void HqlDllGenerator::addLibrariesToCompiler()
 }
 
 
-void HqlDllGenerator::expandCode(const char * templateName, const char * ext, IHqlCppInstance * code, bool multiFile, unsigned pass, CompilerType compiler)
+void HqlDllGenerator::expandCode(StringBuffer & filename, const char * templateName, bool isHeader, IHqlCppInstance * code, bool multiFile, unsigned pass, CompilerType compiler)
 {
+    filename.clear().append(wuname);
+    if (pass != 0)
+        filename.append("_").append(pass);
+    const char * ext = isHeader ? ".hpp" : ".cpp";
+    filename.append(ext);
+
     StringBuffer fullname;
-    addDirectoryPrefix(fullname, targetDir).append(wuname).append(ext);
+    addDirectoryPrefix(fullname, targetDir).append(filename);
 
     Owned<IFile> out = createIFile(fullname.str());
     Owned<ITemplateExpander> expander = createTemplateExpander(out, templateName, template_dir);
@@ -206,7 +212,11 @@ void HqlDllGenerator::expandCode(const char * templateName, const char * ext, IH
     totalGeneratedSize += out->size();
 
     if (!deleteGenerated)
-        addCppName(fullname);
+    {
+        unsigned minActivity, maxActivity;
+        code->getActivityRange(pass, minActivity, maxActivity);
+        addCppName(fullname, minActivity, maxActivity);
+    }
 }
 
 
@@ -499,21 +509,17 @@ void HqlDllGenerator::doExpand(HqlCppTranslator & translator)
     cycle_t startCycles = get_cycles_now();
     unsigned numExtraFiles = translator.getNumExtraCppFiles();
     bool isMultiFile = translator.spanMultipleCppFiles() && (numExtraFiles != 0);
-    expandCode(MAIN_MODULE_TEMPLATE, ".cpp", code, isMultiFile, 0, translator.queryOptions().targetCompiler);
+    CompilerType targetCompiler = translator.queryOptions().targetCompiler;
+
     StringBuffer fullname;
-    fullname.append(wuname).append(".cpp");
+    expandCode(fullname, MAIN_MODULE_TEMPLATE, false, code, isMultiFile, 0, targetCompiler);
     sourceFiles.append(fullname);
     if (isMultiFile)
     {
-        expandCode(HEADER_TEMPLATE, ".hpp", code, true, 0, translator.queryOptions().targetCompiler);
+        expandCode(fullname, HEADER_TEMPLATE, true, code, true, 0, targetCompiler);
         for (unsigned i= 0; i < translator.getNumExtraCppFiles(); i++)
         {
-            StringBuffer fullext;
-            fullext.append("_").append(i+1).append(".cpp");
-            expandCode(CHILD_MODULE_TEMPLATE, fullext, code, true, i+1, translator.queryOptions().targetCompiler);
-
-            StringBuffer fullname;
-            fullname.append(wuname).append("_").append(i+1).append(".cpp");
+            expandCode(fullname, CHILD_MODULE_TEMPLATE, false, code, true, i+1, targetCompiler);
             sourceFiles.append(fullname);
         }
     }

ecl/hqlcpp/hqlhtcpp.cpp

@@ -1357,7 +1357,7 @@ unsigned HqlCppTranslator::getConsistentUID(IHqlExpression * ptr)
 }
 
 
-unsigned HqlCppTranslator::cppIndexNextActivity(bool isChildActivity)
+unsigned HqlCppTranslator::beginFunctionGetCppIndex(unsigned activityId, bool isChildActivity)
 {
     activitiesThisCpp++;
     if (activitiesThisCpp > options.activitiesPerCpp)
@@ -1368,8 +1368,16 @@ unsigned HqlCppTranslator::cppIndexNextActivity(bool isChildActivity)
         {
             curCppFile++;
             activitiesThisCpp = 1;
+            code->cppInfo.append(* new CppFileInfo(activityId));
         }
     }
+    CppFileInfo & curCpp = code->cppInfo.tos();
+    if (activityId)
+    {
+        if (curCpp.minActivityId == 0)
+            curCpp.minActivityId = activityId;
+        curCpp.maxActivityId = activityId;
+    }
     return curCppFile;
 }
 
@@ -2142,6 +2150,7 @@ void ActivityInstance::moveDefinitionToHeader()
         //remove this class from the c++ file and include it in the header file instead
         includedInHeader = true;
         classGroupStmt->setIncluded(false);
+        addAttributeBool("helperinheader", true);
 
         BuildCtx headerctx(*translator.code, parentHelpersAtom);
         headerctx.addAlias(classStmt);
@@ -2163,7 +2172,7 @@ void ActivityInstance::buildPrefix()
 {
     StringBuffer s;
 
-    sourceFileSequence.setown(getSizetConstant(translator.cppIndexNextActivity(isChildActivity())));
+    sourceFileSequence.setown(getSizetConstant(translator.beginFunctionGetCppIndex(activityId, isChildActivity())));
     if (containerActivity && colocalMember)
         containerActivity->noteChildActivityLocation(sourceFileSequence);
 

esp/scm/ws_workunits.ecm

@@ -134,6 +134,8 @@ ESPStruct [nil_remove] ECLHelpFile
     [min_ver("1.32")] string Description;
     [min_ver("1.43")] int64 FileSize;
     [min_ver("1.44")] unsigned PID(0);
+    [min_ver("1.58")] unsigned minActivityId(0);
+    [min_ver("1.58")] unsigned maxActivityId(0);
 };
 
 //  ===========================================================================
@@ -1753,7 +1755,7 @@ ESPresponse [exceptions_inline, nil_remove] WUGetArchiveFileResponse
 };
 
 ESPservice [
-    version("1.57"), default_client_version("1.57"),
+    version("1.58"), default_client_version("1.58"),
     noforms,exceptions_inline("./smc_xslt/exceptions.xslt"),use_method_name] WsWorkunits
 {
     ESPmethod [resp_xsl_default("/esp/xslt/workunits.xslt")]     WUQuery(WUQueryRequest, WUQueryResponse);

esp/services/ws_workunits/ws_workunitsHelpers.cpp

@@ -1673,6 +1673,11 @@ void WsWuInfo::getHelpFiles(IConstWUQuery* query, WUFileType type, IArrayOf<IEsp
                 if (getFileSize(name.str(), Ip.str(), fileSize))
                     h->setFileSize(fileSize);
             }
+            if (version >= 1.58)
+            {
+                h->setMinActivityId(cur.getMinActivityId());
+                h->setMaxActivityId(cur.getMaxActivityId());
+            }
         }
         helpers.append(*h.getLink());
     }

tools/wutool/wutool.cpp

@@ -817,7 +817,7 @@ protected:
             query->setQueryName("qname");
             query->setQueryMainDefinition("fred");
             query->setQueryType(QueryTypeEcl);
-            query->addAssociatedFile(FileTypeCpp, "myfile", "1.2.3.4", "Description", 53);
+            query->addAssociatedFile(FileTypeCpp, "myfile", "1.2.3.4", "Description", 53, 3, 4);
             createWu->setState(WUStateCompleted);
             createWu.clear();
         }
@@ -837,6 +837,8 @@ protected:
         ASSERT(streq(file->getName(s).str(), "myfile"));
         ASSERT(file->getCrc()==53);
         ASSERT(file->getType()==FileTypeCpp);
+        ASSERT(file->getMinActivityId()==3);
+        ASSERT(file->getMaxActivityId()==4);
         ASSERT(streq(file->getIp(s).str(), "1.2.3.4"));
         query.clear();
         wu.clear();