Fix: #549 Commonize similar code into roxiehelper

Several filename resolution/manipulation functions are (almost) the
same between roxie and hthor, and the Roxie ones did not consider stand
alone mode.
Removed them from roxie/hthor, and moved them into roxiehelper. Also
enhance to pass workunit into expandLogicalFilename for optional scope
resolution, and pass in resolveLocally flag to signal building of local
file system path. Modify roxie/hthor to call these modified common functions.
This is a step towards enhancing roxie to handle standAlone diskWrite
functionality, which is a step towards adding indexWrite ability (issue #491)

Signed-off-by: William Whitehead <william.whitehead@lexisnexis.com>

common/roxiehelper/roxiehelper.cpp

@@ -1317,3 +1317,57 @@ ROXIEHELPER_API IOrderedOutputSerializer * createOrderedOutputSerializer(FILE *
 {
     return new COrderedOutputSerializer(_outFile);
 }
+
+//=====================================================================================================
+
+ROXIEHELPER_API StringBuffer & mangleHelperFileName(StringBuffer & out, const char * in, const char * wuid, unsigned int flags)
+{
+    out = in;
+    if (flags & (TDXtemporary | TDXjobtemp))
+        out.append("__").append(wuid);
+    return out;
+}
+
+ROXIEHELPER_API StringBuffer & mangleLocalTempFilename(StringBuffer & out, char const * in)
+{
+    char const * start = in;
+    while(true)
+    {
+        char const * end = strstr(start, "::");
+        if(end)
+        {
+            out.append(end-start, start).append("__scope__");
+            start = end + 2;
+        }
+        else
+        {
+            out.append(start);
+            break;
+        }
+    }
+    return out;
+}
+
+ROXIEHELPER_API StringBuffer & expandLogicalFilename(StringBuffer & logicalName, const char * fname, IConstWorkUnit * wu, bool resolveLocally)
+{
+    if (fname[0]=='~')
+        logicalName.append(fname+1);
+    else if (resolveLocally)
+    {
+        StringBuffer sb(fname);
+        sb.replaceString("::",PATHSEPSTR);
+        makeAbsolutePath(sb.str(), logicalName.clear());
+    }
+    else
+    {
+        SCMStringBuffer lfn;
+        if (wu)
+        {
+            wu->getScope(lfn);
+            if(lfn.length())
+                logicalName.append(lfn.s).append("::");
+        }
+        logicalName.append(fname);
+    }
+    return logicalName;
+}

common/roxiehelper/roxiehelper.hpp

@@ -23,6 +23,7 @@
 #include "roxiehelper.ipp"
 #include "roxiemem.hpp"
 #include "mpbase.hpp"
+#include "workunit.hpp"
 
 #ifdef _WIN32
  #ifdef ROXIEHELPER_EXPORTS
@@ -209,4 +210,8 @@ private:
 
 //==============================================================================================================
 
+ROXIEHELPER_API StringBuffer & mangleHelperFileName(StringBuffer & out, const char * in, const char * wuid, unsigned int flags);
+ROXIEHELPER_API StringBuffer & mangleLocalTempFilename(StringBuffer & out, char const * in);
+ROXIEHELPER_API StringBuffer & expandLogicalFilename(StringBuffer & logicalName, const char * fname, IConstWorkUnit * wu, bool resolveLocally);
+
 #endif // ROXIEHELPER_HPP

ecl/eclagent/agentctx.hpp

@@ -104,6 +104,8 @@ struct IAgentContext : extends IGlobalCodeContext
     virtual IGroup *getHThorGroup(StringBuffer &grpnameout) = 0;
 
     virtual unsigned __int64 queryStopAfter() = 0;
+    
+    virtual const char *queryWuid() = 0;
 };
 
 struct WorkunitUpdate : public Owned<IWorkUnit>

ecl/eclagent/eclagent.cpp

@@ -1357,7 +1357,7 @@ bool EclAgent::expandLogicalName(StringBuffer & fullname, const char * logicalNa
 ILocalOrDistributedFile *EclAgent::resolveLFN(const char *fname, const char *errorTxt, bool optional, bool noteRead, bool isWrite, StringBuffer * expandedlfn)
 {
     StringBuffer lfn;
-    expandLogicalFilename(lfn, fname, *this);
+    expandLogicalFilename(lfn, fname, queryWorkUnit(), resolveFilesLocally);
     if (resolveFilesLocally && *fname != '~')
     {
         StringBuffer name;
@@ -1501,7 +1501,7 @@ __int64 EclAgent::countDiskFile(__int64 id, IHThorCountFileArg & arg)
     }
 
     StringBuffer mangled;
-    mangleHelperFileName(mangled, arg.getFileName(), *this, arg.getFlags());
+    mangleHelperFileName(mangled, arg.getFileName(), queryWuid(), arg.getFlags());
     Owned<ILocalOrDistributedFile> ldFile = resolveLFN(mangled, "CountDisk", 0 != (arg.getFlags() & TDRoptional));
     if (ldFile) 
     {
@@ -2653,6 +2653,11 @@ char *EclAgent::getWuid()
     return strdup(wuid);
 }
 
+const char *EclAgent::queryWuid()
+{
+    return wuid.get();
+}
+
 char * EclAgent::getDaliServers()
 {
     if (!isCovenActive())

ecl/eclagent/eclagent.ipp

@@ -249,10 +249,11 @@ public:
         return ctx->getHThorGroup(name);
     }
     
+    virtual const char *queryWuid()
+    {
+        return ctx->queryWuid();
+    }
     
-    
-    
-
 protected:
     IAgentContext * ctx;
 };
@@ -533,7 +534,7 @@ public:
     virtual char *resolveName(const char *in, char *out, unsigned outlen);
     virtual void logFileAccess(IDistributedFile * file, char const * component, char const * type);
     virtual IRecordLayoutTranslatorCache * queryRecordLayoutTranslatorCache() const { return rltCache; }
-    virtual ILocalOrDistributedFile  *resolveLFN(const char *logicalName, const char *errorTxt=NULL, bool optional=false, bool noteRead=true, bool write=false, StringBuffer * expandedlfn=NULL);//@@
+    virtual ILocalOrDistributedFile  *resolveLFN(const char *logicalName, const char *errorTxt=NULL, bool optional=false, bool noteRead=true, bool write=false, StringBuffer * expandedlfn=NULL);
 
     virtual void executeThorGraph(const char * graphName);
     virtual void executeGraph(const char * graphName, bool realThor, size32_t parentExtractSize, const void * parentExtract);
@@ -580,6 +581,7 @@ public:
     virtual unsigned __int64 getFileOffset(const char *logicalPart) { UNIMPLEMENTED; return 0; }
     virtual char *getOutputDir() { UNIMPLEMENTED; }
     virtual char *getWuid();
+    virtual const char *queryWuid();
     virtual IDistributedFileTransaction *querySuperFileTransaction();
     virtual unsigned getPriority() const { return 0; }
     virtual char *getPlatform() { return strdup(isStandAloneExe ? "standalone" : "hthor"); }
@@ -666,7 +668,6 @@ public:
     
     IGroup *getHThorGroup(StringBuffer &out);
     
-    
 };
 
 //---------------------------------------------------------------------------

ecl/hthor/hthor.cpp

@@ -98,62 +98,6 @@ inline bool checkIsCompressed(unsigned int flags, size32_t fixedSize, bool group
 
 //=====================================================================================================
 
-StringBuffer & mangleHelperFileName(StringBuffer & out, const char * in, IAgentContext &agent, unsigned int flags)
-{
-    out = in;
-    if (flags & (TDXtemporary | TDXjobtemp))
-    {
-        char * wuid = agent.queryCodeContext()->getWuid();
-        out.append("__").append(wuid);
-        free(wuid);
-    }
-    return out;
-}
-
-
-StringBuffer & expandLogicalFilename(StringBuffer & logicalName, const char * fname, IAgentContext &agent)
-{
-    if (fname[0]=='~')
-        logicalName.append(fname+1);
-    else if (agent.queryResolveFilesLocally())
-        logicalName.append(fname);
-    else
-    {
-        SCMStringBuffer lfn;
-        agent.queryWorkUnit()->getScope(lfn);
-        if(lfn.length())
-            logicalName.append(lfn.s).append("::");
-        logicalName.append(fname);
-    }
-    if (agent.queryResolveFilesLocally())
-    {
-        StringBuffer sb(logicalName.str());
-        sb.replaceString("::",PATHSEPSTR);
-        makeAbsolutePath(sb.str(), logicalName.clear());
-    }
-    return logicalName;
-}
-
-StringBuffer & mangleLocalTempFilename(StringBuffer & out, char const * in)
-{
-    char const * start = in;
-    while(true)
-    {
-        char const * end = strstr(start, "::");
-        if(end)
-        {
-            out.append(end-start, start).append("__scope__");
-            start = end + 2;
-        }
-        else
-        {
-            out.append(start);
-            break;
-        }
-    }
-    return out;
-}
-
 //=====================================================================================================
 
 CRowBuffer::CRowBuffer(IRecordSize * _recsize, bool _grouped) : recsize(_recsize), grouped(_grouped)
@@ -455,7 +399,7 @@ void CHThorDiskWriteActivity::done()
 
 void CHThorDiskWriteActivity::resolve()
 {
-    mangleHelperFileName(mangledHelperFileName, helper.getFileName(), agent, helper.getFlags());
+    mangleHelperFileName(mangledHelperFileName, helper.getFileName(), agent.queryWuid(), helper.getFlags());
     assertex(mangledHelperFileName.str());
     if((helper.getFlags() & TDXtemporary) == 0)
     {
@@ -710,7 +654,7 @@ void CHThorDiskWriteActivity::publish()
     properties.setPropInt("@formatCrc", helper.getFormatCrc());
 
     StringBuffer lfn;
-    expandLogicalFilename(lfn, mangledHelperFileName.str(), agent);
+    expandLogicalFilename(lfn, mangledHelperFileName.str(), agent.queryWorkUnit(), agent.queryResolveFilesLocally());
     CDfsLogicalFileName logicalName;
     if (agent.queryResolveFilesLocally())
         logicalName.allowOsPath(true);
@@ -986,7 +930,7 @@ CHThorIndexWriteActivity::CHThorIndexWriteActivity(IAgentContext &_agent, unsign
 {
     incomplete = false;
     StringBuffer lfn;
-    expandLogicalFilename(lfn, helper.getFileName(), agent);
+    expandLogicalFilename(lfn, helper.getFileName(), agent.queryWorkUnit(), agent.queryResolveFilesLocally());
     if (!agent.queryResolveFilesLocally())
     {
         Owned<IDistributedFile> f = queryDistributedFileDirectory().lookup(lfn, agent.queryCodeContext()->queryUserDescriptor(), true);
@@ -1202,7 +1146,7 @@ void CHThorIndexWriteActivity::execute()
     if (!agent.queryResolveFilesLocally())
     {
         dfile.setown(queryDistributedFileDirectory().createNew(desc));
-        expandLogicalFilename(lfn, helper.getFileName(), agent);
+        expandLogicalFilename(lfn, helper.getFileName(), agent.queryWorkUnit(), agent.queryResolveFilesLocally());
         dfile->attach(lfn.str(),NULL, agent.queryCodeContext()->queryUserDescriptor());
         agent.logFileAccess(dfile, "HThor", "CREATED");
     }
@@ -7517,7 +7461,7 @@ void CHThorDiskReadBaseActivity::done()
 
 void CHThorDiskReadBaseActivity::resolve()
 {
-    mangleHelperFileName(mangledHelperFileName, helper.getFileName(), agent, helper.getFlags());
+    mangleHelperFileName(mangledHelperFileName, helper.getFileName(), agent.queryWuid(), helper.getFlags());
     if (helper.getFlags() & TDXtemporary)
     {
         StringBuffer mangledFilename;

ecl/hthor/hthor.hpp

@@ -86,7 +86,6 @@ struct IHThorActivity : implements IActivityBase
     virtual void setBoundGraph(IHThorBoundLoopGraph * graph) = 0;
 };
 
-extern HTHOR_API StringBuffer & expandLogicalFilename(StringBuffer & logicalName, const char * fname, IAgentContext &agent);
 
 extern HTHOR_API IHThorActivity *createDiskWriteActivity(IAgentContext &, unsigned _activityId, unsigned _subgraphId, IHThorDiskWriteArg &arg, ThorActivityKind kind);
 extern HTHOR_API IHThorActivity *createIterateActivity(IAgentContext &, unsigned _activityId, unsigned _subgraphId, IHThorIterateArg &arg, ThorActivityKind kind);
@@ -219,6 +218,5 @@ extern HTHOR_API IHThorException * makeHThorException(ThorActivityKind kind, uns
 extern HTHOR_API IEngineRowAllocator * createHThorRowAllocator(roxiemem::IRowManager & _rowManager, IOutputMetaData * _meta, unsigned _activityId, unsigned _allocatorId);
 extern HTHOR_API IRtlRowCallback * queryHThorRtlRowCallback();
 
-StringBuffer & mangleHelperFileName(StringBuffer & out, const char * in, IAgentContext &agent, unsigned int flags);
 
 #endif // HTHOR_INCL

roxie/ccd/ccdserver.cpp

@@ -10469,43 +10469,6 @@ public:
 
 //==================================================================================
 
-extern StringBuffer & expandLogicalFilename(StringBuffer & logicalName, const char * fname, IConstWorkUnit * wu)
-{
-    if (fname[0]=='~')
-        logicalName.append(fname+1);
-    else
-    {
-        SCMStringBuffer lfn;
-        if (wu)
-            wu->getScope(lfn);
-        if(lfn.length())
-            logicalName.append(lfn.s).append("::");
-        logicalName.append(fname);
-    }
-    return logicalName;
-}
-
-StringBuffer & mangleLocalTempFilename(StringBuffer & out, char const * in)
-{
-    char const * start = in;
-    while(true)
-    {
-        char const * end = strstr(start, "::");
-        if(end)
-        {
-            out.append(end-start, start).append("__scope__");
-            start = end + 2;
-        }
-        else
-        {
-            out.append(start);
-            break;
-        }
-    }
-    return out;
-}
-
-//=====================================================================================================
 class CRoxieServerDiskWriteActivity : public CRoxieServerInternalSinkActivity
 {
 protected:
@@ -10577,7 +10540,7 @@ protected:
         assertex(rawLogicalName);
         if((helper.getFlags() & TDXtemporary) == 0)
         {
-            expandLogicalFilename(lfn, rawLogicalName, NULL); // MORE - should probably pass wu if we have one?
+            expandLogicalFilename(lfn, rawLogicalName, NULL, false); // MORE - should probably pass wu if we have one?
             CDfsLogicalFileName logicalName;
             if (!logicalName.setValidate(lfn.str()))
                 throw MakeStringException(99, "Cannot write %s, invalid logical name", lfn.str());

roxie/ccd/ccdserver.hpp

@@ -511,5 +511,4 @@ extern IRoxieServerActivityFactory *createRoxieServerPrefetchProjectActivityFact
 extern IRoxieServerActivityFactory *createRoxieServerStreamedIteratorActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind);
 extern IRoxieServerActivityFactory *createRoxieServerWhenActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind);
 
-extern StringBuffer & expandLogicalFilename(StringBuffer & logicalName, const char * fname, IConstWorkUnit * wu);
 #endif

roxie/ccd/ccdstate.cpp

@@ -517,7 +517,7 @@ public:
     virtual const IResolvedFile *lookupFileName(const char *_fileName, bool opt, bool cache) const
     {
         StringBuffer fileName;
-        expandLogicalFilename(fileName, _fileName, NULL);   // MORE - if we have a wu, and we have not yet got rid of the concept of scope, we should use it here
+        expandLogicalFilename(fileName, _fileName, NULL, false);   // MORE - if we have a wu, and we have not yet got rid of the concept of scope, we should use it here
 
         const IResolvedFile *result = lookupFile(fileName, cache);
         if (!result)