Merge branch 'candidate-6.4.0'

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,60 @@
+<!-- Thank you for submitting a pull request to the HPCC project
+
+ PLEASE READ the following before proceeding.
+
+ Lines in comments may be deleted from the comment before you submit.
+ Other lines should be modified appropriately and left in place.
+
+ This project only accepts pull requests related to open JIRA issues.
+ If suggesting a new feature or change, please discuss it in a JIRA issue first.
+ If fixing a bug, there should be an issue describing it with steps to reproduce.
+ The title line of the pull request (and of each commit within it) should refer to the
+ associated issue using the format:
+
+ HPCC-nnnnn Short description of issue
+
+ This will allow the Jira ticket to be automatically updated to refer to this pull request,and
+ and will ensure that the automatically-generated changelog is properly formatted.
+ Where a pull request contains a single commit the pull request title will be set automatically,
+ assuming that the commit has followed the proper guidelines.
+
+ Please go over all the following points, and put an `x` in all the boxes that apply. You may find
+ it easier to press the 'Create' button first then click on the checkboxes to edit the comment.
+-->
+
+## Type of change:
+- [ ] This change is a bug fix (non-breaking change which fixes an issue).
+- [ ] This change is a new feature (non-breaking change which adds functionality).
+- [ ] This change is a breaking change (fix or feature that will cause existing behavior to change).
+- [ ] This change alters the query API (existing queries will have to be recompiled)
+
+## Checklist:
+- [ ] My code follows the code style of this project.
+  - [ ] My code does not create any new warnings from compiler, build system, or lint.
+- [ ] My change requires a change to the documentation.
+  - [ ] I have updated the documentation accordingly, or...
+  - [ ] I have created a JIRA ticket to update the documentation.
+  - [ ] Any new interfaces or exported functions are appropriately commented.
+- [ ] I have read the CONTRIBUTORS document.
+- [ ] The change has been fully tested:
+  - [ ] I have added tests to cover my changes.
+  - [ ] All new and existing tests passed.
+  - [ ] I have checked that this change does not introduce memory leaks.
+  - [ ] I have used Valgrind or similar tools to check for potential issues.
+- [ ] I have given due consideration to all of the following potential concerns:
+  - [ ] Scalability
+  - [ ] Performance
+  - [ ] Security
+  - [ ] Thread-safety
+  - [ ] Premature optimization
+  - [ ] Existing deployed queries will not be broken
+  - [ ] This change fixes the problem, not just the symptom
+  - [ ] The target branch of this pull request is appropriate for such a change.
+- [ ] There are no similar instances of the same problem that should be addressed
+  - [ ] I have addressed them here
+  - [ ] I have raised JIRA issues to address them separately
+
+## Testing:
+<!-- Please describe how this change has been tested.-->
+
+<!-- Thank you for taking the time to submit this pull request and to answer all of the above-->

ecl/ecl-bundle/ecl-bundle.cpp

@@ -118,9 +118,10 @@ unsigned doPipeCommand(StringBuffer &output, const char *cmd, const char *args,
         if (input)
             printf("with input %s\n", input);
     }
-    unsigned ret = runExternalCommand(output, runcmd, input);
+    StringBuffer error;
+    unsigned ret = runExternalCommand(output, error, runcmd, input);
     if (optVerbose && (ret > 0))
-        printf("%s return code was %d\n", cmd, ret);
+        printf("%s return code was %d\n%s\n", cmd, ret, error.str());
     return ret;
 }
 
@@ -414,7 +415,11 @@ public:
                                     " [ (UTF8) COUNT(B.dependsOn) ] + B.dependsOn + "
                                     " [ (UTF8) #IFDEFINED(B.platformVersion, '')]", bundleName.str());
             if (doPipeCommand(output, queryEclccPath(), eclOpts.str(), bundleCmd) > 0)
+            {
+                if (optVerbose)
+                    printf("eclcc reported:\n%s", output.str());
                 throw MakeStringException(0, "%s cannot be parsed as a bundle\n", suppliedname);
+            }
             // output should contain [ 'name', 'version', etc ... ]
             if (optVerbose)
                 printf("Bundle info from ECL compiler: %s\n", output.str());

ecl/hql/hqlfold.cpp

@@ -681,11 +681,10 @@ bool checkExternFoldable(IHqlExpression* expr, unsigned foldOptions, StringBuffe
     return true;
 }
 
-IValue * doFoldExternalCall(IHqlExpression* expr, unsigned foldOptions, ITemplateContext *templateContext, const char *library, const char *entrypoint)
+void *loadExternalEntryPoint(IHqlExpression* expr, unsigned foldOptions, ITemplateContext *templateContext, const char *library, const char *entrypoint, HINSTANCE &hDLL)
 {
     IHqlExpression * funcdef = expr->queryExternalDefinition();
     IHqlExpression *body = funcdef->queryChild(0);
-
     // Get the handle to the library and procedure.
 #ifdef __APPLE__
     StringBuffer fullLibraryPath;
@@ -713,13 +712,13 @@ IValue * doFoldExternalCall(IHqlExpression* expr, unsigned foldOptions, ITemplat
 #endif
 #endif
 
-    HINSTANCE hDLL=LoadSharedObject(library, false, false);
+    hDLL=LoadSharedObject(library, false, false);
     if (!LoadSucceeded(hDLL))
     {
         if (body->hasAttribute(templateAtom))
-            throw MakeStringException(ERR_SVC_LOADLIBFAILED, "Error happened when trying to load library %s for template helper function", library);
+            throw MakeStringException(ERR_SVC_LOADLIBFAILED, "Error when trying to load library %s for template helper function", library);
         if (foldOptions & HFOthrowerror)
-            throw MakeStringException(ERR_SVC_LOADLIBFAILED, "Error happened when trying to load library %s", library);
+            throw MakeStringException(ERR_SVC_LOADLIBFAILED, "Error when trying to load library %s", library);
         return NULL;
     }
     void* fh = GetSharedProcedure(hDLL, entrypoint);
@@ -727,9 +726,19 @@ IValue * doFoldExternalCall(IHqlExpression* expr, unsigned foldOptions, ITemplat
     {
         FreeSharedObject(hDLL);
         if (foldOptions & HFOthrowerror)
-            throw MakeStringException(ERR_SVC_LOADFUNCFAILED, "Error happened when trying to load procedure %s from library %s", entrypoint, library);
+            throw MakeStringException(ERR_SVC_LOADFUNCFAILED, "Error when trying to load procedure %s from library %s", entrypoint, library);
         return NULL;
     }
+    return fh;
+}
+
+IValue * doFoldExternalCall(IHqlExpression* expr, unsigned foldOptions, ITemplateContext *templateContext, const char *library, const char *entrypoint, void *fh)
+{
+    // NOTE - on OSX there are compiler bugs that prevent exceptions thrown from within this function from properly unwinding.
+    // Hence anything that can throw an exception should be pre-checked in one of the functions above.
+
+    IHqlExpression * funcdef = expr->queryExternalDefinition();
+    IHqlExpression *body = funcdef->queryChild(0);
 
     // create a FuncCallStack to generate a stack used to pass parameters to 
     // the called function
@@ -1272,7 +1281,6 @@ IValue * doFoldExternalCall(IHqlExpression* expr, unsigned foldOptions, ITemplat
 #endif //win32
     }
     catch (...) {
-        FreeSharedObject(hDLL);
         if(retCharStar || charStarInParam) { // Char* return type, need to free up tgt.
             free(tgt);
         }
@@ -1282,7 +1290,6 @@ IValue * doFoldExternalCall(IHqlExpression* expr, unsigned foldOptions, ITemplat
         return NULL;
     }
 
-    // NOTE - we do not call FreeSharedObject(hDLL); here - the embedded language folding requires that the dll stay loaded, and it's also more efficient for other cases
 
     IValue* result = NULL;
 
@@ -1366,7 +1373,12 @@ IValue * foldExternalCall(IHqlExpression* expr, unsigned foldOptions, ITemplateC
     StringBuffer entry;
     if (!checkExternFoldable(expr, foldOptions, library, entry))
         return NULL;
-    return doFoldExternalCall(expr, foldOptions, templateContext, library.str(), entry.str());
+    // NOTE - we do not call FreeSharedObject(hDLL) - the embedded language folding requires that the dll stay loaded, and it's also more efficient for other cases
+    HINSTANCE hDll;
+    void *funcptr = loadExternalEntryPoint(expr, foldOptions, templateContext, library.str(), entry.str(), hDll);
+    if (!funcptr)
+        return NULL;
+    return doFoldExternalCall(expr, foldOptions, templateContext, library.str(), entry.str(), funcptr);
 }
 
 //------------------------------------------------------------------------------------------

ecl/hqlcpp/hqlcerrors.hpp

@@ -273,6 +273,7 @@
 #define HQLWRN_OnlyLocalMergeJoin               4545
 #define HQLWRN_WorkflowDependParameter          4546
 #define HQLWRN_OutputScalarInsideChildQuery     4547
+#define HQLWRN_GlobalDatasetFromChildQuery      4548
 
 //Temporary errors
 #define HQLERR_OrderOnVarlengthStrings          4601
@@ -564,6 +565,7 @@
 #define HQLWRN_OnlyLocalMergeJoin_Text          "Only LOCAL versions of %s are currently supported on THOR"
 #define HQLWRN_WorkflowDependParameter_Text     "Workflow action %s appears to be dependent upon a parameter"
 #define HQLWRN_OutputScalarInsideChildQuery_Text "Output(%s) of single value inside a child query has undefined behaviour"
+#define HQLWRN_GlobalDatasetFromChildQuery_Text "Global dataset expression (%s) is used in a child query"
 
 #define HQLERR_DistributionVariableLengthX_Text "DISTRIBUTION does not support variable length field '%s'"
 #define HQLERR_DistributionUnsupportedTypeXX_Text "DISTRIBUTION does not support field '%s' with type %s"

ecl/hqlcpp/hqlttcpp.cpp

@@ -8506,6 +8506,15 @@ IHqlExpression * AutoScopeMigrateTransformer::createTransformed(IHqlExpression *
             s.append("[").append(expr->queryName()).append("] ");
         s.append("as an item to hoist");
         DBGLOG("%s", s.str());
+        if (extra->globalInsideChild)
+        {
+            StringBuffer nameText;
+            if (expr->queryName())
+                nameText.append(expr->queryName());
+            else
+                getExprECL(expr, nameText);
+            translator.reportWarning(CategoryEfficiency, SeverityIgnore, queryActiveLocation(expr), HQLWRN_GlobalDatasetFromChildQuery, HQLWRN_GlobalDatasetFromChildQuery_Text, nameText.str());
+        }
 
         GlobalAttributeInfo info("jobtemp::auto","auto", transformed);
         info.extractGlobal(NULL, translator.getTargetClusterType());

esp/bindings/http/client/httpclient.cpp

@@ -60,15 +60,15 @@ IHttpClient* CHttpClientContext::createHttpClient(const char* proxy, const char*
         CriticalBlock b(m_sscrit);
         if(m_ssctx.get() == NULL)
         {
+            StringBuffer libName;
+            IEspPlugin *pplg = loadPlugin(libName.append(SharedObjectPrefix).append(SSLIB).append(SharedObjectExtension));
+            if (!pplg)
+                throw MakeStringException(-1, "dll/shared-object %s can't be loaded", libName.str());
+
             if(m_config.get() == NULL)
             {
                 createSecureSocketContext_t xproc = NULL;
-                IEspPlugin *pplg = loadPlugin(SSLIB);
-                if (pplg)
-                    xproc = (createSecureSocketContext_t) pplg->getProcAddress("createSecureSocketContext");
-                else
-                    throw MakeStringException(-1, "dll/shared-object %s can't be loaded", SSLIB);
-
+                xproc = (createSecureSocketContext_t) pplg->getProcAddress("createSecureSocketContext");
                 if (xproc)
                     m_ssctx.setown(xproc(ClientSocket));
                 else
@@ -77,12 +77,7 @@ IHttpClient* CHttpClientContext::createHttpClient(const char* proxy, const char*
             else
             {
                 createSecureSocketContextEx2_t xproc = NULL;
-                IEspPlugin *pplg = loadPlugin(SSLIB);
-                if (pplg)
-                    xproc = (createSecureSocketContextEx2_t) pplg->getProcAddress("createSecureSocketContextEx2");
-                else
-                    throw MakeStringException(-1, "dll/shared-object %s can't be loaded", SSLIB);
-
+                xproc = (createSecureSocketContextEx2_t) pplg->getProcAddress("createSecureSocketContextEx2");
                 if (xproc)
                     m_ssctx.setown(xproc(m_config.get(),ClientSocket));
                 else

esp/platform/espplugin.ipp

@@ -51,7 +51,7 @@ public:
         SharedObject::load(m_plugin.str(), true);       // I'm not really sure what this should be - if global (as default) there will be clashes between multiple dloads
         if (!loaded())
         {
-            ERRLOG("ESP Failed to load shared object (%s)", m_plugin.str());    
+            ERRLOG("Failed to load shared object (%s)", m_plugin.str());
             return false;
         }
         return true;

esp/scm/ws_workunits.ecm

@@ -690,6 +690,9 @@ ESPrequest WUInfoRequest
     [min_ver("1.16")] bool IncludeWorkflows(true);
     [min_ver("1.39")] bool IncludeXmlSchemas(false);
     [min_ver("1.47")] bool IncludeResourceURLs(false);
+    [min_ver("1.66")] bool IncludeECL(true);
+    [min_ver("1.66")] bool IncludeHelpers(true);
+    [min_ver("1.66")] bool IncludeAllowedClusters(true);
     [min_ver("1.16")] bool SuppressResultSchemas(false);
     [min_ver("1.25")] string ThorSlaveIP;
 };
@@ -1795,7 +1798,7 @@ ESPresponse [exceptions_inline, nil_remove] WUGetNumFileToCopyResponse
 
 ESPservice [
     auth_feature("DEFERRED"), //This declares that the method logic handles feature level authorization
-    version("1.65"), default_client_version("1.65"),
+    version("1.66"), default_client_version("1.66"),
     noforms,exceptions_inline("./smc_xslt/exceptions.xslt"),use_method_name] WsWorkunits
 {
     ESPmethod [resp_xsl_default("/esp/xslt/workunits.xslt")]     WUQuery(WUQueryRequest, WUQueryResponse);

esp/services/ws_dfu/ws_dfuXRefService.cpp

@@ -20,6 +20,7 @@
 #include "ws_dfuXRefService.hpp"
 
 #include "dadfs.hpp"
+#include "daft.hpp"
 #include "wshelpers.hpp"
 #include "exception_util.hpp"
 #include "package.h"
@@ -195,6 +196,55 @@ IXRefFilesNode* CWsDfuXRefEx::getFileNodeInterface(IXRefNode& XRefNode,const cha
     return 0;
 }
 
+void CWsDfuXRefEx::readLostFileQueryResult(IEspContext &context, StringBuffer& buf)
+{
+    Owned<IPropertyTree> lostFilesQueryResult = createPTreeFromXMLString(buf.str());
+    if (!lostFilesQueryResult)
+    {
+        PROGLOG("readLostFileQueryResult() failed in creating PTree.");
+        return;
+    }
+
+    StringBuffer username;
+    Owned<IUserDescriptor> userdesc;
+    context.getUserID(username);
+    if(username.length() > 0)
+    {
+        const char* passwd = context.queryPassword();
+        userdesc.setown(createUserDescriptor());
+        userdesc->set(username.str(), passwd);
+    }
+
+    Owned<IPropertyTreeIterator> iter = lostFilesQueryResult->getElements("File");
+    ForEach(*iter)
+    {
+        IPropertyTree& item = iter->query();
+        const char* fileName = item.queryProp("Name");
+        if (!fileName || !*fileName)
+            continue;
+
+        try
+        {
+            Owned<IDistributedFile> df = queryDistributedFileDirectory().lookup(fileName, userdesc, false, false, false, NULL, 0);
+            if(df)
+                item.addPropInt64("Size", queryDistributedFileSystem().getSize(df));
+        }
+        catch(IException* e)
+        {
+            item.addProp("Status", "Warning: this file may be locked now. It can't be recovered as locked.");
+            StringBuffer eMsg;
+            PROGLOG("Exception in readLostFileQueryResult(): %s", e->errorMessage(eMsg).str());
+            e->Release();
+        }
+    }
+
+    if (context.getResponseFormat() == ESPSerializationJSON)
+        toJSON(lostFilesQueryResult, buf.clear());
+    else
+        toXML(lostFilesQueryResult, buf.clear());
+}
+
+
 bool CWsDfuXRefEx::onDFUXRefLostFiles(IEspContext &context, IEspDFUXRefLostFilesQueryRequest &req, IEspDFUXRefLostFilesQueryResponse &resp)
 {
     try
@@ -202,9 +252,6 @@ bool CWsDfuXRefEx::onDFUXRefLostFiles(IEspContext &context, IEspDFUXRefLostFiles
         if (!context.validateFeatureAccess(FEATURE_URL, SecAccess_Read, false))
             throw MakeStringException(ECLWATCH_DFU_XREF_ACCESS_DENIED, "Failed to read Xref Lost Files. Permission denied.");
 
-        StringBuffer username;
-        context.getUserID(username);
-
         if (!req.getCluster() || !*req.getCluster())
             throw MakeStringExceptionDirect(ECLWATCH_INVALID_INPUT, "Cluster not defined.");
 
@@ -218,11 +265,7 @@ bool CWsDfuXRefEx::onDFUXRefLostFiles(IEspContext &context, IEspDFUXRefLostFiles
         {
             _lost->Serialize(buf);
             if (!buf.isEmpty())
-            {
-                ESPSerializationFormat fmt = context.getResponseFormat();
-                if (fmt == ESPSerializationJSON)
-                    dfuXrefXMLToJSON(buf);
-            }
+                readLostFileQueryResult(context, buf);
         }
         resp.setDFUXRefLostFilesQueryResult(buf.str());
     }

esp/services/ws_dfu/ws_dfuXRefService.hpp

@@ -161,6 +161,7 @@ private:
 private:
     IXRefFilesNode* getFileNodeInterface(IXRefNode& XRefNode,const char* nodeType);
     void addXRefNode(const char* name, IPropertyTree* pXRefNodeTree);
+    void readLostFileQueryResult(IEspContext &context, StringBuffer& buf);
 public:
    IMPLEMENT_IINTERFACE;
 

esp/services/ws_workunits/ws_workunitsHelpers.cpp

@@ -188,7 +188,7 @@ void getSashaNode(SocketEndpoint &ep)
 }
 
 
-void WsWuInfo::getSourceFiles(IEspECLWorkunit &info, unsigned flags)
+void WsWuInfo::getSourceFiles(IEspECLWorkunit &info, unsigned long flags)
 {
     if (!(flags & WUINFO_IncludeSourceFiles))
         return;
@@ -299,7 +299,7 @@ void WsWuInfo::getSourceFiles(IEspECLWorkunit &info, unsigned flags)
     }
 }
 
-void WsWuInfo::getExceptions(IEspECLWorkunit &info, unsigned flags)
+void WsWuInfo::getExceptions(IEspECLWorkunit &info, unsigned long flags)
 {
     if ((flags & WUINFO_IncludeExceptions) || version > 1.16)
     {
@@ -316,7 +316,7 @@ void WsWuInfo::getExceptions(IEspECLWorkunit &info, unsigned flags)
     }
 }
 
-void WsWuInfo::getVariables(IEspECLWorkunit &info, unsigned flags)
+void WsWuInfo::getVariables(IEspECLWorkunit &info, unsigned long flags)
 {
     if (!(flags & WUINFO_IncludeVariables))
         return;
@@ -426,7 +426,7 @@ void WsWuInfo::doGetTimers(IArrayOf<IEspECLTimer>& timers)
     }
 }
 
-void WsWuInfo::getTimers(IEspECLWorkunit &info, unsigned flags)
+void WsWuInfo::getTimers(IEspECLWorkunit &info, unsigned long flags)
 {
     if (!(flags & WUINFO_IncludeTimers))
         return;
@@ -476,7 +476,7 @@ mapEnums queryFileTypes[] = {
    { FileTypeSize,  NULL },
 };
 
-void WsWuInfo::getHelpers(IEspECLWorkunit &info, unsigned flags)
+void WsWuInfo::getHelpers(IEspECLWorkunit &info, unsigned long flags)
 {
     try
     {
@@ -487,20 +487,25 @@ void WsWuInfo::getHelpers(IEspECLWorkunit &info, unsigned flags)
         {
             ERRLOG("Cannot get Query for this workunit.");
             info.setHelpersDesc("Cannot get Query for this workunit.");
+
+            if (!(flags & WUINFO_IncludeHelpers))
+                return;
         }
         else
         {
-            SCMStringBuffer qname;
-            query->getQueryShortText(qname);
-            if(qname.length())
+            if (flags & WUINFO_IncludeECL)
             {
-                if((flags & WUINFO_TruncateEclTo64k) && (qname.length() > 64000))
-                    qname.setLen(qname.str(), 64000);
+                SCMStringBuffer queryText;
+                query->getQueryShortText(queryText);
+                if (queryText.length())
+                {
+                    if((flags & WUINFO_TruncateEclTo64k) && (queryText.length() > 64000))
+                        queryText.setLen(queryText.str(), 64000);
 
-                IEspECLQuery* q=&info.updateQuery();
-                q->setText(qname.str());
+                    IEspECLQuery* q=&info.updateQuery();
+                    q->setText(queryText.str());
+                }
             }
-
             if (version > 1.34)
             {
                 SCMStringBuffer mainDefinition;
@@ -517,6 +522,9 @@ void WsWuInfo::getHelpers(IEspECLWorkunit &info, unsigned flags)
                 info.setHasArchiveQuery(query->hasArchive());
             }
 
+            if (!(flags & WUINFO_IncludeHelpers))
+                return;
+
             for (unsigned i = 0; i < FileTypeSize; i++)
                 getHelpFiles(query, (WUFileType) i, helpers);
         }
@@ -588,7 +596,7 @@ void WsWuInfo::getHelpers(IEspECLWorkunit &info, unsigned flags)
     }
 }
 
-void WsWuInfo::getApplicationValues(IEspECLWorkunit &info, unsigned flags)
+void WsWuInfo::getApplicationValues(IEspECLWorkunit &info, unsigned long flags)
 {
     if (!(flags & WUINFO_IncludeApplicationValues))
         return;
@@ -618,7 +626,7 @@ void WsWuInfo::getApplicationValues(IEspECLWorkunit &info, unsigned flags)
     }
 }
 
-void WsWuInfo::getDebugValues(IEspECLWorkunit &info, unsigned flags)
+void WsWuInfo::getDebugValues(IEspECLWorkunit &info, unsigned long flags)
 {
     if (!(flags & WUINFO_IncludeDebugValues))
     {
@@ -753,7 +761,7 @@ void WsWuInfo::doGetGraphs(IArrayOf<IEspECLGraph>& graphs)
     }
 }
 
-void WsWuInfo::getGraphInfo(IEspECLWorkunit &info, unsigned flags)
+void WsWuInfo::getGraphInfo(IEspECLWorkunit &info, unsigned long flags)
 {
      if (version > 1.01)
      {
@@ -795,7 +803,7 @@ void WsWuInfo::getWUGraphNameAndTypes(WUGraphType graphType, IArrayOf<IEspNameAn
     }
 }
 
-void WsWuInfo::getGraphTimingData(IArrayOf<IConstECLTimingData> &timingData, unsigned flags)
+void WsWuInfo::getGraphTimingData(IArrayOf<IConstECLTimingData> &timingData)
 {
     StatisticsFilter filter(SCTall, SSTsubgraph, SMeasureTimeNs, StTimeElapsed);
     Owned<IConstWUStatisticIterator> times = &cw->getStatistics(&filter);
@@ -829,10 +837,10 @@ void WsWuInfo::getGraphTimingData(IArrayOf<IConstECLTimingData> &timingData, uns
     }
 
     if (!matched)
-        legacyGetGraphTimingData(timingData, flags);
+        legacyGetGraphTimingData(timingData);
 }
 
-void WsWuInfo::legacyGetGraphTimingData(IArrayOf<IConstECLTimingData> &timingData, unsigned flags)
+void WsWuInfo::legacyGetGraphTimingData(IArrayOf<IConstECLTimingData> &timingData)
 {
     StatisticsFilter filter;
     filter.setScopeDepth(1);
@@ -935,7 +943,7 @@ unsigned WsWuInfo::getLegacyTotalThorTime()
     return 0;
 }
 
-void WsWuInfo::getCommon(IEspECLWorkunit &info, unsigned flags)
+void WsWuInfo::getCommon(IEspECLWorkunit &info, unsigned long flags)
 {
     info.setWuid(cw->queryWuid());
     info.setProtected(cw->isProtected() ? 1 : 0);
@@ -997,7 +1005,7 @@ void WsWuInfo::setWUAbortTime(IEspECLWorkunit &info, unsigned __int64 abortTS)
     info.setAbortTime(abortTimeStr.str());
 }
 
-void WsWuInfo::getInfo(IEspECLWorkunit &info, unsigned flags)
+void WsWuInfo::getInfo(IEspECLWorkunit &info, unsigned long flags)
 {
     getCommon(info, flags);
 
@@ -1231,9 +1239,9 @@ unsigned WsWuInfo::getWorkunitThorLogInfo(IArrayOf<IEspECLHelpFile>& helpers, IE
     return countThorLog;
 }
 
-bool WsWuInfo::getClusterInfo(IEspECLWorkunit &info, unsigned flags)
+bool WsWuInfo::getClusterInfo(IEspECLWorkunit &info, unsigned long flags)
 {
-    if (version > 1.04)
+    if ((flags & WUINFO_IncludeAllowedClusters) && (version > 1.04))
     {
         StringArray allowedClusters;
         SCMStringBuffer val;
@@ -1287,7 +1295,7 @@ bool WsWuInfo::getClusterInfo(IEspECLWorkunit &info, unsigned flags)
     return true;
 }
 
-void WsWuInfo::getWorkflow(IEspECLWorkunit &info, unsigned flags)
+void WsWuInfo::getWorkflow(IEspECLWorkunit &info, unsigned long flags)
 {
     if (!(flags & WUINFO_IncludeWorkflows))
         return;
@@ -1449,7 +1457,7 @@ bool WsWuInfo::getResultEclSchemas(IConstWUResult &r, IArrayOf<IEspECLSchemaItem
     return true;
 }
 
-void WsWuInfo::getResult(IConstWUResult &r, IArrayOf<IEspECLResult>& results, unsigned flags)
+void WsWuInfo::getResult(IConstWUResult &r, IArrayOf<IEspECLResult>& results, unsigned long flags)
 {
     SCMStringBuffer name;
     r.getResultName(name);
@@ -1557,7 +1565,7 @@ void WsWuInfo::getResult(IConstWUResult &r, IArrayOf<IEspECLResult>& results, un
 }
 
 
-void WsWuInfo::getResults(IEspECLWorkunit &info, unsigned flags)
+void WsWuInfo::getResults(IEspECLWorkunit &info, unsigned long flags)
 {
     if (!(flags & WUINFO_IncludeResults))
         return;
@@ -1779,7 +1787,7 @@ void WsWuInfo::getSubFiles(IPropertyTreeIterator* f, IEspECLSourceFile* eclSuper
     return;
 }
 
-bool WsWuInfo::getResourceInfo(StringArray &viewnames, StringArray &urls, unsigned flags)
+bool WsWuInfo::getResourceInfo(StringArray &viewnames, StringArray &urls, unsigned long flags)
 {
     if (!(flags & (WUINFO_IncludeResultsViewNames | WUINFO_IncludeResourceURLs)))
         return true;

esp/services/ws_workunits/ws_workunitsHelpers.hpp

@@ -125,7 +125,10 @@ private:
 #define WUINFO_IncludeResultsViewNames  0x0800
 #define WUINFO_IncludeXmlSchema         0x1000
 #define WUINFO_IncludeResourceURLs      0x2000
-#define WUINFO_All                      0xFFFF
+#define WUINFO_IncludeECL               0x4000
+#define WUINFO_IncludeHelpers           0x8000
+#define WUINFO_IncludeAllowedClusters   0x10000
+#define WUINFO_All                      0xFFFFFFFF
 
 class WsWuInfo
 {
@@ -151,37 +154,37 @@ public:
             throw MakeStringException(ECLWATCH_CANNOT_OPEN_WORKUNIT,"Cannot open workunit %s.", wuid_);
     }
 
-    bool getResourceInfo(StringArray &viewnames, StringArray &urls, unsigned flags);
+    bool getResourceInfo(StringArray &viewnames, StringArray &urls, unsigned long flags);
     unsigned getResourceURLCount();
 
-    void getCommon(IEspECLWorkunit &info, unsigned flags);
-    void getInfo(IEspECLWorkunit &info, unsigned flags);
+    void getCommon(IEspECLWorkunit &info, unsigned long flags);
+    void getInfo(IEspECLWorkunit &info, unsigned long flags);
 
-    void getResults(IEspECLWorkunit &info, unsigned flags);
-    void getVariables(IEspECLWorkunit &info, unsigned flags);
-    void getDebugValues(IEspECLWorkunit &info, unsigned flags);
-    bool getClusterInfo(IEspECLWorkunit &info, unsigned flags);
-    void getApplicationValues(IEspECLWorkunit &info, unsigned flags);
-    void getExceptions(IEspECLWorkunit &info, unsigned flags);
-    void getSourceFiles(IEspECLWorkunit &info, unsigned flags);
+    void getResults(IEspECLWorkunit &info, unsigned long flags);
+    void getVariables(IEspECLWorkunit &info, unsigned long flags);
+    void getDebugValues(IEspECLWorkunit &info, unsigned long flags);
+    bool getClusterInfo(IEspECLWorkunit &info, unsigned long flags);
+    void getApplicationValues(IEspECLWorkunit &info, unsigned long flags);
+    void getExceptions(IEspECLWorkunit &info, unsigned long flags);
+    void getSourceFiles(IEspECLWorkunit &info, unsigned long flags);
     unsigned getTimerCount();
-    void getTimers(IEspECLWorkunit &info, unsigned flags);
+    void getTimers(IEspECLWorkunit &info, unsigned long flags);
     void doGetTimers(IArrayOf<IEspECLTimer>& timers);
-    void getHelpers(IEspECLWorkunit &info, unsigned flags);
-    void getGraphInfo(IEspECLWorkunit &info, unsigned flags);
+    void getHelpers(IEspECLWorkunit &info, unsigned long flags);
+    void getGraphInfo(IEspECLWorkunit &info, unsigned long flags);
     void doGetGraphs(IArrayOf<IEspECLGraph>& graphs);
     void getWUGraphNameAndTypes(WUGraphType graphType, IArrayOf<IEspNameAndType>& graphNameAndTypes);
-    void getGraphTimingData(IArrayOf<IConstECLTimingData> &timingData, unsigned flags);
+    void getGraphTimingData(IArrayOf<IConstECLTimingData> &timingData);
     bool getFileSize(const char* fileName, const char* IPAddress, offset_t& fileSize);
 
-    void getWorkflow(IEspECLWorkunit &info, unsigned flags);
+    void getWorkflow(IEspECLWorkunit &info, unsigned long flags);
 
     void getHelpFiles(IConstWUQuery* query, WUFileType type, IArrayOf<IEspECLHelpFile>& helpers);
     void getSubFiles(IPropertyTreeIterator* f, IEspECLSourceFile* eclSuperFile, StringArray& fileNames);
     void getEclSchemaChildFields(IArrayOf<IEspECLSchemaItem>& schemas, IHqlExpression * expr, bool isConditional);
     void getEclSchemaFields(IArrayOf<IEspECLSchemaItem>& schemas, IHqlExpression * expr, bool isConditional);
     bool getResultEclSchemas(IConstWUResult &r, IArrayOf<IEspECLSchemaItem>& schemas);
-    void getResult(IConstWUResult &r, IArrayOf<IEspECLResult>& results, unsigned flags);
+    void getResult(IConstWUResult &r, IArrayOf<IEspECLResult>& results, unsigned long flags);
     void getStats(StatisticsFilter& filter, bool createDescriptions, IArrayOf<IEspWUStatisticItem>& statistics);
 
     void getWorkunitEclAgentLog(const char* eclAgentInstance, const char* agentPid, MemoryBuffer& buf);
@@ -211,7 +214,7 @@ protected:
     unsigned getLegacyTotalThorTime();
     bool hasSubGraphTimings();
     bool legacyHasSubGraphTimings();
-    void legacyGetGraphTimingData(IArrayOf<IConstECLTimingData> &timingData, unsigned flags);
+    void legacyGetGraphTimingData(IArrayOf<IConstECLTimingData> &timingData);
 
 public:
     IEspContext &context;

esp/services/ws_workunits/ws_workunitsService.cpp

@@ -1440,7 +1440,7 @@ bool CWsWorkunitsEx::onWUInfo(IEspContext &context, IEspWUInfoRequest &req, IEsp
                 ensureWsWorkunitAccess(context, wuid.str(), SecAccess_Read);
                 PROGLOG("WUInfo: %s", wuid.str());
 
-                unsigned flags=0;
+                unsigned long flags=0;
                 if (req.getTruncateEclTo64k())
                     flags|=WUINFO_TruncateEclTo64k;
                 if (req.getIncludeExceptions())
@@ -1469,6 +1469,12 @@ bool CWsWorkunitsEx::onWUInfo(IEspContext &context, IEspWUInfoRequest &req, IEsp
                     flags|=WUINFO_IncludeResultsViewNames;
                 if (req.getIncludeResourceURLs())
                     flags|=WUINFO_IncludeResourceURLs;
+                if (req.getIncludeECL())
+                    flags|=WUINFO_IncludeECL;
+                if (req.getIncludeHelpers())
+                    flags|=WUINFO_IncludeHelpers;
+                if (req.getIncludeAllowedClusters())
+                    flags|=WUINFO_IncludeAllowedClusters;
 
                 WsWuInfo winfo(context, wuid.str());
                 winfo.getInfo(resp.updateWorkunit(), flags);
@@ -4004,7 +4010,7 @@ bool CWsWorkunitsEx::onWUGraphTiming(IEspContext &context, IEspWUGraphTimingRequ
 
         WsWuInfo winfo(context, cw);
         IArrayOf<IConstECLTimingData> timingData;
-        winfo.getGraphTimingData(timingData, 0);
+        winfo.getGraphTimingData(timingData);
         resp.updateWorkunit().setTimingData(timingData);
     }
     catch(IException* e)

plugins/fileservices/fileservices.cpp

@@ -500,10 +500,10 @@ FILESERVICES_API void FILESERVICES_CALL fsSendEmailAttachData(ICodeContext * ctx
 
 FILESERVICES_API char * FILESERVICES_CALL fsCmdProcess(const char *prog, const char *src)
 {
-    StringBuffer in, out;
+    StringBuffer in, out, err;
     in.append(src);
 
-    runExternalCommand(out, prog, in);
+    runExternalCommand(out, err, prog, in);
 
     return CTXSTRDUP(parentCtx, out.str());
 }
@@ -511,10 +511,10 @@ FILESERVICES_API char * FILESERVICES_CALL fsCmdProcess(const char *prog, const c
 
 FILESERVICES_API void FILESERVICES_CALL fsCmdProcess2(unsigned & tgtLen, char * & tgt, const char *prog, unsigned srcLen, const char * src)
 {
-    StringBuffer in, out;
+    StringBuffer in, out, err;
     in.append(srcLen, src);
 
-    runExternalCommand(out, prog, in);
+    runExternalCommand(out, err, prog, in);
 
     tgtLen = out.length();
     tgt = (char *)CTXDUP(parentCtx, out.str(), out.length());

system/jlib/jsmartsock.cpp

@@ -430,25 +430,25 @@ void CSmartSocketFactory::setStatus(SocketEndpoint &ep, bool status)
 
 StringBuffer & CSmartSocketFactory::getUrlStr(StringBuffer &url, bool useHostName)
 {
-	SmartSocketEndpoint * sep = nextSmartEndpoint();
-	if (sep)
-	{
-		SocketEndpoint ep;
-		if(useHostName && sep->name.length())
-		{
-			url.append(sep->name.str());
-			ep = sep->ep;
-			if (ep.port)
-				url.append(':').append((unsigned)ep.port);
-		}
-		else
-		{
-			sep->checkHost(dnsInterval);
-			SocketEndpoint ep = sep->ep;
-			ep.getUrlStr(url);
-		}
-	}
-	return url;
+    SmartSocketEndpoint * sep = nextSmartEndpoint();
+    if (sep)
+    {
+        SocketEndpoint ep;
+        if(useHostName && sep->name.length())
+        {
+            url.append(sep->name.str());
+            ep = sep->ep;
+            if (ep.port)
+                url.append(':').append((unsigned)ep.port);
+        }
+        else
+        {
+            sep->checkHost(dnsInterval);
+            SocketEndpoint ep = sep->ep;
+            ep.getUrlStr(url);
+        }
+    }
+    return url;
 }
 
 ISmartSocketFactory *createSmartSocketFactory(const char *_socklist, bool _retry, unsigned _retryInterval, unsigned _dnsInterval) {

system/jlib/jthread.cpp

@@ -1831,7 +1831,6 @@ protected: friend class PipeWriterThread;
             CriticalBlock block(sect); // clear forkthread and stderrbufferthread
             ft.setown(forkthread.getClear());
             et = stderrbufferthread;
-            stderrbufferthread = NULL;
         }
         if (ft)
         {
@@ -1841,7 +1840,7 @@ protected: friend class PipeWriterThread;
         if (et)
         {
             et->stop();
-            delete et;
+            // NOTE - we don't delete it here, since we want to be able to still read the buffered data
         }
     }
 public:
@@ -1870,6 +1869,7 @@ public:
         closeOutput();
         closeError();
         clearUtilityThreads();
+        delete stderrbufferthread;
     }
 
 

system/jlib/jutil.cpp

@@ -1746,7 +1746,7 @@ static const char *findExtension(const char *fn)
     return ret;
 }
 
-unsigned runExternalCommand(StringBuffer &output, const char *cmd, const char *input)
+unsigned runExternalCommand(StringBuffer &output, StringBuffer &error, const char *cmd, const char *input)
 {
     try
     {
@@ -1768,7 +1768,6 @@ unsigned runExternalCommand(StringBuffer &output, const char *cmd, const char *i
                 output.append(read, buf);
             }
             ret = pipe->wait();
-            StringBuffer error;
             while (true)
             {
                 size32_t read = pipe->readError(sizeof(buf), buf);

system/jlib/jutil.hpp

@@ -227,7 +227,7 @@ extern jlib_decl void doStackProbe();
 #define arraysize(T) (sizeof(T)/sizeof(*T))
 #endif
 
-extern jlib_decl unsigned runExternalCommand(StringBuffer &output, const char *cmd, const char *input);
+extern jlib_decl unsigned runExternalCommand(StringBuffer &output, StringBuffer &error, const char *cmd, const char *input);
 
 extern jlib_decl unsigned __int64 greatestCommonDivisor(unsigned __int64 left, unsigned __int64 right);
 

system/security/securesocket/securesocket.hpp

@@ -31,11 +31,7 @@
 #include "jsocket.hpp"
 #include "jptree.hpp"
 
-#ifdef _WIN32
-#define SSLIB "securesocket.dll"
-#else
-#define SSLIB "libsecuresocket.so"
-#endif
+#define SSLIB "securesocket"
 
 enum SecureSocketType
 {

testing/regress/ecl-test

@@ -29,6 +29,7 @@ from hpcc.util import argparse
 from hpcc.regression.regress import Regression
 from hpcc.util.ecl.file import ECLFile
 from hpcc.util.util import setConfig, checkPqParam, getVersionNumbers, checkXParam, convertPath, getRealIPAddress, parentPath, checkClusters, checkHpccStatus
+from hpcc.util.expandcheck import ExpandCheck
 from hpcc.common.error import Error
 from hpcc.common.config import Config
 
@@ -49,7 +50,7 @@ class RegressMain:
     def query(self):
         if not self.args.query:
             print "\nMissing ECL query file!\n"
-            parser_query.print_help()
+            self.parser_query.print_help()
             exit()
         eclfiles=[]   # List for ECL filenames to be executed
         for ecl in self.args.query:
@@ -114,6 +115,7 @@ class RegressMain:
         prog = "ecl-test"
         description = 'HPCC Platform Regression suite'
         pythonVer = getVersionNumbers()
+        defaultConfigFile="ecl-test.json"
 
         if (pythonVer['main'] <= 2) and (pythonVer['minor'] <=6) and (pythonVer['patch'] <6):
             print "\nError!"
@@ -132,7 +134,7 @@ class RegressMain:
 
         helperParser=argparse.ArgumentParser(add_help=False)
         helperParser.add_argument('--config', help="config file to use. Default: ecl-test.json",
-                            nargs='?', default="ecl-test.json")
+                            nargs='?', default=defaultConfigFile)
         helperParser.add_argument('--loglevel', help="set the log level. Use debug for more detailed logfile.",
                             nargs='?', default="info",
                             choices=['info', 'debug'])
@@ -185,17 +187,27 @@ class RegressMain:
         parser_run.add_argument('--publish', '-p', help="Publish compiled query instead of run.",
                                 action='store_true')
 
-        parser_query = subparsers.add_parser('query', help='query help',  parents=[helperParser, commonParser, executionParser])
-        parser_query.set_defaults(func='query')
-        parser_query.add_argument('query', help="One or more ECL file(s). It can contain wildcards. (mandatory).",
+        self.parser_query = subparsers.add_parser('query', help='query help',  parents=[helperParser, commonParser, executionParser])
+        self.parser_query.set_defaults(func='query')
+        self.parser_query.add_argument('query', help="One or more ECL file(s). It can contain wildcards. (mandatory).",
                                   nargs='+', metavar="ECL_query")
-        parser_query.add_argument('--target', '-t', help="Target cluster(s) for query to run. If target = 'all' then run query on all clusters. If not defined then default value(s) come from config (ecl-test.json by default).",
+        self.parser_query.add_argument('--target', '-t', help="Target cluster(s) for query to run. If target = 'all' then run query on all clusters. If not defined then default value(s) come from config (ecl-test.json by default).",
                                 nargs='?', default='', metavar="target_cluster_list | all")
-        parser_query.add_argument('--publish', '-p', help="Publish compiled query instead of run.",
+        self.parser_query.add_argument('--publish', '-p', help="Publish compiled query instead of run.",
                                 action='store_true')
 
         self.args = parser.parse_args()
 
+        regressionSuiteMainDir = os.path.dirname(__file__)
+        regressionSuiteFullPath = os.path.realpath(regressionSuiteMainDir)
+
+        if defaultConfigFile == self.args.config:
+            # Resolve Regression Suite starting path for ecl-test.json config file
+            # It is necessary when Regression Suite doesn't started from its home directory
+            self.args.config = str(os.path.join(regressionSuiteFullPath, self.args.config))
+        else:
+            self.args.config = ExpandCheck.dirExists(self.args.config, True,  True)
+
         # Process config parameter
         self.config = Config(self.args.config).configObj
         if ('server' in self.args) and (self.args.server != None):
@@ -247,12 +259,6 @@ class RegressMain:
                 self.regress = None
                 raise Error(self.args.X[0])
 
-            # Resolve Regression Suite starting path for ecl-test.json config file
-            # It is necessary when Regression Suite doesn't started from its home directory
-            regressionSuiteMainDir = os.path.dirname(__file__)
-            regressionSuiteFullPath = os.path.realpath(regressionSuiteMainDir)
-            self.args.config = str(os.path.join(regressionSuiteFullPath, self.args.config))
-
             self.regressionSuiteHpccMainOsDir = regressionSuiteFullPath
             self.regressionSuiteHpccMainEclDir = convertPath(regressionSuiteFullPath)+"::download"
 

testing/regress/ecl/key/roxiepipe.xml

@@ -0,0 +1,11 @@
+<Dataset name='Result 1'>
+ <Row><name><first>Joe       </first><last>Doe            </last></name><address><city>Fresno    </city><state>CA</state><zipcode>11111</zipcode></address></Row>
+ <Row><name><first>Jason     </first><last>Jones          </last></name><address><city>Tempe     </city><state>AZ</state><zipcode>22222</zipcode></address></Row>
+ <Row><name><first>Becky     </first><last>Lopez          </last></name><address><city>New York  </city><state>NY</state><zipcode>33333</zipcode></address></Row>
+ <Row><name><first>Roderic   </first><last>Lykke          </last></name><address><city>Lubbock   </city><state>TX</state><zipcode>44444</zipcode></address></Row>
+ <Row><name><first>Rina      </first><last>Yonkers        </last></name><address><city>Denver    </city><state>CO</state><zipcode>55555</zipcode></address></Row>
+ <Row><name><first>Laverna   </first><last>Campa          </last></name><address><city>Detroit   </city><state>MI</state><zipcode>66666</zipcode></address></Row>
+ <Row><name><first>Shantell  </first><last>Mattera        </last></name><address><city>Toledo    </city><state>OH</state><zipcode>77777</zipcode></address></Row>
+ <Row><name><first>John      </first><last>Smith          </last></name><address><city>Boise     </city><state>ID</state><zipcode>88888</zipcode></address></Row>
+ <Row><name><first>Jane      </first><last>Smith          </last></name><address><city>Smallville</city><state>KY</state><zipcode>99999</zipcode></address></Row>
+</Dataset>

testing/regress/ecl/roxiepipe.ecl

@@ -0,0 +1,58 @@
+//nothor
+//nohthor
+
+NameRec := RECORD
+  string10 First;
+  string15 Last;
+END;
+
+AddressRec := RECORD
+  string10 City;
+  string2 State;
+  integer4 ZipCode;
+END;
+
+PersonRec := RECORD
+  NameRec Name;
+  AddressRec Address;
+END;
+
+pipe_send := DATASET([{{'Joe', 'Doe'}, {'Fresno', 'CA', 11111}},
+{{'Jason', 'Jones'}, {'Tempe', 'AZ', 22222}},
+{{'Becky', 'Lopez'}, {'New York', 'NY', 33333}},
+{{'Roderic', 'Lykke'}, {'Lubbock', 'TX', 44444}},
+{{'Rina', 'Yonkers'}, {'Denver', 'CO', 55555}},
+{{'Laverna', 'Campa'}, {'Detroit', 'MI', 66666}},
+{{'Shantell', 'Mattera'}, {'Toledo', 'OH', 77777}},
+{{'John', 'Smith'}, {'Boise', 'ID', 88888}},
+{{'Jane','Smith'}, {'Smallville', 'KY', 99999}}], PersonRec);
+
+/*  NOTES:
+	roxiepipe = the executable we're calling
+	iw = Input Width
+	vip = Virtual IP
+	t = # threads
+	ow = output width
+	b = # records in a batch
+	mr = max retries
+	h = ??? -- the URL to the Roxie batch service
+	r = name of the results dataset (case sensitive!)
+	q = query
+*/
+
+
+// This is the remote call to the 'roxie_echo' service.
+pipe_recv := PIPE(pipe_send,
+    'roxiepipe' +
+    ' -iw ' + SIZEOF(PersonRec) +
+    ' -t 1' +
+    ' -ow ' + SIZEOF(PersonRec) +
+    ' -b 3' +
+    ' -mr 2' +
+    ' -h .:9876 ' +
+    ' -r Peeps' +
+    ' -q "<roxie_echo format=\'raw\'><peeps id=\'id\' format=\'raw\'></peeps></roxie_echo>"'
+, PersonRec);
+
+OUTPUT(pipe_recv);
+

testing/regress/ecl/setup/roxie_echo.ecl

@@ -0,0 +1,24 @@
+//nothor
+//nohthor
+//publish
+
+NameRec := RECORD
+  string10 First;
+  string15 Last;
+END;
+
+AddressRec := RECORD
+  string10 City;
+  string2 State;
+  integer4 ZipCode;
+END;
+
+PersonRec := RECORD
+  NameRec Name;
+  AddressRec Address;
+END;
+
+peeps := DATASET([], PersonRec) : STORED('peeps', FEW);
+
+OUTPUT(peeps, NAMED('Peeps'));
+

testing/regress/hpcc/common/config.py

@@ -60,7 +60,10 @@ class Config:
             rC = namedtuple("Regress", js.keys())
             return _dict(getattr(rC(**js), "Regress"))
         except IOError as e:
-            raise(e)
+            if e.errno == 21:
+                raise IOError("Error: "+file+" "+e.strerror+", we need a JSON file!")
+            else:
+                raise(e)
 
     #implement writing out of a config.
     def writeConfig(self, file):

testing/regress/hpcc/regression/regress.py

@@ -71,11 +71,11 @@ class Regression:
             self.keyDir = args.keyDir
             logging.debug("Try to use alternative key directory: %s", self.keyDir)
 
-        self.suiteDir = ExpandCheck.dir_exists(self.suiteDir, True)
-        self.regressionDir = ExpandCheck.dir_exists(self.config.regressionDir, True)
-        self.logDir = ExpandCheck.dir_exists(self.config.logDir, True)
-        self.dir_ec = ExpandCheck.dir_exists(os.path.join(self.suiteDir, self.config.eclDir), True)
-        self.dir_ex = ExpandCheck.dir_exists(os.path.join(self.suiteDir, self.keyDir), True)
+        self.suiteDir = ExpandCheck.dirExists(self.suiteDir, True)
+        self.regressionDir = ExpandCheck.dirExists(self.config.regressionDir, True)
+        self.logDir = ExpandCheck.dirExists(self.config.logDir, True)
+        self.dir_ec = ExpandCheck.dirExists(os.path.join(self.suiteDir, self.config.eclDir), True)
+        self.dir_ex = ExpandCheck.dirExists(os.path.join(self.suiteDir, self.keyDir), True)
         self.dir_a = os.path.join(self.regressionDir, self.config.archiveDir)
         self.dir_r = os.path.join(self.regressionDir, self.config.resultDir)
         self.dir_zap =  os.path.join(self.regressionDir,self.config.zapDir)
@@ -144,7 +144,7 @@ class Regression:
         self.createDirectory(self.dir_r)
         self.createDirectory(self.logDir)
         self.createDirectory(self.dir_zap)
-        self.setupDir = ExpandCheck.dir_exists(os.path.join(self.suiteDir, self.config.setupDir), True)
+        self.setupDir = ExpandCheck.dirExists(os.path.join(self.suiteDir, self.config.setupDir), True)
         logging.debug("Setup Dir      : %s", self.setupDir)
         self.setupSuite = Suite(args.target, self.setupDir, self.dir_a, self.dir_ex, self.dir_r, self.logDir, self.dir_inc, args, True)
         self.maxtasks = len(self.setupSuite.getSuite())

testing/regress/hpcc/util/ecl/file.py

@@ -22,7 +22,6 @@ import logging
 import os
 import traceback
 import re
-import tempfile
 import xml.etree.ElementTree as ET
 import unicodedata
 

testing/regress/hpcc/util/expandcheck.py

@@ -23,22 +23,26 @@ import logging
 class ExpandCheck:
 
     @staticmethod
-    def dir_exists(path, require=False):
-        logging.debug("dir_exists(path: %s, require: %s)", path, require)
+    def dirExists(path, require=False,  silent=False):
+        if not silent:
+            logging.debug("dirExists(path: %s, require: %s)", path, require)
         if '~' in path:
             path = os.path.expanduser(path)
         else:
             path = os.path.abspath(path)
-        logging.debug("path: %s", path)
+        if not silent:
+            logging.debug("path: %s", path)
         if not os.path.exists(path):
             if require:
-                logging.debug("Path: %s not found and it is required!" ,path)
+                if not silent:
+                    logging.debug("Path: %s not found and it is required!" ,path)
                 try:
                     os.mkdir(path)
                 except:
                     raise IOError("REQUIRED DIRECTORY NOT FOUND. " + path)
             else:
-                logging.debug( "DIRECTORY NOT FOUND. " + path)
+                if not silent:
+                    logging.debug( "DIRECTORY NOT FOUND. " + path)
                 path = None
 
         return(path)

thorlcr/msort/tsorts.cpp

@@ -997,7 +997,15 @@ public:
         assertex(transferserver.get()!=NULL);
         if (intercept)
             rowArray.kill(); // don't need samples any more. All merged from disk.
-        transferserver->merge(mapsize,map,mapupper,num,endpoints,partno);
+        try
+        {
+            transferserver->merge(mapsize,map,mapupper,num,endpoints,partno);
+        }
+        catch (IException *e)
+        {
+            startmergesem.interrupt(e);
+            stop();
+        }
     }
     virtual void SingleMerge()
     {