Merge remote-tracking branch 'origin/closedown-5.0.x'

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

ecl/eclagent/eclgraph.cpp

@@ -916,7 +916,6 @@ void EclSubGraph::doExecute(const byte * parentExtract, bool checkDependencies)
 
     if (!prepare(parentExtract, checkDependencies))
     {
-        throwUnexpected();
         executed = true;
         return;
     }

ecl/hql/hqlexpr.hpp

@@ -881,6 +881,7 @@ public:
         expandCallsWhenBound = DEFAULT_EXPAND_CALL;
         ignoreUnknownImport = false;
         _clear(metaState);
+        aborting = false;
     }
 
     void addForwardReference(IHqlScope * owner, IHasUnlinkedOwnerReference * child);
@@ -899,6 +900,8 @@ public:
     inline IPropertyTree * getMetaTree() { return LINK(metaTree); }
     inline IPropertyTree * queryArchive() const { return archive; }
     inline IEclRepository * queryRepository() const { return eclRepository; }
+    inline bool isAborting() const { return aborting; }
+    inline void setAborting() { aborting = true; }
 
 public:
     Linked<IPropertyTree> archive;
@@ -911,6 +914,7 @@ public:
     CIArrayOf<ForwardScopeItem> forwardLinks;
     bool expandCallsWhenBound;
     bool ignoreUnknownImport;
+    bool aborting;
 
 private:
     bool checkBeginMeta();
@@ -939,7 +943,7 @@ public:
     {
         errs.set(other.errs); 
         functionCache = other.functionCache; 
-        curAttrTree.set(other.curAttrTree); 
+        curAttrTree.set(other.curAttrTree);
     }
     HqlLookupContext(HqlParseContext & _parseCtx, IErrorReceiver * _errs)
     : parseCtx(_parseCtx), errs(_errs)
@@ -962,6 +966,8 @@ public:
     inline bool queryExpandCallsWhenBound() const { return parseCtx.expandCallsWhenBound; }
     inline HqlParseContext & queryParseContext() const { return parseCtx; }
     inline unsigned numErrors() const { return errs ? errs->errCount() : 0; }
+    inline bool isAborting() const { return parseCtx.isAborting(); }
+    inline void setAborting() { parseCtx.setAborting(); }
 
 protected:
 

ecl/hql/hqlgram2.cpp

@@ -376,7 +376,7 @@ void HqlGram::init(IHqlScope * _globalScope, IHqlScope * _containerScope)
 
     outerScopeAccessDepth = 0;
     inType = false;
-    aborting = false;
+    aborting = lookupCtx.isAborting();
     errorHandler = NULL;
     moduleName = NULL;
     resolveSymbols = true;
@@ -3267,6 +3267,10 @@ IHqlExpression *HqlGram::lookupSymbol(IIdAtom * searchName, const attribute& err
     if (expectedUnknownId)
         return NULL;
 
+    //Check periodically if parsing a referenced identifier has caused the compile to abort.
+    if (lookupCtx.isAborting())
+        aborting = true;
+
     try
     {
         // If there is a temporary scope, we only look up in that (and it must exist!).
@@ -10894,6 +10898,7 @@ void HqlGram::abortParsing()
 {
     // disable more error report
     disableError();
+    lookupCtx.setAborting();
     aborting = true;
 }
 

esp/scm/ws_topology.ecm

@@ -137,6 +137,13 @@ ESPStruct TpBinding
     string Protocol;
 
 };
+
+ESPStruct TpEspServicePlugin
+{
+    string Name;
+    string Type;
+};
+
 //  ===========================================================================
 ESPStruct TpEspServer
 {
@@ -565,6 +572,15 @@ ESPresponse [exceptions_inline,encode(0)] TpThorStatusResponse
     int AutoRefresh;
 };
 
+ESPrequest TpGetServicePluginsRequest
+{
+};
+
+ESPresponse [exceptions_inline,encode(0)] TpGetServicePluginsResponse
+{
+    ESParray<ESPstruct TpEspServicePlugin, Plugin> Plugins;
+};
+
 ESPservice [noforms, version("1.20"), default_client_version("1.20"), exceptions_inline("./smc_xslt/exceptions.xslt")] WsTopology
 {
     ESPuses ESPStruct TpBinding;
@@ -603,6 +619,7 @@ ESPservice [noforms, version("1.20"), default_client_version("1.20"), exceptions
     ESPmethod [resp_xsl_default("/esp/xslt/tplog.xslt")] TpLogFile(TpLogFileRequest, TpLogFileResponse);
     ESPmethod [resp_xsl_default("/esp/xslt/tplogdisplay.xslt")] TpLogFileDisplay(TpLogFileRequest, TpLogFileResponse);
     ESPmethod TpGetComponentFile(TpGetComponentFileRequest, TpGetComponentFileResponse);
+    ESPmethod TpGetServicePlugins(TpGetServicePluginsRequest, TpGetServicePluginsResponse);
     ESPmethod TpListTargetClusters(TpListTargetClustersRequest, TpListTargetClustersResponse);
 
     ESPmethod SystemLog(SystemLogRequest, SystemLogResponse);

esp/services/ws_smc/ws_smcService.cpp

@@ -541,6 +541,7 @@ void CWsSMCEx::readDFUWUs(IEspContext &context, const char* queueName, const cha
 {
     StringAttrArray wulist;
     unsigned running = queuedJobs(queueName, wulist);
+    Owned<IDFUWorkUnitFactory> factory = getDFUWorkUnitFactory();
     ForEachItemIn(i, wulist)
     {
         StringBuffer jname, uname, state, error;
@@ -552,7 +553,7 @@ void CWsSMCEx::readDFUWUs(IEspContext &context, const char* queueName, const cha
 
         try
         {
-            Owned<IConstDFUWorkUnit> dfuwu = getDFUWorkUnitFactory()->openWorkUnit(wuid, false);
+            Owned<IConstDFUWorkUnit> dfuwu = factory->openWorkUnit(wuid, false);
             dfuwu->getUser(uname);
             dfuwu->getJobName(jname);
         }

esp/services/ws_topology/ws_topologyService.cpp

@@ -83,6 +83,23 @@ void CWsTopologyEx::init(IPropertyTree *cfg, const char *process, const char *se
             defaultTargetClusterPrefix.set(cfg->queryProp(xpath.str()));
     }
 
+    xpath.clear().appendf("Software/EspProcess[@name=\"%s\"]/EspService[@servicePlugin]", process);
+    Owned<IPropertyTreeIterator> it= cfg->getElements(xpath.str());
+    ForEach(*it)
+    {
+        IPropertyTree& espService = it->query();
+        const char* servicePlugin = espService.queryProp("@servicePlugin");
+        const char* servicePluginType = espService.queryProp("@servicePluginType");
+        if (!servicePlugin || !*servicePlugin)
+            continue;
+        Owned<IEspTpEspServicePlugin> espServicePlugin= createTpEspServicePlugin();
+        espServicePlugin->setName(servicePlugin);
+        if (servicePluginType && *servicePluginType)
+            espServicePlugin->setType(servicePluginType);
+
+        espServicePlugins.append(*espServicePlugin.getClear());
+    }
+
     m_enableSNMP = false;
 }
 
@@ -1715,3 +1732,29 @@ bool CWsTopologyEx::onTpThorStatus(IEspContext &context, IEspTpThorStatusRequest
     }
     return false;
 }
+
+bool CWsTopologyEx::onTpGetServicePlugins(IEspContext &context, IEspTpGetServicePluginsRequest &req, IEspTpGetServicePluginsResponse &resp)
+{
+    try
+    {
+        if (!context.validateFeatureAccess(FEATURE_URL, SecAccess_Read, false))
+            throw MakeStringException(ECLWATCH_TOPOLOGY_ACCESS_DENIED, "Failed to get Service Plugins. Permission denied.");
+
+        IArrayOf<IEspTpEspServicePlugin> plugins;
+        ForEachItemIn(i,espServicePlugins)
+        {
+            IEspTpEspServicePlugin& servicePlugin = espServicePlugins.item(i);
+
+            Owned<IEspTpEspServicePlugin> plugin= createTpEspServicePlugin();
+            plugin->setName(servicePlugin.getName());
+            plugin->setType(servicePlugin.getType());
+            plugins.append(*plugin.getClear());
+        }
+        resp.setPlugins(plugins);
+    }
+    catch(IException* e)
+    {
+        FORWARDEXCEPTION(context, e,  ECLWATCH_INTERNAL_ERROR);
+    }
+    return false;
+}

esp/services/ws_topology/ws_topologyService.hpp

@@ -89,6 +89,7 @@ private:
     bool                                m_enableSNMP;
     StringAttr                          defaultTargetClusterName;
     StringAttr                          defaultTargetClusterPrefix;
+    IArrayOf<IEspTpEspServicePlugin>    espServicePlugins;
 
     void getThorXml(const char *cluster,StringBuffer& strBuff);
     void getThorLog(const char *cluster,MemoryBuffer& returnbuff);
@@ -144,7 +145,7 @@ public:
     bool onTpLogFileDisplay(IEspContext &context,IEspTpLogFileRequest  &req, IEspTpLogFileResponse &resp);
 
     bool onTpServiceQuery(IEspContext &context, IEspTpServiceQueryRequest &req, IEspTpServiceQueryResponse &resp);
-
+    bool onTpGetServicePlugins(IEspContext &context, IEspTpGetServicePluginsRequest &req, IEspTpGetServicePluginsResponse &resp);
     bool onTpGetComponentFile(IEspContext &context, IEspTpGetComponentFileRequest &req, IEspTpGetComponentFileResponse &resp);
 
     bool onTpThorStatus(IEspContext &context, IEspTpThorStatusRequest &req, IEspTpThorStatusResponse &resp);

esp/services/ws_workunits/ws_workunitsHelpers.cpp

@@ -800,7 +800,7 @@ void WsWuInfo::getEventScheduleFlag(IEspECLWorkunit &info)
                 if (!r)
                     continue;
 
-                IWorkflowEvent *wfevent = r->getScheduleEvent();
+                Owned<IWorkflowEvent> wfevent = r->getScheduleEvent();
                 if (!wfevent)
                     continue;
 

esp/src/eclwatch/LZBrowseWidget.js

@@ -171,7 +171,7 @@ define([
         _onCheckUploadSubmit: function () {
             var context = this;
             var fileList = registry.byId(this.id + "Upload").getFileList();
-
+            var list = this.arrayToList(fileList, "name");
             if (this.overwriteCheckbox.checked) {
                 this._onUploadSubmit();
                 this.fileListDialog.hide();
@@ -197,7 +197,7 @@ define([
                         context._onUploadSubmit();
                         context.fileListDialog.hide();
                     } else {
-                        alert(context.i18n.OverwriteMessage);
+                        alert(context.i18n.OverwriteMessage + "\n" + list );
                     }
                 });
             }

esp/src/eclwatch/LogsWidget.js

@@ -84,6 +84,10 @@ define([
                     case "hpp":
                         params = "/WUFile?Wuid=" + this.wu.Wuid + "&Name=" + item.Orig.Name + "&IPAddress=" + item.Orig.IPAddress + "&Description=" + item.Orig.Description + "&Type=" + item.Orig.Type;
                         break;
+                    case "xml":
+                        if (option !== undefined)
+                            params = "/WUFile?Wuid=" + this.wu.Wuid + "&Name=" + item.Orig.Name + "&IPAddress=" + item.Orig.IPAddress + "&Description=" + item.Orig.Description + "&Type=" + item.Orig.Type;
+                        break;
                 }
 
                 return ESPRequest.getBaseURL() + params + (option ? "&Option=" + option : "&Option=1");

esp/src/eclwatch/_Widget.js

@@ -239,7 +239,7 @@ define([
                     retVal += "..." + (arr.length - 10) + " " + this.i18n.More + "...";
                     return true;                    
                 }
-                retVal += item[field];
+                retVal += field ? item[field] : item;
             }, this);
             return retVal;
         },

roxie/ccd/ccdfile.cpp

@@ -141,6 +141,13 @@ public:
     virtual void setCopying(bool _copying)
     {
         CriticalBlock b(crit);
+        if (remote && currentIdx)
+        {
+            // The current location is not our preferred location. Recheck whether we can now access our preferred location
+            setFailure();
+            currentIdx = 0;
+            _checkOpen();
+        }
         copying = _copying; 
     }
 
@@ -559,6 +566,7 @@ typedef StringArray *StringArrayPtr;
 
 class CRoxieFileCache : public CInterface, implements ICopyFileProgress, implements IRoxieFileCache
 {
+    friend class CcdFileTest;
     mutable ICopyArrayOf<ILazyFileIO> todo; // Might prefer a queue but probably doesn't really matter.
     InterruptableSemaphore toCopy;
     InterruptableSemaphore toClose;
@@ -568,6 +576,7 @@ class CRoxieFileCache : public CInterface, implements ICopyFileProgress, impleme
     bool started;
     bool aborting;
     bool closing;
+    bool testMode;
     bool closePending[2];
     StringAttrMapping fileErrorList;
     Semaphore bctStarted;
@@ -615,7 +624,7 @@ class CRoxieFileCache : public CInterface, implements ICopyFileProgress, impleme
                            offset_t size, const CDateTime &modified, unsigned crc)
     {
         Owned<IFile> local = createIFile(localLocation);
-        bool isCompressed = pdesc->queryOwner().isCompressed();
+        bool isCompressed = testMode ? false : pdesc->queryOwner().isCompressed();
         Owned<CLazyFileIO> ret = new CLazyFileIO(local.getLink(), size, modified, crc, isCompressed);
         RoxieFileStatus fileStatus = fileUpToDate(local, size, modified, crc, isCompressed);
         if (fileStatus == FileIsValid)
@@ -631,7 +640,10 @@ class CRoxieFileCache : public CInterface, implements ICopyFileProgress, impleme
 
             // put the peerRoxieLocations next in the list
             StringArray localLocations;
-            appendRemoteLocations(pdesc, localLocations, localLocation, roxieName, true);  // Adds all locations on the same cluster
+            if (testMode)
+                localLocations.append("test.buddy");
+            else
+                appendRemoteLocations(pdesc, localLocations, localLocation, roxieName, true);  // Adds all locations on the same cluster
             ForEachItemIn(roxie_idx, localLocations)
             {
                 try
@@ -646,6 +658,15 @@ class CRoxieFileCache : public CInterface, implements ICopyFileProgress, impleme
                         ret->addSource(remote.getClear());
                         addedOne = true;
                     }
+                    else if (status==FileNotFound)
+                    {
+                        // Even though it's not on the buddy (yet), add it to the locations since it may well be there
+                        // by the time we come to copy (and if it is, we want to copy from there)
+                        if (miscDebugTraceLevel > 5)
+                            DBGLOG("adding missing peer location %s", remoteName);
+                        ret->addSource(remote.getClear());
+                        // Don't set addedOne - we need to go to remote too
+                    }
                     else if (miscDebugTraceLevel > 10)
                         DBGLOG("Checked peer roxie location %s, status=%d", remoteName, (int) status);
                 }
@@ -656,7 +677,7 @@ class CRoxieFileCache : public CInterface, implements ICopyFileProgress, impleme
                 }
             }
 
-            if (!addedOne && (copyResources || useRemoteResources))  // If no peer locations available, go to remote
+            if (!addedOne && (copyResources || useRemoteResources || testMode))  // If no peer locations available, go to remote
             {
                 ForEachItemIn(idx, remoteLocationInfo)
                 {
@@ -891,7 +912,7 @@ class CRoxieFileCache : public CInterface, implements ICopyFileProgress, impleme
 public:
     IMPLEMENT_IINTERFACE;
 
-    CRoxieFileCache(bool testmode = false) : bct(*this), hct(*this)
+    CRoxieFileCache(bool _testMode = false) : bct(*this), hct(*this), testMode(_testMode)
     {
         aborting = false;
         closing = false;
@@ -2643,3 +2664,105 @@ extern IRoxieWriteHandler *createRoxieWriteHandler(IRoxieDaliHelper *_daliHelper
 {
     return new CRoxieWriteHandler(_daliHelper, _dFile, _clusters);
 }
+
+//================================================================================================================
+
+#ifdef _USE_CPPUNIT
+#include "unittests.hpp"
+
+class CcdFileTest : public CppUnit::TestFixture
+{
+    CPPUNIT_TEST_SUITE(CcdFileTest);
+        CPPUNIT_TEST(testCopy);
+    CPPUNIT_TEST_SUITE_END();
+protected:
+
+    class DummyPartDescriptor : public CInterfaceOf<IPartDescriptor>
+    {
+        virtual unsigned queryPartIndex() { UNIMPLEMENTED; }
+
+        virtual unsigned numCopies() { UNIMPLEMENTED; }
+        virtual INode *getNode(unsigned copy=0) { UNIMPLEMENTED; }
+        virtual INode *queryNode(unsigned copy=0) { UNIMPLEMENTED; }
+
+        virtual IPropertyTree &queryProperties() { UNIMPLEMENTED; }
+        virtual IPropertyTree *getProperties() { UNIMPLEMENTED; }
+
+        virtual RemoteFilename &getFilename(unsigned copy, RemoteFilename &rfn) { UNIMPLEMENTED; }
+        virtual StringBuffer &getTail(StringBuffer &name) { UNIMPLEMENTED; }
+        virtual StringBuffer &getDirectory(StringBuffer &name,unsigned copy = 0) { UNIMPLEMENTED; }
+        virtual StringBuffer &getPath(StringBuffer &name,unsigned copy = 0) { UNIMPLEMENTED; }
+
+        virtual void serialize(MemoryBuffer &tgt) { UNIMPLEMENTED; }
+
+        virtual bool isMulti() { UNIMPLEMENTED; }
+        virtual RemoteMultiFilename &getMultiFilename(unsigned copy, RemoteMultiFilename &rfn) { UNIMPLEMENTED; }
+
+        virtual bool getCrc(unsigned &crc) { UNIMPLEMENTED; }
+        virtual IFileDescriptor &queryOwner() { UNIMPLEMENTED; }
+        virtual const char *queryOverrideName() { UNIMPLEMENTED; }
+        virtual unsigned copyClusterNum(unsigned copy,unsigned *replicate=NULL) { UNIMPLEMENTED; }
+        virtual IReplicatedFile *getReplicatedFile() { UNIMPLEMENTED; }
+    };
+
+    void testCopy()
+    {
+        CRoxieFileCache cache(true);
+        StringArray remotes;
+        DummyPartDescriptor pdesc;
+        CDateTime dummy;
+        remotes.append("test.remote");
+
+        int f = open("test.remote", _O_WRONLY | _O_CREAT | _O_TRUNC, _S_IREAD | _S_IWRITE);
+        int val = 1;
+        int wrote = write(f, &val, sizeof(int));
+        CPPUNIT_ASSERT(wrote==sizeof(int));
+        close(f);
+
+        Owned<ILazyFileIO> io = cache.openFile("test.local", 0, "test.local", NULL, remotes, sizeof(int), dummy, 0);
+        CPPUNIT_ASSERT(io != NULL);
+
+        // Reading it should read 1
+        val = 0;
+        io->read(0, sizeof(int), &val);
+        CPPUNIT_ASSERT(val==1);
+
+        // Now create the buddy
+
+        f = open("test.buddy", _O_WRONLY | _O_CREAT | _O_TRUNC, _S_IREAD | _S_IWRITE);
+        val = 2;
+        write(f, &val, sizeof(int));
+        close(f);
+
+        // Reading it should still read 1...
+        val = 0;
+        io->read(0, sizeof(int), &val);
+        CPPUNIT_ASSERT(val==1);
+
+        // Now copy it - should copy the buddy
+        cache.doCopy(io, false, false);
+
+        // Reading it should read 2...
+        val = 0;
+        io->read(0, sizeof(int), &val);
+        CPPUNIT_ASSERT(val==2);
+
+        // And the data in the file should be 2
+        f = open("test.local", _O_RDONLY);
+        val = 0;
+        read(f, &val, sizeof(int));
+        close(f);
+        CPPUNIT_ASSERT(val==2);
+
+        io.clear();
+        remove("test.local");
+        remove("test.remote");
+        remove("test.buddy");
+    }
+
+};
+
+CPPUNIT_TEST_SUITE_REGISTRATION( CcdFileTest );
+CPPUNIT_TEST_SUITE_NAMED_REGISTRATION( CcdFileTest, "CcdFileTest" );
+
+#endif

system/jlib/jbuff.cpp

@@ -67,7 +67,7 @@
 
 //-----------------------------------------------------------------------
 
-jlib_decl void *checked_realloc(void *orig, size32_t newlen, size32_t origlen,int errcode)
+jlib_decl void *checked_realloc(void *orig, size_t newlen, size_t origlen,int errcode)
 {
     if (newlen==0) {
         free(orig);
@@ -84,13 +84,13 @@ jlib_decl void *checked_realloc(void *orig, size32_t newlen, size32_t origlen,in
 class jlib_thrown_decl COutOfMemException: public CInterface, implements IOutOfMemException
 {
     int errcode;
-    size32_t wanted;
-    size32_t got;
+    size_t wanted;
+    size_t got;
     static int recursion;
     bool expected;
 public:
     IMPLEMENT_IINTERFACE;
-    COutOfMemException(int _errcode,size32_t _wanted,memsize_t _got,bool _expected) 
+    COutOfMemException(int _errcode,size_t _wanted,size_t _got,bool _expected)
     {
         errcode = _errcode;
         wanted = _wanted;
@@ -102,7 +102,7 @@ public:
 // Bit risky if *very* out of memory so protect against recursion and catch exceptions
             try { 
                 // try to log
-                PROGLOG("Jbuff: Out of Memory (%d,%d,%"I64F"dk)",_errcode,_wanted,(unsigned __int64) (got/1024));
+                PROGLOG("Jbuff: Out of Memory (%d,%"I64F"d,%"I64F"dk)",_errcode,(unsigned __int64)wanted,(unsigned __int64) (got/1024));
                 PrintStackReport();
                 PrintMemoryReport();
             }
@@ -115,9 +115,9 @@ public:
     int             errorCode() const { return errcode; }
     StringBuffer &  errorMessage(StringBuffer &str) const
     { 
-        str.append("Jbuff: Out of Memory (").append(wanted);
+        str.append("Jbuff: Out of Memory (").append((unsigned __int64)wanted);
         if (got) 
-            str.append(',').append(got/1024);
+            str.append(',').append((unsigned __int64)(got/1024));
         return str.append("k)");
     }
     MessageAudience errorAudience() const { return MSGAUD_user; }
@@ -125,12 +125,12 @@ public:
 
 int COutOfMemException::recursion=0;
 
-IOutOfMemException *createOutOfMemException(int errcode,size32_t wanted,memsize_t got,bool expected)
+IOutOfMemException *createOutOfMemException(int errcode,size_t wanted,size_t got,bool expected)
 {
     return new COutOfMemException(errcode,wanted,got,expected);
 }
 
-void RaiseOutOfMemException(int errcode, size32_t wanted, size32_t got,bool expected)
+void RaiseOutOfMemException(int errcode, size_t wanted, size_t got,bool expected)
 {
     throw createOutOfMemException(errcode, wanted, got,expected);
 }

system/jlib/jbuff.hpp

@@ -620,8 +620,8 @@ class CLargeMemorySequentialReader
 
 
 interface IOutOfMemException;
-jlib_decl IOutOfMemException *createOutOfMemException(int errcode, size32_t wanted, memsize_t got=0,bool expected=false);
-jlib_decl void RaiseOutOfMemException(int errcode, size32_t wanted, size32_t got=0,bool expected=false);
+jlib_decl IOutOfMemException *createOutOfMemException(int errcode, size_t wanted, size_t got=0,bool expected=false);
+jlib_decl void RaiseOutOfMemException(int errcode, size_t wanted, size_t got=0,bool expected=false);
 
 interface ILargeMemLimitNotify: extends IInterface
 {
@@ -632,7 +632,7 @@ interface ILargeMemLimitNotify: extends IInterface
 
 extern jlib_decl void setLargeMemLimitNotify(memsize_t size,ILargeMemLimitNotify *notify);
 
-inline void *checked_malloc(size32_t len,int errcode)
+inline void *checked_malloc(size_t len,int errcode)
 {
     if (len==0)
         return NULL;
@@ -642,7 +642,7 @@ inline void *checked_malloc(size32_t len,int errcode)
     return ret;
 }
 
-jlib_decl void *checked_realloc(void *orig, size32_t newlen, size32_t origlen,int errcode);
+jlib_decl void *checked_realloc(void *orig, size_t newlen, size_t origlen,int errcode);
 
 class NonReentrantSpinLock;
 

system/jlib/jsuperhash.cpp

@@ -18,7 +18,7 @@
 
 #include "jlib.hpp"
 #include "jsuperhash.hpp"
-#include <assert.h>
+#include "jexcept.hpp"
 
 #ifndef HASHSIZE_POWER2
 #define HASHSIZE_POWER2
@@ -278,6 +278,8 @@ void SuperHashTable::expand()
     else
         newsize += newsize+1;
 #endif
+    if (newsize < tablesize)
+        throw MakeStringException(0, "HashTable expanded beyond 2^32 items");
     void * *newtable = (void * *) checked_malloc(newsize*sizeof(void *),-603);
     memset(newtable,0,newsize*sizeof(void *));
     void * *oldtable = table;

testing/CMakeLists.txt

@@ -14,7 +14,9 @@
 #    limitations under the License.
 ################################################################################
 HPCC_ADD_SUBDIRECTORY (unittests)
+HPCC_ADD_SUBDIRECTORY (regress)
 install( DIRECTORY regress DESTINATION "./testing" COMPONENT Runtime
          USE_SOURCE_PERMISSIONS
-         PATTERN regress/ecl EXCLUDE )
-
+         PATTERN regress/ecl EXCLUDE
+         PATTERN regress/environment.xml.in EXCLUDE
+         PATTERN regress/CMakeLists.txt EXCLUDE )

testing/regress/CMakeLists.txt

@@ -0,0 +1,19 @@
+################################################################################
+#    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+#
+#    All rights reserved. This program is free software: you can redistribute it
+#    and/or modify
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+################################################################################
+configure_file("environment.xml.in" "environment.xml")
+
+Install ( FILES ${CMAKE_CURRENT_BINARY_DIR}/environment.xml DESTINATION "./testing/regress" COMPONENT Runtime )

testing/regress/README.rst

@@ -21,6 +21,7 @@ Result:
 |                       [--ignoreResult]
 |                       [-X name1=value1[,name2=value2...]]
 |                       [-f optionA=valueA[,optionB=valueB...]]
+|                       [--dynamic source=cluster_list|all]
 |                       {list,setup,run,query} ...
 | 
 |       HPCC Platform Regression suite
@@ -48,6 +49,8 @@ Result:
 |                                 sets the stored input value (stored('name')).
 |        -f optionA=valueA[,optionB=valueB...]
 |                                 set an ECL option (equivalent to #option).
+|        --dynamic source=cluster_list|all
+|                                 execute ECL query through a generated stub with source cluster(s).
 
 Important!
     There is a bug in Python argparse library whichis impacts the quoted parameters. So either in -X or -f or both contains a value with space(s) inside then the whole argument should be put in double quote!
@@ -461,7 +464,48 @@ The format of the output is the same as 'run', except there is a log, result and
 |
 |         End.
 
-6. Tags used in testcases:
+
+6. Use --dynamic parameter:
+---------------------------
+
+Example command:
+
+        ./ecl-test  --dynamic source='all' run -t hthor --runclass=dynamic,dfu
+
+This command executes all ECL tests which are belongs to dynamic and dfu classes. If any test contains //dynamic:source tag then that will executed with all sources via a generated stub for each.
+
+The format of the output is the same as 'run', except all dynamic executed ECL file marked with source:
+
+|         [Action] Suite: hthor
+|         [Action] Queries: 5
+|         [Action]   1. Test: dynamic_test2.ecl ( source: hthor )
+|         [Pass]   1. Pass W20140917-103147 (1 sec)
+|         [Pass]   1. URL http://127.0.0.1:8010/WsWorkunits/WUInfo?Wuid=W20140917-103147
+|         [Action]   2. Test: dynamic_test2.ecl ( source: thor )
+|         [Pass]   2. Pass W20140917-103149 (1 sec)
+|         [Pass]   2. URL http://127.0.0.1:8010/WsWorkunits/WUInfo?Wuid=W20140917-103149
+|         [Action]   3. Test: dynamic_test2.ecl ( source: roxie )
+|         [Pass]   3. Pass W20140917-103151 (1 sec)
+|         [Pass]   3. URL http://127.0.0.1:8010/WsWorkunits/WUInfo?Wuid=W20140917-103151
+|         [Action]   4. Test: spray_test.ecl
+|         [Pass]   4. Pass W20140917-103153 (2 sec)
+|         [Pass]   4. URL http://127.0.0.1:8010/WsWorkunits/WUInfo?Wuid=W20140917-103153
+|         [Action]   5. Test: spray_test2.ecl
+|         [Pass]   5. Pass W20140917-103156 (2 sec)
+|         [Pass]   5. URL http://127.0.0.1:8010/WsWorkunits/WUInfo?Wuid=W20140917-103156
+|         [Action]
+|            Results
+|            -------------------------------------------------
+|             Passing: 5
+|             Failure: 0
+|             -------------------------------------------------
+|             Log: /home/ati/HPCCSystems-regression/log/hthor.14-09-17-10-31-46.log
+|             -------------------------------------------------
+|             Elapsed time: 14 sec  (00:00:14)
+|             -------------------------------------------------
+
+
+7. Tags used in testcases:
 --------------------------
 
     To exclude testcase from cluster or clusters, the tag is:
@@ -486,8 +530,11 @@ The format of the output is the same as 'run', except there is a log, result and
     To define a class to be executed/excluded in run mode.
 //class=<class_name>
 
+    To use dynamic source to execute same ECL with different source
+//dynamic:source
+
 
-7. Key file handling:
+8. Key file handling:
 ---------------------
 
 After an ECL test case execution finished and all output collected the result checking follows these steps:
@@ -555,7 +602,7 @@ If we execute setup on any other target:
 Then the RS executes all ecl files from setup directory and 
     - the test cases results compared with corresponding file in ecl/key directory.
 
-8. Key file generation:
+9. Key file generation:
 -----------------------
 
 The regression suite stores every test case output into ~/HPCCSystems-regression/result directory. This is the latest version of result. (The previous version can be found in ~/HPCCSystems-regression/archives directory.) When a test case execution finished Regression Suite compares this output file with the relevant key file to verify the result.
@@ -568,7 +615,7 @@ So if you have a new test case and it works well on all clusters (or some of the
 
 (To check everything is fine, repeat the step 1 and the query should now pass. )
 
-9. Configuration setting in ecl-test.json file:
+10. Configuration setting in ecl-test.json file:
 -------------------------------------------------------------
 
         "IpAddress":{
@@ -698,7 +745,7 @@ Important!
 
 
 
-10. Authentication:
+11. Authentication:
 -------------------
 
 If your HPCC System is configured to use LDAP authentication you should change value of "username" and "password" fields in ecl-test.json file to yours.

testing/regress/ecl-test

@@ -160,6 +160,8 @@ class RegressMain:
                             nargs=1, type=checkXParam,  default='None',  metavar="name1=value1[,name2=value2...]")
         parser.add_argument('-f', help="set an ECL option (equivalent to #option).",
                             nargs=1, type=checkXParam,  default='None',  metavar="optionA=valueA[,optionB=valueB...]")
+        parser.add_argument('--dynamic', help="execute ECL query through a generated stub with source cluster(s).",
+                            nargs=1, type=checkXParam,  default='None',  metavar="source=cluster_list|all")
 
         subparsers = parser.add_subparsers(help='sub-command help')
 

testing/regress/ecl/dynamicsource_tag.ecl

@@ -0,0 +1,5 @@
+//class=dynamic
+//dynamic:source
+export dynamicsource_tag := MODULE
+   EXPORT execute(string source = '', boolean useLocal = false) := output('Source = fine');
+END;

testing/regress/ecl/key/dynamicsource_tag.xml

@@ -0,0 +1,3 @@
+<Dataset name='Result 1'>
+ <Row><Result_1>Source = fine</Result_1></Row>
+</Dataset>

testing/regress/hpcc/regression/regress.py

@@ -307,6 +307,7 @@ class Regression:
             logging.warn('%s','' , extra={'filebuffer':True,  'filesort':True})
             suite.setEndTime(time.time())
             Regression.displayReport(report, suite.getElapsTime())
+            suite.close()
             self.closeLogging()
 
         except Exception as e:
@@ -385,6 +386,7 @@ class Regression:
 
             suite.setEndTime(time.time())
             Regression.displayReport(report, suite.getElapsTime())
+            suite.close()
             self.closeLogging()
 
         except Exception as e:
@@ -423,6 +425,7 @@ class Regression:
 
             self.StopTimeoutThread()
             Regression.displayReport(report,  time.time()-start)
+            eclfile.close()
             self.closeLogging()
 
         except Exception as e:
@@ -435,7 +438,7 @@ class Regression:
         self.exitmutexes[th].acquire()
 
         logging.debug("runQuery(cluster: '%s', query: '%s', cnt: %d, publish: %s, thread id: %d" % ( cluster, query.ecl, cnt, publish,  th))
-        logging.warn("%3d. Test: %s" % (cnt, query.ecl),  extra={'taskId':cnt})
+        logging.warn("%3d. Test: %s" % (cnt, query.getBaseEclRealName()),  extra={'taskId':cnt})
 
         self.loggermutex.release()
         res = 0

testing/regress/hpcc/regression/suite.py

@@ -20,9 +20,11 @@
 import os
 import sys
 import time
+import glob
 
 from ..util.ecl.file import ECLFile
 from ..common.error import Error
+from ..util.util import checkClusters
 
 class Suite:
     def __init__(self, name, dir_ec, dir_a, dir_ex, dir_r, logDir, args, isSetup=False,  fileList = None):
@@ -30,6 +32,7 @@ class Suite:
             self.name = 'setup_'+name
         else:
             self.name = name
+        self.args=args
         self.suite = []
         self.dir_ec = dir_ec
         self.dir_a = dir_a
@@ -38,6 +41,16 @@ class Suite:
         self.logDir = logDir
         self.exclude = []
         self.publish = []
+        self.isDynamicSource=False
+        if args.dynamic != 'None':
+            self.isDynamicSource=True
+            self.dynamicSources=args.dynamic[0].replace('source=', '').replace('\'','').split(',')
+            self.dynamicSources=checkClusters(self.dynamicSources,  "Dynamic source")
+            pass
+
+        # If there are some temprary files left, then remove them
+        for file in glob.glob(self.dir_ec+'/_tmp*.ecl'):
+            os.unlink(file)
 
         self.buildSuite(args, isSetup, fileList)
 
@@ -51,6 +64,10 @@ class Suite:
                 self.log.write(item+"\n")
             self.log.close();
 
+    def __del__(self):
+        print "Suite destructor."
+        pass
+
     def buildSuite(self, args, isSetup,  fileList):
         if fileList == None:
             if not os.path.isdir(self.dir_ec):
@@ -74,7 +91,7 @@ class Suite:
             if file.endswith(".ecl"):
                 ecl = os.path.join(self.dir_ec, file)
                 eclfile = ECLFile(ecl, self.dir_a, self.dir_ex,
-                                  self.dir_r,  self.name, args)
+                                  self.dir_r,  self.args.target,  args)
                 if isSetup:
                     skipResult = eclfile.testSkip('setup')
                 else:
@@ -99,7 +116,7 @@ class Suite:
                         exclusionReason=' ECL excluded'
 
                     if not exclude:
-                        self.suite.append(eclfile)
+                        self.addFileToSuite(eclfile)
                     else:
                         self.exclude.append(format(file, "30")+exclusionReason)
                 else:
@@ -108,6 +125,21 @@ class Suite:
                 if eclfile.testPublish():
                     self.publish.append(eclfile.getBaseEcl())
 
+    def addFileToSuite(self, eclfile):
+        if eclfile.testDynamicSource() and self.isDynamicSource:
+            # going through the source lists
+            basename = eclfile.getEcl()
+            for source in self.dynamicSources:
+                # generates ECLs based on sources
+                eclfile = ECLFile(basename, self.dir_a, self.dir_ex,
+                                  self.dir_r,  self.name, self.args)
+                eclfile.setDynamicSource(source)
+                # add newly generated ECL to suite
+                self.suite.append(eclfile)
+            pass
+        else:
+            self.suite.append(eclfile)
+
     def testPublish(self, ecl):
         if ecl in self.publish:
             return True
@@ -127,3 +159,7 @@ class Suite:
 
     def getSuiteName(self):
         return self.name
+
+    def close(self):
+        for ecl in self.suite:
+            ecl.close()

testing/regress/hpcc/util/ecl/cc.py

@@ -34,9 +34,15 @@ class ECLCC(Shell):
     def __ECLCC(self):
         return self.command(self.cmd, *self.defaults)
 
-    def getArchive(self, file):
+    def getArchive(self, ecl):
         try:
-            return self.__ECLCC()('-E', file)
+            if ecl.testDynamicSource():
+                stub = ecl.getRealEclSource()
+                # do eclcc with stdin
+                return self.__ECLCC()('-E', stub)
+            else:
+                file = ecl.getEcl()
+                return self.__ECLCC()('-E', file)
         except Error as err:
             logging.debug("getArchive exception:'%s'",  repr(err))
             self.makeArchiveError = str(err)
@@ -50,7 +56,7 @@ class ECLCC(Shell):
             os.mkdir(dirname)
         if os.path.isfile(filename):
             os.unlink(filename)
-        result = self.getArchive(ecl.getEcl())
+        result = self.getArchive(ecl)
 
         if result.startswith( 'Error()'):
             retVal = False
@@ -68,6 +74,7 @@ class ECLCC(Shell):
                 ecl.diff += repr(self.makeArchiveError)
             self.makeArchiveError=''
         else:
+            logging.debug("%3d. makeArchive (filename:'%s')", ecl.getTaskId(), filename )
             FILE = open(filename, "w")
             FILE.write(result)
             FILE.close()

testing/regress/hpcc/util/ecl/file.py

@@ -22,6 +22,7 @@ import logging
 import os
 import traceback
 import re
+import tempfile
 
 from ...util.util import isPositiveIntNum, getConfig
 
@@ -42,6 +43,16 @@ class ECLFile:
     abortReason = ''
     taskId = -1
     ignoreResult=False
+    isDynamicSource=None
+    dynamicSource='test'
+
+    def __del__(self):
+        logging.debug("%3d. File destructor (file:%s).", self.taskId, self.ecl )
+
+    def close(self):
+        if self.tempFile:
+            self.tempFile.close()
+            pass
 
     def __init__(self, ecl, dir_a, dir_ex, dir_r,  cluster, args):
         self.dir_ec = os.path.dirname(ecl)
@@ -49,16 +60,18 @@ class ECLFile:
         self.dir_r = dir_r
         self.dir_a = dir_a
         self.cluster = cluster;
-        baseEcl = os.path.basename(ecl)
-        self.basename = os.path.splitext(baseEcl)[0]
-        baseXml = self.basename + '.xml'
-        self.ecl = baseEcl
-        self.xml_e = baseXml
-        self.xml_r = baseXml
-        self.xml_a = 'archive_' + baseXml
+        self.baseEcl = os.path.basename(ecl)
+        self.basename = os.path.splitext(self.baseEcl)[0]
+        self.baseXml = self.basename + '.xml'
+        self.ecl = self.baseEcl
+        self.xml_e = self.baseXml
+        self.xml_r = self.baseXml
+        self.xml_a = 'archive_' + self.baseXml
         self.jobname = self.basename
         self.diff = ''
         self.abortReason =''
+        self.tags={}
+        self.tempFile=None
 
         #If there is a --publish CL parameter then force publish this ECL file
         self.forcePublish=False
@@ -77,7 +90,7 @@ class ECLFile:
                     testSpec = testSpec.replace('*',  '\w+')
 
                 testSpec = testSpec.replace('.',  '\.')
-                match = re.match(testSpec,  baseEcl)
+                match = re.match(testSpec,  self.baseEcl)
                 if match:
                     optXs = ("-X"+val.replace(',',  ',-X')).split(',')
                     self.processKeyValPairs(optXs,  self.optXHash)
@@ -158,17 +171,43 @@ class ECLFile:
         return os.path.join(self.dir_r, self.xml_r)
 
     def getArchive(self):
-        return os.path.join(self.dir_a, self.xml_a)
+        logging.debug("%3d. getArchive (isDynamicSource:'%s')", self.taskId, self.isDynamicSource )
+        if self.isDynamicSource:
+            dynamicFilename='archive_' + self.basename + '_'+ self.dynamicSource+'.xml'
+            return os.path.join(self.dir_a, dynamicFilename)
+        else:
+            return os.path.join(self.dir_a, self.xml_a)
 
     def getEcl(self):
         return os.path.join(self.dir_ec, self.ecl)
 
+    def getRealEclSource(self):
+        logging.debug("%3d. getRealEclSource (isDynamicSource:'%s')", self.taskId, self.isDynamicSource )
+        if self.isDynamicSource:
+            # generate stub and return with it
+            self.tempFile = tempfile.NamedTemporaryFile(prefix='_temp',  suffix='.ecl', dir=self.dir_ec)
+            self.tempFile.write('import '+self.basename+';\n')
+            self.tempFile.write(self.basename+'.execute(source := \''+self.dynamicSource+'\');\n')
+            self.tempFile.flush()
+            # now return with the generated
+            return os.path.join(self.dir_ec, self.tempFile.name)
+        else:
+            return os.path.join(self.dir_ec, self.ecl)
+
     def getBaseEcl(self):
         return self.ecl
 
     def getBaseEclName(self):
         return self.basename
 
+    def getBaseEclRealName(self):
+        logging.debug("%3d. getBaseEclRealName (isDynamicSource:'%s')", self.taskId, self.isDynamicSource )
+        if self.isDynamicSource:
+            realName = self.basename + '.ecl ( source: ' + self.dynamicSource + ' )'
+        else:
+            realName = self.getBaseEcl()
+        return realName
+
     def getWuid(self):
         return self.wuid
 
@@ -278,6 +317,16 @@ class ECLFile:
         logging.debug("%3d. testInClass() returns with: %s",  self.taskId,  retVal)
         return retVal
 
+    def testDynamicSource(self):
+        if self.isDynamicSource == None:
+            # Standard string has a problem with unicode characters
+            # use byte arrays and binary file open instead
+            tag = b'//dynamic:source'
+            logging.debug("%3d. testDynamicSource (ecl:'%s', tag:'%s')", self.taskId, self.ecl,  tag)
+            self.isDynamicSource = self.__checkTag(tag)
+            logging.debug("%3d. testDynamicSource() returns with: %s",  self.taskId,  self.isDynamicSource)
+        return self.isDynamicSource
+
     def getTimeout(self):
         timeout = 0
         # Standard string has a problem with unicode characters
@@ -359,3 +408,10 @@ class ECLFile:
     def getFParameters(self):
         return self.optF
 
+    def setDynamicSource(self,  source):
+        self.dynamicSource = source
+        self.isDynamicSource=True
+
+    def getDynamicSource(self):
+        return self.dynamicSource
+