Merge branch 'closedown-4.2.x' into candidate-5.0.2

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

ecl/eclagent/eclgraph.cpp

@@ -916,7 +916,6 @@ void EclSubGraph::doExecute(const byte * parentExtract, bool checkDependencies)
 
     if (!prepare(parentExtract, checkDependencies))
     {
-        throwUnexpected();
         executed = true;
         return;
     }

ecl/hql/hqlexpr.hpp

@@ -879,6 +879,7 @@ public:
         expandCallsWhenBound = DEFAULT_EXPAND_CALL;
         ignoreUnknownImport = false;
         _clear(metaState);
+        aborting = false;
     }
 
     void addForwardReference(IHqlScope * owner, IHasUnlinkedOwnerReference * child);
@@ -897,6 +898,8 @@ public:
     inline IPropertyTree * getMetaTree() { return LINK(metaTree); }
     inline IPropertyTree * queryArchive() const { return archive; }
     inline IEclRepository * queryRepository() const { return eclRepository; }
+    inline bool isAborting() const { return aborting; }
+    inline void setAborting() { aborting = true; }
 
 public:
     Linked<IPropertyTree> archive;
@@ -909,6 +912,7 @@ public:
     CIArrayOf<ForwardScopeItem> forwardLinks;
     bool expandCallsWhenBound;
     bool ignoreUnknownImport;
+    bool aborting;
 
 private:
     bool checkBeginMeta();
@@ -937,7 +941,7 @@ public:
     {
         errs.set(other.errs); 
         functionCache = other.functionCache; 
-        curAttrTree.set(other.curAttrTree); 
+        curAttrTree.set(other.curAttrTree);
     }
     HqlLookupContext(HqlParseContext & _parseCtx, IErrorReceiver * _errs)
     : parseCtx(_parseCtx), errs(_errs)
@@ -960,6 +964,8 @@ public:
     inline bool queryExpandCallsWhenBound() const { return parseCtx.expandCallsWhenBound; }
     inline HqlParseContext & queryParseContext() const { return parseCtx; }
     inline unsigned numErrors() const { return errs ? errs->errCount() : 0; }
+    inline bool isAborting() const { return parseCtx.isAborting(); }
+    inline void setAborting() { parseCtx.setAborting(); }
 
 protected:
 

ecl/hql/hqlgram2.cpp

@@ -376,7 +376,7 @@ void HqlGram::init(IHqlScope * _globalScope, IHqlScope * _containerScope)
 
     outerScopeAccessDepth = 0;
     inType = false;
-    aborting = false;
+    aborting = lookupCtx.isAborting();
     errorHandler = NULL;
     moduleName = NULL;
     resolveSymbols = true;
@@ -3267,6 +3267,10 @@ IHqlExpression *HqlGram::lookupSymbol(IIdAtom * searchName, const attribute& err
     if (expectedUnknownId)
         return NULL;
 
+    //Check periodically if parsing a referenced identifier has caused the compile to abort.
+    if (lookupCtx.isAborting())
+        aborting = true;
+
     try
     {
         // If there is a temporary scope, we only look up in that (and it must exist!).
@@ -10892,6 +10896,7 @@ void HqlGram::abortParsing()
 {
     // disable more error report
     disableError();
+    lookupCtx.setAborting();
     aborting = true;
 }
 

roxie/ccd/ccdfile.cpp

@@ -141,6 +141,13 @@ public:
     virtual void setCopying(bool _copying)
     {
         CriticalBlock b(crit);
+        if (remote && currentIdx)
+        {
+            // The current location is not our preferred location. Recheck whether we can now access our preferred location
+            setFailure();
+            currentIdx = 0;
+            _checkOpen();
+        }
         copying = _copying; 
     }
 
@@ -559,6 +566,7 @@ typedef StringArray *StringArrayPtr;
 
 class CRoxieFileCache : public CInterface, implements ICopyFileProgress, implements IRoxieFileCache
 {
+    friend class CcdFileTest;
     mutable ICopyArrayOf<ILazyFileIO> todo; // Might prefer a queue but probably doesn't really matter.
     InterruptableSemaphore toCopy;
     InterruptableSemaphore toClose;
@@ -568,6 +576,7 @@ class CRoxieFileCache : public CInterface, implements ICopyFileProgress, impleme
     bool started;
     bool aborting;
     bool closing;
+    bool testMode;
     bool closePending[2];
     StringAttrMapping fileErrorList;
     Semaphore bctStarted;
@@ -615,7 +624,7 @@ class CRoxieFileCache : public CInterface, implements ICopyFileProgress, impleme
                            offset_t size, const CDateTime &modified, unsigned crc)
     {
         Owned<IFile> local = createIFile(localLocation);
-        bool isCompressed = pdesc->queryOwner().isCompressed();
+        bool isCompressed = testMode ? false : pdesc->queryOwner().isCompressed();
         Owned<CLazyFileIO> ret = new CLazyFileIO(local.getLink(), size, modified, crc, isCompressed);
         RoxieFileStatus fileStatus = fileUpToDate(local, size, modified, crc, isCompressed);
         if (fileStatus == FileIsValid)
@@ -631,7 +640,10 @@ class CRoxieFileCache : public CInterface, implements ICopyFileProgress, impleme
 
             // put the peerRoxieLocations next in the list
             StringArray localLocations;
-            appendRemoteLocations(pdesc, localLocations, localLocation, roxieName, true);  // Adds all locations on the same cluster
+            if (testMode)
+                localLocations.append("test.buddy");
+            else
+                appendRemoteLocations(pdesc, localLocations, localLocation, roxieName, true);  // Adds all locations on the same cluster
             ForEachItemIn(roxie_idx, localLocations)
             {
                 try
@@ -646,6 +658,15 @@ class CRoxieFileCache : public CInterface, implements ICopyFileProgress, impleme
                         ret->addSource(remote.getClear());
                         addedOne = true;
                     }
+                    else if (status==FileNotFound)
+                    {
+                        // Even though it's not on the buddy (yet), add it to the locations since it may well be there
+                        // by the time we come to copy (and if it is, we want to copy from there)
+                        if (miscDebugTraceLevel > 5)
+                            DBGLOG("adding missing peer location %s", remoteName);
+                        ret->addSource(remote.getClear());
+                        // Don't set addedOne - we need to go to remote too
+                    }
                     else if (miscDebugTraceLevel > 10)
                         DBGLOG("Checked peer roxie location %s, status=%d", remoteName, (int) status);
                 }
@@ -656,7 +677,7 @@ class CRoxieFileCache : public CInterface, implements ICopyFileProgress, impleme
                 }
             }
 
-            if (!addedOne && (copyResources || useRemoteResources))  // If no peer locations available, go to remote
+            if (!addedOne && (copyResources || useRemoteResources || testMode))  // If no peer locations available, go to remote
             {
                 ForEachItemIn(idx, remoteLocationInfo)
                 {
@@ -891,7 +912,7 @@ class CRoxieFileCache : public CInterface, implements ICopyFileProgress, impleme
 public:
     IMPLEMENT_IINTERFACE;
 
-    CRoxieFileCache(bool testmode = false) : bct(*this), hct(*this)
+    CRoxieFileCache(bool _testMode = false) : bct(*this), hct(*this), testMode(_testMode)
     {
         aborting = false;
         closing = false;
@@ -2643,3 +2664,105 @@ extern IRoxieWriteHandler *createRoxieWriteHandler(IRoxieDaliHelper *_daliHelper
 {
     return new CRoxieWriteHandler(_daliHelper, _dFile, _clusters);
 }
+
+//================================================================================================================
+
+#ifdef _USE_CPPUNIT
+#include "unittests.hpp"
+
+class CcdFileTest : public CppUnit::TestFixture
+{
+    CPPUNIT_TEST_SUITE(CcdFileTest);
+        CPPUNIT_TEST(testCopy);
+    CPPUNIT_TEST_SUITE_END();
+protected:
+
+    class DummyPartDescriptor : public CInterfaceOf<IPartDescriptor>
+    {
+        virtual unsigned queryPartIndex() { UNIMPLEMENTED; }
+
+        virtual unsigned numCopies() { UNIMPLEMENTED; }
+        virtual INode *getNode(unsigned copy=0) { UNIMPLEMENTED; }
+        virtual INode *queryNode(unsigned copy=0) { UNIMPLEMENTED; }
+
+        virtual IPropertyTree &queryProperties() { UNIMPLEMENTED; }
+        virtual IPropertyTree *getProperties() { UNIMPLEMENTED; }
+
+        virtual RemoteFilename &getFilename(unsigned copy, RemoteFilename &rfn) { UNIMPLEMENTED; }
+        virtual StringBuffer &getTail(StringBuffer &name) { UNIMPLEMENTED; }
+        virtual StringBuffer &getDirectory(StringBuffer &name,unsigned copy = 0) { UNIMPLEMENTED; }
+        virtual StringBuffer &getPath(StringBuffer &name,unsigned copy = 0) { UNIMPLEMENTED; }
+
+        virtual void serialize(MemoryBuffer &tgt) { UNIMPLEMENTED; }
+
+        virtual bool isMulti() { UNIMPLEMENTED; }
+        virtual RemoteMultiFilename &getMultiFilename(unsigned copy, RemoteMultiFilename &rfn) { UNIMPLEMENTED; }
+
+        virtual bool getCrc(unsigned &crc) { UNIMPLEMENTED; }
+        virtual IFileDescriptor &queryOwner() { UNIMPLEMENTED; }
+        virtual const char *queryOverrideName() { UNIMPLEMENTED; }
+        virtual unsigned copyClusterNum(unsigned copy,unsigned *replicate=NULL) { UNIMPLEMENTED; }
+        virtual IReplicatedFile *getReplicatedFile() { UNIMPLEMENTED; }
+    };
+
+    void testCopy()
+    {
+        CRoxieFileCache cache(true);
+        StringArray remotes;
+        DummyPartDescriptor pdesc;
+        CDateTime dummy;
+        remotes.append("test.remote");
+
+        int f = open("test.remote", _O_WRONLY | _O_CREAT | _O_TRUNC, _S_IREAD | _S_IWRITE);
+        int val = 1;
+        int wrote = write(f, &val, sizeof(int));
+        CPPUNIT_ASSERT(wrote==sizeof(int));
+        close(f);
+
+        Owned<ILazyFileIO> io = cache.openFile("test.local", 0, "test.local", NULL, remotes, sizeof(int), dummy, 0);
+        CPPUNIT_ASSERT(io != NULL);
+
+        // Reading it should read 1
+        val = 0;
+        io->read(0, sizeof(int), &val);
+        CPPUNIT_ASSERT(val==1);
+
+        // Now create the buddy
+
+        f = open("test.buddy", _O_WRONLY | _O_CREAT | _O_TRUNC, _S_IREAD | _S_IWRITE);
+        val = 2;
+        write(f, &val, sizeof(int));
+        close(f);
+
+        // Reading it should still read 1...
+        val = 0;
+        io->read(0, sizeof(int), &val);
+        CPPUNIT_ASSERT(val==1);
+
+        // Now copy it - should copy the buddy
+        cache.doCopy(io, false, false);
+
+        // Reading it should read 2...
+        val = 0;
+        io->read(0, sizeof(int), &val);
+        CPPUNIT_ASSERT(val==2);
+
+        // And the data in the file should be 2
+        f = open("test.local", _O_RDONLY);
+        val = 0;
+        read(f, &val, sizeof(int));
+        close(f);
+        CPPUNIT_ASSERT(val==2);
+
+        io.clear();
+        remove("test.local");
+        remove("test.remote");
+        remove("test.buddy");
+    }
+
+};
+
+CPPUNIT_TEST_SUITE_REGISTRATION( CcdFileTest );
+CPPUNIT_TEST_SUITE_NAMED_REGISTRATION( CcdFileTest, "CcdFileTest" );
+
+#endif

system/jlib/jbuff.cpp

@@ -67,7 +67,7 @@
 
 //-----------------------------------------------------------------------
 
-jlib_decl void *checked_realloc(void *orig, size32_t newlen, size32_t origlen,int errcode)
+jlib_decl void *checked_realloc(void *orig, size_t newlen, size_t origlen,int errcode)
 {
     if (newlen==0) {
         free(orig);
@@ -84,13 +84,13 @@ jlib_decl void *checked_realloc(void *orig, size32_t newlen, size32_t origlen,in
 class jlib_thrown_decl COutOfMemException: public CInterface, implements IOutOfMemException
 {
     int errcode;
-    size32_t wanted;
-    size32_t got;
+    size_t wanted;
+    size_t got;
     static int recursion;
     bool expected;
 public:
     IMPLEMENT_IINTERFACE;
-    COutOfMemException(int _errcode,size32_t _wanted,memsize_t _got,bool _expected) 
+    COutOfMemException(int _errcode,size_t _wanted,size_t _got,bool _expected)
     {
         errcode = _errcode;
         wanted = _wanted;
@@ -102,7 +102,7 @@ public:
 // Bit risky if *very* out of memory so protect against recursion and catch exceptions
             try { 
                 // try to log
-                PROGLOG("Jbuff: Out of Memory (%d,%d,%"I64F"dk)",_errcode,_wanted,(unsigned __int64) (got/1024));
+                PROGLOG("Jbuff: Out of Memory (%d,%"I64F"d,%"I64F"dk)",_errcode,(unsigned __int64)wanted,(unsigned __int64) (got/1024));
                 PrintStackReport();
                 PrintMemoryReport();
             }
@@ -115,9 +115,9 @@ public:
     int             errorCode() const { return errcode; }
     StringBuffer &  errorMessage(StringBuffer &str) const
     { 
-        str.append("Jbuff: Out of Memory (").append(wanted);
+        str.append("Jbuff: Out of Memory (").append((unsigned __int64)wanted);
         if (got) 
-            str.append(',').append(got/1024);
+            str.append(',').append((unsigned __int64)(got/1024));
         return str.append("k)");
     }
     MessageAudience errorAudience() const { return MSGAUD_user; }
@@ -125,12 +125,12 @@ public:
 
 int COutOfMemException::recursion=0;
 
-IOutOfMemException *createOutOfMemException(int errcode,size32_t wanted,memsize_t got,bool expected)
+IOutOfMemException *createOutOfMemException(int errcode,size_t wanted,size_t got,bool expected)
 {
     return new COutOfMemException(errcode,wanted,got,expected);
 }
 
-void RaiseOutOfMemException(int errcode, size32_t wanted, size32_t got,bool expected)
+void RaiseOutOfMemException(int errcode, size_t wanted, size_t got,bool expected)
 {
     throw createOutOfMemException(errcode, wanted, got,expected);
 }

system/jlib/jbuff.hpp

@@ -620,8 +620,8 @@ class CLargeMemorySequentialReader
 
 
 interface IOutOfMemException;
-jlib_decl IOutOfMemException *createOutOfMemException(int errcode, size32_t wanted, memsize_t got=0,bool expected=false);
-jlib_decl void RaiseOutOfMemException(int errcode, size32_t wanted, size32_t got=0,bool expected=false);
+jlib_decl IOutOfMemException *createOutOfMemException(int errcode, size_t wanted, size_t got=0,bool expected=false);
+jlib_decl void RaiseOutOfMemException(int errcode, size_t wanted, size_t got=0,bool expected=false);
 
 interface ILargeMemLimitNotify: extends IInterface
 {
@@ -632,7 +632,7 @@ interface ILargeMemLimitNotify: extends IInterface
 
 extern jlib_decl void setLargeMemLimitNotify(memsize_t size,ILargeMemLimitNotify *notify);
 
-inline void *checked_malloc(size32_t len,int errcode)
+inline void *checked_malloc(size_t len,int errcode)
 {
     if (len==0)
         return NULL;
@@ -642,7 +642,7 @@ inline void *checked_malloc(size32_t len,int errcode)
     return ret;
 }
 
-jlib_decl void *checked_realloc(void *orig, size32_t newlen, size32_t origlen,int errcode);
+jlib_decl void *checked_realloc(void *orig, size_t newlen, size_t origlen,int errcode);
 
 class NonReentrantSpinLock;
 

system/jlib/jsuperhash.cpp

@@ -18,7 +18,7 @@
 
 #include "jlib.hpp"
 #include "jsuperhash.hpp"
-#include <assert.h>
+#include "jexcept.hpp"
 
 #ifndef HASHSIZE_POWER2
 #define HASHSIZE_POWER2
@@ -278,6 +278,8 @@ void SuperHashTable::expand()
     else
         newsize += newsize+1;
 #endif
+    if (newsize < tablesize)
+        throw MakeStringException(0, "HashTable expanded beyond 2^32 items");
     void * *newtable = (void * *) checked_malloc(newsize*sizeof(void *),-603);
     memset(newtable,0,newsize*sizeof(void *));
     void * *oldtable = table;