Merge remote-tracking branch 'origin/candidate-6.4.0'

Signed-off-by: Gavin Halliday <gavin.halliday@lexisnexis.com>

common/remote/remoteerr.hpp

@@ -94,7 +94,9 @@ interface REMOTE_API IDAFS_Exception: extends IException
 enum DAFS_ERROR_CODES {
     DAFSERR_connection_failed               = -1,   
     DAFSERR_authenticate_failed             = -2,
-    DAFSERR_protocol_failure                = -3
+    DAFSERR_protocol_failure                = -3,
+    DAFSERR_serveraccept_failed             = -4,
+    DAFSERR_serverinit_failed               = -5
 };
 
 

common/remote/sockfile.cpp

@@ -179,7 +179,7 @@ static unsigned maxReceiveTime = 0;
 static class _securitySettings
 {
 public:
-    SSLCfg          useSSL;
+    DAFSConnectCfg  connectMethod;
     unsigned short  daFileSrvPort;
     unsigned short  daFileSrvSSLPort;
     const char *    certificate;
@@ -188,7 +188,7 @@ public:
 
     _securitySettings()
     {
-        queryDafsSecSettings(&useSSL, &daFileSrvPort, &daFileSrvSSLPort, &certificate, &privateKey, &passPhrase);
+        queryDafsSecSettings(&connectMethod, &daFileSrvPort, &daFileSrvSSLPort, &certificate, &privateKey, &passPhrase);
     }
 } securitySettings;
 
@@ -549,7 +549,7 @@ void setDafsEndpointPort(SocketEndpoint &ep)
     }
     if (ep.port==0)
     {
-        if ( (securitySettings.useSSL == SSLNone) || (securitySettings.useSSL == UnsecureFirst) )
+        if ( (securitySettings.connectMethod == SSLNone) || (securitySettings.connectMethod == UnsecureFirst) )
             ep.port = securitySettings.daFileSrvPort;
         else
             ep.port = securitySettings.daFileSrvSSLPort;
@@ -958,7 +958,7 @@ class CRemoteBase: public CInterface
     Owned<ISocket>          socket;
     static  SocketEndpoint  lastfailep;
     static unsigned         lastfailtime;
-    SSLCfg                  useSSL;
+    DAFSConnectCfg          connectMethod;
 
     void connectSocket(SocketEndpoint &ep, unsigned localConnectTime=0, unsigned localRetries=0)
     {
@@ -1159,7 +1159,7 @@ protected: friend class CRemoteFileIO;
             if (!socket)
             {
                 bool doConnect = true;
-                if (useSSL == SSLFirst || useSSL == UnsecureFirst)
+                if (connectMethod == SSLFirst || connectMethod == UnsecureFirst)
                 {
                     // MCK - could maintain a list of 100 or so previous endpoints and if connection failed
                     // then mark port down for a delay (like 15 min above) to avoid having to try every time ...
@@ -1329,7 +1329,7 @@ public:
         : filename(_filename)
     {
         ep = _ep;
-        useSSL = securitySettings.useSSL;
+        connectMethod = securitySettings.connectMethod;
     }
 
     void connect()
@@ -2638,7 +2638,7 @@ void CRemoteFile::copyTo(IFile *dest, size32_t buffersize, ICopyFileProgress *pr
 
 ISocket *checkSocketSecure(ISocket *socket)
 {
-    if (securitySettings.useSSL == SSLNone)
+    if (securitySettings.connectMethod == SSLNone)
         return LINK(socket);
 
     char pname[256];
@@ -2666,7 +2666,7 @@ ISocket *connectDafs(SocketEndpoint &ep, unsigned timeoutms)
 {
     Owned<ISocket> socket;
 
-    if ( (securitySettings.useSSL == SSLNone) || (securitySettings.useSSL == SSLOnly) )
+    if ( (securitySettings.connectMethod == SSLNone) || (securitySettings.connectMethod == SSLOnly) )
     {
         socket.setown(ISocket::connect_timeout(ep, timeoutms));
         return socket.getClear();
@@ -5721,7 +5721,29 @@ public:
         return new cCommandProcessor();
     }
 
-    void run(SSLCfg _useSSL, SocketEndpoint &listenep, unsigned sslPort)
+    void cleanupSocket(ISocket *sock)
+    {
+        if (!sock)
+            return;
+        try
+        {
+            sock->shutdown();
+        }
+        catch (IException *e)
+        {
+            e->Release();
+        }
+        try
+        {
+            sock->close();
+        }
+        catch (IException *e)
+        {
+            e->Release();
+        }
+    }
+
+    void run(DAFSConnectCfg _connectMethod, SocketEndpoint &listenep, unsigned sslPort)
     {
         SocketEndpoint sslep(listenep);
         if (sslPort)
@@ -5730,10 +5752,10 @@ public:
             sslep.port = securitySettings.daFileSrvSSLPort;
         Owned<ISocket> acceptSocket, acceptSSLSocket;
 
-        if (_useSSL != SSLOnly)
+        if (_connectMethod != SSLOnly)
         {
             if (listenep.port == 0)
-                throw createDafsException(DAFSERR_connection_failed, "dafilesrv port not specified");
+                throw createDafsException(DAFSERR_serverinit_failed, "dafilesrv port not specified");
 
             if (listenep.isNull())
                 acceptSocket.setown(ISocket::create(listenep.port));
@@ -5745,13 +5767,40 @@ public:
             }
         }
 
-        if (_useSSL)
+        if (_connectMethod == SSLOnly || _connectMethod == SSLFirst || _connectMethod == UnsecureFirst)
         {
             if (sslep.port == 0)
-                throw createDafsException(DAFSERR_connection_failed, "Secure dafilesrv port not specified");
+                throw createDafsException(DAFSERR_serverinit_failed, "Secure dafilesrv port not specified");
 
-            if ( (_useSSL != UnsecureFirst) && (!securitySettings.certificate || !securitySettings.privateKey) )
-                throw createDafsException(DAFSERR_connection_failed, "SSL Certificate and/or Key file information not found in environment.conf");
+            if (_connectMethod == UnsecureFirst)
+            {
+                // don't fail, but warn - this allows for fast SSL client rejections
+                if (!securitySettings.certificate)
+                    WARNLOG("SSL Certificate information not found in environment.conf, cannot accept SSL connections");
+                else if ( !checkFileExists(securitySettings.certificate) )
+                {
+                    WARNLOG("SSL Certificate File not found in environment.conf, cannot accept SSL connections");
+                    securitySettings.certificate = nullptr;
+                }
+                if (!securitySettings.privateKey)
+                    WARNLOG("SSL Key information not found in environment.conf, cannot accept SSL connections");
+                else if ( !checkFileExists(securitySettings.privateKey) )
+                {
+                    WARNLOG("SSL Key File not found in environment.conf, cannot accept SSL connections");
+                    securitySettings.privateKey = nullptr;
+                }
+            }
+            else
+            {
+                if (!securitySettings.certificate)
+                    throw createDafsException(DAFSERR_serverinit_failed, "SSL Certificate information not found in environment.conf");
+                if (!checkFileExists(securitySettings.certificate))
+                    throw createDafsException(DAFSERR_serverinit_failed, "SSL Certificate File not found in environment.conf");
+                if (!securitySettings.privateKey)
+                    throw createDafsException(DAFSERR_serverinit_failed, "SSL Key information not found in environment.conf");
+                if (!checkFileExists(securitySettings.privateKey))
+                    throw createDafsException(DAFSERR_serverinit_failed, "SSL Key File not found in environment.conf");
+            }
 
             if (sslep.isNull())
                 acceptSSLSocket.setown(ISocket::create(sslep.port));
@@ -5763,25 +5812,25 @@ public:
             }
         }
 
-        run(_useSSL, acceptSocket.getClear(), acceptSSLSocket.getClear());
+        run(_connectMethod, acceptSocket.getClear(), acceptSSLSocket.getClear());
     }
 
-    void run(SSLCfg _useSSL, ISocket *regSocket, ISocket *secureSocket)
+    void run(DAFSConnectCfg _connectMethod, ISocket *regSocket, ISocket *secureSocket)
     {
-        if (_useSSL != SSLOnly)
+        if (_connectMethod != SSLOnly)
         {
             if (regSocket)
                 acceptsock.setown(regSocket);
             else
-                throw createDafsException(DAFSERR_connection_failed, "Invalid non-secure socket");
+                throw createDafsException(DAFSERR_serverinit_failed, "Invalid non-secure socket");
         }
 
-        if (_useSSL)
+        if (_connectMethod == SSLOnly || _connectMethod == SSLFirst || _connectMethod == UnsecureFirst)
         {
             if (secureSocket)
                 securesock.setown(secureSocket);
             else
-                throw createDafsException(DAFSERR_connection_failed, "Invalid secure socket");
+                throw createDafsException(DAFSERR_serverinit_failed, "Invalid secure socket");
         }
 
         selecthandler->start();
@@ -5792,9 +5841,9 @@ public:
             Owned<ISocket> sockSSL;
             bool sockavail = false;
             bool securesockavail = false;
-            if (_useSSL == SSLNone)
+            if (_connectMethod == SSLNone)
                 sockavail = acceptsock->wait_read(1000*60*1)!=0;
-            else if (_useSSL == SSLOnly)
+            else if (_connectMethod == SSLOnly)
                 securesockavail = securesock->wait_read(1000*60*1)!=0;
             else
             {
@@ -5855,30 +5904,30 @@ public:
 
                 if (securesockavail)
                 {
+                    Owned<ISecureSocket> ssock;
                     try
                     {
                         sockSSL.setown(securesock->accept(true));
                         if (!sockSSL||stopping)
                             break;
 
-                        if ( (_useSSL == UnsecureFirst) && (!securitySettings.certificate || !securitySettings.privateKey) )
+                        if ( (_connectMethod == UnsecureFirst) && (!securitySettings.certificate || !securitySettings.privateKey) )
                         {
                             // for client secure_connect() to fail quickly ...
-                            sockSSL->shutdown();
-                            sockSSL->close();
+                            cleanupSocket(sockSSL);
                             sockSSL.clear();
                             securesockavail = false;
                         }
                         else
                         {
 #ifdef _USE_OPENSSL
-                            Owned<ISecureSocket> ssock = createSecureSocket(sockSSL.getClear(), ServerSocket);
+                            ssock.setown(createSecureSocket(sockSSL.getClear(), ServerSocket));
                             int status = ssock->secure_accept();
                             if (status < 0)
-                                throw createDafsException(DAFSERR_connection_failed,"Failure to establish secure connection");
+                                throw createDafsException(DAFSERR_serveraccept_failed,"Failure to establish secure connection");
                             sockSSL.setown(ssock.getLink());
 #else
-                            throw createDafsException(DAFSERR_connection_failed,"Failure to establish secure connection: OpenSSL disabled in build");
+                            throw createDafsException(DAFSERR_serveraccept_failed,"Failure to establish secure connection: OpenSSL disabled in build");
 #endif
 #ifdef _DEBUG
                             SocketEndpoint eps;
@@ -5889,12 +5938,23 @@ public:
 #endif
                         }
                     }
-                    catch (IException *e)
+                    catch (IJSOCK_Exception *e)
                     {
+                        // accept failed ...
                         EXCLOG(e,"CRemoteFileServer (secure)");
                         e->Release();
                         break;
                     }
+                    catch (IException *e) // IDAFS_Exception also ...
+                    {
+                        EXCLOG(e,"CRemoteFileServer1 (secure)");
+                        e->Release();
+                        cleanupSocket(sockSSL);
+                        sockSSL.clear();
+                        cleanupSocket(ssock);
+                        ssock.clear();
+                        securesockavail = false;
+                    }
                 }
 
                 if (sockavail)
@@ -6270,7 +6330,7 @@ protected:
         // IThreaded
             virtual void main()
             {
-                SSLCfg sslCfg = SSLNone;
+                DAFSConnectCfg sslCfg = SSLNone;
                 server->run(sslCfg, socket, nullptr);
             }
         };

common/remote/sockfile.hpp

@@ -53,7 +53,7 @@ enum ThrottleClass
 
 interface IRemoteFileServer : extends IInterface
 {
-    virtual void run(SSLCfg useSSL, SocketEndpoint &listenep, unsigned sslPort=0) = 0;
+    virtual void run(DAFSConnectCfg connectMethod, SocketEndpoint &listenep, unsigned sslPort=0) = 0;
     virtual void stop() = 0;
     virtual unsigned idleTime() = 0; // in ms
     virtual void setThrottle(ThrottleClass throttleClass, unsigned limit, unsigned delayMs=DEFAULT_STDCMD_THROTTLEDELAYMS, unsigned cpuThreshold=DEFAULT_STDCMD_THROTTLECPULIMIT, unsigned queueLimit=DEFAULT_STDCMD_THROTTLEQUEUELIMIT) = 0;

common/thorhelper/thorcommon.cpp

@@ -1134,7 +1134,7 @@ protected:
     CThorStreamDeserializerSource source;
     Owned<ISourceRowPrefetcher> prefetcher;
     CThorContiguousRowBuffer prefetchBuffer; // used if prefetcher set
-    bool grouped;
+    EmptyRowSemantics emptyRowSemantics;
     bool eoi;
     bool eos;
     bool eog;
@@ -1156,13 +1156,12 @@ protected:
 public:
     IMPLEMENT_IINTERFACE_USING(CSimpleInterface);
 
-    CRowStreamReader(IFileIO *_fileio, IMemoryMappedFile *_mmfile, IRowInterfaces *rowif, offset_t _ofs, offset_t _len, bool _tallycrc, bool _grouped)
-        : fileio(_fileio), mmfile(_mmfile), allocator(rowif->queryRowAllocator()), prefetchBuffer(NULL) 
+    CRowStreamReader(IFileIO *_fileio, IMemoryMappedFile *_mmfile, IRowInterfaces *rowif, offset_t _ofs, offset_t _len, bool _tallycrc, EmptyRowSemantics _emptyRowSemantics)
+        : fileio(_fileio), mmfile(_mmfile), allocator(rowif->queryRowAllocator()), prefetchBuffer(NULL), emptyRowSemantics(_emptyRowSemantics)
     {
 #ifdef TRACE_CREATE
         PROGLOG("CRowStreamReader %d = %p",++rdnum,this);
 #endif
-        grouped = _grouped;
         eoi = false;
         eos = false;
         eog = false;
@@ -1175,7 +1174,7 @@ public:
         if (prefetcher)
             prefetchBuffer.setStream(strm);
         source.setStream(strm);
-        deserializer.set(rowif->queryRowDeserializer());            
+        deserializer.set(rowif->queryRowDeserializer());
     }
 
     ~CRowStreamReader()
@@ -1198,35 +1197,60 @@ public:
 
     const void *nextRow()
     {
-        if (eog) {
+        if (eog)
+        {
             eog = false;
-            return NULL;
+            return nullptr;
         }
         if (eos)
-            return NULL;
-        if (source.eos()) {
+            return nullptr;
+        if (source.eos())
+        {
             eos = true;
-            return NULL;
+            return nullptr;
         }
-        RtlDynamicRowBuilder rowBuilder(*allocator);
-        size_t size = deserializer->deserialize(rowBuilder,source);
-        if (grouped) {
+        if (ers_allow == emptyRowSemantics)
+        {
             byte b;
-            source.read(sizeof(b),&b);
-            eog = (b==1);
+            source.read(sizeof(b), &b);
+            if (1==b)
+                return nullptr;
+            RtlDynamicRowBuilder rowBuilder(*allocator);
+            size_t size = deserializer->deserialize(rowBuilder, source);
+            return rowBuilder.finalizeRowClear(size);
+        }
+        else
+        {
+            RtlDynamicRowBuilder rowBuilder(*allocator);
+            size_t size = deserializer->deserialize(rowBuilder, source);
+            if (ers_eogonly == emptyRowSemantics)
+            {
+                byte b;
+                source.read(sizeof(b), &b);
+                eog = (b==1);
+            }
+            return rowBuilder.finalizeRowClear(size);
         }
-        return rowBuilder.finalizeRowClear(size);
     }
 
     const void *prefetchRow(size32_t *sz)
     {
         if (eog) 
             eog = false;
-        else if (!eos) {
+        else if (!eos)
+        {
             if (source.eos()) 
                 eos = true;
-            else {
+            else
+            {
                 assertex(prefetcher);
+                if (ers_allow == emptyRowSemantics)
+                {
+                    byte b;
+                    strm->get(sizeof(b),&b);
+                    if (1==b)
+                        return nullptr;
+                }
                 prefetcher->readAhead(prefetchBuffer);
                 const byte * ret = prefetchBuffer.queryRow();
                 if (sz)
@@ -1242,7 +1266,8 @@ public:
     void prefetchDone()
     {
         prefetchBuffer.finishedRow();
-        if (grouped) {
+        if (ers_eogonly == emptyRowSemantics)
+        {
             byte b;
             strm->get(sizeof(b),&b);
             eog = (b==1);
@@ -1293,8 +1318,8 @@ class CLimitedRowStreamReader : public CRowStreamReader
     unsigned __int64 rownum;
 
 public:
-    CLimitedRowStreamReader(IFileIO *_fileio, IMemoryMappedFile *_mmfile, IRowInterfaces *rowif, offset_t _ofs, offset_t _len, unsigned __int64 _maxrows, bool _tallycrc, bool _grouped)
-        : CRowStreamReader(_fileio, _mmfile, rowif, _ofs, _len, _tallycrc, _grouped)
+    CLimitedRowStreamReader(IFileIO *_fileio, IMemoryMappedFile *_mmfile, IRowInterfaces *rowif, offset_t _ofs, offset_t _len, unsigned __int64 _maxrows, bool _tallycrc, EmptyRowSemantics _emptyRowSemantics)
+        : CRowStreamReader(_fileio, _mmfile, rowif, _ofs, _len, _tallycrc, _emptyRowSemantics)
     {
         maxrows = _maxrows;
         rownum = 0;
@@ -1329,6 +1354,7 @@ bool UseMemoryMappedRead = false;
 IExtRowStream *createRowStreamEx(IFile *file, IRowInterfaces *rowIf, offset_t offset, offset_t len, unsigned __int64 maxrows, unsigned rwFlags, IExpander *eexp)
 {
     bool compressed = TestRwFlag(rwFlags, rw_compress);
+    EmptyRowSemantics emptyRowSemantics = extractESRFromRWFlags(rwFlags);
     if (UseMemoryMappedRead && !compressed)
     {
         PROGLOG("Memory Mapped read of %s",file->queryFilename());
@@ -1336,9 +1362,9 @@ IExtRowStream *createRowStreamEx(IFile *file, IRowInterfaces *rowIf, offset_t of
         if (!mmfile)
             return NULL;
         if (maxrows == (unsigned __int64)-1)
-            return new CRowStreamReader(NULL, mmfile, rowIf, offset, len, TestRwFlag(rwFlags, rw_crc), TestRwFlag(rwFlags, rw_grouped));
+            return new CRowStreamReader(NULL, mmfile, rowIf, offset, len, TestRwFlag(rwFlags, rw_crc), emptyRowSemantics);
         else
-            return new CLimitedRowStreamReader(NULL, mmfile, rowIf, offset, len, maxrows, TestRwFlag(rwFlags, rw_crc), TestRwFlag(rwFlags, rw_grouped));
+            return new CLimitedRowStreamReader(NULL, mmfile, rowIf, offset, len, maxrows, TestRwFlag(rwFlags, rw_crc), emptyRowSemantics);
     }
     else
     {
@@ -1354,9 +1380,9 @@ IExtRowStream *createRowStreamEx(IFile *file, IRowInterfaces *rowIf, offset_t of
         if (!fileio)
             return NULL;
         if (maxrows == (unsigned __int64)-1)
-            return new CRowStreamReader(fileio, NULL, rowIf, offset, len, TestRwFlag(rwFlags, rw_crc), TestRwFlag(rwFlags, rw_grouped));
+            return new CRowStreamReader(fileio, NULL, rowIf, offset, len, TestRwFlag(rwFlags, rw_crc), emptyRowSemantics);
         else
-            return new CLimitedRowStreamReader(fileio, NULL, rowIf, offset, len, maxrows, TestRwFlag(rwFlags, rw_crc), TestRwFlag(rwFlags, rw_grouped));
+            return new CLimitedRowStreamReader(fileio, NULL, rowIf, offset, len, maxrows, TestRwFlag(rwFlags, rw_crc), emptyRowSemantics);
     }
 }
 
@@ -1380,7 +1406,7 @@ class CRowStreamWriter : private IRowSerializerTarget, implements IExtRowWriter,
     Linked<IOutputRowSerializer> serializer;
     Linked<IEngineRowAllocator> allocator;
     CRC32 crc;
-    bool grouped;
+    EmptyRowSemantics emptyRowSemantics;
     bool tallycrc;
     unsigned nested;
     MemoryAttr ma;
@@ -1441,13 +1467,12 @@ class CRowStreamWriter : private IRowSerializerTarget, implements IExtRowWriter,
 public:
     IMPLEMENT_IINTERFACE_USING(CSimpleInterface);
 
-    CRowStreamWriter(IFileIOStream *_stream,IOutputRowSerializer *_serializer,IEngineRowAllocator *_allocator,bool _grouped, bool _tallycrc, bool _autoflush)
-        : stream(_stream), serializer(_serializer), allocator(_allocator)
+    CRowStreamWriter(IFileIOStream *_stream, IOutputRowSerializer *_serializer, IEngineRowAllocator *_allocator, EmptyRowSemantics _emptyRowSemantics, bool _tallycrc, bool _autoflush)
+        : stream(_stream), serializer(_serializer), allocator(_allocator), emptyRowSemantics(_emptyRowSemantics)
     {
 #ifdef TRACE_CREATE
         PROGLOG("createRowWriter %d = %p",++wrnum,this);
 #endif
-        grouped = _grouped;
         tallycrc = _tallycrc;
         nested = 0;
         buf = (byte *)ma.allocate(ROW_WRITER_BUFFERSIZE);
@@ -1472,26 +1497,44 @@ public:
 
     void putRow(const void *row)
     {
-        if (row) {
-            serializer->serialize(*this,(const byte *)row);
-            if (grouped) {
+        if (row)
+        {
+            if (ers_allow == emptyRowSemantics)
+            {
                 byte b = 0;
-                if (bufpos<ROW_WRITER_BUFFERSIZE) 
-                    buf[bufpos++] = b;
-                else 
-                    extbuf.append(b);
+                put(1, &b);
+                serializer->serialize(*this, (const byte *)row);
+            }
+            else
+            {
+                serializer->serialize(*this, (const byte *)row);
+                if (ers_eogonly == emptyRowSemantics)
+                {
+                    byte b = 0;
+                    if (bufpos<ROW_WRITER_BUFFERSIZE)
+                        buf[bufpos++] = b;
+                    else
+                        extbuf.append(b);
+                }
             }
             allocator->releaseRow(row);
         }
-        else if (grouped) { // backpatch
+        else if (ers_eogonly == emptyRowSemantics) // backpatch
+        {
             byte b = 1;
             if (extbuf.length())
-                extbuf.writeDirect(extbuf.length()-1,sizeof(b),&b);
-            else {
+                extbuf.writeDirect(extbuf.length()-1, sizeof(b), &b);
+            else
+            {
                 assertex(bufpos);
                 buf[bufpos-1] = b;
             }
         }
+        else if (ers_allow == emptyRowSemantics)
+        {
+            byte b = 1;
+            put(1, &b);
+        }
     }
 
     void flush()
@@ -1617,7 +1660,8 @@ IExtRowWriter *createRowWriter(IFileIOStream *strm, IRowInterfaces *rowIf, unsig
 {
     if (0 != (flags & (rw_extend|rw_buffered|COMP_MASK)))
         throw MakeStringException(0, "Unsupported createRowWriter flags");
-    Owned<CRowStreamWriter> writer = new CRowStreamWriter(strm, rowIf->queryRowSerializer(), rowIf->queryRowAllocator(), TestRwFlag(flags, rw_grouped), TestRwFlag(flags, rw_crc), TestRwFlag(flags, rw_autoflush));
+    EmptyRowSemantics emptyRowSemantics = extractESRFromRWFlags(flags);
+    Owned<CRowStreamWriter> writer = new CRowStreamWriter(strm, rowIf->queryRowSerializer(), rowIf->queryRowAllocator(), emptyRowSemantics, TestRwFlag(flags, rw_crc), TestRwFlag(flags, rw_autoflush));
     return writer.getClear();
 }
 

common/thorhelper/thorcommon.hpp

@@ -93,7 +93,8 @@ enum RowReaderWriterFlags
     rw_autoflush      = 0x40,
     rw_buffered       = 0x80,
     rw_lzw            = 0x100, // if rw_compress
-    rw_lz4            = 0x200  // if rw_compress
+    rw_lz4            = 0x200, // if rw_compress
+    rw_sparse         = 0x400  // NB: mutually exclusive with rw_grouped
 };
 #define DEFAULT_RWFLAGS (rw_buffered|rw_autoflush|rw_compressblkcrc)
 inline bool TestRwFlag(unsigned flags, RowReaderWriterFlags flag) { return 0 != (flags & flag); }
@@ -159,6 +160,30 @@ interface IExtRowWriter: extends IRowWriter
     virtual void flush(CRC32 *crcout=NULL) = 0;
 };
 
+enum EmptyRowSemantics { ers_forbidden, ers_allow, ers_eogonly };
+inline unsigned mapESRToRWFlags(EmptyRowSemantics emptyRowSemantics)
+{
+    switch (emptyRowSemantics)
+    {
+        case ers_allow:
+            return rw_sparse;
+        case ers_eogonly:
+            return rw_grouped;
+        default:
+            return 0;
+    }
+}
+
+inline EmptyRowSemantics extractESRFromRWFlags(unsigned rwFlags)
+{
+    if (TestRwFlag(rwFlags, rw_sparse))
+        return ers_allow;
+    else if (TestRwFlag(rwFlags, rw_grouped))
+        return ers_eogonly;
+    else
+        return ers_forbidden;
+}
+
 interface IExpander;
 extern THORHELPER_API IExtRowStream *createRowStream(IFile *file, IRowInterfaces *rowif, unsigned flags=DEFAULT_RWFLAGS, IExpander *eexp=NULL);
 extern THORHELPER_API IExtRowStream *createRowStreamEx(IFile *file, IRowInterfaces *rowif, offset_t offset=0, offset_t len=(offset_t)-1, unsigned __int64 maxrows=(unsigned __int64)-1, unsigned flags=DEFAULT_RWFLAGS, IExpander *eexp=NULL);

common/thorhelper/thorstrand.cpp

@@ -277,7 +277,7 @@ RowBlockAllocator::RowBlockAllocator(roxiemem::IRowManager & rowManager, size32_
 
     size_t classSize = sizeof(RoxieRowBlock) - RoxieRowBlock::numDummyDynamicRows * sizeof(void *);
     size_t requestedSize = classSize + minRowsPerBlock * sizeof(void*);
-    unsigned heapFlags = roxiemem::RHFunique|roxiemem::RHFnofragment;
+    roxiemem::RoxieHeapFlags heapFlags = roxiemem::RHFunique|roxiemem::RHFnofragment;
     heap.setown(rowManager.createFixedRowHeap(requestedSize, 0, heapFlags, 0));
     rowsPerBlock = (rowManager.getExpectedCapacity(requestedSize, heapFlags) - classSize ) / sizeof(void*);
     assertex(rowsPerBlock >= minRowsPerBlock);

dali/base/dadiags.cpp

@@ -205,14 +205,27 @@ public:
                 }
                 else if (0 == stricmp(id, "settracetransactions")) {
                     PROGLOG("Dalidiag requests Trace Transactions");
-                    if(traceAllTransactions(true))
+                    if (traceAllTransactions())
+                        mb.append("OK - transaction tracing enabled");
+                    else
                         mb.append("OK - no change");
+                }
+                else if (0 == stricmp(id, "settraceslowtransactions")) {
+                    unsigned thresholdMs;
+                    params.read(thresholdMs);
+                    PROGLOG("Dalidiag requests Trace Slow Transactions (ms = %u)", thresholdMs);
+                    if (traceSlowTransactions(thresholdMs)) {
+                        if (thresholdMs)
+                            mb.append("OK - slow transaction tracing enabled");
+                        else
+                            mb.append("OK - transaction tracing disabled");
+                    }
                     else
-                        mb.append("OK - transaction tracing enabled");
+                        mb.append("OK - no change");
                 }
                 else if (0 == stricmp(id, "cleartracetransactions")) {
                     PROGLOG("Dalidiag requests Trace Transactions stopped");
-                    if(traceAllTransactions(false))
+                    if (clearAllTransactions())
                         mb.append("OK - transaction tracing disabled");
                     else
                         mb.append("OK - no change");

dali/base/dasds.hpp

@@ -272,7 +272,9 @@ enum
 };
 extern da_decl IStoreHelper *createStoreHelper(const char *storeName, const char *location, const char *remoteBackupLocation, unsigned configFlags, unsigned keepStores=0, unsigned delay=5000, const bool *abort=NULL);
 extern da_decl bool applyXmlDeltas(IPropertyTree &root, IIOStream &stream, bool stopOnError=false);
-extern da_decl bool traceAllTransactions(bool on); // server only 
+extern da_decl bool traceAllTransactions(); // server only
+extern da_decl bool traceSlowTransactions(unsigned thresholdMs); // server only
+extern da_decl bool clearAllTransactions(); // server only
 
 extern da_decl void LogRemoteConn(IRemoteConnection *conn); // logs info about a connection
 

dali/base/dautils.cpp

@@ -3181,12 +3181,44 @@ ILocalOrDistributedFile* createLocalOrDistributedFile(const char *fname,IUserDes
 }
 
 static bool transactionLoggingOn=false;
+static cycle_t slowTransactionThreshold=0;
 const bool &queryTransactionLogging() { return transactionLoggingOn; }
-bool traceAllTransactions(bool on)
+cycle_t querySlowTransactionThreshold() { return slowTransactionThreshold; }
+bool traceAllTransactions()
 {
-    bool ret = transactionLoggingOn;
-    transactionLoggingOn = on;
-    return ret;
+    if (transactionLoggingOn)
+        return false; // no change
+    transactionLoggingOn = true;
+    return true;
+}
+
+bool clearAllTransactions()
+{
+    if (!transactionLoggingOn)
+        return false; // no change
+    transactionLoggingOn = false;
+    slowTransactionThreshold = 0;
+    return true;
+}
+
+bool traceSlowTransactions(unsigned thresholdMs)
+{
+    if (thresholdMs)
+    {
+        cycle_t newSlowTransactionThreshold = nanosec_to_cycle(((unsigned __int64)thresholdMs)*1000000);
+        bool changed = !transactionLoggingOn || (slowTransactionThreshold != newSlowTransactionThreshold);
+        slowTransactionThreshold = newSlowTransactionThreshold;
+        transactionLoggingOn = true;
+        return changed;
+    }
+    else if (transactionLoggingOn) // was on, turning off
+    {
+        transactionLoggingOn = false;
+        slowTransactionThreshold = 0;
+        return true; // changed
+    }
+    else // was already off
+        return false;
 }
 
 class CLockInfo : public CSimpleInterfaceOf<ILockInfo>

dali/base/dautils.hpp

@@ -179,10 +179,11 @@ public:
 };
 
 extern da_decl const bool &queryTransactionLogging();
+extern da_decl cycle_t querySlowTransactionThreshold();
 struct da_decl TransactionLog
 {
     CTransactionLogTracker &owner;
-    unsigned startMs, extraStartMs;
+    cycle_t startCycles, extraStartCycles;
     StringBuffer msg;
     unsigned cmd;
     const SocketEndpoint &ep;
@@ -192,36 +193,49 @@ struct da_decl TransactionLog
         {
             if (cmd > owner.getMax())
                 cmd = owner.getMax(); // unknown
-            extraStartMs = 0;
             owner.startTransaction(cmd);
             owner.getCmdText(cmd, msg);
             msg.append(", endpoint=");
             ep.getUrlStr(msg);
-            startMs = msTick();
+            startCycles = get_cycles_now();
         }
         else
-            startMs = 0;
+            startCycles = 0;
+        extraStartCycles = 0;
     }
     inline ~TransactionLog()
     {
-        if (startMs)
+        if (startCycles)
         {
-            unsigned now = msTick();
-            unsigned transCount = owner.getTransactionCount(cmd);
             owner.endTransaction(cmd);
-            if (extraStartMs)
-                PROGLOG("<<<[%d] (Timing: total=%d, from mark=%d) %s", transCount, now-startMs, extraStartMs-startMs, msg.str());
+            cycle_t slowTransactionThreshold = querySlowTransactionThreshold();
+            cycle_t cyclesNow = get_cycles_now();
+            if (slowTransactionThreshold)
+            {
+                cycle_t elapsedCycles = cyclesNow - startCycles;
+                if (elapsedCycles < slowTransactionThreshold)
+                    return;
+            }
+            unsigned transCount = owner.getTransactionCount(cmd)+1; // +1 = this one
+            unsigned elapsedSinceStartMs = static_cast<unsigned>(cycle_to_millisec(cyclesNow - startCycles));
+            if (extraStartCycles)
+            {
+                unsigned elapsedSinceExtraMs = static_cast<unsigned>(cycle_to_millisec(extraStartCycles - startCycles));
+                PROGLOG("<<<[%d] (Timing: total=%u, from mark=%u) %s", transCount, elapsedSinceStartMs, elapsedSinceExtraMs, msg.str());
+            }
             else
-                PROGLOG("<<<[%d] (Timing: total=%d) %s", transCount, now-startMs, msg.str());
+                PROGLOG("<<<[%d] (Timing: total=%u) %s", transCount, elapsedSinceStartMs, msg.str());
         }
     }
     inline void log()
     {
+        if (querySlowTransactionThreshold()) return; // suppress if only logging exit of slow transactions
         unsigned transCount = owner.getTransactionCount(cmd);
         PROGLOG(">>>[%d] %s", transCount, msg.str());
     }
     inline void log(const char *formatMsg, ...) __attribute__((format(printf, 2, 3)))
     {
+        if (querySlowTransactionThreshold()) return; // suppress if only logging exit of slow transactions
         va_list args;
         va_start(args, formatMsg);
         msg.append(" ");
@@ -231,8 +245,8 @@ struct da_decl TransactionLog
     }
     inline void markExtra()
     {
-        if (!extraStartMs)
-            extraStartMs = msTick();
+        if (!extraStartCycles)
+            extraStartCycles = get_cycles_now();
     }
     inline void extra(const char *formatMsg, ...) __attribute__((format(printf, 2, 3)))
     {

dali/dafilesrv/dafilesrv.cpp

@@ -32,6 +32,7 @@
 #endif
 
 
+#include "remoteerr.hpp"
 #include "sockfile.hpp"
 
 void usage()
@@ -352,12 +353,12 @@ int main(int argc,char **argv)
     StringBuffer instanceName;
 
     // Get SSL Settings
-    SSLCfg          useSSL;
+    DAFSConnectCfg  connectMethod;
     unsigned short  port;
     unsigned short  sslport;
     const char *    sslCertFile;
     const char *    sslKeyFile;
-    queryDafsSecSettings(&useSSL, &port, &sslport, &sslCertFile, &sslKeyFile, nullptr);
+    queryDafsSecSettings(&connectMethod, &port, &sslport, &sslCertFile, &sslKeyFile, nullptr);
 
     unsigned maxThreads = DEFAULT_THREADLIMIT;
     unsigned maxThreadsDelayMs = DEFAULT_THREADLIMITDELAYMS;
@@ -489,35 +490,16 @@ int main(int argc,char **argv)
         }
         else if (stricmp(argv[i],"-NOSSL")==0) { // overrides config setting
             i++;
-            if (useSSL)
+            if (connectMethod == SSLOnly || connectMethod == SSLFirst || connectMethod == UnsecureFirst)
             {
                 PROGLOG("DaFileSrv SSL specified in config but overridden by -NOSSL in command line");
-                useSSL = SSLNone;
+                connectMethod = SSLNone;
             }
         }
         else
             break;
     }
 
-    if ( (useSSL == SSLOnly) || (useSSL == SSLFirst) )
-    {
-        if ( !sslCertFile || !sslKeyFile )
-        {
-            ERRLOG("DaFileSrv SSL specified but certificate and/or key file information missing from environment.conf");
-            exit(-1);
-        }
-        if ( !checkFileExists(sslCertFile) )
-        {
-            ERRLOG("DaFileSrv SSL specified but certificate file (%s) not found", sslCertFile);
-            exit(-1);
-        }
-        if ( !checkFileExists(sslKeyFile) )
-        {
-            ERRLOG("DaFileSrv SSL specified but key file (%s) not found", sslKeyFile);
-            exit(-1);
-        }
-    }
-
     if (0 == logDir.length())
     {
         getConfigurationDirectory(NULL,"log","dafilesrv",instanceName.str(),logDir);
@@ -555,32 +537,42 @@ int main(int argc,char **argv)
         recvbufsize = (argc>i+2)?(atoi(argv[i+2])*1024):0;
     }
 
-    if ( (useSSL == SSLNone) && (listenep.port == 0) )
+    if ( (connectMethod == SSLNone) && (listenep.port == 0) )
     {
         printf("\nError, port must not be 0\n");
         usage();
         exit(-1);
     }
-    else if ( (useSSL == SSLOnly) && (sslport == 0) )
+    else if ( (connectMethod == SSLOnly) && (sslport == 0) )
     {
         printf("\nError, secure port must not be 0\n");
         usage();
         exit(-1);
     }
-    else if ( ((useSSL == SSLFirst) || (useSSL == UnsecureFirst)) && ((listenep.port == 0) || (sslport == 0)) )
+    else if ( ((connectMethod == SSLFirst) || (connectMethod == UnsecureFirst)) && ((listenep.port == 0) || (sslport == 0)) )
     {
         printf("\nError, both port and secure port must not be 0\n");
         usage();
         exit(-1);
     }
 
+    StringBuffer secMethod;
+    if (connectMethod == SSLNone)
+        secMethod.append("SSLNone");
+    else if (connectMethod == SSLOnly)
+        secMethod.append("SSLOnly");
+    else if (connectMethod == SSLFirst)
+        secMethod.append("SSLFirst");
+    else if (connectMethod == UnsecureFirst)
+        secMethod.append("UnsecureFirst");
+
     if (isdaemon) {
 #ifdef _WIN32
         class cserv: public CService
         {
             bool stopped;
             bool started;
-            SSLCfg useSSL;
+            DAFSConnectCfg connectMethod;
             SocketEndpoint listenep;
             bool requireauthenticate;
             unsigned maxThreads;
@@ -613,12 +605,12 @@ int main(int argc,char **argv)
 
         public:
 
-            cserv(SSLCfg _useSSL, SocketEndpoint _listenep,
+            cserv(DAFSConnectCfg _connectMethod, SocketEndpoint _listenep,
                         unsigned _maxThreads, unsigned _maxThreadsDelayMs, unsigned _maxAsyncCopy,
                         unsigned _parallelRequestLimit, unsigned _throttleDelayMs, unsigned _throttleCPULimit,
                         unsigned _parallelSlowRequestLimit, unsigned _throttleSlowDelayMs, unsigned _throttleSlowCPULimit,
                         unsigned _sslport)
-            : useSSL(_useSSL), listenep(_listenep), pollthread(this),
+            : connectMethod(_connectMethod), listenep(_listenep), pollthread(this),
                   maxThreads(_maxThreads), maxThreadsDelayMs(_maxThreadsDelayMs), maxAsyncCopy(_maxAsyncCopy),
                   parallelRequestLimit(_parallelRequestLimit), throttleDelayMs(_throttleDelayMs), throttleCPULimit(_throttleCPULimit),
                   parallelSlowRequestLimit(_parallelSlowRequestLimit), throttleSlowDelayMs(_throttleSlowDelayMs), throttleSlowCPULimit(_throttleSlowCPULimit),
@@ -686,9 +678,9 @@ int main(int argc,char **argv)
                 else
                     listenep.getUrlStr(eps);
 
-                if (useSSL != SSLOnly)
+                if (connectMethod != SSLOnly)
                     PROGLOG("Opening " DAFS_SERVICE_DISPLAY_NAME " on %s", eps.str());
-                if (useSSL)
+                if (connectMethod == SSLOnly || connectMethod == SSLFirst || connectMethod == UnsecureFirst)
                 {
                     SocketEndpoint sslep(listenep);
                     sslep.port = sslport;
@@ -700,15 +692,6 @@ int main(int argc,char **argv)
                     PROGLOG("Opening " DAFS_SERVICE_DISPLAY_NAME " on SECURE %s", eps.str());
                 }
 
-                StringBuffer secMethod;
-                if (useSSL == SSLNone)
-                    secMethod.append("SSLNone");
-                else if (useSSL == SSLOnly)
-                    secMethod.append("SSLOnly");
-                else if (useSSL == SSLFirst)
-                    secMethod.append("SSLFirst");
-                else if (useSSL == UnsecureFirst)
-                    secMethod.append("UnsecureFirst");
                 PROGLOG("Dali File Server socket security model: %s", secMethod.str());
 
                 const char * verstring = remoteServerVersionString();
@@ -719,7 +702,7 @@ int main(int argc,char **argv)
                 server->setThrottle(ThrottleStd, parallelRequestLimit, throttleDelayMs, throttleCPULimit);
                 server->setThrottle(ThrottleSlow, parallelSlowRequestLimit, throttleSlowDelayMs, throttleSlowCPULimit);
                 try {
-                    server->run(useSSL, listenep, sslport);
+                    server->run(connectMethod, listenep, sslport);
                 }
                 catch (IException *e) {
                     EXCLOG(e,DAFS_SERVICE_NAME);
@@ -728,7 +711,7 @@ int main(int argc,char **argv)
                 PROGLOG(DAFS_SERVICE_DISPLAY_NAME " Stopped");
                 stopped = true;
             }
-        } service(useSSL, listenep,
+        } service(connectMethod, listenep,
                 maxThreads, maxThreadsDelayMs, maxAsyncCopy,
                 parallelRequestLimit, throttleDelayMs, throttleCPULimit,
                 parallelSlowRequestLimit, throttleSlowDelayMs, throttleSlowCPULimit, sslport);
@@ -759,9 +742,9 @@ int main(int argc,char **argv)
         eps.append(listenep.port);
     else
         listenep.getUrlStr(eps);
-    if (useSSL != SSLOnly)
+    if (connectMethod != SSLOnly)
         PROGLOG("Opening Dali File Server on %s", eps.str());
-    if (useSSL)
+    if (connectMethod == SSLOnly || connectMethod == SSLFirst || connectMethod == UnsecureFirst)
     {
         SocketEndpoint sslep(listenep);
         sslep.port = sslport;
@@ -773,15 +756,6 @@ int main(int argc,char **argv)
         PROGLOG("Opening Dali File Server on SECURE %s", eps.str());
     }
 
-    StringBuffer secMethod;
-    if (useSSL == SSLNone)
-        secMethod.append("SSLNone");
-    else if (useSSL == SSLOnly)
-        secMethod.append("SSLOnly");
-    else if (useSSL == SSLFirst)
-        secMethod.append("SSLFirst");
-    else if (useSSL == UnsecureFirst)
-        secMethod.append("UnsecureFirst");
     PROGLOG("Dali File Server socket security model: %s", secMethod.str());
 
     PROGLOG("Version: %s", verstring);
@@ -808,11 +782,13 @@ int main(int argc,char **argv)
     writeSentinelFile(sentinelFile);
     try
     {
-        server->run(useSSL, listenep, sslport);
+        server->run(connectMethod, listenep, sslport);
     }
     catch (IException *e)
     {
         EXCLOG(e,"DAFILESRV");
+        if (e->errorCode() == DAFSERR_serverinit_failed)
+            removeSentinelFile(sentinelFile); // so init does not keep trying to start it ...
         e->Release();
     }
     stopPerformanceMonitor();

dali/dalidiag/dalidiag.cpp

@@ -64,6 +64,7 @@ void usage(const char *exe)
     printf("-unlock <connection_id> [close] -- forcibly disconnect an sds lock\n"); 
     printf("                                   (use id's given by '-locks'\n");
     printf("-settracetransactions    -- trace dali transactions\n");
+    printf("-settraceslowtransactions <millisecond-threshold> -- trace slow dali transactions\n");
     printf("-cleartracetransactions  -- stop tracing dali transactions\n");
     printf("-setldapflags <val>      -- set LDAP flags\n");
     printf("-getldapflags            -- get LDAP flags\n");
@@ -694,6 +695,17 @@ int main(int _argc, char* argv[])
                     printf("\n%s:\n%s",arg,buf.str());
                     break;
                 }
+                if ((stricmp(arg,"settraceslowtransactions")==0)) {
+                    MemoryBuffer mb;
+                    mb.append("settraceslowtransactions");
+                    unsigned slowThresholdMs = atoi(argv[++i]);
+                    mb.append(slowThresholdMs);
+                    getDaliDiagnosticValue(mb);
+                    StringAttr response;
+                    mb.read(response);
+                    printf("\nsettraceslowtransactions:\n%s", response.get());
+                    break;
+                }
                 else {
                     for (;;) {
                         getDaliDiagnosticValue(arg,buf.clear());

dali/dfuplus/dfuplus.cpp

@@ -36,16 +36,16 @@
 
 static class CSecuritySettings
 {
-    SSLCfg useSSL;
+    DAFSConnectCfg connectMethod;
     unsigned short daliServixPort;
     unsigned short daliServixSSLPort;
 public:
     CSecuritySettings()
     {
-        queryDafsSecSettings(&useSSL, &daliServixPort, &daliServixSSLPort, nullptr, nullptr, nullptr);
+        queryDafsSecSettings(&connectMethod, &daliServixPort, &daliServixSSLPort, nullptr, nullptr, nullptr);
     }
 
-    SSLCfg querySSLCfg() { return useSSL; }
+    DAFSConnectCfg queryDAFSConnectCfg() { return connectMethod; }
     unsigned short queryDaliServixPort() { return daliServixPort; }
     unsigned short queryDaliServixSSLPort() { return daliServixSSLPort; }
 } securitySettings;
@@ -78,7 +78,7 @@ public:
     int run()
     {
         try {
-            server->run(securitySettings.querySSLCfg(), listenep);
+            server->run(securitySettings.queryDAFSConnectCfg(), listenep);
         }
         catch (IException *e) {
             EXCLOG(e,"dfuplus(dafilesrv)");
@@ -122,17 +122,17 @@ bool CDfuPlusHelper::runLocalDaFileSvr(SocketEndpoint &listenep,bool requireauth
 
     unsigned sslport = securitySettings.queryDaliServixSSLPort();
 
-    SSLCfg useSSL = securitySettings.querySSLCfg();
+    DAFSConnectCfg connectMethod = securitySettings.queryDAFSConnectCfg();
 
     StringBuffer addlPort;
     SocketEndpoint printep(listenep);
     if (printep.isNull())
     {
-        if (useSSL == SSLNone)
+        if (connectMethod == SSLNone)
             addlPort.appendf("%u", port);
-        else if (useSSL == SSLOnly)
+        else if (connectMethod == SSLOnly)
             addlPort.appendf("%u", sslport);
-        else if (useSSL == SSLFirst)
+        else if (connectMethod == SSLFirst)
             addlPort.appendf("%u:%u", sslport, port);
         else
             addlPort.appendf("%u:%u", port, sslport);
@@ -140,11 +140,11 @@ bool CDfuPlusHelper::runLocalDaFileSvr(SocketEndpoint &listenep,bool requireauth
     }
     else
     {
-        if (useSSL == SSLNone)
+        if (connectMethod == SSLNone)
             printep.port = port;
-        else if (useSSL == SSLOnly)
+        else if (connectMethod == SSLOnly)
             printep.port = sslport;
-        else if (useSSL == SSLFirst)
+        else if (connectMethod == SSLFirst)
         {
             printep.port = sslport;
             addlPort.appendf(":%u", port);

docs/ECLReference/ECLReference.xml

@@ -5,13 +5,11 @@
   <title>ECL Reference</title>
 
   <setinfo>
-    <corpauthor>HPCC Systems<superscript>®</superscript>
-      <para>
-        <xi:include href="common/Version.xml" xpointer="xpointer(//*[@id='CHMVer'])"
-                xmlns:xi="http://www.w3.org/2001/XInclude" />
-      </para>
-    </corpauthor>
-
+    <corpauthor>HPCC Systems<superscript>®</superscript> <para>
+        <xi:include href="common/Version.xml"
+                    xpointer="xpointer(//*[@id='CHMVer'])"
+                    xmlns:xi="http://www.w3.org/2001/XInclude" />
+      </para></corpauthor>
   </setinfo>
 
   <xi:include href="../ECLLanguageReference/ECLR-includer.xml"
@@ -22,6 +20,10 @@
               xpointer="element(/1)"
               xmlns:xi="http://www.w3.org/2001/XInclude" />
 
+  <xi:include href="../DynamicESDL/DynamicESDL_Includer.xml"
+              xpointer="element(/1)"
+              xmlns:xi="http://www.w3.org/2001/XInclude" />
+
   <xi:include href="../ECLProgrammersGuide/PrGd-Includer.xml"
               xpointer="element(/1)"
               xmlns:xi="http://www.w3.org/2001/XInclude" />

docs/ECLWatch/ECLWa_mods/ECLWatchQueries.xml

@@ -282,7 +282,7 @@ depending on where they wind up.-->
                   </listitem>
                 </itemizedlist>
 
-                <para>The Filter also also supports wild cards. </para>
+                <para>The Filter also supports wild cards.</para>
               </listitem>
             </varlistentry>
 
@@ -573,12 +573,20 @@ depending on where they wind up.-->
         </sect3>
       </sect2>
 
-      <sect2 id="Package_Maps">
+      <sect2 id="Package_Maps" role="brk">
         <title>Package Maps</title>
 
-        <para>From the Queries icon link, you can access the package maps
+        <para>A package map provides a reference to the contents of a superkey
+        used in queries that overrides the original definition. Package map
+        file mappings can be organized into a collection of files defining
+        some subsets of queries or to organize by various groupings such as
+        functions, files, developers, etc. These subsets are called <emphasis
+        role="bold">parts</emphasis>. For more information about Package Maps
+        see the <emphasis>Roxie Reference</emphasis> guide.</para>
+
+        <para>From the Queries icon link, you can access the Package Maps
         page. Press the <emphasis role="bold">Package Maps</emphasis> button
-        on the navigation sub-menu bar, to access to the Package Maps on your
+        on the navigation sub-menu bar, to access the Package Maps on your
         cluster. <figure>
             <title>Package Maps</title>
 
@@ -589,33 +597,301 @@ depending on where they wind up.-->
             </mediaobject>
           </figure></para>
 
-        <para>The package maps page, displays all the package maps loaded on
+        <para>The package maps page displays all the package maps loaded on
         your cluster. You can Add, Activate, Deactivate, Delete, or Open a
         package map. To examine a package map, select a package map from the
         list.</para>
 
-        <para>You can select the <emphasis role="bold">Validate Package
-        Map</emphasis> tab to validate a package map.</para>
+        <para>To update the package maps you are using, you would either edit
+        the package map file or add a new one and then activate it. You could
+        later delete the old one.</para>
 
-        <para><figure>
-            <title>Validate Package Maps</title>
+        <sect3 id="Package_Map_Actions">
+          <title id="Packagemap_Actions">Package Map Actions</title>
 
-            <mediaobject>
-              <imageobject>
-                <imagedata fileref="../../images/ECLWaPkMgr.jpg" />
-              </imageobject>
-            </mediaobject>
-          </figure></para>
+          <para>You can perform actions on your package maps from the Package
+          Maps tab in ECL Watch.</para>
+
+          <sect4 id="Open_Pkg_Map">
+            <title>Package Map Open</title>
+
+            <para>To examine a package map, select the package map and press
+            the <emphasis role="bold">Open</emphasis> action button. This
+            opens a new tab where you can access additional tabs with the
+            package information, the XML, and validate the package map.</para>
+          </sect4>
+
+          <sect4 id="Add_PKG_Map">
+            <title>Package Map Add</title>
+
+            <para>To Add a package map to the target cluster:</para>
+
+            <orderedlist>
+              <listitem>
+                <para>Select the package map to add by checking the box next
+                to it.</para>
+              </listitem>
+
+              <listitem>
+                <para>Press the <emphasis role="bold">Add</emphasis> action
+                button and open the Add Package Map dialog.</para>
+
+                <para><figure>
+                    <title>Add Package Maps</title>
+
+                    <mediaobject>
+                      <imageobject>
+                        <imagedata fileref="../../images/ECLWA470A.jpg"
+                                   vendor="ECLWatchSS" />
+                      </imageobject>
+                    </mediaobject>
+                  </figure></para>
+              </listitem>
+
+              <listitem>
+                <para>Press the <emphasis role="bold">Select Package
+                File</emphasis> button and select the package files to
+                add.</para>
+              </listitem>
+
+              <listitem>
+                <para>Choose the <emphasis role="bold">Target </emphasis>to
+                associate the package map with.</para>
+              </listitem>
+
+              <listitem>
+                <para>Select a <emphasis role="bold">Process Filter</emphasis>
+                from the drop list. The process filter determines which
+                physical Roxie clusters will actually load the package
+                map.</para>
+              </listitem>
+
+              <listitem>
+                <para>Enter the IP address or hostname of the remote Dali to
+                use for logical file lookups for the <emphasis
+                role="bold">Remote Dali IP Address</emphasis> field.</para>
+              </listitem>
+
+              <listitem>
+                <para>Check the boxes to Activate or Overwrite as
+                desired.</para>
+              </listitem>
+            </orderedlist>
+          </sect4>
+
+          <sect4 id="Activate_Package_Map">
+            <title>Activate Package Map</title>
+
+            <para>Press the <emphasis role="bold">Activate</emphasis> button
+            to deactivate the currently active package map and make the
+            selected package map active.</para>
+          </sect4>
+
+          <sect4 id="Deactivate_Package_Map">
+            <title>Deactivate Package Map</title>
+
+            <para>Press the <emphasis role="bold">Deactivate</emphasis> button
+            to deactivate the currently active package map.</para>
+          </sect4>
+
+          <sect4 id="PackageMap_Delete">
+            <title>Package Map Delete</title>
+
+            <para>To delete a package map:</para>
+
+            <orderedlist>
+              <listitem>
+                <para>Select the package map to delete by checking the box
+                next to it.</para>
+              </listitem>
+
+              <listitem>
+                <para>Press the <emphasis role="bold">Delete</emphasis> action
+                button.</para>
+              </listitem>
+
+              <listitem>
+                <para>Press <emphasis role="bold">OK</emphasis> when prompted
+                to confirm.</para>
+              </listitem>
+            </orderedlist>
+          </sect4>
+        </sect3>
+
+        <sect3 id="Pacakge_Map_Parts">
+          <title>Package Map Parts</title>
+
+          <para>You can see more information and perform some action on
+          package map parts. Open the package map to see the package parts
+          tab.</para>
+
+          <para><figure>
+              <title>Package Parts</title>
+
+              <mediaobject>
+                <imageobject>
+                  <imagedata fileref="../../images/ECLWAPkgParts01.jpg"
+                             vendor="ECLWatchSS" />
+                </imageobject>
+              </mediaobject>
+            </figure></para>
+
+          <para>You can examine the individual parts, add parts, or remove
+          parts through this interface in ECLWatch.</para>
+
+          <sect4 id="Add_Part">
+            <title>Add Part</title>
+
+            <para>To add a part to the package map:</para>
 
-        <para>Choose the <emphasis role="bold">Target</emphasis> and <emphasis
-        role="bold">Process</emphasis> from the drop lists on the Validate
-        Package Map tab.</para>
+            <orderedlist>
+              <listitem>
+                <para>Select the <emphasis role="bold">Package
+                Parts</emphasis> tab.</para>
+              </listitem>
+
+              <listitem>
+                <para>Press the <emphasis role="bold">Add</emphasis>
+                button.</para>
+              </listitem>
+
+              <listitem>
+                <para>Fill in the appropriate information.</para>
+              </listitem>
+
+              <listitem>
+                <para>Press <emphasis role="bold">Apply</emphasis>.</para>
+              </listitem>
+            </orderedlist>
+          </sect4>
+
+          <sect4 id="RemovePart">
+            <title>Remove Part</title>
+
+            <para>To remove a part from the package map:</para>
+
+            <orderedlist>
+              <listitem>
+                <para>Select the <emphasis role="bold">Package
+                Parts</emphasis> tab.</para>
+              </listitem>
 
-        <para>Press the <emphasis role="bold">Validate</emphasis> button to
-        validate the package map. The result is shown on the <emphasis
-        role="bold">Validate Active Package Map</emphasis> tab. You can also
-        Validate the package content, on the V<emphasis role="bold">alidate
-        Package Content</emphasis> tab.</para>
+              <listitem>
+                <para>Check the box next to the part to remove.</para>
+              </listitem>
+
+              <listitem>
+                <para>Press the <emphasis role="bold">Remove Part</emphasis>
+                button.</para>
+              </listitem>
+
+              <listitem>
+                <para>Press <emphasis role="bold">OK</emphasis> when prompted
+                to confirm.</para>
+              </listitem>
+            </orderedlist>
+          </sect4>
+
+          <sect4 id="GetPart">
+            <title>Get Part</title>
+
+            <para><figure>
+                <title>Get Part</title>
+
+                <mediaobject>
+                  <imageobject>
+                    <imagedata fileref="../../images/ECLWaGetPart01.jpg"
+                               vendor="ECLWatchSS" />
+                  </imageobject>
+                </mediaobject>
+              </figure></para>
+
+            <para>Press the <emphasis role="bold">Get Part</emphasis> button
+            to view the contents of the selected part. <figure>
+                <title>Package Part Contents</title>
+
+                <mediaobject>
+                  <imageobject>
+                    <imagedata fileref="../../images/ECLWaGetPart02.jpg"
+                               vendor="ECLWatchSS" />
+                  </imageobject>
+                </mediaobject>
+              </figure></para>
+          </sect4>
+        </sect3>
+
+        <sect3 id="Packagemap_Validation">
+          <title>Validate Package Map</title>
+
+          <para>The <emphasis role="bold">Validate Package Map</emphasis> tab
+          is used to validate active package maps. The <emphasis
+          role="bold">Validate Package Content</emphasis> tab is used to
+          validate package map content that is not yet loaded. To validate an
+          active package map:</para>
+
+          <para><figure>
+              <title>Validate Package Maps</title>
+
+              <mediaobject>
+                <imageobject>
+                  <imagedata fileref="../../images/ECLWaPkMgr.jpg" />
+                </imageobject>
+              </mediaobject>
+            </figure></para>
+
+          <orderedlist>
+            <listitem>
+              <para>Select the <emphasis role="bold">Validate Package
+              Map</emphasis> tab</para>
+            </listitem>
+
+            <listitem>
+              <para>Choose the <emphasis role="bold">Target</emphasis> and
+              <emphasis role="bold">Process</emphasis> from the drop lists on
+              the <emphasis role="bold">Validate Package Map</emphasis>
+              tab.</para>
+            </listitem>
+
+            <listitem>
+              <para>Press the <emphasis role="bold">Validate</emphasis> button
+              to validate the package map.</para>
+            </listitem>
+          </orderedlist>
+
+          <para>The result is shown on the <emphasis role="bold">Validate
+          Active Package Map</emphasis> tab.</para>
+
+          <para>You can validate any package map, active, inactive, external
+          or one not even uploaded onto the environment.</para>
+
+          <para>To validate an external package map:</para>
+
+          <para><orderedlist>
+              <listitem>
+                <para>Go to the <emphasis role="bold">Package Maps</emphasis>
+                tab.</para>
+              </listitem>
+
+              <listitem>
+                <para>Select the package map to validate.</para>
+              </listitem>
+
+              <listitem>
+                <para>Press the <emphasis role="bold">Open</emphasis> action
+                button.</para>
+              </listitem>
+
+              <listitem>
+                <para>Select the <emphasis role="bold">Validate</emphasis>
+                tab.</para>
+              </listitem>
+            </orderedlist></para>
+
+          <para>The Validate Package Content tab allows you to open any
+          package file, or insert any package content into the form and
+          validate it. The content does not have to be published onto the
+          system.</para>
+        </sect3>
       </sect2>
     </sect1>
   </chapter>

ecl/eclcc/eclcc.cpp

@@ -1202,7 +1202,7 @@ void EclCC::processSingleQuery(EclCompileInstance & instance,
                 if (instance.legacyImport)
                     importRootModulesToScope(scope, ctx);
 
-                instance.query.setown(parseQuery(scope, queryContents, ctx, NULL, NULL, true));
+                instance.query.setown(parseQuery(scope, queryContents, ctx, NULL, NULL, true, true));
 
                 if (instance.archive)
                 {
@@ -2493,11 +2493,12 @@ void EclCC::usage()
 // The following methods are concerned with running eclcc in batch mode (primarily to aid regression testing)
 void EclCC::processBatchedFile(IFile & file, bool multiThreaded)
 {
-    StringBuffer basename, logFilename, xmlFilename, outFilename;
+    StringBuffer basename, logFilename, xmlFilename, outFilename, metaFilename;
 
     splitFilename(file.queryFilename(), NULL, NULL, &basename, &basename);
     addNonEmptyPathSepChar(logFilename.append(optOutputDirectory)).append(basename).append(".log");
     addNonEmptyPathSepChar(xmlFilename.append(optOutputDirectory)).append(basename).append(".xml");
+    addNonEmptyPathSepChar(metaFilename.append(optOutputDirectory)).append(basename).append(".meta");
 
     splitFilename(file.queryFilename(), NULL, NULL, &outFilename, &outFilename);
 
@@ -2536,6 +2537,9 @@ void EclCC::processBatchedFile(IFile & file, bool multiThreaded)
                 info.stats.xmlSize = xml->size();
             }
 
+            if (info.generatedMeta)
+                saveXML(metaFilename, info.generatedMeta, 0, XML_Embed|XML_LineBreak);
+
             info.logStats();
         }
     }

ecl/hql/hqldesc.cpp

@@ -150,6 +150,7 @@ void expandSymbolMeta(IPropertyTree * metaTree, IHqlExpression * expr)
 
         if (expr->isScope() && !isImport(expr))
         {
+            def->setProp("@type", "module");
             expandScopeSymbolsMeta(def, expr->queryScope());
         }
         else if (expr->isRecord())
@@ -157,6 +158,13 @@ void expandSymbolMeta(IPropertyTree * metaTree, IHqlExpression * expr)
             def->setProp("@type", "record");
             expandRecordSymbolsMeta(def, expr);
         }
+        else
+        {
+            StringBuffer ecltype;
+            expr->queryType()->getECLType(ecltype);
+            def->setProp("@type", ecltype);
+        }
+
     }
 }
 
@@ -166,6 +174,18 @@ void expandScopeSymbolsMeta(IPropertyTree * meta, IHqlScope * scope)
     if (!scope)
         return;
 
+    //The following symbols will not have parsed all their members, and can cause recursive dependency errors trying.
+    switch (queryExpression(scope)->getOperator())
+    {
+    case no_forwardscope:
+        meta->setPropBool("@forward", true);
+        return;
+    case no_remotescope:
+        //Strange e.g. shared me := myModule;
+        meta->setPropBool("@global", true);
+        return;
+    }
+
     HqlExprArray symbols;
     scope->getSymbols(symbols);
     symbols.sort(compareSymbolsByPosition);

ecl/hql/hqlexpr.hpp

@@ -1395,7 +1395,7 @@ extern HQL_API IHqlExpression * getCastExpr(IHqlExpression * expr, ITypeInfo * t
 
 extern HQL_API void parseModule(IHqlScope *scope, IFileContents * contents, HqlLookupContext & ctx, IXmlScope *xmlScope, bool loadImplicit);
 extern HQL_API IHqlExpression *parseQuery(IHqlScope *scope, IFileContents * contents, 
-                                          HqlLookupContext & ctx, IXmlScope *xmlScope, IProperties * macroParams, bool loadImplicit);
+                                          HqlLookupContext & ctx, IXmlScope *xmlScope, IProperties * macroParams, bool loadImplicit, bool isRoot);
 extern HQL_API IHqlExpression *parseQuery(const char *in, IErrorReceiver * errs);
 
 extern HQL_API IPropertyTree * gatherAttributeDependencies(IEclRepository * dataServer, const char * items = NULL);

ecl/hql/hqlgram2.cpp

@@ -12045,12 +12045,13 @@ IHqlExpression * PseudoPatternScope::lookupSymbol(IIdAtom * name, unsigned looku
 
 //---------------------------------------------------------------------------------------------------------------------
 
-extern HQL_API IHqlExpression * parseQuery(IHqlScope *scope, IFileContents * contents, HqlLookupContext & ctx, IXmlScope *xmlScope, IProperties * macroParams, bool loadImplicit)
+extern HQL_API IHqlExpression * parseQuery(IHqlScope *scope, IFileContents * contents, HqlLookupContext & ctx, IXmlScope *xmlScope, IProperties * macroParams, bool loadImplicit, bool isRoot)
 {
     assertex(scope);
     try
     {
-        ctx.noteBeginQuery(scope, contents);
+        if (isRoot)
+            ctx.noteBeginQuery(scope, contents);
 
         HqlGram parser(scope, scope, contents, ctx, xmlScope, false, loadImplicit);
         parser.setQuery(true);
@@ -12058,7 +12059,8 @@ extern HQL_API IHqlExpression * parseQuery(IHqlScope *scope, IFileContents * con
         parser.getLexer()->set_yyColumn(1);
         parser.getLexer()->setMacroParams(macroParams);
         OwnedHqlExpr ret = parser.yyParse(false, true);
-        ctx.noteEndQuery();
+        if (isRoot)
+            ctx.noteEndQuery();
         return parser.clearFieldMap(ret.getClear());
     }
     catch (IException *E)
@@ -12125,7 +12127,7 @@ extern HQL_API IHqlExpression * parseQuery(const char * text, IErrorReceiver * e
     Owned<IHqlScope> scope = createScope();
     HqlDummyLookupContext ctx(errs);
     Owned<IFileContents> contents = createFileContentsFromText(text, NULL, false, NULL);
-    return parseQuery(scope, contents, ctx, NULL, NULL, true);
+    return parseQuery(scope, contents, ctx, NULL, NULL, true, true);
 }
 
 

ecl/hql/hqlparse.cpp

@@ -1211,7 +1211,7 @@ void HqlLex::doExport(YYSTYPE & returnToken, bool toXml)
         {
             HqlLookupContext ctx(yyParser->lookupCtx);
             Owned<IFileContents> exportContents = createFileContentsFromText(curParam.str(), sourcePath, yyParser->inSignedModule, yyParser->gpgSignature);
-            expr.setown(parseQuery(scope, exportContents, ctx, xmlScope, NULL, true));
+            expr.setown(parseQuery(scope, exportContents, ctx, xmlScope, NULL, true, false));
 
             if (expr && (expr->getOperator() == no_sizeof))
             {
@@ -1644,7 +1644,7 @@ void HqlLex::doIsValid(YYSTYPE & returnToken)
         HqlLookupContext ctx(yyParser->lookupCtx);
         ctx.errs.clear();   //Deliberately ignore any errors
         Owned<IFileContents> contents = createFileContentsFromText(curParam.str(), sourcePath, yyParser->inSignedModule, yyParser->gpgSignature);
-        expr = parseQuery(scope, contents, ctx, xmlScope, NULL, true);
+        expr = parseQuery(scope, contents, ctx, xmlScope, NULL, true, false);
 
         if(expr)
         {

ecl/hql/hqlutil.cpp

@@ -9155,7 +9155,7 @@ IHqlExpression * expandMacroDefinition(IHqlExpression * expr, HqlLookupContext &
     Owned<IHqlScope> scope = createPrivateScope();
     if (queryLegacyImportSemantics())
         importRootModulesToScope(scope, ctx);
-    return parseQuery(scope, mappedContents, ctx, NULL, macroParms, true);
+    return parseQuery(scope, mappedContents, ctx, NULL, macroParms, true, true);
 }
 
 static IHqlExpression * transformAttributeToQuery(IHqlExpression * expr, HqlLookupContext & ctx, bool syntaxCheck)

ecl/hqlcpp/hqlcpp.cpp

@@ -1420,7 +1420,7 @@ HqlCppTranslator::HqlCppTranslator(IErrorReceiver * _errors, const char * _soNam
             cppSystemScope = createScope();
             Owned<ISourcePath> sysPath = createSourcePath("<system-definitions>");
             Owned<IFileContents> systemContents = createFileContentsFromText(systemText.str(), sysPath, true, NULL);
-            OwnedHqlExpr query = parseQuery(cppSystemScope, systemContents, ctx, NULL, NULL, false);
+            OwnedHqlExpr query = parseQuery(cppSystemScope, systemContents, ctx, NULL, NULL, false, false);
             if (errs.errCount())
             {
                 StringBuffer errtext;

ecl/regress/regress.sh

@@ -113,7 +113,7 @@ fi
 
 if [[ $eclcc != '' ]]; then
     ## Set flags
-    default_flags="-P$target_dir -legacy -platform=thorlcr -fforceGenerate -fregressionTest -b -S -shared"
+    default_flags="-P$target_dir -legacy -platform=thorlcr -fforceGenerate -fregressionTest -b -S -shared -meta+"
     flags="$default_flags $include_dir -fshowMetaInGraph -fspanMultipleCpp- $userflags"
 
     ## Prepare target directory

esp/src/eclwatch/nls/zh/hpcc.js

@@ -24,7 +24,9 @@
     AddFile: "添加文件",
     AddGroup: "添加用户组",
     AdditionalResources: "附加资源",
+    AddPart: "添加部件",
     AddProcessMap: "添加文件包",
+    AddTheseFilesToDali: "添加这些文件到Dali",
     AddtionalProcessesToFilter: "过滤附加进程",
     AddToExistingSuperfile: "添加到现有文件集",
     AddToSuperfile: "加入文件集",
@@ -40,9 +42,11 @@
     ANY: "任何一个",
     AnyAdditionalProcessesToFilter: "任何需过滤的附加进程",
     Append: "添加",
+    AppendCluster: "添加集群",
     Apply: "使用",
     ArchivedOnly: "仅限已存档的工作单元",
     ArchivedWarning: "警告:请指定一个小的日期范围. 否则, 检索时间可能较长,网页浏览器可能超时",
+    Attach: "附加",
     Attribute: "属性",
     AttributesAreRequired: "所需属性",
     AutoRefresh: "自动更新",
@@ -104,6 +108,7 @@
     Critical: "关键的",
     CSV: "CSV",
     Dali: "Dali",
+    DaliIP: "Dali IP",
     dataset: ":=数据集*",
     Date: "日期",
     Day: "日",
@@ -112,12 +117,14 @@
     DEF: "DEF",
     Defaults: "默认",
     Definition: "定义",
+    Definitions: "定义",
     Delete: "删除",
     DeleteBinding: "删除连接",
     Deleted: "已删除",
     DeletedBinding: "已删除的连接",
     DeleteDirectories: "将要删除空目录。你要继续吗？",
     DeleteEmptyDirectories: "删除空目录",
+    DeletePrevious: "删除前面",
     DeleteSelectedFiles: "删除所选择的文件?",
     DeleteSelectedGroups: "删除所选择的用户组?",
     DeleteSelectedPermissions: "删除所选的权限?",
@@ -175,6 +182,8 @@
     EnglishQ: "英文?",
     EnterAPercentage: "输入百分比",
     EnterAPercentageOrMB: "输入百分比或MB",
+    EraseHistory: "擦除历史",
+    EraseHistoryQ: "擦除相关历史:",
     Error: "错误",
     Errorparsingserverresult: "服务器结果解析错误",
     Errors: "错误",
@@ -206,6 +215,7 @@
     FilePath: "文件路径",
     FilePermission: "文件权限",
     Files: "文件",
+    FilesPending: "文件添加",
     FileScopeDefaultPermissions: "文件默认权限",
     FileScopes: "文件范围",
     FileSize: "文件长度",
@@ -234,6 +244,7 @@
     FromTime: "起始时间",
     FullName: "姓名",
     Generate: "生成",
+    GetPart: "获取部件",
     GetSoftwareInformation: "获取软件信息",
     Graph: "图形",
     Graphs: "图形",
@@ -337,6 +348,7 @@
     MaxSize: "最大规格",
     MemberOf: "隶属",
     Members: "成员",
+    MethodConfiguration: "方法配置",
     Message: "信息",
     Methods: "方法",
     Min: "最小",
@@ -394,6 +406,7 @@
     OpenSafeMode: "打开 (安全模式)",
     OpenSource: "开源",
     OpenTreeMode: "打开 (树状模式)",
+    Operation: "操作",
     Operations: "操作",
     Options: "选项",
     OriginalFile: "原文件",
@@ -412,6 +425,7 @@
     PackagesNoQuery: "与查询程序不匹配的文件包",
     ParameterXML: "参数XML",
     Part: "分块",
+    PartName: "部件名称",
     PartMask: "组成部分掩模",
     Parts: "文件分块",
     PartsFound: "找到组成部分",
@@ -428,6 +442,7 @@
     Pause: "暂停",
     PauseNow: "马上暂停",
     PctComplete: "任务完成%",
+    PercentCompressed: "压缩百分比",
     PercentDone: "任务完成%",
     PerformingLayout: "图形布局中...",
     Permission: "权限",
@@ -450,12 +465,14 @@
     Prefix: "前缀",
     PrefixPlaceholder: "文件名{:长度}, 文件大小{:[B|L][1-8]}",
     Preflight: "预检",
+    PreloadAllPackages: "预载所有软件包",
     PreserveCompression: "保留压缩模式",
     Preview: "预览",
     PrimaryLost: "主要丢失",
     PrimaryMonitoring: "主监控",
     Priority: "优先级",
     Process: "进程",
+    Processes: "流程",
     ProcessFilter: "进程过滤器",
     ProcessorInformation: "处理器信息",
     ProgressMessage: "进展信息",
@@ -503,6 +520,7 @@
     Remove: "删除",
     RemoveAttributeQ: "你将要删除此属性。你确定要继续？ ",
     RemoveAtttributes: "删除属性",
+    RemovePart: "删除部件",
     RemoveSubfiles2: "删除子文件？",
     RemoveSubfiles: "删除子文件",
     RemoveUser: "将把你的用户从用户组里删除：",
@@ -533,6 +551,7 @@
     RetainSuperfileStructure: "保留文件集的结构",
     RetypePassword: "验证密码",
     Reverse: "反转",
+    RoxieFileCopy: "Roxie 文件备份的状态",
     RowPath: "行路径",
     Rows: "行",
     RowTag: "行标记",
@@ -549,6 +568,7 @@
     SelectPackageFile: "选择文件包文件",
     Separators: "分割符",
     Services: "服务",
+    Server: "服务器",
     SetBanner: "标语设置",
     SetTextError: "文本太大无法显示. 请用‘帮助’下载",
     SetToFailed: "设为失误状态",
@@ -591,6 +611,7 @@
     Subgraphs: "子图",
     Submit: "提交",
     Subtype: "子类",
+    SuccessfullySaved: "成功保存",
     Summary: "总结",
     SummaryMessage: "总结信息",
     SuperFile: "文件集",
@@ -599,6 +620,7 @@
     SuperFiles: "文件集",
     SuperFilesBelongsTo: "文件集里的文件",
     SuperfilesOnly: "仅含文件集",
+    SuperOwner: "超级拥有者",
     Suspend: "暂停使用",
     Suspended: "已暂停使用",
     SuspendedBy: "暂停使用者",
@@ -627,6 +649,7 @@
     Time: "时间",
     Timers: "定时器",
     TimeSeconds: "时间(秒)",
+    TimeStamp: "时间戳",
     TimeStarted: "起始时间",
     TimeStopped: "终止时间",
     Timings: "时间表",
@@ -653,6 +676,7 @@
     title_GraphPage: "标题",
     title_Graphs: "图形",
     title_GridDetails: "修改",
+    title_History: "历史",
     title_HPCCPlatformECL: "ECL Watch - 首页",
     title_HPCCPlatformFiles: "ECL Watch - 文件",
     title_HPCCPlatformMain: "ECL Watch - 首页",
@@ -667,7 +691,9 @@
     title_LZBrowse: "文件停放区",
     title_MemberOf: "隶属",
     title_Members: "成员",
+    title_Methods: "方法",
     title_OrphanFilesFor: "孤立文件为",
+    title_PackageParts: "软件包组分",
     title_Permissions: "权限",
     title_PreflightResults: "预检结果",
     title_QuerySetDetails: "查询程序的详细说明",
@@ -709,6 +735,7 @@
     Unsuspend: "未暂停使用",
     Unsuspended: "未暂停使用",
     Up: "上",
+    UpdateCloneFrom: "更新复制,从",
     UpdateSuperFiles: "更新文件集",
     Upload: "上传",
     Usage: "使用率",

esp/xslt/esdl2monitor.xslt

@@ -24,6 +24,7 @@
     <xsl:param name="platform" select="'roxie'"/>
     <xsl:param name="diffmode" select="'Monitor'"/>
     <xsl:param name="diffaction" select="'Run'"/>
+    <xsl:param name="listCategories" select="false()"/>
     <xsl:variable name="docname" select="/esxdl/@name"/>
     <xsl:template match="/">
         <xsl:apply-templates select="esxdl"/>
@@ -347,7 +348,7 @@ END;
 //  output(executedAction.prior, NAMED('PRIOR'));
   updateMonitor(DATASET([{executedAction.id, executedAction.responseXML}], monitorStoreRec));
 
-
+<xsl:if test="$listCategories">
   <xsl:if test="Template//*[@diff_monitor]">
   CategoryPathsRec := RECORD
     string categories {xpath('@categories')};
@@ -367,6 +368,7 @@ END;
       </xsl:for-each>
     <xsl:text>], CategoryPathsRec), NAMED('Categories'));</xsl:text>
   </xsl:if>
+</xsl:if>
 
   </xsl:when>
   <xsl:when test="$diffmode='Compare'">

roxie/ccd/ccdprotocol.cpp

@@ -271,6 +271,28 @@ public:
 
     virtual void runOnce(const char *query);
 
+    virtual void cleanupSocket(ISocket *sock)
+    {
+        if (!sock)
+            return;
+        try
+        {
+            sock->shutdown();
+        }
+        catch (IException *e)
+        {
+            e->Release();
+        }
+        try
+        {
+            sock->close();
+        }
+        catch (IException *e)
+        {
+            e->Release();
+        }
+    }
+
     virtual int run()
     {
         DBGLOG("ProtocolSocketListener (%d threads) listening to socket on port %d", sink->getPoolSize(), port);
@@ -298,7 +320,7 @@ public:
                         if (status < 0)
                         {
                             // secure_accept may also DBGLOG() errors ...
-                            WARNLOG("ProtocolSocketListener failure to establish secure connection");
+                            cleanupSocket(ssock);
                             continue;
                         }
                     }
@@ -308,12 +330,19 @@ public:
                         E->errorMessage(s);
                         WARNLOG("%s", s.str());
                         E->Release();
+                        cleanupSocket(ssock);
+                        ssock.clear();
+                        cleanupSocket(client);
+                        client.clear();
                         continue;
                     }
                     catch (...)
                     {
-                        StringBuffer s;
                         WARNLOG("ProtocolSocketListener failure to establish secure connection");
+                        cleanupSocket(ssock);
+                        ssock.clear();
+                        cleanupSocket(client);
+                        client.clear();
                         continue;
                     }
                     client.setown(ssock.getClear());

roxie/roxiemem/roxiemem.cpp

@@ -2420,7 +2420,7 @@ public:
     {
         const ActivityEntry *l = *(const ActivityEntry **) _l;
         const ActivityEntry *r = *(const ActivityEntry **) _r;
-        return l->usage - r->usage;
+        return (l->usage > r->usage) ? +1 : (l->usage < r->usage) ? -1 : 0;
     }
 
     virtual void report(const IContextLogger &logctx, const IRowAllocatorCache *allocatorCache)

roxie/roxiemem/roxiemem.hpp

@@ -422,6 +422,7 @@ enum RoxieHeapFlags
     //internal flags
     RHForphaned         = 0x80000000,   // heap will no longer be used, can be deleted
 };
+inline RoxieHeapFlags operator | (RoxieHeapFlags l, RoxieHeapFlags r) { return (RoxieHeapFlags)((unsigned)l | (unsigned)r); }
 
 //This interface is here to allow atomic updates to allocations when they are being resized.  There are a few complications:
 //- If a new block is allocated, we just need to update the capacity/pointer atomically

system/jlib/jlib.hpp

@@ -204,7 +204,7 @@ public:
     inline bool zap(TYPE * x)                   { return PointerArray::zap(x); }
 };
 
-enum SSLCfg { SSLNone = 0, SSLOnly, SSLFirst, UnsecureFirst };
+enum DAFSConnectCfg { SSLNone = 0, SSLOnly, SSLFirst, UnsecureFirst };
 
 #include "jstring.hpp"
 #include "jarray.hpp"

system/jlib/jutil.cpp

@@ -2397,12 +2397,12 @@ jlib_decl const IProperties &queryEnvironmentConf()
 }
 
 static CriticalSection securitySettingsCrit;
-static SSLCfg useSSL = SSLNone;
+static DAFSConnectCfg connectMethod = SSLNone;
 static StringAttr certificate;
 static StringAttr privateKey;
 static StringAttr passPhrase;
 static bool retrieved = false;
-jlib_decl bool querySecuritySettings(SSLCfg *        _useSSL,
+jlib_decl bool querySecuritySettings(DAFSConnectCfg *_connectMethod,
                                      unsigned short *_port,
                                      const char * *  _certificate,
                                      const char * *  _privateKey,
@@ -2426,15 +2426,15 @@ jlib_decl bool querySecuritySettings(SSLCfg *        _useSSL,
                 {
                     // checking for true | false for backward compatibility
                     if ( strieq(sslMethod.str(), "SSLOnly") || strieq(sslMethod.str(), "true") )
-                        useSSL = SSLOnly;
+                        connectMethod = SSLOnly;
                     else if ( strieq(sslMethod.str(), "SSLFirst") )
-                        useSSL = SSLFirst;
+                        connectMethod = SSLFirst;
                     else if ( strieq(sslMethod.str(), "UnsecureFirst") )
-                        useSSL = UnsecureFirst;
+                        connectMethod = UnsecureFirst;
                     else // SSLNone or false or ...
-                        useSSL = SSLNone;
+                        connectMethod = SSLNone;
                 }
-                if (useSSL)
+                if (connectMethod == SSLOnly || connectMethod == SSLFirst || connectMethod == UnsecureFirst)
                 {
                     certificate.set(conf->queryProp("dfsSSLCertFile"));
                     privateKey.set(conf->queryProp("dfsSSLPrivateKeyFile"));
@@ -2457,12 +2457,12 @@ jlib_decl bool querySecuritySettings(SSLCfg *        _useSSL,
     }
     if (retrieved)
     {
-        if (_useSSL)
-            *_useSSL = useSSL;
+        if (_connectMethod)
+            *_connectMethod = connectMethod;
         if (_port)
         {
             // port to try first (or only) ...
-            if ( (useSSL == SSLNone) || (useSSL == UnsecureFirst) )
+            if ( (connectMethod == SSLNone) || (connectMethod == UnsecureFirst) )
                 *_port = DAFILESRV_PORT;
             else
                 *_port = SECURE_DAFILESRV_PORT;
@@ -2476,22 +2476,22 @@ jlib_decl bool querySecuritySettings(SSLCfg *        _useSSL,
     }
     else
     {
-        if (_useSSL)
-            *_useSSL = SSLNone;
+        if (_connectMethod)
+            *_connectMethod = SSLNone;
         if (_port)
             *_port = DAFILESRV_PORT;
     }
     return retrieved;
 }
 
-jlib_decl bool queryDafsSecSettings(SSLCfg *        _useSSL,
+jlib_decl bool queryDafsSecSettings(DAFSConnectCfg *_connectMethod,
                                     unsigned short *_port,
                                     unsigned short *_sslport,
                                     const char * *  _certificate,
                                     const char * *  _privateKey,
                                     const char * *  _passPhrase)
 {
-    bool ret = querySecuritySettings(_useSSL, nullptr, _certificate, _privateKey, _passPhrase);
+    bool ret = querySecuritySettings(_connectMethod, nullptr, _certificate, _privateKey, _passPhrase);
     // these should really be in env, but currently they are not ...
     if (_port)
         *_port = DAFILESRV_PORT;

system/jlib/jutil.hpp

@@ -360,13 +360,13 @@ extern jlib_decl bool getConfigurationDirectory(const IPropertyTree *dirtree, //
                                                 const char *instance, 
                                                 StringBuffer &dirout);
 
-extern jlib_decl bool querySecuritySettings(SSLCfg *        _useSSL,
+extern jlib_decl bool querySecuritySettings(DAFSConnectCfg *_connectMethod,
                                             unsigned short *_port,
                                             const char * *  _certificate,
                                             const char * *  _privateKey,
                                             const char * *  _passPhrase);
 
-extern jlib_decl bool queryDafsSecSettings(SSLCfg *        _useSSL,
+extern jlib_decl bool queryDafsSecSettings(DAFSConnectCfg *_connectMethod,
                                            unsigned short *_port,
                                            unsigned short *_sslport,
                                            const char * *  _certificate,

system/security/securesocket/securesocket.cpp

@@ -568,9 +568,18 @@ int CSecureSocket::secure_accept(int logLevel)
     err = SSL_accept(m_ssl);
     if(err == 0)
     {
-        char errbuf[512];
-        ERR_error_string_n(ERR_get_error(), errbuf, 512);
-        DBGLOG("SSL_accept returned 0, error - %s", errbuf);
+        int ret = SSL_get_error(m_ssl, err);
+        // if err == 0 && ret == SSL_ERROR_SYSCALL
+        // then client closed connection gracefully before ssl neg
+        // which can happen with port scan / VIP ...
+        // NOTE: ret could also be SSL_ERROR_ZERO_RETURN if client closed
+        // gracefully after ssl neg initiated ...
+        if ( (logLevel >= 5) || (ret != SSL_ERROR_SYSCALL) )
+        {
+            char errbuf[512];
+            ERR_error_string_n(ERR_get_error(), errbuf, 512);
+            DBGLOG("SSL_accept returned 0, error - %s", errbuf);
+        }
         return -1;
     }
     else if(err < 0)

thorlcr/activities/aggregate/thaggregateslave.cpp

@@ -49,7 +49,7 @@ protected:
         if (1 == numPartialResults)
             return firstRow;
 
-        CThorExpandingRowArray partialResults(*this, this, true, stableSort_none, true, numPartialResults);
+        CThorExpandingRowArray partialResults(*this, this, ers_allow, stableSort_none, true, numPartialResults);
         if (hadElement)
             partialResults.setRow(0, firstRow);
         --numPartialResults;

thorlcr/activities/catch/thcatchslave.cpp

@@ -155,7 +155,7 @@ class CSkipCatchSlaveActivity : public CCatchSlaveActivityBase
         try
         {
             gathered = true;
-            Owned<IRowWriterMultiReader> overflowBuf = createOverflowableBuffer(*this, queryRowInterfaces(input), true);
+            Owned<IRowWriterMultiReader> overflowBuf = createOverflowableBuffer(*this, queryRowInterfaces(input), ers_eogonly);
             running = true;
             while (running)
             {

thorlcr/activities/hashdistrib/thhashdistribslave.cpp

@@ -3146,7 +3146,7 @@ friend class CBucket;
 };
 
 CHashTableRowTable::CHashTableRowTable(HashDedupSlaveActivityBase &_activity, IThorRowInterfaces *rowIf, IHash *_iRowHash, IHash *_iKeyHash, ICompare *_iCompare)
-    : CThorExpandingRowArray(_activity, rowIf, true),
+    : CThorExpandingRowArray(_activity, rowIf, ers_allow),
       activity(_activity), iRowHash(_iRowHash), iKeyHash(_iKeyHash), iCompare(_iCompare)
 {
     htMax = htElements = 0;

thorlcr/activities/join/thjoinslave.cpp

@@ -551,7 +551,7 @@ public:
             ActPrintLog("JOIN barrier.1 raised");
 
             // primaryWriter will keep as much in memory as possible.
-            Owned<IRowWriterMultiReader> primaryWriter = createOverflowableBuffer(*this, primaryRowIf, false);
+            Owned<IRowWriterMultiReader> primaryWriter = createOverflowableBuffer(*this, primaryRowIf, ers_forbidden);
             primaryStream.setown(sorter->startMerge(totalrows));
             copyRowStream(primaryStream, primaryWriter);
             primaryStream.setown(primaryWriter->getReader()); // NB: rhsWriter no longer needed after this point

thorlcr/activities/limit/thlimitslave.cpp

@@ -238,7 +238,7 @@ public:
     virtual void start() override
     {
         CLimitSlaveActivityBase::start();
-        buf.setown(createOverflowableBuffer(*this, this, true));
+        buf.setown(createOverflowableBuffer(*this, this, ers_eogonly));
     }
     CATCH_NEXTROW()
     {

thorlcr/activities/lookupjoin/thlookupjoinslave.cpp

@@ -774,14 +774,12 @@ class CThorRowArrayWithFlushMarker : public CThorSpillableRowArray
 public:
     CThorRowArrayWithFlushMarker(CActivityBase &activity) : CThorSpillableRowArray(activity)
     {
-        flushMarker = 0;
     }
-    CThorRowArrayWithFlushMarker(CActivityBase &activity, IThorRowInterfaces *rowIf, bool allowNulls=false, StableSortFlag stableSort=stableSort_none, rowidx_t initialSize=InitialSortElements, size32_t commitDelta=CommitStep)
-        : CThorSpillableRowArray(activity, rowIf, allowNulls, stableSort, initialSize, commitDelta)
+    CThorRowArrayWithFlushMarker(CActivityBase &activity, IThorRowInterfaces *rowIf, EmptyRowSemantics emptyRowSemantics=ers_forbidden, StableSortFlag stableSort=stableSort_none, rowidx_t initialSize=InitialSortElements, size32_t commitDelta=CommitStep)
+        : CThorSpillableRowArray(activity, rowIf, emptyRowSemantics, stableSort, initialSize, commitDelta)
     {
-        flushMarker = 0;
     }
-    rowidx_t flushMarker;
+    rowidx_t flushMarker = 0;
 };
 
 
@@ -1400,7 +1398,7 @@ public:
         leftITDL = queryInput(0);
         rightITDL = queryInput(1);
         rightOutputMeta = rightITDL->queryFromActivity()->queryContainer().queryHelper()->queryOutputMeta();
-        rightAllocator.setown(rightThorAllocator->getRowAllocator(rightOutputMeta, container.queryId(), (roxiemem::RoxieHeapFlags)(roxiemem::RHFpacked|roxiemem::RHFunique)));
+        rightAllocator.setown(rightThorAllocator->getRowAllocator(rightOutputMeta, container.queryId(), (roxiemem::RHFpacked|roxiemem::RHFunique)));
 
         allocator.set(queryRowAllocator());
         leftAllocator.set(::queryRowAllocator(leftITDL));
@@ -1436,7 +1434,7 @@ public:
             rhs.setup(sharedRightRowInterfaces);
             // NB: use sharedRightRowInterfaces, so that expanding ptr array is using shared allocator
             for (unsigned s=0; s<container.queryJob().querySlaves(); s++)
-                rhsSlaveRows.item(s)->setup(sharedRightRowInterfaces, false, stableSort_none, true);
+                rhsSlaveRows.item(s)->setup(sharedRightRowInterfaces, ers_forbidden, stableSort_none, true);
         }
     }
     virtual void setInputStream(unsigned index, CThorInput &_input, bool consumerOrdered) override
@@ -2025,7 +2023,7 @@ protected:
             if (!hasFailedOverToLocal())
             {
                 if (stable && !globallySorted)
-                    rhs.setup(sharedRightRowInterfaces, false, stableSort_earlyAlloc);
+                    rhs.setup(sharedRightRowInterfaces, ers_forbidden, stableSort_earlyAlloc);
                 bool success=false;
                 try
                 {
@@ -2079,7 +2077,7 @@ protected:
                     if (stable && !globallySorted)
                     {
                         ActPrintLog("Clearing rhs stable ptr table");
-                        rhs.setup(sharedRightRowInterfaces, false, stableSort_none); // don't need stable ptr table anymore
+                        rhs.setup(sharedRightRowInterfaces, ers_forbidden, stableSort_none); // don't need stable ptr table anymore
                     }
                 }
             }
@@ -2518,6 +2516,7 @@ protected:
             if (rightStream)
             {
                 ActPrintLog("Performing STANDARD JOIN");
+                setFailoverToStandard(true);
                 setupStandardJoin(rightStream); // NB: rightStream is sorted
             }
             else
@@ -2681,7 +2680,7 @@ public:
             if (isGlobal())
             {
                 for (unsigned s=0; s<container.queryJob().querySlaves(); s++)
-                    rhsSlaveRows.item(s)->setup(sharedRightRowInterfaces, false, stableSort_none, false);
+                    rhsSlaveRows.item(s)->setup(sharedRightRowInterfaces, ers_forbidden, stableSort_none, false);
             }
         }
     }

thorlcr/activities/loop/thloop.cpp

@@ -296,7 +296,7 @@ public:
         unsigned maxIterations = helper->numIterations();
         if ((int)maxIterations < 0) maxIterations = 0;
         Owned<IThorGraphResults> loopResults = queryGraph().createThorGraphResults(maxIterations);
-        IThorResult *result = loopResults->createResult(*this, 0, this, true);
+        IThorResult *result = loopResults->createResult(*this, 0, this, mergeResultTypes(thorgraphresult_distributed, thorgraphresult_grouped));
 
         helper->createParentExtract(extractBuilder);
 
@@ -347,11 +347,11 @@ public:
     CLocalResultActivityMaster(CMasterGraphElement *info) : CLocalResultActivityMasterBase(info)
     {
     }
-    virtual void createResult()
+    virtual void createResult() override
     {
         IHThorLocalResultWriteArg *helper = (IHThorLocalResultWriteArg *)queryHelper();
         CGraphBase *graph = container.queryResultsGraph();
-        graph->createResult(*this, helper->querySequence(), this, true); // NB graph owns result
+        graph->createResult(*this, helper->querySequence(), this, mergeResultTypes(thorgraphresult_distributed, thorgraphresult_grouped)); // NB graph owns result
     }
 };
 
@@ -370,7 +370,7 @@ public:
     {
         IHThorGraphLoopResultWriteArg *helper = (IHThorGraphLoopResultWriteArg *)queryHelper();
         CGraphBase *graph = container.queryResultsGraph();
-        graph->createGraphLoopResult(*this, inputRowIf, true); // NB graph owns result
+        graph->createGraphLoopResult(*this, inputRowIf, mergeResultTypes(thorgraphresult_distributed, thorgraphresult_grouped)); // NB graph owns result
     }
 };
 
@@ -390,7 +390,7 @@ public:
     {
         IHThorDictionaryResultWriteArg *helper = (IHThorDictionaryResultWriteArg *)queryHelper();
         CGraphBase *graph = container.queryResultsGraph();
-        graph->createResult(*this, helper->querySequence(), this, true); // NB graph owns result
+        graph->createResult(*this, helper->querySequence(), this, mergeResultTypes(thorgraphresult_distributed, thorgraphresult_sparse)); // NB graph owns result
     }
 };
 

thorlcr/activities/loop/thloopslave.cpp

@@ -275,7 +275,7 @@ public:
         helper->createParentExtract(extractBuilder);
         maxIterations = helper->numIterations();
         if ((int)maxIterations < 0) maxIterations = 0;
-        loopPending.setown(createOverflowableBuffer(*this, this, false, true));
+        loopPending.setown(createOverflowableBuffer(*this, this, ers_forbidden, true));
         loopPendingCount = 0;
         finishedLooping = ((container.getKind() == TAKloopcount) && (maxIterations == 0));
         if ((flags & IHThorLoopArg::LFnewloopagain) && !helper->loopFirstTime())
@@ -412,7 +412,7 @@ public:
                     if (!((const bool *)row.get())[0])
                         finishedLooping = true; // NB: will finish when loopPending has been consumed
                 }
-                loopPending.setown(createOverflowableBuffer(*this, this, false, true));
+                loopPending.setown(createOverflowableBuffer(*this, this, ers_forbidden, true));
                 loopPendingCount = 0;
                 ++loopCounter;
                 if ((container.getKind() == TAKloopcount) && (loopCounter > maxIterations))
@@ -470,7 +470,12 @@ public:
         if (!executed)
         {
             executed = true;
-            IThorResult *result = loopResults->createResult(*this, 0, this, !queryGraph().isLocalChild());
+            ThorGraphResultType resultType = thorgraphresult_nul;
+            if (!queryGraph().isLocalChild())
+                resultType = mergeResultTypes(resultType, thorgraphresult_distributed);
+            if (input->isGrouped())
+                resultType = mergeResultTypes(resultType, thorgraphresult_grouped);
+            IThorResult *result = loopResults->createResult(*this, 0, this, resultType);
             Owned<IRowWriter> resultWriter = result->getWriter();
             for (;;)
             {
@@ -629,7 +634,12 @@ public:
         abortSoon = false;
         assertex(container.queryResultsGraph());
         CGraphBase *graph = container.queryResultsGraph();
-        IThorResult *result = graph->createResult(*this, helper->querySequence(), this, !queryGraph().isLocalChild());  // NB graph owns result
+        ThorGraphResultType resultType = thorgraphresult_nul;
+        if (!queryGraph().isLocalChild())
+            resultType = mergeResultTypes(resultType, thorgraphresult_distributed);
+        if (input->isGrouped())
+            resultType = mergeResultTypes(resultType, thorgraphresult_grouped);
+        IThorResult *result = graph->createResult(*this, helper->querySequence(), this, resultType);  // NB graph owns result
         resultWriter.setown(result->getWriter());
     }
     CATCH_NEXTROW()
@@ -718,7 +728,12 @@ public:
     {
         IHThorLocalResultWriteArg *helper = (IHThorLocalResultWriteArg *)queryHelper();
         CGraphBase *graph = container.queryResultsGraph();
-        return graph->createResult(*this, helper->querySequence(), this, !queryGraph().isLocalChild());
+        ThorGraphResultType resultType = thorgraphresult_nul;
+        if (!queryGraph().isLocalChild())
+            resultType = mergeResultTypes(resultType, thorgraphresult_distributed);
+        if (input->isGrouped())
+            resultType = mergeResultTypes(resultType, thorgraphresult_grouped);
+        return graph->createResult(*this, helper->querySequence(), this, resultType);
     }
 };
 
@@ -760,7 +775,11 @@ public:
             builder.appendOwn(row);
         }
         CGraphBase *graph = container.queryResultsGraph();
-        IThorResult *result = graph->createResult(*this, helper->querySequence(), this, !queryGraph().isLocalChild());
+        ThorGraphResultType resultType = thorgraphresult_nul;
+        if (!queryGraph().isLocalChild())
+            resultType = mergeResultTypes(resultType, thorgraphresult_distributed);
+        resultType = mergeResultTypes(resultType, thorgraphresult_sparse);
+        IThorResult *result = graph->createResult(*this, helper->querySequence(), this, resultType);
         Owned<IRowWriter> resultWriter = result->getWriter();
         size32_t dictSize = builder.getcount();
         byte ** dictRows = builder.queryrows();
@@ -1384,7 +1403,12 @@ public:
     {
         IHThorGraphLoopResultWriteArg *helper = (IHThorGraphLoopResultWriteArg *)queryHelper();
         CGraphBase *graph = container.queryResultsGraph();
-        return graph->createGraphLoopResult(*this, input->queryFromActivity(), !queryGraph().isLocalChild());
+        ThorGraphResultType resultType = thorgraphresult_nul;
+        if (!queryGraph().isLocalChild())
+            resultType = mergeResultTypes(resultType, thorgraphresult_distributed);
+        if (input->isGrouped())
+            resultType = mergeResultTypes(resultType, thorgraphresult_grouped);
+        return graph->createGraphLoopResult(*this, input->queryFromActivity(), resultType);
     }
 };
 

thorlcr/activities/msort/thsortu.cpp

@@ -1680,7 +1680,7 @@ class CMultiCoreJoinHelper: public CMultiCoreJoinHelperBase
             PROGLOG("CMultiCoreJoinHelper::cWorker started");
 
             Owned<IThorRowInterfaces> rowIf = parent->activity.getRowInterfaces();
-            Owned<IEngineRowAllocator> allocator = parent->activity.getRowAllocator(rowIf->queryRowMetaData(), (roxiemem::RoxieHeapFlags)(roxiemem::RHFpacked|roxiemem::RHFunique));
+            Owned<IEngineRowAllocator> allocator = parent->activity.getRowAllocator(rowIf->queryRowMetaData(), (roxiemem::RHFpacked|roxiemem::RHFunique));
 
             IRowWriter *rowWriter = rowStream->queryWriter();
             for (;;)
@@ -1879,7 +1879,7 @@ class CMultiCoreUnorderedJoinHelper: public CMultiCoreJoinHelperBase
         int run()
         {
             Owned<IThorRowInterfaces> rowIf = parent->activity.getRowInterfaces();
-            Owned<IEngineRowAllocator> allocator = parent->activity.getRowAllocator(rowIf->queryRowMetaData(), (roxiemem::RoxieHeapFlags)(roxiemem::RHFpacked|roxiemem::RHFunique));
+            Owned<IEngineRowAllocator> allocator = parent->activity.getRowAllocator(rowIf->queryRowMetaData(), (roxiemem::RHFpacked|roxiemem::RHFunique));
 
             Owned<IRowWriter> rowWriter = parent->multiWriter->getWriter();
             PROGLOG("CMulticoreUnorderedJoinHelper::cWorker started");

thorlcr/activities/project/thprojectslave.cpp

@@ -31,7 +31,7 @@ public:
     {
         helper = static_cast <IHThorProjectArg *> (queryHelper());
         Owned<IRowInterfaces> rowIf = parent.getRowInterfaces();
-        allocator.setown(parent.getRowAllocator(rowIf->queryRowMetaData(), (roxiemem::RoxieHeapFlags)(parent.queryHeapFlags()|roxiemem::RHFpacked|roxiemem::RHFunique)));
+        allocator.setown(parent.getRowAllocator(rowIf->queryRowMetaData(), (parent.queryHeapFlags()|roxiemem::RHFpacked|roxiemem::RHFunique)));
     }
     STRAND_CATCH_NEXTROW()
     {

thorlcr/activities/thdiskbase.cpp

@@ -455,7 +455,7 @@ rowcount_t getCount(CActivityBase &activity, unsigned partialResults, rowcount_t
 const void *getAggregate(CActivityBase &activity, unsigned partialResults, IThorRowInterfaces &rowIf, IHThorCompoundAggregateExtra &aggHelper, mptag_t mpTag)
 {
     // JCSMORE - pity this isn't common routine with similar one in aggregate, but helper is not common
-    CThorExpandingRowArray slaveResults(activity, &activity, true, stableSort_none, true, partialResults);
+    CThorExpandingRowArray slaveResults(activity, &activity, ers_allow, stableSort_none, true, partialResults);
     unsigned _partialResults = partialResults;
     while (_partialResults--)
     {

thorlcr/graph/thgraph.cpp

@@ -47,7 +47,8 @@ class CThorGraphResult : implements IThorResult, implements IRowWriter, public C
     Owned<IRowWriterMultiReader> rowBuffer;
     IThorRowInterfaces *rowIf;
     IEngineRowAllocator *allocator;
-    bool stopped, readers, distributed;
+    bool stopped, readers;
+    ThorGraphResultType resultType;
 
     void init()
     {
@@ -63,7 +64,7 @@ class CThorGraphResult : implements IThorResult, implements IRowWriter, public C
     public:
         IMPLEMENT_IINTERFACE_USING(CSimpleInterface);
 
-        CStreamWriter(CThorGraphResult &_owner) : owner(_owner), rows(owner.activity, owner.rowIf, true)
+        CStreamWriter(CThorGraphResult &_owner, EmptyRowSemantics emptyRowSemantics) : owner(_owner), rows(owner.activity, owner.rowIf, emptyRowSemantics)
         {
         }
 
@@ -81,13 +82,20 @@ class CThorGraphResult : implements IThorResult, implements IRowWriter, public C
 public:
     IMPLEMENT_IINTERFACE;
 
-    CThorGraphResult(CActivityBase &_activity, IThorRowInterfaces *_rowIf, bool _distributed, unsigned spillPriority) : activity(_activity), rowIf(_rowIf), distributed(_distributed)
+    CThorGraphResult(CActivityBase &_activity, IThorRowInterfaces *_rowIf, ThorGraphResultType _resultType, unsigned spillPriority) : activity(_activity), rowIf(_rowIf), resultType(_resultType)
     {
         init();
+        EmptyRowSemantics emptyRowSemantics;
+        if (isGrouped())
+            emptyRowSemantics = ers_eogonly;
+        else if (isSparse())
+            emptyRowSemantics = ers_allow;
+        else
+            emptyRowSemantics = ers_forbidden;
         if (SPILL_PRIORITY_DISABLE == spillPriority)
-            rowBuffer.setown(new CStreamWriter(*this));
+            rowBuffer.setown(new CStreamWriter(*this, emptyRowSemantics));
         else
-            rowBuffer.setown(createOverflowableBuffer(activity, rowIf, true, true));
+            rowBuffer.setown(createOverflowableBuffer(activity, rowIf, emptyRowSemantics, true));
     }
 
 // IRowWriter
@@ -118,7 +126,9 @@ public:
     }
     virtual IThorRowInterfaces *queryRowInterfaces() { return rowIf; }
     virtual CActivityBase *queryActivity() { return &activity; }
-    virtual bool isDistributed() const { return distributed; }
+    virtual bool isDistributed() const { return resultType & thorgraphresult_distributed; }
+    virtual bool isSparse() const { return resultType & thorgraphresult_sparse; }
+    virtual bool isGrouped() const { return resultType & thorgraphresult_grouped; }
     virtual void serialize(MemoryBuffer &mb)
     {
         Owned<IRowStream> stream = getRowStream();
@@ -191,18 +201,18 @@ public:
 
 /////
 
-IThorResult *CThorGraphResults::createResult(CActivityBase &activity, unsigned id, IThorRowInterfaces *rowIf, bool distributed, unsigned spillPriority)
+IThorResult *CThorGraphResults::createResult(CActivityBase &activity, unsigned id, IThorRowInterfaces *rowIf, ThorGraphResultType resultType, unsigned spillPriority)
 {
-    Owned<IThorResult> result = ::createResult(activity, rowIf, distributed, spillPriority);
+    Owned<IThorResult> result = ::createResult(activity, rowIf, resultType, spillPriority);
     setResult(id, result);
     return result;
 }
 
 /////
 
-IThorResult *createResult(CActivityBase &activity, IThorRowInterfaces *rowIf, bool distributed, unsigned spillPriority)
+IThorResult *createResult(CActivityBase &activity, IThorRowInterfaces *rowIf, ThorGraphResultType resultType, unsigned spillPriority)
 {
-    return new CThorGraphResult(activity, rowIf, distributed, spillPriority);
+    return new CThorGraphResult(activity, rowIf, resultType, spillPriority);
 }
 
 /////
@@ -230,7 +240,7 @@ public:
         thor_loop_counter_t * res = (thor_loop_counter_t *)counterRow.ensureCapacity(sizeof(thor_loop_counter_t),NULL);
         *res = loopCounter;
         OwnedConstThorRow counterRowFinal = counterRow.finalizeRowClear(sizeof(thor_loop_counter_t));
-        IThorResult *counterResult = results->createResult(activity, pos, countRowIf, false, SPILL_PRIORITY_DISABLE);
+        IThorResult *counterResult = results->createResult(activity, pos, countRowIf, thorgraphresult_nul, SPILL_PRIORITY_DISABLE);
         Owned<IRowWriter> counterResultWriter = counterResult->getWriter();
         counterResultWriter->putRow(counterRowFinal.getClear());
     }
@@ -238,14 +248,15 @@ public:
     {
         if (!loopAgainRowIf)
             loopAgainRowIf.setown(activity.createRowInterfaces(loopAgainMeta));
-        activity.queryGraph().createResult(activity, pos, results, loopAgainRowIf, !activity.queryGraph().isLocalChild(), SPILL_PRIORITY_DISABLE);
+        activity.queryGraph().createResult(activity, pos, results, loopAgainRowIf, activity.queryGraph().isLocalChild() ? thorgraphresult_nul : thorgraphresult_distributed, SPILL_PRIORITY_DISABLE);
     }
     virtual void prepareLoopResults(CActivityBase &activity, IThorGraphResults *results)
     {
         if (!resultRowIf)
             resultRowIf.setown(activity.createRowInterfaces(resultMeta));
-        IThorResult *loopResult = results->createResult(activity, 0, resultRowIf, !activity.queryGraph().isLocalChild()); // loop output
-        IThorResult *inputResult = results->createResult(activity, 1, resultRowIf, !activity.queryGraph().isLocalChild()); // loop input
+        ThorGraphResultType resultType = activity.queryGraph().isLocalChild() ? thorgraphresult_nul : thorgraphresult_distributed;
+        IThorResult *loopResult = results->createResult(activity, 0, resultRowIf, resultType); // loop output
+        IThorResult *inputResult = results->createResult(activity, 1, resultRowIf, resultType); // loop input
     }
     virtual void execute(CActivityBase &activity, unsigned counter, IThorGraphResults *results, IRowWriterMultiReader *inputStream, rowcount_t rowStreamCount, size32_t parentExtractSz, const byte *parentExtract)
     {
@@ -2076,19 +2087,19 @@ IThorResult *CGraphBase::getGraphLoopResult(unsigned id, bool distributed)
     return graphLoopResults->getResult(id, distributed);
 }
 
-IThorResult *CGraphBase::createResult(CActivityBase &activity, unsigned id, IThorGraphResults *results, IThorRowInterfaces *rowIf, bool distributed, unsigned spillPriority)
+IThorResult *CGraphBase::createResult(CActivityBase &activity, unsigned id, IThorGraphResults *results, IThorRowInterfaces *rowIf, ThorGraphResultType resultType, unsigned spillPriority)
 {
-    return results->createResult(activity, id, rowIf, distributed, spillPriority);
+    return results->createResult(activity, id, rowIf, resultType, spillPriority);
 }
 
-IThorResult *CGraphBase::createResult(CActivityBase &activity, unsigned id, IThorRowInterfaces *rowIf, bool distributed, unsigned spillPriority)
+IThorResult *CGraphBase::createResult(CActivityBase &activity, unsigned id, IThorRowInterfaces *rowIf, ThorGraphResultType resultType, unsigned spillPriority)
 {
-    return localResults->createResult(activity, id, rowIf, distributed, spillPriority);
+    return localResults->createResult(activity, id, rowIf, resultType, spillPriority);
 }
 
-IThorResult *CGraphBase::createGraphLoopResult(CActivityBase &activity, IThorRowInterfaces *rowIf, bool distributed, unsigned spillPriority)
+IThorResult *CGraphBase::createGraphLoopResult(CActivityBase &activity, IThorRowInterfaces *rowIf, ThorGraphResultType resultType, unsigned spillPriority)
 {
-    return graphLoopResults->createResult(activity, rowIf, distributed, spillPriority);
+    return graphLoopResults->createResult(activity, rowIf, resultType, spillPriority);
 }
 
 // IEclGraphResults

thorlcr/graph/thgraph.hpp

@@ -121,14 +121,22 @@ interface IThorResult : extends IInterface
     virtual const void * getLinkedRowResult() = 0;
 };
 
+enum ThorGraphResultType:unsigned
+{
+    thorgraphresult_nul = 0x00,
+    thorgraphresult_distributed = 0x01,
+    thorgraphresult_grouped = 0x02,
+    thorgraphresult_sparse = 0x04
+};
+inline ThorGraphResultType mergeResultTypes(ThorGraphResultType l, ThorGraphResultType r) { return (ThorGraphResultType) (l|r); }
 class CActivityBase;
 // JCSMORE - based on IHThorGraphResults
 interface IThorGraphResults : extends IEclGraphResults
 {
     virtual void clear() = 0;
     virtual IThorResult *getResult(unsigned id, bool distributed=false) = 0;
-    virtual IThorResult *createResult(CActivityBase &activity, unsigned id, IThorRowInterfaces *rowIf, bool distributed, unsigned spillPriority=SPILL_PRIORITY_RESULT) = 0;
-    virtual IThorResult *createResult(CActivityBase &activity, IThorRowInterfaces *rowIf, bool distributed, unsigned spillPriority=SPILL_PRIORITY_RESULT) = 0;
+    virtual IThorResult *createResult(CActivityBase &activity, unsigned id, IThorRowInterfaces *rowIf, ThorGraphResultType resultType, unsigned spillPriority=SPILL_PRIORITY_RESULT) = 0;
+    virtual IThorResult *createResult(CActivityBase &activity, IThorRowInterfaces *rowIf, ThorGraphResultType resultType, unsigned spillPriority=SPILL_PRIORITY_RESULT) = 0;
     virtual unsigned addResult(IThorResult *result) = 0;
     virtual void setResult(unsigned id, IThorResult *result) = 0;
     virtual unsigned count() = 0;
@@ -740,9 +748,9 @@ public:
 
     virtual IThorResult *getResult(unsigned id, bool distributed=false);
     virtual IThorResult *getGraphLoopResult(unsigned id, bool distributed=false);
-    virtual IThorResult *createResult(CActivityBase &activity, unsigned id, IThorGraphResults *results, IThorRowInterfaces *rowIf, bool distributed, unsigned spillPriority=SPILL_PRIORITY_RESULT);
-    virtual IThorResult *createResult(CActivityBase &activity, unsigned id, IThorRowInterfaces *rowIf, bool distributed, unsigned spillPriority=SPILL_PRIORITY_RESULT);
-    virtual IThorResult *createGraphLoopResult(CActivityBase &activity, IThorRowInterfaces *rowIf, bool distributed, unsigned spillPriority=SPILL_PRIORITY_RESULT);
+    virtual IThorResult *createResult(CActivityBase &activity, unsigned id, IThorGraphResults *results, IThorRowInterfaces *rowIf, ThorGraphResultType resultType, unsigned spillPriority=SPILL_PRIORITY_RESULT);
+    virtual IThorResult *createResult(CActivityBase &activity, unsigned id, IThorRowInterfaces *rowIf, ThorGraphResultType resultType, unsigned spillPriority=SPILL_PRIORITY_RESULT);
+    virtual IThorResult *createGraphLoopResult(CActivityBase &activity, IThorRowInterfaces *rowIf, ThorGraphResultType resultType, unsigned spillPriority=SPILL_PRIORITY_RESULT);
 
 // IEclGraphResults
     virtual void getDictionaryResult(unsigned & count, byte * * & ret, unsigned id);
@@ -1195,10 +1203,10 @@ public:
         // NB: stream static after this, i.e. nothing can be added to this result
         return LINK(&results.item(id));
     }
-    virtual IThorResult *createResult(CActivityBase &activity, unsigned id, IThorRowInterfaces *rowIf, bool distributed, unsigned spillPriority=SPILL_PRIORITY_RESULT);
-    virtual IThorResult *createResult(CActivityBase &activity, IThorRowInterfaces *rowIf, bool distributed, unsigned spillPriority=SPILL_PRIORITY_RESULT)
+    virtual IThorResult *createResult(CActivityBase &activity, unsigned id, IThorRowInterfaces *rowIf, ThorGraphResultType resultType, unsigned spillPriority=SPILL_PRIORITY_RESULT);
+    virtual IThorResult *createResult(CActivityBase &activity, IThorRowInterfaces *rowIf, ThorGraphResultType resultType, unsigned spillPriority=SPILL_PRIORITY_RESULT)
     {
-        return createResult(activity, results.ordinality(), rowIf, distributed, spillPriority);
+        return createResult(activity, results.ordinality(), rowIf, resultType, spillPriority);
     }
     virtual unsigned addResult(IThorResult *result)
     {
@@ -1237,7 +1245,7 @@ public:
     virtual activity_id queryOwnerId() const { return ownerId; }
 };
 
-extern graph_decl IThorResult *createResult(CActivityBase &activity, IThorRowInterfaces *rowIf, bool distributed, unsigned spillPriority=SPILL_PRIORITY_RESULT);
+extern graph_decl IThorResult *createResult(CActivityBase &activity, IThorRowInterfaces *rowIf, ThorGraphResultType resulType, unsigned spillPriority=SPILL_PRIORITY_RESULT);
 
 
 class CGraphElementBase;

thorlcr/graph/thgraphmaster.cpp

@@ -2014,7 +2014,7 @@ class CCollatedResult : implements IThorResult, public CSimpleInterface
                 }
             }
         }
-        Owned<IThorResult> _result = ::createResult(activity, rowIf, false, spillPriority);
+        Owned<IThorResult> _result = ::createResult(activity, rowIf, thorgraphresult_nul, spillPriority);
         Owned<IRowWriter> resultWriter = _result->getWriter();
         for (unsigned s=0; s<numSlaves; s++)
         {
@@ -2748,21 +2748,21 @@ bool CMasterGraph::deserializeStats(unsigned node, MemoryBuffer &mb)
     return true;
 }
 
-IThorResult *CMasterGraph::createResult(CActivityBase &activity, unsigned id, IThorGraphResults *results, IThorRowInterfaces *rowIf, bool distributed, unsigned spillPriority)
+IThorResult *CMasterGraph::createResult(CActivityBase &activity, unsigned id, IThorGraphResults *results, IThorRowInterfaces *rowIf, ThorGraphResultType resultType, unsigned spillPriority)
 {
     Owned<CCollatedResult> result = new CCollatedResult(*this, activity, rowIf, id, results->queryOwnerId(), spillPriority);
     results->setResult(id, result);
     return result;
 }
 
-IThorResult *CMasterGraph::createResult(CActivityBase &activity, unsigned id, IThorRowInterfaces *rowIf, bool distributed, unsigned spillPriority)
+IThorResult *CMasterGraph::createResult(CActivityBase &activity, unsigned id, IThorRowInterfaces *rowIf, ThorGraphResultType resultType, unsigned spillPriority)
 {
     Owned<CCollatedResult> result = new CCollatedResult(*this, activity, rowIf, id, localResults->queryOwnerId(), spillPriority);
     localResults->setResult(id, result);
     return result;
 }
 
-IThorResult *CMasterGraph::createGraphLoopResult(CActivityBase &activity, IThorRowInterfaces *rowIf, bool distributed, unsigned spillPriority)
+IThorResult *CMasterGraph::createGraphLoopResult(CActivityBase &activity, IThorRowInterfaces *rowIf, ThorGraphResultType resultType, unsigned spillPriority)
 {
     Owned<CCollatedResult> result = new CCollatedResult(*this, activity, rowIf, 0, localResults->queryOwnerId(), spillPriority);
     unsigned id = graphLoopResults->addResult(result);

thorlcr/graph/thgraphmaster.ipp

@@ -76,9 +76,9 @@ public:
     virtual void done() override;
     virtual void reset() override;
     virtual void abort(IException *e) override;
-    IThorResult *createResult(CActivityBase &activity, unsigned id, IThorGraphResults *results, IThorRowInterfaces *rowIf, bool distributed, unsigned spillPriority=SPILL_PRIORITY_RESULT);
-    IThorResult *createResult(CActivityBase &activity, unsigned id, IThorRowInterfaces *rowIf, bool distributed, unsigned spillPriority=SPILL_PRIORITY_RESULT);
-    IThorResult *createGraphLoopResult(CActivityBase &activity, IThorRowInterfaces *rowIf, bool distributed, unsigned spillPriority=SPILL_PRIORITY_RESULT);
+    IThorResult *createResult(CActivityBase &activity, unsigned id, IThorGraphResults *results, IThorRowInterfaces *rowIf, ThorGraphResultType resultType, unsigned spillPriority=SPILL_PRIORITY_RESULT);
+    IThorResult *createResult(CActivityBase &activity, unsigned id, IThorRowInterfaces *rowIf, ThorGraphResultType resultType, unsigned spillPriority=SPILL_PRIORITY_RESULT);
+    IThorResult *createGraphLoopResult(CActivityBase &activity, IThorRowInterfaces *rowIf, ThorGraphResultType resultType, unsigned spillPriority=SPILL_PRIORITY_RESULT);
 
 // IExceptionHandler
     virtual bool fireException(IException *e);

thorlcr/graph/thgraphslave.cpp

@@ -1277,7 +1277,7 @@ IThorResult *CSlaveGraph::getGlobalResult(CActivityBase &activity, IThorRowInter
     if (!queryJobChannel().queryJobComm().send(msg, 0, queryJob().querySlaveMpTag(), LONGTIMEOUT))
         throwUnexpected();
 
-    Owned<IThorResult> result = ::createResult(activity, rowIf, false);
+    Owned<IThorResult> result = ::createResult(activity, rowIf, thorgraphresult_nul);
     Owned<IRowWriter> resultWriter = result->getWriter();
 
     MemoryBuffer mb;

thorlcr/msort/tsortm.cpp

@@ -186,7 +186,7 @@ struct PartitionInfo
     size32_t guard;
     Linked<IThorRowInterfaces> prowif;
     PartitionInfo(CActivityBase *_activity, IThorRowInterfaces *rowif)
-        : splitkeys(*_activity, rowif, true), prowif(rowif)
+        : splitkeys(*_activity, rowif, ers_allow), prowif(rowif)
     {
         nodes = NULL;
         mpports = NULL;
@@ -222,42 +222,6 @@ struct PartitionInfo
         // should be more defensive here
         return (numnodes!=0)&&(splitkeys.ordinality()!=0);
     }
-
-    void serialize(MemoryBuffer &mb)
-    {
-        mb.append(numnodes);
-        unsigned i;
-        for (i=0;i<numnodes;i++)
-            nodes[i].serialize(mb);
-        for (i=0;i<numnodes;i++)
-            mb.append((unsigned short)mpports[i]);
-        mb.append((unsigned)mpTagRPC);
-        mb.append(guard);
-        splitkeys.serialize(mb);
-    }   
-    void deserialize(size32_t len,void *src)
-    {
-        kill();
-        MemoryBuffer mb(len,src);
-        mb.read(numnodes);
-        nodes = (SocketEndpoint *)malloc(numnodes*sizeof(SocketEndpoint));
-        unsigned i;
-        for (i=0;i<numnodes;i++)
-            nodes[i].deserialize(mb);
-        mpports = (unsigned short *)malloc(numnodes*sizeof(unsigned short));
-        for (i=0;i<numnodes;i++) 
-            mb.read(mpports[i]);
-        unsigned t;
-        mb.read(t);
-        mpTagRPC = (mptag_t)t;
-        size32_t left = mb.remaining();
-        size32_t dsguard;
-        mb.read(dsguard);
-        if (guard!=dsguard)
-            throw MakeStringException(-1,"SORT: PartitionInfo meta info mismatch(%d,%d)",guard,dsguard);
-        splitkeys.kill();
-        splitkeys.deserialize(left, mb.readDirect(left));
-    }
 };
 
 
@@ -595,7 +559,7 @@ public:
         unsigned averagesamples = OVERSAMPLE*numnodes;  
         rowcount_t averagerecspernode = (rowcount_t)(total/numnodes);
         CriticalSection asect;
-        CThorExpandingRowArray sample(*activity, keyIf, true);
+        CThorExpandingRowArray sample(*activity, keyIf, ers_allow);
         class casyncfor1: public CAsyncFor
         {
             CSortMaster &owner;
@@ -651,7 +615,7 @@ public:
         offset_t ts=sample.serializedSize();
         estrecsize = numsamples?((size32_t)(ts/numsamples)):100;
         sample.sort(*icompare, activity->queryMaxCores());
-        CThorExpandingRowArray mid(*activity, keyIf, true);
+        CThorExpandingRowArray mid(*activity, keyIf, ers_allow);
         if (numsamples) // could shuffle up empty nodes here
         {
             for (unsigned i=0;i<numsplits;i++)
@@ -757,12 +721,12 @@ public:
             return splitmap.getClear();
         }
         unsigned numsplits=numnodes-1;
-        CThorExpandingRowArray emin(*activity, keyIf, true);
-        CThorExpandingRowArray emax(*activity, keyIf, true);
-        CThorExpandingRowArray totmid(*activity, keyIf, true);
+        CThorExpandingRowArray emin(*activity, keyIf, ers_allow);
+        CThorExpandingRowArray emax(*activity, keyIf, ers_allow);
+        CThorExpandingRowArray totmid(*activity, keyIf, ers_allow);
         ECFarray = &totmid;
         ECFcompare = icompare;
-        CThorExpandingRowArray mid(*activity, keyIf, true);
+        CThorExpandingRowArray mid(*activity, keyIf, ers_allow);
         unsigned i;
         unsigned j;
         for(i=0;i<numsplits;i++)
@@ -925,8 +889,8 @@ public:
                     }
                 }
 
-                CThorExpandingRowArray newmin(*activity, keyIf, true);
-                CThorExpandingRowArray newmax(*activity, keyIf, true);
+                CThorExpandingRowArray newmin(*activity, keyIf, ers_allow);
+                CThorExpandingRowArray newmax(*activity, keyIf, ers_allow);
                 unsigned __int64 maxerror=0;
                 unsigned __int64 nodewanted = (stotal/numnodes); // Note scaled total
                 unsigned __int64 variancelimit = estrecsize?maxdeviance/estrecsize:0;
@@ -1088,7 +1052,7 @@ public:
         // I think this dependent on row being same format as meta
 
         unsigned numsplits=numnodes-1;
-        CThorExpandingRowArray splits(*activity, keyIf, true);
+        CThorExpandingRowArray splits(*activity, keyIf, ers_allow);
         char *s=cosortfilenames;
         unsigned i;
         for(i=0;i<numnodes;i++)
@@ -1122,7 +1086,7 @@ public:
     {
         ActPrintLog(activity, "Previous partition");
         unsigned numsplits=numnodes-1;
-        CThorExpandingRowArray splits(*activity, keyIf, true);
+        CThorExpandingRowArray splits(*activity, keyIf, ers_allow);
         unsigned i;
         for(i=1;i<numnodes;i++)
         {

thorlcr/msort/tsorts.cpp

@@ -167,7 +167,7 @@ public:
     IMPLEMENT_IINTERFACE_USING(CSimpleInterface);
 
     CWriteIntercept(CActivityBase &_activity, IThorRowInterfaces *_rowIf, unsigned _interval)
-        : activity(_activity), rowIf(_rowIf), interval(_interval), sampleRows(activity, rowIf, true)
+        : activity(_activity), rowIf(_rowIf), interval(_interval), sampleRows(activity, rowIf, ers_forbidden)
     {
         interval = _interval;
         idx = 0;
@@ -867,9 +867,9 @@ public:
     {
         // finds the keys within the ranges specified
         // uses empty keys (0 size) if none found
-        CThorExpandingRowArray low(*activity, keyIf, true);
-        CThorExpandingRowArray high(*activity, keyIf, true);
-        CThorExpandingRowArray mid(*activity, keyIf, true);
+        CThorExpandingRowArray low(*activity, keyIf, ers_allow);
+        CThorExpandingRowArray high(*activity, keyIf, ers_allow);
+        CThorExpandingRowArray mid(*activity, keyIf, ers_allow);
         low.deserializeExpand(lbufsize, lkeybuf);
         high.deserializeExpand(hbufsize, hkeybuf);
         unsigned n=low.ordinality();
@@ -936,13 +936,13 @@ public:
     }
     virtual void MultiBinChop(size32_t keybufsize, const byte * keybuf, unsigned num, rowcount_t * pos, byte cmpfn)
     {
-        CThorExpandingRowArray keys(*activity, keyIf, true);
+        CThorExpandingRowArray keys(*activity, keyIf, ers_allow);
         keys.deserialize(keybufsize, keybuf);
         doBinChop(keys, pos, num, cmpfn);
     }
     virtual void MultiBinChopStart(size32_t keybufsize, const byte * keybuf, byte cmpfn)
     {
-        CThorExpandingRowArray keys(*activity, keyIf, true);
+        CThorExpandingRowArray keys(*activity, keyIf, ers_allow);
         keys.deserializeExpand(keybufsize, keybuf);
         assertex(multibinchoppos==NULL); // check for reentrancy
         multibinchopnum = keys.ordinality();
@@ -971,7 +971,7 @@ public:
         for (i=0;i<mapsize;i++)
             ActPrintLog(activity, "%" RCPF "d ",overflowmap[i]);
 #endif
-        CThorExpandingRowArray keys(*activity, keyIf, true);
+        CThorExpandingRowArray keys(*activity, keyIf, ers_allow);
         keys.deserialize(keybufsize, keybuf);
         for (i=0;i<mapsize-1;i++)
             AdjustOverflow(overflowmap[i], keys.query(i), cmpfn);

thorlcr/thorutil/thbuf.cpp

@@ -631,10 +631,10 @@ class COverflowableBuffer : public CSimpleInterface, implements IRowWriterMultiR
 public:
     IMPLEMENT_IINTERFACE_USING(CSimpleInterface);
 
-    COverflowableBuffer(CActivityBase &_activity, IThorRowInterfaces *_rowIf, bool grouped, bool _shared, unsigned spillPriority)
+    COverflowableBuffer(CActivityBase &_activity, IThorRowInterfaces *_rowIf, EmptyRowSemantics emptyRowSemantics, bool _shared, unsigned spillPriority)
         : activity(_activity), rowIf(_rowIf), shared(_shared)
     {
-        collector.setown(createThorRowCollector(activity, rowIf, NULL, stableSort_none, rc_mixed, spillPriority, grouped));
+        collector.setown(createThorRowCollector(activity, rowIf, NULL, stableSort_none, rc_mixed, spillPriority, emptyRowSemantics));
         writer.setown(collector->getWriter());
         eoi = false;
     }
@@ -662,9 +662,9 @@ public:
     }
 };
 
-IRowWriterMultiReader *createOverflowableBuffer(CActivityBase &activity, IThorRowInterfaces *rowIf, bool grouped, bool shared, unsigned spillPriority)
+IRowWriterMultiReader *createOverflowableBuffer(CActivityBase &activity, IThorRowInterfaces *rowIf, EmptyRowSemantics emptyRowSemantics, bool shared, unsigned spillPriority)
 {
-    return new COverflowableBuffer(activity, rowIf, grouped, shared, spillPriority);
+    return new COverflowableBuffer(activity, rowIf, emptyRowSemantics, shared, spillPriority);
 }
 
 
@@ -1169,7 +1169,7 @@ friend class CRowSet;
 };
 
 CRowSet::CRowSet(CSharedWriteAheadBase &_sharedWriteAhead, unsigned _chunk, unsigned maxRows)
-    : sharedWriteAhead(_sharedWriteAhead), rows(*_sharedWriteAhead.activity, _sharedWriteAhead.activity, true, stableSort_none, true, maxRows), chunk(_chunk)
+    : sharedWriteAhead(_sharedWriteAhead), rows(*_sharedWriteAhead.activity, _sharedWriteAhead.activity, ers_eogonly, stableSort_none, true, maxRows), chunk(_chunk)
 {
 }
 
@@ -1745,7 +1745,7 @@ public:
         eos = eow = readerBlocked = false;
         rowPos = rowsToRead = 0;
         writersComplete = writersBlocked = 0;
-        rows.setup(rowIf, false, stableSort_none, true); // turning on throwOnOom;
+        rows.setup(rowIf, ers_forbidden, stableSort_none, true); // turning on throwOnOom;
     }
     ~CRowMultiWriterReader()
     {

thorlcr/thorutil/thbuf.hpp

@@ -81,7 +81,7 @@ interface IRowWriterMultiReader : extends IRowWriter
     virtual IRowStream *getReader() = 0;
 };
 
-extern graph_decl IRowWriterMultiReader *createOverflowableBuffer(CActivityBase &activity, IThorRowInterfaces *rowif, bool grouped, bool shared=false, unsigned spillPriority=SPILL_PRIORITY_OVERFLOWABLE_BUFFER);
+extern graph_decl IRowWriterMultiReader *createOverflowableBuffer(CActivityBase &activity, IThorRowInterfaces *rowif, EmptyRowSemantics emptyRowSemantics, bool shared=false, unsigned spillPriority=SPILL_PRIORITY_OVERFLOWABLE_BUFFER);
 // NB first write all then read (not interleaved!)
 
 // Multiple writers, one reader

thorlcr/thorutil/thmem.cpp

@@ -229,7 +229,8 @@ public:
 class CSpillableStreamBase : public CSpillable
 {
 protected:
-    bool preserveNulls, ownsRows;
+    bool ownsRows;
+    EmptyRowSemantics emptyRowSemantics;
     unsigned spillCompInfo;
     CThorSpillableRowArray rows;
     OwnedIFile spillFile;
@@ -252,13 +253,13 @@ protected:
         return true;
     }
 public:
-    CSpillableStreamBase(CActivityBase &_activity, CThorSpillableRowArray &inRows, IThorRowInterfaces *_rowIf, bool _preserveNulls, unsigned _spillPriority)
-        : CSpillable(_activity, _rowIf, _spillPriority), rows(_activity), preserveNulls(_preserveNulls)
+    CSpillableStreamBase(CActivityBase &_activity, CThorSpillableRowArray &inRows, IThorRowInterfaces *_rowIf, EmptyRowSemantics _emptyRowSemantics, unsigned _spillPriority)
+        : CSpillable(_activity, _rowIf, _spillPriority), rows(_activity), emptyRowSemantics(_emptyRowSemantics)
     {
         assertex(inRows.isFlushed());
         ownsRows = false;
         spillCompInfo = 0x0;
-        rows.setup(rowIf, _preserveNulls);
+        rows.setup(rowIf, emptyRowSemantics);
         rows.swap(inRows);
     }
     ~CSpillableStreamBase()
@@ -337,9 +338,7 @@ class CSharedSpillableRowSet : public CSpillableStreamBase
                     {
                         block.clearCB = true;
                         assertex(((offset_t)-1) != outputOffset);
-                        unsigned rwFlags = DEFAULT_RWFLAGS;
-                        if (owner->preserveNulls)
-                            rwFlags |= rw_grouped;
+                        unsigned rwFlags = DEFAULT_RWFLAGS | mapESRToRWFlags(owner->emptyRowSemantics);
                         spillStream.setown(::createRowStreamEx(owner->spillFile, owner->rowIf, outputOffset, (offset_t)-1, (unsigned __int64)-1, rwFlags));
                         owner->rows.unregisterWriteCallback(*this); // no longer needed
                         ret = spillStream->nextRow();
@@ -358,7 +357,7 @@ class CSharedSpillableRowSet : public CSpillableStreamBase
                 }
                 if (ret)
                     return ret;
-                if (!owner->preserveNulls)
+                if (ers_forbidden == owner->emptyRowSemantics)
                     eos = true;
             }
             return nullptr;
@@ -380,8 +379,8 @@ class CSharedSpillableRowSet : public CSpillableStreamBase
     };
 
 public:
-    CSharedSpillableRowSet(CActivityBase &_activity, CThorSpillableRowArray &inRows, IThorRowInterfaces *_rowIf, bool _preserveNulls, unsigned _spillPriority)
-        : CSpillableStreamBase(_activity, inRows, _rowIf, _preserveNulls, _spillPriority)
+    CSharedSpillableRowSet(CActivityBase &_activity, CThorSpillableRowArray &inRows, IThorRowInterfaces *_rowIf, EmptyRowSemantics _emptyRowSemantics, unsigned _spillPriority)
+        : CSpillableStreamBase(_activity, inRows, _rowIf, _emptyRowSemantics, _spillPriority)
     {
         activateSpillingCallback();
     }
@@ -391,9 +390,7 @@ public:
         if (spillFile) // already spilled?
         {
             block.clearCB = true;
-            unsigned rwFlags = DEFAULT_RWFLAGS;
-            if (preserveNulls)
-                rwFlags |= rw_grouped;
+            unsigned rwFlags = DEFAULT_RWFLAGS | mapESRToRWFlags(emptyRowSemantics);
             return ::createRowStream(spillFile, rowIf, rwFlags);
         }
         rowidx_t toRead = rows.numCommitted();
@@ -414,8 +411,8 @@ class CSpillableStream : public CSpillableStreamBase, implements IRowStream
 public:
     IMPLEMENT_IINTERFACE_USING(CSpillableStreamBase);
 
-    CSpillableStream(CActivityBase &_activity, CThorSpillableRowArray &inRows, IThorRowInterfaces *_rowIf, bool _preserveNulls, unsigned _spillPriority, unsigned _spillCompInfo)
-        : CSpillableStreamBase(_activity, inRows, _rowIf, _preserveNulls, _spillPriority)
+    CSpillableStream(CActivityBase &_activity, CThorSpillableRowArray &inRows, IThorRowInterfaces *_rowIf, EmptyRowSemantics _emptyRowSemantics, unsigned _spillPriority, unsigned _spillCompInfo)
+        : CSpillableStreamBase(_activity, inRows, _rowIf, _emptyRowSemantics, _spillPriority)
     {
         spillCompInfo = _spillCompInfo;
         pos = numReadRows = 0;
@@ -449,8 +446,7 @@ public:
                     rwFlags |= rw_compress;
                     rwFlags |= spillCompInfo;
                 }
-                if (preserveNulls)
-                    rwFlags |= rw_grouped;
+                rwFlags |= mapESRToRWFlags(emptyRowSemantics);
                 spillStream.setown(createRowStream(spillFile, rowIf, rwFlags));
                 return spillStream->nextRow();
             }
@@ -668,18 +664,15 @@ inline bool CThorExpandingRowArray::_resize(rowidx_t requiredRows, unsigned maxS
     return true;
 }
 
-CThorExpandingRowArray::CThorExpandingRowArray(CActivityBase &_activity)
-    : activity(_activity)
+CThorExpandingRowArray::CThorExpandingRowArray(CActivityBase &_activity) : activity(_activity)
 {
-    initCommon();
-    setup(NULL, false, stableSort_none, true);
+    rowManager = activity.queryRowManager();
 }
 
-CThorExpandingRowArray::CThorExpandingRowArray(CActivityBase &_activity, IThorRowInterfaces *_rowIf, bool _allowNulls, StableSortFlag _stableSort, bool _throwOnOom, rowidx_t initialSize)
+CThorExpandingRowArray::CThorExpandingRowArray(CActivityBase &_activity, IThorRowInterfaces *_rowIf, EmptyRowSemantics _emptyRowSemantics, StableSortFlag _stableSort, bool _throwOnOom, rowidx_t initialSize)
     : activity(_activity)
 {
-    initCommon();
-    setup(_rowIf, _allowNulls, _stableSort, _throwOnOom);
+    setup(_rowIf, _emptyRowSemantics, _stableSort, _throwOnOom);
     if (initialSize)
     {
         rows = static_cast<const void * *>(rowManager->allocate(initialSize * sizeof(void*), activity.queryContainer().queryId(), defaultMaxSpillCost));
@@ -697,18 +690,10 @@ CThorExpandingRowArray::~CThorExpandingRowArray()
     ReleaseThorRow(stableTable);
 }
 
-void CThorExpandingRowArray::initCommon()
-{
-    stableTable = NULL;
-    rows = NULL;
-    maxRows = 0;
-    numRows = 0;
-}
-
-void CThorExpandingRowArray::setup(IThorRowInterfaces *_rowIf, bool _allowNulls, StableSortFlag _stableSort, bool _throwOnOom)
+void CThorExpandingRowArray::setup(IThorRowInterfaces *_rowIf, EmptyRowSemantics _emptyRowSemantics, StableSortFlag _stableSort, bool _throwOnOom)
 {
     rowIf = _rowIf;
-    allowNulls = _allowNulls;
+    emptyRowSemantics = _emptyRowSemantics;
     stableSort = _stableSort;
     throwOnOom = _throwOnOom;
     if (rowIf)
@@ -784,7 +769,7 @@ void CThorExpandingRowArray::swap(CThorExpandingRowArray &other)
     IThorRowInterfaces *otherRowIf = other.rowIf;
     const void **otherRows = other.rows;
     void **otherStableTable = other.stableTable;
-    bool otherAllowNulls = other.allowNulls;
+    EmptyRowSemantics otherEmptyRowSemantics = other.emptyRowSemantics;
     StableSortFlag otherStableSort = other.stableSort;
     bool otherThrowOnOom = other.throwOnOom;
     unsigned otherDefaultMaxSpillCost = other.defaultMaxSpillCost;
@@ -796,7 +781,7 @@ void CThorExpandingRowArray::swap(CThorExpandingRowArray &other)
     other.stableTable = stableTable;
     other.maxRows = maxRows;
     other.numRows = numRows;
-    other.setup(rowIf, allowNulls, stableSort, throwOnOom);
+    other.setup(rowIf, emptyRowSemantics, stableSort, throwOnOom);
     other.setDefaultMaxSpillCost(defaultMaxSpillCost);
 
     rowManager = otherRowManager;
@@ -804,7 +789,7 @@ void CThorExpandingRowArray::swap(CThorExpandingRowArray &other)
     stableTable = otherStableTable;
     maxRows = otherMaxRows;
     numRows = otherNumRows;
-    setup(otherRowIf, otherAllowNulls, otherStableSort, otherThrowOnOom);
+    setup(otherRowIf, otherEmptyRowSemantics, otherStableSort, otherThrowOnOom);
     setDefaultMaxSpillCost(otherDefaultMaxSpillCost);
 }
 
@@ -1160,7 +1145,7 @@ void CThorExpandingRowArray::serialize(MemoryBuffer &mb)
 {
     assertex(serializer);
     CMemoryRowSerializer s(mb);
-    if (!allowNulls)
+    if (emptyRowSemantics == ers_forbidden)
         serialize(s);
     else
     {
@@ -1234,7 +1219,7 @@ void CThorExpandingRowArray::deserializeRow(IRowDeserializerSource &in)
 
 void CThorExpandingRowArray::deserialize(size32_t sz, const void *buf)
 {
-    if (allowNulls)
+    if (emptyRowSemantics != ers_forbidden)
     {
         ASSERTEX((sz>=sizeof(short))&&(*(unsigned short *)buf==0x7631)); // check for mismatch
         buf = (const byte *)buf+sizeof(unsigned short);
@@ -1243,7 +1228,7 @@ void CThorExpandingRowArray::deserialize(size32_t sz, const void *buf)
     CThorStreamDeserializerSource d(sz,buf);
     while (!d.eos())
     {
-        if (allowNulls)
+        if (emptyRowSemantics != ers_forbidden)
         {
             bool nullrow;
             d.read(sizeof(bool),&nullrow);
@@ -1291,15 +1276,12 @@ void CThorSpillableRowArray::safeUnregisterWriteCallback(IWritePosCallback &cb)
 CThorSpillableRowArray::CThorSpillableRowArray(CActivityBase &activity)
     : CThorExpandingRowArray(activity)
 {
-    initCommon();
-    commitDelta = CommitStep;
     throwOnOom = false;
 }
 
-CThorSpillableRowArray::CThorSpillableRowArray(CActivityBase &activity, IThorRowInterfaces *rowIf, bool allowNulls, StableSortFlag stableSort, rowidx_t initialSize, size32_t _commitDelta)
-    : CThorExpandingRowArray(activity, rowIf, false, stableSort, false, initialSize), commitDelta(_commitDelta)
+CThorSpillableRowArray::CThorSpillableRowArray(CActivityBase &activity, IThorRowInterfaces *rowIf, EmptyRowSemantics emptyRowSemantics, StableSortFlag stableSort, rowidx_t initialSize, size32_t _commitDelta)
+    : CThorExpandingRowArray(activity, rowIf, ers_forbidden, stableSort, false, initialSize), commitDelta(_commitDelta)
 {
-    initCommon();
 }
 
 CThorSpillableRowArray::~CThorSpillableRowArray()
@@ -1307,12 +1289,6 @@ CThorSpillableRowArray::~CThorSpillableRowArray()
     clearRows();
 }
 
-void CThorSpillableRowArray::initCommon()
-{
-    commitRows = 0;
-    firstRow = 0;
-}
-
 void CThorSpillableRowArray::clearRows()
 {
     roxiemem::ReleaseRoxieRowRange(rows, firstRow, numRows);
@@ -1361,7 +1337,7 @@ rowidx_t CThorSpillableRowArray::save(IFile &iFile, unsigned _spillCompInfo, boo
     rowidx_t n = numCommitted();
     if (0 == n)
         return 0;
-    ActPrintLog(&activity, "%s: CThorSpillableRowArray::save (skipNulls=%s, allowNulls=%s) max rows = %"  RIPF "u", tracingPrefix, boolToStr(skipNulls), boolToStr(allowNulls), n);
+    ActPrintLog(&activity, "%s: CThorSpillableRowArray::save (skipNulls=%s, emptyRowSemantics=%u) max rows = %"  RIPF "u", tracingPrefix, boolToStr(skipNulls), emptyRowSemantics, n);
 
     if (_spillCompInfo)
         assertex(0 == writeCallbacks.ordinality()); // incompatible
@@ -1372,8 +1348,7 @@ rowidx_t CThorSpillableRowArray::save(IFile &iFile, unsigned _spillCompInfo, boo
         rwFlags |= rw_compress;
         rwFlags |= _spillCompInfo;
     }
-    if (allowNulls)
-        rwFlags |= rw_grouped;
+    rwFlags |= mapESRToRWFlags(emptyRowSemantics);
 
     // NB: This is always called within a CThorArrayLockBlock, as such no writebacks are added or updating
     rowidx_t nextCBI = RCIDXMAX; // indicates none
@@ -1420,7 +1395,7 @@ rowidx_t CThorSpillableRowArray::save(IFile &iFile, unsigned _spillCompInfo, boo
             }
             else if (!skipNulls)
             {
-                assertex(allowNulls);
+                assertex(emptyRowSemantics != ers_forbidden);
                 writer->putRow(NULL);
             }
             ++i;
@@ -1589,7 +1564,7 @@ void CThorSpillableRowArray::transferRowsCopy(const void **outRows, bool takeOwn
 IRowStream *CThorSpillableRowArray::createRowStream(unsigned spillPriority, unsigned spillCompInfo)
 {
     assertex(rowIf);
-    return new CSpillableStream(activity, *this, rowIf, allowNulls, spillPriority, spillCompInfo);
+    return new CSpillableStream(activity, *this, rowIf, emptyRowSemantics, spillPriority, spillCompInfo);
 }
 
 
@@ -1608,7 +1583,7 @@ protected:
     offset_t sizeSpill;
     ICompare *iCompare;
     StableSortFlag stableSort;
-    bool preserveGrouping;
+    EmptyRowSemantics emptyRowSemantics = ers_forbidden;
     CriticalSection readerLock;
     Owned<CSharedSpillableRowSet> spillableRowSet;
     unsigned options;
@@ -1646,10 +1621,10 @@ protected:
         spillCycles += spillTimer.elapsedCycles();
         return true;
     }
-    void setPreserveGrouping(bool _preserveGrouping)
+    void setEmptyRowSemantics(EmptyRowSemantics _emptyRowSemantics)
     {
-        preserveGrouping = _preserveGrouping;
-        spillableRows.setAllowNulls(preserveGrouping);
+        emptyRowSemantics = _emptyRowSemantics;
+        spillableRows.setEmptyRowSemantics(emptyRowSemantics);
     }
     bool flush()
     {
@@ -1734,8 +1709,7 @@ protected:
             rwFlags |= rw_compress;
             rwFlags |= spillCompInfo;
         }
-        if (preserveGrouping)
-            rwFlags |= rw_grouped;
+        rwFlags |= mapESRToRWFlags(emptyRowSemantics);
         IArrayOf<IRowStream> instrms;
         ForEachItemIn(f, spillFiles)
         {
@@ -1778,7 +1752,7 @@ protected:
                     instrms.append(*spillableRows.createRowStream(spillPriority, spillCompInfo)); // NB: stream will take ownership of rows in spillableRows
                 else
                 {
-                    spillableRowSet.setown(new CSharedSpillableRowSet(activity, spillableRows, rowIf, preserveGrouping, spillPriority));
+                    spillableRowSet.setown(new CSharedSpillableRowSet(activity, spillableRows, rowIf, emptyRowSemantics, spillPriority));
                     instrms.append(*spillableRowSet->createRowStream());
                 }
             }
@@ -1815,7 +1789,6 @@ public:
           iCompare(_iCompare), stableSort(_stableSort), diskMemMix(_diskMemMix),
           spillableRows(_activity)
     {
-        preserveGrouping = false;
         totalRows = 0;
         overflowCount = outStreams = 0;
         sizeSpill = 0;
@@ -1825,7 +1798,7 @@ public:
             activateSpillingCallback();
         maxCores = activity.queryMaxCores();
         options = 0;
-        spillableRows.setup(rowIf, false, stableSort);
+        spillableRows.setup(rowIf, ers_forbidden, stableSort);
         if (activity.getOptBool(THOROPT_COMPRESS_SPILLS, true))
         {
             StringBuffer compType;
@@ -1914,7 +1887,7 @@ public:
             mmRegistered = false;
             activity.queryRowManager()->removeRowBuffer(this);
         }
-        spillableRows.setup(rowIf, false, stableSort);
+        spillableRows.setup(rowIf, ers_forbidden, stableSort);
     }
     virtual void resize(rowidx_t max)
     {
@@ -1967,7 +1940,7 @@ class CThorRowLoader : public CThorRowCollectorBase, implements IThorRowLoader
         if (doReset)
             reset();
         activateSpillingCallback();
-        setPreserveGrouping(trl_preserveGrouping == grouping);
+        setEmptyRowSemantics(trl_preserveGrouping == grouping ? ers_eogonly : ers_forbidden);
         while (!abort)
         {
             const void *next = in->nextRow();
@@ -2050,10 +2023,10 @@ public:
     {
     }
 // IThorRowCollectorCommon
-    virtual void setPreserveGrouping(bool tf)
+    virtual void setEmptyRowSemantics(EmptyRowSemantics emptyGroupSemantics)
     {
-        assertex(!iCompare || !tf); // can't sort if group preserving
-        CThorRowCollectorBase::setPreserveGrouping(tf);
+        assertex(!iCompare || (ers_forbidden == emptyGroupSemantics)); // can't sort if preserving end of groups or nulls
+        CThorRowCollectorBase::setEmptyRowSemantics(emptyGroupSemantics);
     }
     virtual rowcount_t numRows() const { return CThorRowCollectorBase::numRows(); }
     virtual unsigned numOverflows() const { return CThorRowCollectorBase::numOverflows(); }
@@ -2118,10 +2091,10 @@ public:
     virtual bool shrink(StringBuffer *traceInfo) { return CThorRowCollectorBase::shrink(traceInfo); }
 };
 
-IThorRowCollector *createThorRowCollector(CActivityBase &activity, IThorRowInterfaces *rowIf, ICompare *iCompare, StableSortFlag stableSort, RowCollectorSpillFlags diskMemMix, unsigned spillPriority, bool preserveGrouping)
+IThorRowCollector *createThorRowCollector(CActivityBase &activity, IThorRowInterfaces *rowIf, ICompare *iCompare, StableSortFlag stableSort, RowCollectorSpillFlags diskMemMix, unsigned spillPriority, EmptyRowSemantics emptyRowSemantics)
 {
     Owned<IThorRowCollector> collector = new CThorRowCollector(activity, rowIf, iCompare, stableSort, diskMemMix, spillPriority);
-    collector->setPreserveGrouping(preserveGrouping);
+    collector->setEmptyRowSemantics(emptyRowSemantics);
     return collector.getClear();
 }
 

thorlcr/thorutil/thmem.hpp

@@ -263,7 +263,6 @@ class graph_decl CThorExpandingRowArray : public CSimpleInterface
         virtual void unlock() const {  }
     } dummyLock;
 
-    void initCommon();
     bool resizeRowTable(void **&_rows, rowidx_t requiredRows, bool copy, unsigned maxSpillCost, memsize_t &newCapacity, const char *errMsg);
     bool _resize(rowidx_t requiredRows, unsigned maxSpillCost);
     const void **_allocateRowTable(rowidx_t num, unsigned maxSpillCost);
@@ -273,18 +272,18 @@ class graph_decl CThorExpandingRowArray : public CSimpleInterface
 
 protected:
     CActivityBase &activity;
-    IThorRowInterfaces *rowIf;
-    IEngineRowAllocator *allocator;
-    IOutputRowSerializer *serializer;
-    IOutputRowDeserializer *deserializer;
-    roxiemem::IRowManager *rowManager;
-    const void **rows;
-    void **stableTable;
-    bool throwOnOom; // tested during array expansion (resize())
-    bool allowNulls;
-    StableSortFlag stableSort;
-    rowidx_t maxRows;  // Number of rows that can fit in the allocated memory.
-    rowidx_t numRows;  // High water mark of rows added
+    IThorRowInterfaces *rowIf = nullptr;
+    IEngineRowAllocator *allocator = nullptr;
+    IOutputRowSerializer *serializer = nullptr;
+    IOutputRowDeserializer *deserializer = nullptr;
+    roxiemem::IRowManager *rowManager = nullptr;
+    const void **rows = nullptr;
+    void **stableTable = nullptr;
+    bool throwOnOom = true; // tested during array expansion (resize())
+    EmptyRowSemantics emptyRowSemantics = ers_forbidden;
+    StableSortFlag stableSort = stableSort_none;
+    rowidx_t maxRows = 0;  // Number of rows that can fit in the allocated memory.
+    rowidx_t numRows = 0;  // High water mark of rows added
     unsigned defaultMaxSpillCost = roxiemem::SpillAllCost;
 
     const void **allocateRowTable(rowidx_t num);
@@ -295,12 +294,12 @@ protected:
     inline rowidx_t getRowsCapacity() const { return rows ? RoxieRowCapacity(rows) / sizeof(void *) : 0; }
 public:
     CThorExpandingRowArray(CActivityBase &activity);
-    CThorExpandingRowArray(CActivityBase &activity, IThorRowInterfaces *rowIf, bool allowNulls=false, StableSortFlag stableSort=stableSort_none, bool throwOnOom=true, rowidx_t initialSize=InitialSortElements);
+    CThorExpandingRowArray(CActivityBase &activity, IThorRowInterfaces *rowIf, EmptyRowSemantics emptyRowSemantics=ers_forbidden, StableSortFlag stableSort=stableSort_none, bool throwOnOom=true, rowidx_t initialSize=InitialSortElements);
     ~CThorExpandingRowArray();
     CActivityBase &queryActivity() { return activity; }
     // NB: throws error on OOM by default
-    void setup(IThorRowInterfaces *rowIf, bool allowNulls=false, StableSortFlag stableSort=stableSort_none, bool throwOnOom=true);
-    inline void setAllowNulls(bool b) { allowNulls = b; }
+    void setup(IThorRowInterfaces *rowIf, EmptyRowSemantics emptyRowSemantics=ers_forbidden, StableSortFlag stableSort=stableSort_none, bool throwOnOom=true);
+    inline void setEmptyRowSemantics(EmptyRowSemantics _emptyRowSemantics) { emptyRowSemantics = _emptyRowSemantics; }
     inline void setDefaultMaxSpillCost(unsigned _defaultMaxSpillCost) { defaultMaxSpillCost = _defaultMaxSpillCost; }
     inline unsigned queryDefaultMaxSpillCost() const { return defaultMaxSpillCost; }
     void clearRows();
@@ -320,7 +319,7 @@ public:
     }
     inline bool append(const void *row) // NB: takes ownership on success
     {
-        assertex(row || allowNulls);
+        assertex(row || (emptyRowSemantics != ers_forbidden));
         if (numRows >= maxRows)
         {
             if (!resize(numRows+1))
@@ -404,14 +403,13 @@ interface IWritePosCallback : extends IInterface
 
 class graph_decl CThorSpillableRowArray : private CThorExpandingRowArray, implements IThorArrayLock
 {
-    size32_t commitDelta;  // How many rows need to be written before they are added to the committed region?
-    rowidx_t firstRow; // Only rows firstRow..numRows are considered initialized.  Only read/write within cs.
-    rowidx_t commitRows;  // can only be updated by writing thread within a critical section
+    size32_t commitDelta = CommitStep;  // How many rows need to be written before they are added to the committed region?
+    rowidx_t firstRow = 0; // Only rows firstRow..numRows are considered initialized.  Only read/write within cs.
+    rowidx_t commitRows = 0;  // can only be updated by writing thread within a critical section
     mutable CriticalSection cs;
     ICopyArrayOf<IWritePosCallback> writeCallbacks;
     size32_t compBlkSz = 0; // means use default
 
-    void initCommon();
     bool _flush(bool force);
     void doFlush();
     inline bool needToMoveRows(bool force) { return (firstRow != 0 && (force || (firstRow >= commitRows/2))); }
@@ -419,18 +417,18 @@ class graph_decl CThorSpillableRowArray : private CThorExpandingRowArray, implem
 public:
 
     CThorSpillableRowArray(CActivityBase &activity);
-    CThorSpillableRowArray(CActivityBase &activity, IThorRowInterfaces *rowIf, bool allowNulls=false, StableSortFlag stableSort=stableSort_none, rowidx_t initialSize=InitialSortElements, size32_t commitDelta=CommitStep);
+    CThorSpillableRowArray(CActivityBase &activity, IThorRowInterfaces *rowIf, EmptyRowSemantics emptyRowSemantics=ers_forbidden, StableSortFlag stableSort=stableSort_none, rowidx_t initialSize=InitialSortElements, size32_t commitDelta=CommitStep);
     ~CThorSpillableRowArray();
     // NB: default throwOnOom to false
-    void setup(IThorRowInterfaces *rowIf, bool allowNulls=false, StableSortFlag stableSort=stableSort_none, bool throwOnOom=false)
+    void setup(IThorRowInterfaces *rowIf, EmptyRowSemantics emptyRowSemantics=ers_forbidden, StableSortFlag stableSort=stableSort_none, bool throwOnOom=false)
     {
-        CThorExpandingRowArray::setup(rowIf, allowNulls, stableSort, throwOnOom);
+        CThorExpandingRowArray::setup(rowIf, emptyRowSemantics, stableSort, throwOnOom);
     }
     void registerWriteCallback(IWritePosCallback &cb);
     void unregisterWriteCallback(IWritePosCallback &cb);
     void safeRegisterWriteCallback(IWritePosCallback &cb);
     void safeUnregisterWriteCallback(IWritePosCallback &cb);
-    inline void setAllowNulls(bool b) { CThorExpandingRowArray::setAllowNulls(b); }
+    inline void setEmptyRowSemantics(EmptyRowSemantics _emptyRowSemantics) { CThorExpandingRowArray::setEmptyRowSemantics(_emptyRowSemantics); }
     inline void setDefaultMaxSpillCost(unsigned defaultMaxSpillCost) { CThorExpandingRowArray::setDefaultMaxSpillCost(defaultMaxSpillCost); }
     inline void setCompBlockSize(size32_t sz) { compBlkSz = sz; }
     inline unsigned queryDefaultMaxSpillCost() const { return CThorExpandingRowArray::queryDefaultMaxSpillCost(); }
@@ -444,7 +442,7 @@ public:
     inline bool append(const void *row) __attribute__((warn_unused_result))
     {
         //GH->JCS Should this really be inline?
-        assertex(row || allowNulls);
+        assertex(row || (emptyRowSemantics != ers_forbidden));
         if (numRows >= maxRows)
         {
             if (!resize(numRows+1))
@@ -553,7 +551,7 @@ interface IThorRowLoader : extends IThorRowCollectorCommon
 
 interface IThorRowCollector : extends IThorRowCollectorCommon
 {
-    virtual void setPreserveGrouping(bool tf) = 0;
+    virtual void setEmptyRowSemantics(EmptyRowSemantics emptyRowSemantics) = 0;
     virtual IRowWriter *getWriter() = 0;
     virtual void reset() = 0;
     virtual IRowStream *getStream(bool shared=false, CThorExpandingRowArray *allMemRows=NULL) = 0;
@@ -564,7 +562,7 @@ interface IThorRowCollector : extends IThorRowCollectorCommon
 
 extern graph_decl IThorRowLoader *createThorRowLoader(CActivityBase &activity, IThorRowInterfaces *rowIf, ICompare *iCompare=NULL, StableSortFlag stableSort=stableSort_none, RowCollectorSpillFlags diskMemMix=rc_mixed, unsigned spillPriority=SPILL_PRIORITY_DEFAULT);
 extern graph_decl IThorRowLoader *createThorRowLoader(CActivityBase &activity, ICompare *iCompare=NULL, StableSortFlag stableSort=stableSort_none, RowCollectorSpillFlags diskMemMix=rc_mixed, unsigned spillPriority=SPILL_PRIORITY_DEFAULT);
-extern graph_decl IThorRowCollector *createThorRowCollector(CActivityBase &activity, IThorRowInterfaces *rowIf, ICompare *iCompare=NULL, StableSortFlag stableSort=stableSort_none, RowCollectorSpillFlags diskMemMix=rc_mixed, unsigned spillPriority=SPILL_PRIORITY_DEFAULT, bool preserveGrouping=false);
+extern graph_decl IThorRowCollector *createThorRowCollector(CActivityBase &activity, IThorRowInterfaces *rowIf, ICompare *iCompare=NULL, StableSortFlag stableSort=stableSort_none, RowCollectorSpillFlags diskMemMix=rc_mixed, unsigned spillPriority=SPILL_PRIORITY_DEFAULT, EmptyRowSemantics emptyRowSemantics=ers_forbidden);
 
 
 

tools/esdlcmd/esdlcmd_common.hpp

@@ -74,6 +74,7 @@ typedef IEsdlCommand *(*EsdlCommandFactory)(const char *cmdname);
 #define ESDLOPT_NUMBER                  "-n"
 #define ESDLOPT_NO_COLLAPSE             "--show-inheritance"
 #define ESDLOPT_NO_ARRAYOF              "--no-arrayof"
+#define ESDLOPT_OUTPUT_CATEGORIES       "--output-categories"
 
 #define ESDLOPT_WSDL_ADDRESS            "--wsdl-address"
 

tools/esdlcmd/esdlcmd_monitor.cpp

@@ -28,6 +28,62 @@
 #include "esdl-publish.cpp"
 #include "xsdparser.hpp"
 
+bool isUTF16Bom(const char *check)
+{
+    return (check[0]=='\xfe' && check[1]=='\xff');
+}
+
+bool isUTF8Bom(const char *check)
+{
+    return (check[0]=='\xef' && check[1]=='\xbb' && check[2]=='\xbf');
+}
+
+static const char *skipBOM(const char *content)
+{
+    if (isUTF8Bom(content))
+        return content+3;
+
+    if (isUTF16Bom(content))
+        return content+2;
+
+    return content;
+}
+
+StringBuffer &appendEscapedEclString(StringBuffer &s, const char *content)
+{
+    if (!content || !*content)
+        return s;
+
+    content=skipBOM(content);
+
+    for (;*content;content++)
+    {
+        switch (*content)
+        {
+        case '\'':
+            s.append("\\'");
+            break;
+        case '\t':
+            s.append("\\t");
+            break;
+        case '\n':
+            s.append("\\n");
+            break;
+        case '\r':
+            s.append("\\r");
+            break;
+        case '\\':
+            s.append("\\\\");
+            break;
+        default:
+            s.append(*content);
+            break;
+        }
+    }
+    return s;
+}
+
+
 StringBuffer &getEsdlCmdComponentFilesPath(StringBuffer & path)
 {
     if (getComponentFilesRelPathFromBin(path))
@@ -275,6 +331,8 @@ public:
             {
                 if (iter.matchOption(optXsltPath, ESDLOPT_XSLT_PATH))
                     continue;
+                if (iter.matchFlag(optOutputCategoryList, ESDLOPT_OUTPUT_CATEGORIES))
+                    continue;
                 if (EsdlConvertCmd::parseCommandLineOption(iter))
                     continue;
                 if (EsdlConvertCmd::matchCommandLineOption(iter, true)!=EsdlCmdOptionMatch)
@@ -971,11 +1029,23 @@ public:
 
         StringBuffer ecl;
 
+        StringBuffer escapedTemplate;
+        appendEscapedEclString(escapedTemplate, diffTemplateContent);
+
+
+
+        ecl.appendf("STRING monitoringTemplate :='%s';\nBOOLEAN IncludeTemplate := false : STORED('IncludeTemplate');\nIF (IncludeTemplate, OUTPUT(monitoringTemplate, NAMED('MonitoringTemplate')));\nOUTPUT(HASHMD5(monitoringTemplate), NAMED('Hash'));\n", escapedTemplate.str());
+        filename.setf("Monitor_ActiveTemplate_%s.ecl", optMethod.str());
+        saveAsFile(".", filename, ecl);
+
+        if (optOutputCategoryList)
+            xform->setParameter("listCategories", "true()");
+
         xform->setParameter("diffmode", "'Monitor'");
         xform->setParameter("diffaction", "'Create'");
 
         xform->setParameter("platform", "'roxie'");
-        xform->transform(ecl);
+        xform->transform(ecl.clear());
         filename.setf("MonitorRoxie_create_%s.ecl", optMethod.str());
         saveAsFile(".", filename, ecl);
 
@@ -1036,6 +1106,7 @@ public:
     StringAttr optXsltPath;
     StringAttr optMethod;
     unsigned optFlags;
+    bool optOutputCategoryList=false;  //hidden option, do not document
 };